def test_train_eval_model(self):
    'Tests that a simple model trains and exported models are valid.'
    gin.bind_parameter('tf.estimator.RunConfig.save_checkpoints_steps', 100)
    model_dir = self.create_tempdir().full_path
    mock_t2r_model = mocks.MockT2RModel(preprocessor_cls=noop_preprocessor.NoOpPreprocessor)
    mock_input_generator_train = mocks.MockInputGenerator(batch_size=_BATCH_SIZE)
    mock_input_generator_eval = mocks.MockInputGenerator(batch_size=1)
    fake_hook_builder = FakeHookBuilder()
    train_eval.train_eval_model(t2r_model=mock_t2r_model, input_generator_train=mock_input_generator_train, input_generator_eval=mock_input_generator_eval, max_train_steps=_MAX_TRAIN_STEPS, model_dir=model_dir, train_hook_builders=[fake_hook_builder], eval_hook_builders=[fake_hook_builder], eval_steps=_EVAL_STEPS, eval_throttle_secs=_EVAL_THROTTLE_SECS, create_exporters_fn=train_eval.create_default_exporters)
    self.assertTrue(fake_hook_builder.hook_mock.begin.called)
    best_exporter_numpy_path = os.path.join(model_dir, 'export', 'best_exporter_numpy', '*')
    numpy_model_paths = sorted(tf.io.gfile.glob(best_exporter_numpy_path))
    self.assertGreater(len(numpy_model_paths), 0)
    self.assertLessEqual(len(numpy_model_paths), 5)
    best_exporter_tf_example_path = os.path.join(model_dir, 'export', 'best_exporter_tf_example', '*')
    tf_example_model_paths = sorted(tf.io.gfile.glob(best_exporter_tf_example_path))
    self.assertGreater(len(tf_example_model_paths), 0)
    self.assertLessEqual(len(tf_example_model_paths), 5)
    estimator_predict = tf.estimator.Estimator(model_fn=mock_t2r_model.model_fn, config=tf.estimator.RunConfig(model_dir=model_dir))
    prediction_ref = estimator_predict.predict(input_fn=mock_input_generator_eval.create_dataset_input_fn(mode=tf.estimator.ModeKeys.EVAL))
    numpy_predictor_fn = contrib_predictor.from_saved_model(numpy_model_paths[(- 1)])
    (features, labels) = mock_input_generator_eval.create_numpy_data()
    ref_error = self._compute_total_loss(labels, [val['logit'].flatten() for val in prediction_ref])
    numpy_predictions = []
    for (feature, label) in zip(features, labels):
        predicted = numpy_predictor_fn({'x': feature.reshape(1, (- 1))})['logit'].flatten()
        numpy_predictions.append(predicted)
        if (label > 0):
            self.assertGreater(predicted[0], 0)
        else:
            self.assertLess(predicted[0], 0)
    numpy_error = self._compute_total_loss(labels, numpy_predictions)
    tf_example_predictor_fn = contrib_predictor.from_saved_model(tf_example_model_paths[(- 1)])
    tf_example_predictions = []
    for (feature, label) in zip(features, labels):
        example = tf.train.Example()
        example.features.feature['measured_position'].float_list.value.extend(feature)
        feed_dict = {'input_example_tensor': np.array(example.SerializeToString()).reshape(1)}
        predicted = tf_example_predictor_fn(feed_dict)['logit'].flatten()
        tf_example_predictions.append(predicted)
        if (label > 0):
            self.assertGreater(predicted[0], 0)
        else:
            self.assertLess(predicted[0], 0)
    tf_example_error = self._compute_total_loss(labels, tf_example_predictions)
    np.testing.assert_almost_equal(tf_example_error, numpy_error)
    np.testing.assert_almost_equal(ref_error, tf_example_error, decimal=3)
Tests that a simple model trains and exported models are valid.
def test_train_eval_model(self):
    
    gin.bind_parameter('tf.estimator.RunConfig.save_checkpoints_steps', 100)
    model_dir = self.create_tempdir().full_path
    mock_t2r_model = mocks.MockT2RModel(preprocessor_cls=noop_preprocessor.NoOpPreprocessor)
    mock_input_generator_train = mocks.MockInputGenerator(batch_size=_BATCH_SIZE)
    mock_input_generator_eval = mocks.MockInputGenerator(batch_size=1)
    fake_hook_builder = FakeHookBuilder()
    train_eval.train_eval_model(t2r_model=mock_t2r_model, input_generator_train=mock_input_generator_train, input_generator_eval=mock_input_generator_eval, max_train_steps=_MAX_TRAIN_STEPS, model_dir=model_dir, train_hook_builders=[fake_hook_builder], eval_hook_builders=[fake_hook_builder], eval_steps=_EVAL_STEPS, eval_throttle_secs=_EVAL_THROTTLE_SECS, create_exporters_fn=train_eval.create_default_exporters)
    self.assertTrue(fake_hook_builder.hook_mock.begin.called)
    best_exporter_numpy_path = os.path.join(model_dir, 'export', 'best_exporter_numpy', '*')
    numpy_model_paths = sorted(tf.io.gfile.glob(best_exporter_numpy_path))
    self.assertGreater(len(numpy_model_paths), 0)
    self.assertLessEqual(len(numpy_model_paths), 5)
    best_exporter_tf_example_path = os.path.join(model_dir, 'export', 'best_exporter_tf_example', '*')
    tf_example_model_paths = sorted(tf.io.gfile.glob(best_exporter_tf_example_path))
    self.assertGreater(len(tf_example_model_paths), 0)
    self.assertLessEqual(len(tf_example_model_paths), 5)
    estimator_predict = tf.estimator.Estimator(model_fn=mock_t2r_model.model_fn, config=tf.estimator.RunConfig(model_dir=model_dir))
    prediction_ref = estimator_predict.predict(input_fn=mock_input_generator_eval.create_dataset_input_fn(mode=tf.estimator.ModeKeys.EVAL))
    numpy_predictor_fn = contrib_predictor.from_saved_model(numpy_model_paths[(- 1)])
    (features, labels) = mock_input_generator_eval.create_numpy_data()
    ref_error = self._compute_total_loss(labels, [val['logit'].flatten() for val in prediction_ref])
    numpy_predictions = []
    for (feature, label) in zip(features, labels):
        predicted = numpy_predictor_fn({'x': feature.reshape(1, (- 1))})['logit'].flatten()
        numpy_predictions.append(predicted)
        if (label > 0):
            self.assertGreater(predicted[0], 0)
        else:
            self.assertLess(predicted[0], 0)
    numpy_error = self._compute_total_loss(labels, numpy_predictions)
    tf_example_predictor_fn = contrib_predictor.from_saved_model(tf_example_model_paths[(- 1)])
    tf_example_predictions = []
    for (feature, label) in zip(features, labels):
        example = tf.train.Example()
        example.features.feature['measured_position'].float_list.value.extend(feature)
        feed_dict = {'input_example_tensor': np.array(example.SerializeToString()).reshape(1)}
        predicted = tf_example_predictor_fn(feed_dict)['logit'].flatten()
        tf_example_predictions.append(predicted)
        if (label > 0):
            self.assertGreater(predicted[0], 0)
        else:
            self.assertLess(predicted[0], 0)
    tf_example_error = self._compute_total_loss(labels, tf_example_predictions)
    np.testing.assert_almost_equal(tf_example_error, numpy_error)
    np.testing.assert_almost_equal(ref_error, tf_example_error, decimal=3)