code: # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """TensorFlow estimators for Linear and DNN joined training models."""
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import math
  
  import six
  
  from tensorflow.python.estimator import estimator
  from tensorflow.python.estimator.canned import dnn
  from tensorflow.python.estimator.canned import head as head_lib
  from tensorflow.python.estimator.canned import linear
  from tensorflow.python.estimator.canned import optimizers
  from tensorflow.python.feature_column import feature_column_v2
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import control_flow_ops
  from tensorflow.python.ops import nn
  from tensorflow.python.ops import partitioned_variables
  from tensorflow.python.ops import state_ops
  from tensorflow.python.ops import variable_scope
  from tensorflow.python.ops.losses import losses
  from tensorflow.python.summary import summary
  from tensorflow.python.training import sync_replicas_optimizer
  from tensorflow.python.training import training_util
  from tensorflow.python.util.tf_export import estimator_export
  
  # The default learning rates are a historical artifact of the initial
  # implementation.
  _DNN_LEARNING_RATE = 0.001
  _LINEAR_LEARNING_RATE = 0.005
  
  
  def _check_no_sync_replicas_optimizer(optimizer):
    if isinstance(optimizer, sync_replicas_optimizer.SyncReplicasOptimizer):
      raise ValueError(
          'SyncReplicasOptimizer does not support multi optimizers case. '
          'Therefore, it is not supported in DNNLinearCombined model. '
          'If you want to use this optimizer, please use either DNN or Linear '
          'model.')
  
  
  def _linear_learning_rate(num_linear_feature_columns):
    """Returns the default learning rate of the linear model.
  
    The calculation is a historical artifact of this initial implementation, but
    has proven a reasonable choice.
  
    Args:
      num_linear_feature_columns: The number of feature columns of the linear
        model.
  
    Returns:
      A float.
    """
    default_learning_rate = 1. / math.sqrt(num_linear_feature_columns)
    return min(_LINEAR_LEARNING_RATE, default_learning_rate)
  
  
  def _add_layer_summary(value, tag):
    summary.scalar('%s/fraction_of_zero_values' % tag, nn.zero_fraction(value))
    summary.histogram('%s/activation' % tag, value)
  
  
  def _dnn_linear_combined_model_fn(features,
                                    labels,
                                    mode,
                                    head,
                                    linear_feature_columns=None,
                                    linear_optimizer='Ftrl',
                                    dnn_feature_columns=None,
                                    dnn_optimizer='Adagrad',
                                    dnn_hidden_units=None,
                                    dnn_activation_fn=nn.relu,
                                    dnn_dropout=None,
                                    input_layer_partitioner=None,
                                    config=None,
                                    batch_norm=False,
                                    linear_sparse_combiner='sum'):
    """Deep Neural Net and Linear combined model_fn.
  
    Args:
      features: dict of `Tensor`.
      labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype
        `int32` or `int64` in the range `[0, n_classes)`.
      mode: Defines whether this is training, evaluation or prediction.
        See `ModeKeys`.
      head: A `Head` instance.
      linear_feature_columns: An iterable containing all the feature columns used
        by the Linear model.
      linear_optimizer: string, `Optimizer` object, or callable that defines the
        optimizer to use for training the Linear model. Defaults to the Ftrl
        optimizer.
      dnn_feature_columns: An iterable containing all the feature columns used by
        the DNN model.
      dnn_optimizer: string, `Optimizer` object, or callable that defines the
        optimizer to use for training the DNN model. Defaults to the Adagrad
        optimizer.
      dnn_hidden_units: List of hidden units per DNN layer.
      dnn_activation_fn: Activation function applied to each DNN layer. If `None`,
        will use `tf.nn.relu`.
      dnn_dropout: When not `None`, the probability we will drop out a given DNN
        coordinate.
      input_layer_partitioner: Partitioner for input layer.
      config: `RunConfig` object to configure the runtime settings.
      batch_norm: Whether to use batch normalization after each hidden layer.
      linear_sparse_combiner: A string specifying how to reduce the linear model
        if a categorical column is multivalent.  One of "mean", "sqrtn", and
        "sum".
    Returns:
      An `EstimatorSpec` instance.
  
    Raises:
      ValueError: If both `linear_feature_columns` and `dnn_features_columns`
        are empty at the same time, or `input_layer_partitioner` is missing,
        or features has the wrong type.
    """
    if not isinstance(features, dict):
      raise ValueError('features should be a dictionary of `Tensor`s. '
                       'Given type: {}'.format(type(features)))
    if not linear_feature_columns and not dnn_feature_columns:
      raise ValueError(
          'Either linear_feature_columns or dnn_feature_columns must be defined.')
  
    num_ps_replicas = config.num_ps_replicas if config else 0
    input_layer_partitioner = input_layer_partitioner or (
        partitioned_variables.min_max_variable_partitioner(
            max_partitions=num_ps_replicas,
            min_slice_size=64 << 20))
  
    shared_state_manager = feature_column_v2.maybe_create_shared_state_manager(
        list(linear_feature_columns) + list(dnn_feature_columns))
  
    # Build DNN Logits.
    dnn_parent_scope = 'dnn'
  
    if not dnn_feature_columns:
      dnn_logits = None
    else:
      dnn_optimizer = optimizers.get_optimizer_instance(
          dnn_optimizer, learning_rate=_DNN_LEARNING_RATE)
      _check_no_sync_replicas_optimizer(dnn_optimizer)
      if not dnn_hidden_units:
        raise ValueError(
            'dnn_hidden_units must be defined when dnn_feature_columns is '
            'specified.')
      dnn_partitioner = (
          partitioned_variables.min_max_variable_partitioner(
              max_partitions=num_ps_replicas))
      with variable_scope.variable_scope(
          dnn_parent_scope,
          values=tuple(six.itervalues(features)),
          partitioner=dnn_partitioner) as scope:
        dnn_absolute_scope = scope.name
        dnn_logit_fn = dnn._dnn_logit_fn_builder(  # pylint: disable=protected-access
            units=head.logits_dimension,
            hidden_units=dnn_hidden_units,
            feature_columns=dnn_feature_columns,
            activation_fn=dnn_activation_fn,
            dropout=dnn_dropout,
            batch_norm=batch_norm,
            input_layer_partitioner=input_layer_partitioner,
            shared_state_manager=shared_state_manager)
        dnn_logits = dnn_logit_fn(features=features, mode=mode)
  
    linear_parent_scope = 'linear'
  
    if not linear_feature_columns:
      linear_logits = None
    else:
      linear_optimizer = optimizers.get_optimizer_instance(
          linear_optimizer,
          learning_rate=_linear_learning_rate(len(linear_feature_columns)))
      _check_no_sync_replicas_optimizer(linear_optimizer)
      with variable_scope.variable_scope(
          linear_parent_scope,
          values=tuple(six.itervalues(features)),
          partitioner=input_layer_partitioner) as scope:
        linear_absolute_scope = scope.name
        logit_fn = linear._linear_logit_fn_builder(  # pylint: disable=protected-access
            units=head.logits_dimension,
            feature_columns=linear_feature_columns,
            sparse_combiner=linear_sparse_combiner)
        linear_logits = logit_fn(features=features)
        _add_layer_summary(linear_logits, scope.name)
  
    # Combine logits and build full model.
    if dnn_logits is not None and linear_logits is not None:
      logits = dnn_logits + linear_logits
    elif dnn_logits is not None:
      logits = dnn_logits
    else:
      logits = linear_logits
  
    def _train_op_fn(loss):
      """Returns the op to optimize the loss."""
      train_ops = []
      global_step = training_util.get_global_step()
      if dnn_logits is not None:
        train_ops.append(
            dnn_optimizer.minimize(
                loss,
                var_list=ops.get_collection(
                    ops.GraphKeys.TRAINABLE_VARIABLES,
                    scope=dnn_absolute_scope)))
      if linear_logits is not None:
        train_ops.append(
            linear_optimizer.minimize(
                loss,
                var_list=ops.get_collection(
                    ops.GraphKeys.TRAINABLE_VARIABLES,
                    scope=linear_absolute_scope)))
  
      train_op = control_flow_ops.group(*train_ops)
      with ops.control_dependencies([train_op]):
        return state_ops.assign_add(global_step, 1).op
  
    return head.create_estimator_spec(
        features=features,
        mode=mode,
        labels=labels,
        train_op_fn=_train_op_fn,
        logits=logits)
  
  
  @estimator_export('estimator.DNNLinearCombinedClassifier')
  class DNNLinearCombinedClassifier(estimator.Estimator):
    """An estimator for TensorFlow Linear and DNN joined classification models.
  
    Note: This estimator is also known as wide-n-deep.
  
    Example:
  
    ```python
    numeric_feature = numeric_column(...)
    categorical_column_a = categorical_column_with_hash_bucket(...)
    categorical_column_b = categorical_column_with_hash_bucket(...)
  
    categorical_feature_a_x_categorical_feature_b = crossed_column(...)
    categorical_feature_a_emb = embedding_column(
        categorical_column=categorical_feature_a, ...)
    categorical_feature_b_emb = embedding_column(
        categorical_id_column=categorical_feature_b, ...)
  
    estimator = DNNLinearCombinedClassifier(
        # wide settings
        linear_feature_columns=[categorical_feature_a_x_categorical_feature_b],
        linear_optimizer=tf.train.FtrlOptimizer(...),
        # deep settings
        dnn_feature_columns=[
            categorical_feature_a_emb, categorical_feature_b_emb,
            numeric_feature],
        dnn_hidden_units=[1000, 500, 100],
        dnn_optimizer=tf.train.ProximalAdagradOptimizer(...),
        # warm-start settings
        warm_start_from="/path/to/checkpoint/dir")
  
    # To apply L1 and L2 regularization, you can set dnn_optimizer to:
    tf.train.ProximalAdagradOptimizer(
        learning_rate=0.1,
        l1_regularization_strength=0.001,
        l2_regularization_strength=0.001)
    # To apply learning rate decay, you can set dnn_optimizer to a callable:
    lambda: tf.AdamOptimizer(
        learning_rate=tf.exponential_decay(
            learning_rate=0.1,
            global_step=tf.get_global_step(),
            decay_steps=10000,
            decay_rate=0.96)
    # It is the same for linear_optimizer.
  
    # Input builders
    def input_fn_train: # returns x, y
      pass
    estimator.train(input_fn=input_fn_train, steps=100)
  
    def input_fn_eval: # returns x, y
      pass
    metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10)
    def input_fn_predict: # returns x, None
      pass
    predictions = estimator.predict(input_fn=input_fn_predict)
    ```
  
    Input of `train` and `evaluate` should have following features,
    otherwise there will be a `KeyError`:
  
    * for each `column` in `dnn_feature_columns` + `linear_feature_columns`:
      - if `column` is a `_CategoricalColumn`, a feature with `key=column.name`
        whose `value` is a `SparseTensor`.
      - if `column` is a `_WeightedCategoricalColumn`, two features: the first
        with `key` the id column name, the second with `key` the weight column
        name. Both features' `value` must be a `SparseTensor`.
      - if `column` is a `_DenseColumn`, a feature with `key=column.name`
        whose `value` is a `Tensor`.
  
    Loss is calculated by using softmax cross entropy.
  
    @compatibility(eager)
    Estimators can be used while eager execution is enabled. Note that `input_fn`
    and all hooks are executed inside a graph context, so they have to be written
    to be compatible with graph mode. Note that `input_fn` code using `tf.data`
    generally works in both graph and eager modes.
    @end_compatibility
    """
  
    def __init__(self,
                 model_dir=None,
                 linear_feature_columns=None,
                 linear_optimizer='Ftrl',
                 dnn_feature_columns=None,
                 dnn_optimizer='Adagrad',
                 dnn_hidden_units=None,
                 dnn_activation_fn=nn.relu,
                 dnn_dropout=None,
                 n_classes=2,
                 weight_column=None,
                 label_vocabulary=None,
                 input_layer_partitioner=None,
                 config=None,
                 warm_start_from=None,
                 loss_reduction=losses.Reduction.SUM,
                 batch_norm=False,
                 linear_sparse_combiner='sum'):
      """Initializes a DNNLinearCombinedClassifier instance.
  
      Args:
        model_dir: Directory to save model parameters, graph and etc. This can
          also be used to load checkpoints from the directory into a estimator
          to continue training a previously saved model.
        linear_feature_columns: An iterable containing all the feature columns
          used by linear part of the model. All items in the set must be
          instances of classes derived from `FeatureColumn`.
        linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
          the linear part of the model. Can also be a string (one of 'Adagrad',
          'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to FTRL
          optimizer.
        dnn_feature_columns: An iterable containing all the feature columns used
          by deep part of the model. All items in the set must be instances of
          classes derived from `FeatureColumn`.
        dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to
          the deep part of the model. Can also be a string (one of 'Adagrad',
          'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to Adagrad
          optimizer.
        dnn_hidden_units: List of hidden units per layer. All layers are fully
          connected.
        dnn_activation_fn: Activation function applied to each layer. If None,
          will use `tf.nn.relu`.
        dnn_dropout: When not None, the probability we will drop out
          a given coordinate.
        n_classes: Number of label classes. Defaults to 2, namely binary
          classification. Must be > 1.
        weight_column: A string or a `_NumericColumn` created by
          `tf.feature_column.numeric_column` defining feature column representing
          weights. It is used to down weight or boost examples during training. It
          will be multiplied by the loss of the example. If it is a string, it is
          used as a key to fetch weight tensor from the `features`. If it is a
          `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
          then weight_column.normalizer_fn is applied on it to get weight tensor.
        label_vocabulary: A list of strings represents possible label values. If
          given, labels must be string type and have any value in
          `label_vocabulary`. If it is not given, that means labels are
          already encoded as integer or float within [0, 1] for `n_classes=2` and
          encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 .
          Also there will be errors if vocabulary is not provided and labels are
          string.
        input_layer_partitioner: Partitioner for input layer. Defaults to
          `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
        config: RunConfig object to configure the runtime settings.
        warm_start_from: A string filepath to a checkpoint to warm-start from, or
          a `WarmStartSettings` object to fully configure warm-starting.  If the
          string filepath is provided instead of a `WarmStartSettings`, then all
          weights are warm-started, and it is assumed that vocabularies and Tensor
          names are unchanged.
        loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
          to reduce training loss over batch. Defaults to `SUM`.
        batch_norm: Whether to use batch normalization after each hidden layer.
        linear_sparse_combiner: A string specifying how to reduce the linear model
          if a categorical column is multivalent.  One of "mean", "sqrtn", and
          "sum" -- these are effectively different ways to do example-level
          normalization, which can be useful for bag-of-words features.  For more
          details, see `tf.feature_column.linear_model`.
  
      Raises:
        ValueError: If both linear_feature_columns and dnn_features_columns are
          empty at the same time.
      """
      linear_feature_columns = linear_feature_columns or []
      dnn_feature_columns = dnn_feature_columns or []
      self._feature_columns = (
          list(linear_feature_columns) + list(dnn_feature_columns))
      if not self._feature_columns:
        raise ValueError('Either linear_feature_columns or dnn_feature_columns '
                         'must be defined.')
      if n_classes == 2:
        head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(  # pylint: disable=protected-access
            weight_column=weight_column,
            label_vocabulary=label_vocabulary,
            loss_reduction=loss_reduction)
      else:
        head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(  # pylint: disable=protected-access
            n_classes,
            weight_column=weight_column,
            label_vocabulary=label_vocabulary,
            loss_reduction=loss_reduction)
  
      def _model_fn(features, labels, mode, config):
        """Call the _dnn_linear_combined_model_fn."""
        return _dnn_linear_combined_model_fn(
            features=features,
            labels=labels,
            mode=mode,
            head=head,
            linear_feature_columns=linear_feature_columns,
            linear_optimizer=linear_optimizer,
            dnn_feature_columns=dnn_feature_columns,
            dnn_optimizer=dnn_optimizer,
            dnn_hidden_units=dnn_hidden_units,
            dnn_activation_fn=dnn_activation_fn,
            dnn_dropout=dnn_dropout,
            input_layer_partitioner=input_layer_partitioner,
            config=config,
            batch_norm=batch_norm,
            linear_sparse_combiner=linear_sparse_combiner)
  
      super(DNNLinearCombinedClassifier, self).__init__(
          model_fn=_model_fn, model_dir=model_dir, config=config,
          warm_start_from=warm_start_from)
  
  
  @estimator_export('estimator.DNNLinearCombinedRegressor')
  class DNNLinearCombinedRegressor(estimator.Estimator):
    """An estimator for TensorFlow Linear and DNN joined models for regression.
  
    Note: This estimator is also known as wide-n-deep.
  
    Example:
  
    ```python
    numeric_feature = numeric_column(...)
    categorical_column_a = categorical_column_with_hash_bucket(...)
    categorical_column_b = categorical_column_with_hash_bucket(...)
  
    categorical_feature_a_x_categorical_feature_b = crossed_column(...)
    categorical_feature_a_emb = embedding_column(
        categorical_column=categorical_feature_a, ...)
    categorical_feature_b_emb = embedding_column(
        categorical_column=categorical_feature_b, ...)
  
    estimator = DNNLinearCombinedRegressor(
        # wide settings
        linear_feature_columns=[categorical_feature_a_x_categorical_feature_b],
        linear_optimizer=tf.train.FtrlOptimizer(...),
        # deep settings
        dnn_feature_columns=[
            categorical_feature_a_emb, categorical_feature_b_emb,
            numeric_feature],
        dnn_hidden_units=[1000, 500, 100],
        dnn_optimizer=tf.train.ProximalAdagradOptimizer(...),
        # warm-start settings
        warm_start_from="/path/to/checkpoint/dir")
  
    # To apply L1 and L2 regularization, you can set dnn_optimizer to:
    tf.train.ProximalAdagradOptimizer(
        learning_rate=0.1,
        l1_regularization_strength=0.001,
        l2_regularization_strength=0.001)
    # To apply learning rate decay, you can set dnn_optimizer to a callable:
    lambda: tf.AdamOptimizer(
        learning_rate=tf.exponential_decay(
            learning_rate=0.1,
            global_step=tf.get_global_step(),
            decay_steps=10000,
            decay_rate=0.96)
    # It is the same for linear_optimizer.
  
    # Input builders
    def input_fn_train: # returns x, y
      pass
    estimator.train(input_fn=input_fn_train, steps=100)
  
    def input_fn_eval: # returns x, y
      pass
    metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10)
    def input_fn_predict: # returns x, None
      pass
    predictions = estimator.predict(input_fn=input_fn_predict)
    ```
  
    Input of `train` and `evaluate` should have following features,
    otherwise there will be a `KeyError`:
  
    * for each `column` in `dnn_feature_columns` + `linear_feature_columns`:
      - if `column` is a `_CategoricalColumn`, a feature with `key=column.name`
        whose `value` is a `SparseTensor`.
      - if `column` is a `_WeightedCategoricalColumn`, two features: the first
        with `key` the id column name, the second with `key` the weight column
        name. Both features' `value` must be a `SparseTensor`.
      - if `column` is a `_DenseColumn`, a feature with `key=column.name`
        whose `value` is a `Tensor`.
  
    Loss is calculated by using mean squared error.
  
    @compatibility(eager)
    Estimators can be used while eager execution is enabled. Note that `input_fn`
    and all hooks are executed inside a graph context, so they have to be written
    to be compatible with graph mode. Note that `input_fn` code using `tf.data`
    generally works in both graph and eager modes.
    @end_compatibility
    """
  
    def __init__(self,
                 model_dir=None,
                 linear_feature_columns=None,
                 linear_optimizer='Ftrl',
                 dnn_feature_columns=None,
                 dnn_optimizer='Adagrad',
                 dnn_hidden_units=None,
                 dnn_activation_fn=nn.relu,
                 dnn_dropout=None,
                 label_dimension=1,
                 weight_column=None,
                 input_layer_partitioner=None,
                 config=None,
                 warm_start_from=None,
                 loss_reduction=losses.Reduction.SUM,
                 batch_norm=False,
                 linear_sparse_combiner='sum'):
      """Initializes a DNNLinearCombinedRegressor instance.
  
      Args:
        model_dir: Directory to save model parameters, graph and etc. This can
          also be used to load checkpoints from the directory into a estimator
          to continue training a previously saved model.
        linear_feature_columns: An iterable containing all the feature columns
          used by linear part of the model. All items in the set must be
          instances of classes derived from `FeatureColumn`.
        linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
          the linear part of the model. Can also be a string (one of 'Adagrad',
          'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to FTRL
          optimizer.
        dnn_feature_columns: An iterable containing all the feature columns used
          by deep part of the model. All items in the set must be instances of
          classes derived from `FeatureColumn`.
        dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to
          the deep part of the model. Can also be a string (one of 'Adagrad',
          'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to Adagrad
          optimizer.
        dnn_hidden_units: List of hidden units per layer. All layers are fully
          connected.
        dnn_activation_fn: Activation function applied to each layer. If None,
          will use `tf.nn.relu`.
        dnn_dropout: When not None, the probability we will drop out
          a given coordinate.
        label_dimension: Number of regression targets per example. This is the
          size of the last dimension of the labels and logits `Tensor` objects
          (typically, these have shape `[batch_size, label_dimension]`).
        weight_column: A string or a `_NumericColumn` created by
          `tf.feature_column.numeric_column` defining feature column representing
          weights. It is used to down weight or boost examples during training. It
          will be multiplied by the loss of the example. If it is a string, it is
          used as a key to fetch weight tensor from the `features`. If it is a
          `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
          then weight_column.normalizer_fn is applied on it to get weight tensor.
        input_layer_partitioner: Partitioner for input layer. Defaults to
          `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
        config: RunConfig object to configure the runtime settings.
        warm_start_from: A string filepath to a checkpoint to warm-start from, or
          a `WarmStartSettings` object to fully configure warm-starting.  If the
          string filepath is provided instead of a `WarmStartSettings`, then all
          weights are warm-started, and it is assumed that vocabularies and Tensor
          names are unchanged.
        loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
          to reduce training loss over batch. Defaults to `SUM`.
        batch_norm: Whether to use batch normalization after each hidden layer.
        linear_sparse_combiner: A string specifying how to reduce the linear model
          if a categorical column is multivalent.  One of "mean", "sqrtn", and
          "sum" -- these are effectively different ways to do example-level
          normalization, which can be useful for bag-of-words features.  For more
          details, see `tf.feature_column.linear_model`.
  
      Raises:
        ValueError: If both linear_feature_columns and dnn_features_columns are
          empty at the same time.
      """
      linear_feature_columns = linear_feature_columns or []
      dnn_feature_columns = dnn_feature_columns or []
      self._feature_columns = (
          list(linear_feature_columns) + list(dnn_feature_columns))
      if not self._feature_columns:
        raise ValueError('Either linear_feature_columns or dnn_feature_columns '
                         'must be defined.')
  
      def _model_fn(features, labels, mode, config):
        """Call the _dnn_linear_combined_model_fn."""
        return _dnn_linear_combined_model_fn(
            features=features,
            labels=labels,
            mode=mode,
            head=head_lib._regression_head(  # pylint: disable=protected-access
                label_dimension=label_dimension, weight_column=weight_column,
                loss_reduction=loss_reduction),
            linear_feature_columns=linear_feature_columns,
            linear_optimizer=linear_optimizer,
            dnn_feature_columns=dnn_feature_columns,
            dnn_optimizer=dnn_optimizer,
            dnn_hidden_units=dnn_hidden_units,
            dnn_activation_fn=dnn_activation_fn,
            dnn_dropout=dnn_dropout,
            input_layer_partitioner=input_layer_partitioner,
            config=config,
            batch_norm=batch_norm,
            linear_sparse_combiner=linear_sparse_combiner)
  
      super(DNNLinearCombinedRegressor, self).__init__(
          model_fn=_model_fn, model_dir=model_dir, config=config,
          warm_start_from=warm_start_from)
   # Copyright 2018 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Tests for manip_ops."""
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import numpy as np
  
  from tensorflow.python.framework import constant_op
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import errors_impl
  from tensorflow.python.framework import test_util
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import gradient_checker
  from tensorflow.python.ops import manip_ops
  from tensorflow.python.platform import test as test_lib
  
  # pylint: disable=g-import-not-at-top
  try:
    from distutils.version import StrictVersion as Version
    # numpy.roll for multiple shifts was introduced in numpy version 1.12.0
    NP_ROLL_CAN_MULTISHIFT = Version(np.version.version) >= Version("1.12.0")
  except ImportError:
    NP_ROLL_CAN_MULTISHIFT = False
  # pylint: enable=g-import-not-at-top
  
  
  class RollTest(test_util.TensorFlowTestCase):
  
    def _testRoll(self, np_input, shift, axis):
      expected_roll = np.roll(np_input, shift, axis)
      with self.cached_session():
        roll = manip_ops.roll(np_input, shift, axis)
        self.assertAllEqual(roll.eval(), expected_roll)
  
    def _testGradient(self, np_input, shift, axis):
      with self.cached_session():
        inx = constant_op.constant(np_input.tolist())
        xs = list(np_input.shape)
        y = manip_ops.roll(inx, shift, axis)
        # Expected y's shape to be the same
        ys = xs
        jacob_t, jacob_n = gradient_checker.compute_gradient(
            inx, xs, y, ys, x_init_value=np_input)
        self.assertAllClose(jacob_t, jacob_n, rtol=1e-5, atol=1e-5)
  
    def _testAll(self, np_input, shift, axis):
      self._testRoll(np_input, shift, axis)
      if np_input.dtype == np.float32:
        self._testGradient(np_input, shift, axis)
  
    def testIntTypes(self):
      for t in [np.int32, np.int64]:
        self._testAll(np.random.randint(-100, 100, (5)).astype(t), 3, 0)
        if NP_ROLL_CAN_MULTISHIFT:
          self._testAll(
              np.random.randint(-100, 100, (4, 4, 3)).astype(t), [1, -2, 3],
              [0, 1, 2])
          self._testAll(
              np.random.randint(-100, 100, (4, 2, 1, 3)).astype(t), [0, 1, -2],
              [1, 2, 3])
  
    def testFloatTypes(self):
      for t in [np.float32, np.float64]:
        self._testAll(np.random.rand(5).astype(t), 2, 0)
        if NP_ROLL_CAN_MULTISHIFT:
          self._testAll(np.random.rand(3, 4).astype(t), [1, 2], [1, 0])
          self._testAll(np.random.rand(1, 3, 4).astype(t), [1, 0, -3], [0, 1, 2])
  
    def testComplexTypes(self):
      for t in [np.complex64, np.complex128]:
        x = np.random.rand(4, 4).astype(t)
        self._testAll(x + 1j * x, 2, 0)
        if NP_ROLL_CAN_MULTISHIFT:
          x = np.random.rand(2, 5).astype(t)
          self._testAll(x + 1j * x, [1, 2], [1, 0])
          x = np.random.rand(3, 2, 1, 1).astype(t)
          self._testAll(x + 1j * x, [2, 1, 1, 0], [0, 3, 1, 2])
  
    def testNegativeAxis(self):
      self._testAll(np.random.randint(-100, 100, (5)).astype(np.int32), 3, -1)
      self._testAll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), 3, -2)
      # Make sure negative axis should be 0 <= axis + dims < dims
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "is out of range"):
          manip_ops.roll(np.random.randint(-100, 100, (4, 4)).astype(np.int32),
                         3, -10).eval()
  
    def testInvalidInputShape(self):
      # The input should be 1-D or higher, checked in shape function.
      with self.assertRaisesRegexp(
          ValueError, "Shape must be at least rank 1 but is rank 0"):
        manip_ops.roll(7, 1, 0)
  
    def testRollInputMustVectorHigherRaises(self):
      # The input should be 1-D or higher, checked in kernel.
      tensor = array_ops.placeholder(dtype=dtypes.int32)
      shift = 1
      axis = 0
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "input must be 1-D or higher"):
          manip_ops.roll(tensor, shift, axis).eval(feed_dict={tensor: 7})
  
    def testInvalidAxisShape(self):
      # The axis should be a scalar or 1-D, checked in shape function.
      with self.assertRaisesRegexp(
          ValueError, "Shape must be at most rank 1 but is rank 2"):
        manip_ops.roll([[1, 2], [3, 4]], 1, [[0, 1]])
  
    def testRollAxisMustBeScalarOrVectorRaises(self):
      # The axis should be a scalar or 1-D, checked in kernel.
      tensor = [[1, 2], [3, 4]]
      shift = 1
      axis = array_ops.placeholder(dtype=dtypes.int32)
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "axis must be a scalar or a 1-D vector"):
          manip_ops.roll(tensor, shift, axis).eval(feed_dict={axis: [[0, 1]]})
  
    def testInvalidShiftShape(self):
      # The shift should be a scalar or 1-D, checked in shape function.
      with self.assertRaisesRegexp(
          ValueError, "Shape must be at most rank 1 but is rank 2"):
        manip_ops.roll([[1, 2], [3, 4]], [[0, 1]], 1)
  
    def testRollShiftMustBeScalarOrVectorRaises(self):
      # The shift should be a scalar or 1-D, checked in kernel.
      tensor = [[1, 2], [3, 4]]
      shift = array_ops.placeholder(dtype=dtypes.int32)
      axis = 1
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "shift must be a scalar or a 1-D vector"):
          manip_ops.roll(tensor, shift, axis).eval(feed_dict={shift: [[0, 1]]})
  
    def testInvalidShiftAndAxisNotEqualShape(self):
      # The shift and axis must be same size, checked in shape function.
      with self.assertRaisesRegexp(ValueError, "both shapes must be equal"):
        manip_ops.roll([[1, 2], [3, 4]], [1], [0, 1])
  
    def testRollShiftAndAxisMustBeSameSizeRaises(self):
      # The shift and axis must be same size, checked in kernel.
      tensor = [[1, 2], [3, 4]]
      shift = array_ops.placeholder(dtype=dtypes.int32)
      axis = [0, 1]
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "shift and axis must have the same size"):
          manip_ops.roll(tensor, shift, axis).eval(feed_dict={shift: [1]})
  
    def testRollAxisOutOfRangeRaises(self):
      tensor = [1, 2]
      shift = 1
      axis = 1
      with self.cached_session():
        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                     "is out of range"):
          manip_ops.roll(tensor, shift, axis).eval()
  
  
  if __name__ == "__main__":
    test_lib.main()
   # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Tests for `tf.data.experimental.prefetch_to_device()`."""
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  from tensorflow.core.protobuf import config_pb2
  from tensorflow.python.data.experimental.ops import prefetching_ops
  from tensorflow.python.data.kernel_tests import test_base
  from tensorflow.python.data.ops import dataset_ops
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import errors
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import sparse_tensor
  from tensorflow.python.framework import test_util
  from tensorflow.python.platform import test
  
  
  class PrefetchToDeviceTest(test_base.DatasetTestBase):
  
    def testPrefetchToDevice(self):
      host_dataset = dataset_ops.Dataset.range(10)
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/cpu:1"))
  
      # NOTE(mrry): This device block creates the "host" dataset and iterator on
      # /cpu:0, and ensures that the prefetching is across devices. In typical use
      # this would not be necessary, because the GPU device would not support any
      # of the dataset-related ops.
      with ops.device("/cpu:0"):
        iterator = device_dataset.make_one_shot_iterator()
  
      self.assertEqual(host_dataset.output_types, device_dataset.output_types)
      self.assertEqual(host_dataset.output_types, iterator.output_types)
      self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
      self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
      self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
      self.assertEqual(host_dataset.output_classes, iterator.output_classes)
  
      next_element = iterator.get_next()
      self.assertEqual(dtypes.int64, next_element.dtype)
      self.assertEqual([], next_element.shape)
  
      worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
      with self.test_session(config=worker_config) as sess:
        for i in range(10):
          self.assertEqual(i, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchToSameDevice(self):
      host_dataset = dataset_ops.Dataset.range(10)
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device(
              "/job:localhost/replica:0/task:0/device:CPU:0"))
  
      # NOTE(mrry): This device block creates the "host" dataset and iterator on
      # /cpu:0, and ensures that the prefetching is across devices. In typical use
      # this would not be necessary, because the GPU device would not support any
      # of the dataset-related ops.
      with ops.device("/cpu:0"):
        iterator = device_dataset.make_one_shot_iterator()
  
      self.assertEqual(host_dataset.output_types, device_dataset.output_types)
      self.assertEqual(host_dataset.output_types, iterator.output_types)
      self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
      self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
      self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
      self.assertEqual(host_dataset.output_classes, iterator.output_classes)
  
      next_element = iterator.get_next()
      self.assertEqual(dtypes.int64, next_element.dtype)
      self.assertEqual([], next_element.shape)
  
      with self.cached_session() as sess:
        for i in range(10):
          self.assertEqual(i, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchDictToDevice(self):
      host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/cpu:1"))
  
      # NOTE(mrry): This device block creates the "host" dataset and iterator on
      # /cpu:0, and ensures that the prefetching is across devices. In typical use
      # this would not be necessary, because the GPU device would not support any
      # of the dataset-related ops.
      with ops.device("/cpu:0"):
        iterator = device_dataset.make_one_shot_iterator()
  
      self.assertEqual(host_dataset.output_types, device_dataset.output_types)
      self.assertEqual(host_dataset.output_types, iterator.output_types)
      self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
      self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
      self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
      self.assertEqual(host_dataset.output_classes, iterator.output_classes)
  
      next_element = iterator.get_next()
      self.assertEqual(dtypes.int64, next_element["a"].dtype)
      self.assertEqual([], next_element["a"].shape)
  
      worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
      with self.test_session(config=worker_config) as sess:
        for i in range(10):
          self.assertEqual({"a": i}, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchSparseTensorsToDevice(self):
      def make_tensor(i):
        return sparse_tensor.SparseTensorValue(
            indices=[[0, 0]], values=(i*[1]), dense_shape=[2, 2])
      host_dataset = dataset_ops.Dataset.range(10).map(make_tensor)
  
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/cpu:1"))
  
      # NOTE(mrry): This device block creates the "host" dataset and iterator on
      # /cpu:0, and ensures that the prefetching is across devices. In typical use
      # this would not be necessary, because the GPU device would not support any
      # of the dataset-related ops.
      with ops.device("/cpu:0"):
        iterator = device_dataset.make_one_shot_iterator()
  
      self.assertEqual(host_dataset.output_types, device_dataset.output_types)
      self.assertEqual(host_dataset.output_types, iterator.output_types)
      self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
      self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
      self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
      self.assertEqual(host_dataset.output_classes, iterator.output_classes)
  
      next_element = iterator.get_next()
      self.assertEqual(dtypes.int64, next_element.dtype)
  
      worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
      with self.test_session(config=worker_config) as sess:
        for i in range(10):
          actual = sess.run(next_element)
          self.assertAllEqual([i], actual.values)
          self.assertAllEqual([[0, 0]], actual.indices)
          self.assertAllEqual([2, 2], actual.dense_shape)
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchToDeviceGpu(self):
      if not test_util.is_gpu_available():
        self.skipTest("No GPU available")
  
      host_dataset = dataset_ops.Dataset.range(10)
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/gpu:0"))
  
      iterator = device_dataset.make_one_shot_iterator()
      next_element = iterator.get_next()
  
      with self.cached_session() as sess:
        for i in range(10):
          self.assertEqual(i, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchToDeviceWithReInit(self):
      host_dataset = dataset_ops.Dataset.range(10)
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/cpu:1"))
  
      # NOTE(mrry): This device block creates the "host" dataset and iterator on
      # /cpu:0, and ensures that the prefetching is across devices. In typical use
      # this would not be necessary, because the GPU device would not support any
      # of the dataset-related ops.
      with ops.device("/cpu:0"):
        iterator = device_dataset.make_initializable_iterator()
  
      self.assertEqual(host_dataset.output_types, device_dataset.output_types)
      self.assertEqual(host_dataset.output_types, iterator.output_types)
      self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
      self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
      self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
      self.assertEqual(host_dataset.output_classes, iterator.output_classes)
  
      next_element = iterator.get_next()
      self.assertEqual(dtypes.int64, next_element.dtype)
      self.assertEqual([], next_element.shape)
  
      worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
      with self.test_session(config=worker_config) as sess:
        sess.run(iterator.initializer)
        for i in range(5):
          self.assertEqual(i, sess.run(next_element))
        sess.run(iterator.initializer)
        for i in range(10):
          self.assertEqual(i, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
    def testPrefetchToDeviceGpuWithReInit(self):
      if not test_util.is_gpu_available():
        self.skipTest("No GPU available")
  
      host_dataset = dataset_ops.Dataset.range(10)
      device_dataset = host_dataset.apply(
          prefetching_ops.prefetch_to_device("/gpu:0"))
  
      iterator = device_dataset.make_initializable_iterator()
      next_element = iterator.get_next()
  
      with self.cached_session() as sess:
        sess.run(iterator.initializer)
        for i in range(5):
          self.assertEqual(i, sess.run(next_element))
        sess.run(iterator.initializer)
        for i in range(10):
          self.assertEqual(i, sess.run(next_element))
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(next_element)
  
  
  if __name__ == "__main__":
    test.main()
   # Copyright 2015 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Functional tests for Proximal Gradient Descent operations."""
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import numpy as np
  
  from tensorflow.python.framework import constant_op
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import embedding_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import resource_variable_ops
  from tensorflow.python.ops import variables
  from tensorflow.python.platform import test
  from tensorflow.python.training import gradient_descent
  from tensorflow.python.training import proximal_gradient_descent
  
  
  class ProximalGradientDescentOptimizerTest(test.TestCase):
  
    def doTestProximalGradientDescentwithoutRegularization(
        self, use_resource=False):
      with self.cached_session() as sess:
        if use_resource:
          var0 = resource_variable_ops.ResourceVariable([0.0, 0.0])
          var1 = resource_variable_ops.ResourceVariable([0.0, 0.0])
        else:
          var0 = variables.Variable([0.0, 0.0])
          var1 = variables.Variable([0.0, 0.0])
        grads0 = constant_op.constant([0.1, 0.2])
        grads1 = constant_op.constant([0.01, 0.02])
        opt = proximal_gradient_descent.ProximalGradientDescentOptimizer(
            3.0, l1_regularization_strength=0.0, l2_regularization_strength=0.0)
        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
        variables.global_variables_initializer().run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose([0.0, 0.0], v0_val)
        self.assertAllClose([0.0, 0.0], v1_val)
  
        # Run 3 steps Proximal Gradient Descent.
        for _ in range(3):
          update.run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose(np.array([-0.9, -1.8]), v0_val)
        self.assertAllClose(np.array([-0.09, -0.18]), v1_val)
  
    def testProximalGradientDescentwithoutRegularization(self):
      self.doTestProximalGradientDescentwithoutRegularization(use_resource=False)
  
    def testResourceProximalGradientDescentwithoutRegularization(self):
      self.doTestProximalGradientDescentwithoutRegularization(use_resource=True)
  
    def testProximalGradientDescentwithoutRegularization2(self):
      with self.cached_session() as sess:
        var0 = variables.Variable([1.0, 2.0])
        var1 = variables.Variable([4.0, 3.0])
        grads0 = constant_op.constant([0.1, 0.2])
        grads1 = constant_op.constant([0.01, 0.02])
  
        opt = proximal_gradient_descent.ProximalGradientDescentOptimizer(
            3.0, l1_regularization_strength=0.0, l2_regularization_strength=0.0)
        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
        variables.global_variables_initializer().run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose([1.0, 2.0], v0_val)
        self.assertAllClose([4.0, 3.0], v1_val)
  
        # Run 3 steps Proximal Gradient Descent
        for _ in range(3):
          update.run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose(np.array([0.1, 0.2]), v0_val)
        self.assertAllClose(np.array([3.91, 2.82]), v1_val)
  
    def testMinimizeSparseResourceVariable(self):
      for dtype in [dtypes.float32, dtypes.float64]:
        with self.cached_session():
          var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
          x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
          pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
          loss = pred * pred
          sgd_op = proximal_gradient_descent.ProximalGradientDescentOptimizer(
              1.0).minimize(loss)
          variables.global_variables_initializer().run()
          # Fetch params to validate initial values
          self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
          # Run 1 step of sgd
          sgd_op.run()
          # Validate updated params
          self.assertAllCloseAccordingToType(
              [[-111, -138]], var0.eval(), atol=0.01)
  
    def testProximalGradientDescentWithL1_L2(self):
      with self.cached_session() as sess:
        var0 = variables.Variable([1.0, 2.0])
        var1 = variables.Variable([4.0, 3.0])
        grads0 = constant_op.constant([0.1, 0.2])
        grads1 = constant_op.constant([0.01, 0.02])
  
        opt = proximal_gradient_descent.ProximalGradientDescentOptimizer(
            3.0, l1_regularization_strength=0.001, l2_regularization_strength=2.0)
        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
        variables.global_variables_initializer().run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose([1.0, 2.0], v0_val)
        self.assertAllClose([4.0, 3.0], v1_val)
  
        # Run 10 steps Proximal Gradient Descent
        for _ in range(10):
          update.run()
  
        v0_val, v1_val = sess.run([var0, var1])
        self.assertAllClose(np.array([-0.0495, -0.0995]), v0_val)
        self.assertAllClose(np.array([-0.0045, -0.0095]), v1_val)
  
    def applyOptimizer(self, opt, steps=5, is_sparse=False):
      if is_sparse:
        var0 = variables.Variable([[1.0], [2.0]])
        var1 = variables.Variable([[3.0], [4.0]])
        grads0 = ops.IndexedSlices(
            constant_op.constant(
                [0.1], shape=[1, 1]),
            constant_op.constant([0]),
            constant_op.constant([2, 1]))
        grads1 = ops.IndexedSlices(
            constant_op.constant(
                [0.02], shape=[1, 1]),
            constant_op.constant([1]),
            constant_op.constant([2, 1]))
      else:
        var0 = variables.Variable([1.0, 2.0])
        var1 = variables.Variable([3.0, 4.0])
        grads0 = constant_op.constant([0.1, 0.2])
        grads1 = constant_op.constant([0.01, 0.02])
  
      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
      variables.global_variables_initializer().run()
  
      sess = ops.get_default_session()
      v0_val, v1_val = sess.run([var0, var1])
      if is_sparse:
        self.assertAllClose([[1.0], [2.0]], v0_val)
        self.assertAllClose([[3.0], [4.0]], v1_val)
      else:
        self.assertAllClose([1.0, 2.0], v0_val)
        self.assertAllClose([3.0, 4.0], v1_val)
  
      # Run ProximalAdagrad for a few steps
      for _ in range(steps):
        update.run()
  
      v0_val, v1_val = sess.run([var0, var1])
      return v0_val, v1_val
  
    def testEquivSparseGradientDescentwithoutRegularization(self):
      with self.cached_session():
        val0, val1 = self.applyOptimizer(
            proximal_gradient_descent.ProximalGradientDescentOptimizer(
                3.0,
                l1_regularization_strength=0.0,
                l2_regularization_strength=0.0),
            is_sparse=True)
  
      with self.cached_session():
        val2, val3 = self.applyOptimizer(
            gradient_descent.GradientDescentOptimizer(3.0), is_sparse=True)
  
      self.assertAllClose(val0, val2)
      self.assertAllClose(val1, val3)
  
    def testEquivGradientDescentwithoutRegularization(self):
      with self.cached_session():
        val0, val1 = self.applyOptimizer(
            proximal_gradient_descent.ProximalGradientDescentOptimizer(
                3.0,
                l1_regularization_strength=0.0,
                l2_regularization_strength=0.0))
  
      with self.cached_session():
        val2, val3 = self.applyOptimizer(
            gradient_descent.GradientDescentOptimizer(3.0))
  
      self.assertAllClose(val0, val2)
      self.assertAllClose(val1, val3)
  
  
  if __name__ == "__main__":
    test.main()
  
apis: tensorflow.python.estimator.canned.head._multi_class_head_with_softmax_cross_entropy_loss tensorflow.python.estimator.canned.head._regression_head tensorflow.python.estimator.canned.optimizers.get_optimizer_instance tensorflow.python.util.tf_export.estimator_export tensorflow.python.training.training_util.get_global_step tensorflow.python.estimator.canned.linear._linear_logit_fn_builder tensorflow.python.estimator.canned.head._binary_logistic_head_with_sigmoid_cross_entropy_loss tensorflow.python.framework.ops.get_collection tensorflow.python.estimator.canned.dnn._dnn_logit_fn_builder tensorflow.python.summary.summary.histogram tensorflow.python.framework.ops.control_dependencies tensorflow.python.ops.state_ops.assign_add tensorflow.python.ops.nn.zero_fraction tensorflow.python.ops.partitioned_variables.min_max_variable_partitioner tensorflow.python.ops.control_flow_ops.group
apis: numpy.roll tensorflow.python.ops.manip_ops.roll tensorflow.python.ops.array_ops.placeholder tensorflow.python.platform.test.main numpy.random.rand tensorflow.python.ops.gradient_checker.compute_gradient numpy.random.randint
apis: tensorflow.python.data.experimental.ops.prefetching_ops.prefetch_to_device tensorflow.core.protobuf.config_pb2.ConfigProto tensorflow.python.data.ops.dataset_ops.Dataset.range tensorflow.python.platform.test.main tensorflow.python.framework.sparse_tensor.SparseTensorValue tensorflow.python.framework.test_util.is_gpu_available tensorflow.python.framework.ops.device
apis: tensorflow.python.framework.ops.get_default_session tensorflow.python.training.gradient_descent.GradientDescentOptimizer tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer tensorflow.python.ops.variables.global_variables_initializer tensorflow.python.ops.embedding_ops.embedding_lookup tensorflow.python.platform.test.main tensorflow.python.ops.resource_variable_ops.ResourceVariable numpy.array tensorflow.python.ops.variables.Variable tensorflow.python.framework.constant_op.constant