code: # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  
  """Inception V2 Faster R-CNN implementation.
  
  See "Rethinking the Inception Architecture for Computer Vision"
  https://arxiv.org/abs/1512.00567
  """
  import tensorflow as tf
  
  from object_detection.meta_architectures import faster_rcnn_meta_arch
  from nets import inception_v2
  
  slim = tf.contrib.slim
  
  
  def _batch_norm_arg_scope(list_ops,
                            use_batch_norm=True,
                            batch_norm_decay=0.9997,
                            batch_norm_epsilon=0.001,
                            batch_norm_scale=False,
                            train_batch_norm=False):
      """Slim arg scope for InceptionV2 batch norm."""
      if use_batch_norm:
          batch_norm_params = {
              'is_training': train_batch_norm,
              'scale': batch_norm_scale,
              'decay': batch_norm_decay,
              'epsilon': batch_norm_epsilon
          }
          normalizer_fn = slim.batch_norm
      else:
          normalizer_fn = None
          batch_norm_params = None
  
      return slim.arg_scope(list_ops,
                            normalizer_fn=normalizer_fn,
                            normalizer_params=batch_norm_params)
  
  
  class FasterRCNNInceptionV2FeatureExtractor(
      faster_rcnn_meta_arch.FasterRCNNFeatureExtractor):
      """Faster R-CNN Inception V2 feature extractor implementation."""
  
      def __init__(self,
                   is_training,
                   first_stage_features_stride,
                   batch_norm_trainable=False,
                   reuse_weights=None,
                   weight_decay=0.0,
                   depth_multiplier=1.0,
                   min_depth=16):
          """Constructor.
  
          Args:
            is_training: See base class.
            first_stage_features_stride: See base class.
            batch_norm_trainable: See base class.
            reuse_weights: See base class.
            weight_decay: See base class.
            depth_multiplier: float depth multiplier for feature extractor.
            min_depth: minimum feature extractor depth.
  
          Raises:
            ValueError: If `first_stage_features_stride` is not 8 or 16.
          """
          if first_stage_features_stride != 8 and first_stage_features_stride != 16:
              raise ValueError('`first_stage_features_stride` must be 8 or 16.')
          self._depth_multiplier = depth_multiplier
          self._min_depth = min_depth
          super(FasterRCNNInceptionV2FeatureExtractor, self).__init__(
              is_training, first_stage_features_stride, batch_norm_trainable,
              reuse_weights, weight_decay)
  
      def preprocess(self, resized_inputs):
          """Faster R-CNN Inception V2 preprocessing.
  
          Maps pixel values to the range [-1, 1].
  
          Args:
            resized_inputs: a [batch, height, width, channels] float tensor
              representing a batch of images.
  
          Returns:
            preprocessed_inputs: a [batch, height, width, channels] float tensor
              representing a batch of images.
          """
          return (2.0 / 255.0) * resized_inputs - 1.0
  
      def _extract_proposal_features(self, preprocessed_inputs, scope):
          """Extracts first stage RPN features.
  
          Args:
            preprocessed_inputs: A [batch, height, width, channels] float32 tensor
              representing a batch of images.
            scope: A scope name.
  
          Returns:
            rpn_feature_map: A tensor with shape [batch, height, width, depth]
          Raises:
            InvalidArgumentError: If the spatial size of `preprocessed_inputs`
              (height or width) is less than 33.
            ValueError: If the created network is missing the required activation.
          """
  
          preprocessed_inputs.get_shape().assert_has_rank(4)
          shape_assert = tf.Assert(
              tf.logical_and(tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
                             tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
              ['image size must at least be 33 in both height and width.'])
  
          with tf.control_dependencies([shape_assert]):
              with tf.variable_scope('InceptionV2', reuse=self._reuse_weights):
                  # with slim.arg_scope(inception_v2.inception_v2_arg_scope()) as scope:
                  with _batch_norm_arg_scope([slim.conv2d, slim.separable_conv2d],
                                             batch_norm_scale=True,
                                             train_batch_norm=self._train_batch_norm):
                      _, activations = inception_v2.inception_v2_base(
                          preprocessed_inputs,
                          final_endpoint='Mixed_4e',
                          min_depth=self._min_depth,
                          depth_multiplier=self._depth_multiplier,
                          scope=scope)
  
          return activations['Mixed_4e']
  
      def _extract_box_classifier_features(self, proposal_feature_maps, scope):
          """Extracts second stage box classifier features.
  
          Args:
            proposal_feature_maps: A 4-D float tensor with shape
              [batch_size * self.max_num_proposals, crop_height, crop_width, depth]
              representing the feature map cropped to each proposal.
            scope: A scope name (unused).
  
          Returns:
            proposal_classifier_features: A 4-D float tensor with shape
              [batch_size * self.max_num_proposals, height, width, depth]
              representing box classifier features for each proposal.
          """
          net = proposal_feature_maps
  
          depth = lambda d: max(int(d * self._depth_multiplier), self._min_depth)
          trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev)
  
          data_format = 'NHWC'
          concat_dim = 3 if data_format == 'NHWC' else 1
  
          with tf.variable_scope('InceptionV2', reuse=self._reuse_weights):
              # with tf.variable_scope(inception_v2.inception_v2_arg_scope()):
              with slim.arg_scope(
                      [slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
                      stride=1,
                      padding='SAME',
                      data_format=data_format):
                  with _batch_norm_arg_scope([slim.conv2d, slim.separable_conv2d],
                                             batch_norm_scale=True,
                                             train_batch_norm=self._train_batch_norm):
                      with tf.variable_scope('Mixed_5a'):
                          with tf.variable_scope('Branch_0'):
                              branch_0 = slim.conv2d(
                                  net, depth(128), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_0 = slim.conv2d(branch_0, depth(192), [3, 3], stride=2,
                                                     scope='Conv2d_1a_3x3')
                          with tf.variable_scope('Branch_1'):
                              branch_1 = slim.conv2d(
                                  net, depth(192), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_1 = slim.conv2d(branch_1, depth(256), [3, 3],
                                                     scope='Conv2d_0b_3x3')
                              branch_1 = slim.conv2d(branch_1, depth(256), [3, 3], stride=2,
                                                     scope='Conv2d_1a_3x3')
                          with tf.variable_scope('Branch_2'):
                              branch_2 = slim.max_pool2d(net, [3, 3], stride=2,
                                                         scope='MaxPool_1a_3x3')
                          net = tf.concat([branch_0, branch_1, branch_2], concat_dim)
  
                      with tf.variable_scope('Mixed_5b'):
                          with tf.variable_scope('Branch_0'):
                              branch_0 = slim.conv2d(net, depth(352), [1, 1],
                                                     scope='Conv2d_0a_1x1')
                          with tf.variable_scope('Branch_1'):
                              branch_1 = slim.conv2d(
                                  net, depth(192), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_1 = slim.conv2d(branch_1, depth(320), [3, 3],
                                                     scope='Conv2d_0b_3x3')
                          with tf.variable_scope('Branch_2'):
                              branch_2 = slim.conv2d(
                                  net, depth(160), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_2 = slim.conv2d(branch_2, depth(224), [3, 3],
                                                     scope='Conv2d_0b_3x3')
                              branch_2 = slim.conv2d(branch_2, depth(224), [3, 3],
                                                     scope='Conv2d_0c_3x3')
                          with tf.variable_scope('Branch_3'):
                              branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3')
                              branch_3 = slim.conv2d(
                                  branch_3, depth(128), [1, 1],
                                  weights_initializer=trunc_normal(0.1),
                                  scope='Conv2d_0b_1x1')
                          net = tf.concat([branch_0, branch_1, branch_2, branch_3],
                                          concat_dim)
  
                      with tf.variable_scope('Mixed_5c'):
                          with tf.variable_scope('Branch_0'):
                              branch_0 = slim.conv2d(net, depth(352), [1, 1],
                                                     scope='Conv2d_0a_1x1')
                          with tf.variable_scope('Branch_1'):
                              branch_1 = slim.conv2d(
                                  net, depth(192), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_1 = slim.conv2d(branch_1, depth(320), [3, 3],
                                                     scope='Conv2d_0b_3x3')
                          with tf.variable_scope('Branch_2'):
                              branch_2 = slim.conv2d(
                                  net, depth(192), [1, 1],
                                  weights_initializer=trunc_normal(0.09),
                                  scope='Conv2d_0a_1x1')
                              branch_2 = slim.conv2d(branch_2, depth(224), [3, 3],
                                                     scope='Conv2d_0b_3x3')
                              branch_2 = slim.conv2d(branch_2, depth(224), [3, 3],
                                                     scope='Conv2d_0c_3x3')
                          with tf.variable_scope('Branch_3'):
                              branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3')
                              branch_3 = slim.conv2d(
                                  branch_3, depth(128), [1, 1],
                                  weights_initializer=trunc_normal(0.1),
                                  scope='Conv2d_0b_1x1')
                          proposal_classifier_features = tf.concat(
                              [branch_0, branch_1, branch_2, branch_3], concat_dim)
  
          return proposal_classifier_features
   # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  
  """SSDFeatureExtractor for MobilenetV1 features."""
  
  import tensorflow as tf
  
  from object_detection.meta_architectures import ssd_meta_arch
  from object_detection.models import feature_map_generators
  from object_detection.utils import ops
  from nets import mobilenet_v1
  
  slim = tf.contrib.slim
  
  
  class SSDMobileNetV1FeatureExtractor(ssd_meta_arch.SSDFeatureExtractor):
      """SSD Feature Extractor using MobilenetV1 features."""
  
      def __init__(self,
                   is_training,
                   depth_multiplier,
                   min_depth,
                   pad_to_multiple,
                   conv_hyperparams,
                   batch_norm_trainable=True,
                   reuse_weights=None):
          """MobileNetV1 Feature Extractor for SSD Models.
  
          Args:
            is_training: whether the network is in training mode.
            depth_multiplier: float depth multiplier for feature extractor.
            min_depth: minimum feature extractor depth.
            pad_to_multiple: the nearest multiple to zero pad the input height and
              width dimensions to.
            conv_hyperparams: tf slim arg_scope for conv2d and separable_conv2d ops.
            batch_norm_trainable: Whether to update batch norm parameters during
              training or not. When training with a small batch size
              (e.g. 1), it is desirable to disable batch norm update and use
              pretrained batch norm params.
            reuse_weights: Whether to reuse variables. Default is None.
          """
          super(SSDMobileNetV1FeatureExtractor, self).__init__(
              is_training, depth_multiplier, min_depth, pad_to_multiple,
              conv_hyperparams, batch_norm_trainable, reuse_weights)
  
      def preprocess(self, resized_inputs):
          """SSD preprocessing.
  
          Maps pixel values to the range [-1, 1].
  
          Args:
            resized_inputs: a [batch, height, width, channels] float tensor
              representing a batch of images.
  
          Returns:
            preprocessed_inputs: a [batch, height, width, channels] float tensor
              representing a batch of images.
          """
          return (2.0 / 255.0) * resized_inputs - 1.0
  
      def extract_features(self, preprocessed_inputs):
          """Extract features from preprocessed inputs.
  
          Args:
            preprocessed_inputs: a [batch, height, width, channels] float tensor
              representing a batch of images.
  
          Returns:
            feature_maps: a list of tensors where the ith tensor has shape
              [batch, height_i, width_i, depth_i]
          """
          preprocessed_inputs.get_shape().assert_has_rank(4)
          shape_assert = tf.Assert(
              tf.logical_and(tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
                             tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
              ['image size must at least be 33 in both height and width.'])
  
          feature_map_layout = {
              'from_layer': ['Conv2d_11_pointwise', 'Conv2d_13_pointwise', '', '',
                             '', ''],
              'layer_depth': [-1, -1, 512, 256, 256, 128],
          }
  
          with tf.control_dependencies([shape_assert]):
              with slim.arg_scope(self._conv_hyperparams):
                  with slim.arg_scope([slim.batch_norm], fused=False):
                      with tf.variable_scope('MobilenetV1',
                                             reuse=self._reuse_weights) as scope:
                          _, image_features = mobilenet_v1.mobilenet_v1_base(
                              ops.pad_to_multiple(preprocessed_inputs, self._pad_to_multiple),
                              final_endpoint='Conv2d_13_pointwise',
                              min_depth=self._min_depth,
                              depth_multiplier=self._depth_multiplier,
                              scope=scope)
                          feature_maps = feature_map_generators.multi_resolution_feature_maps(
                              feature_map_layout=feature_map_layout,
                              depth_multiplier=self._depth_multiplier,
                              min_depth=self._min_depth,
                              insert_1x1_conv=True,
                              image_features=image_features)
  
          return feature_maps.values()
   # Copyright 2017 Google Inc. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  
  """Network units wrapping TensorFlows' tf.contrib.rnn cells.
  
  Please put all wrapping logic for tf.contrib.rnn in this module; this will help
  collect common subroutines that prove useful.
  """
  
  import abc
  
  import tensorflow as tf
  
  from dragnn.python import network_units as dragnn
  from syntaxnet.util import check
  
  
  def capture_variables(function, scope_name):
      """Captures and returns variables created by a function.
  
      Runs |function| in a scope of name |scope_name| and returns the list of
      variables created by |function|.
  
      Args:
        function: Function whose variables should be captured.  The function should
            take one argument, its enclosing variable scope.
        scope_name: Variable scope in which the |function| is evaluated.
  
      Returns:
        List of created variables.
      """
      # Use a dict to dedupe captured variables.
      created_vars = {}
  
      def _custom_getter(getter, *args, **kwargs):
          """Calls the real getter and captures its result in |created_vars|."""
          real_variable = getter(*args, **kwargs)
          created_vars[real_variable.name] = real_variable
          return real_variable
  
      with tf.variable_scope(
              scope_name, reuse=None, custom_getter=_custom_getter) as scope:
          function(scope)
      return created_vars.values()
  
  
  def apply_with_captured_variables(function, scope_name, component):
      """Applies a function using previously-captured variables.
  
      The counterpart to capture_variables(); invokes |function| in a scope of name
      |scope_name|, extracting captured variables from the |component|.
  
      Args:
        function: Function to apply using captured variables.  The function should
            take one argument, its enclosing variable scope.
        scope_name: Variable scope in which the |function| is evaluated.  Must match
            the scope passed to capture_variables().
        component: Component from which to extract captured variables.
  
      Returns:
        Results of function application.
      """
  
      def _custom_getter(getter, *args, **kwargs):
          """Retrieves the normal or moving-average variables."""
          return component.get_variable(var_params=getter(*args, **kwargs))
  
      with tf.variable_scope(
              scope_name, reuse=True, custom_getter=_custom_getter) as scope:
          return function(scope)
  
  
  class BaseLSTMNetwork(dragnn.NetworkUnitInterface):
      """Base class for wrapped LSTM networks.
  
      This LSTM network unit supports multiple layers with layer normalization.
      Because it is imported from tf.contrib.rnn, we need to capture the created
      variables during initialization time.
  
      Layers:
        ...subclass-specific layers...
        last_layer: Alias for the activations of the last hidden layer.
        logits: Logits associated with component actions.
      """
  
      def __init__(self, component, additional_attr_defaults=None):
          """Initializes the LSTM base class.
  
          Parameters used:
            hidden_layer_sizes: Comma-delimited number of hidden units for each layer.
            input_dropout_rate (-1.0): Input dropout rate for each layer.  If < 0.0,
                use the global |dropout_rate| hyperparameter.
            recurrent_dropout_rate (0.8): Recurrent dropout rate.  If < 0.0, use the
                global |recurrent_dropout_rate| hyperparameter.
            layer_norm (True): Whether or not to use layer norm.
  
          Hyperparameters used:
            dropout_rate: Input dropout rate.
            recurrent_dropout_rate: Recurrent dropout rate.
  
          Args:
            component: parent ComponentBuilderBase object.
            additional_attr_defaults: Additional attributes for use by derived class.
          """
          attr_defaults = additional_attr_defaults or {}
          attr_defaults.update({
              'layer_norm': True,
              'input_dropout_rate': -1.0,
              'recurrent_dropout_rate': 0.8,
              'hidden_layer_sizes': '256',
          })
          self._attrs = dragnn.get_attrs_with_defaults(
              component.spec.network_unit.parameters,
              defaults=attr_defaults)
  
          self._hidden_layer_sizes = map(int,
                                         self._attrs['hidden_layer_sizes'].split(','))
  
          self._input_dropout_rate = self._attrs['input_dropout_rate']
          if self._input_dropout_rate < 0.0:
              self._input_dropout_rate = component.master.hyperparams.dropout_rate
  
          self._recurrent_dropout_rate = self._attrs['recurrent_dropout_rate']
          if self._recurrent_dropout_rate < 0.0:
              self._recurrent_dropout_rate = (
                  component.master.hyperparams.recurrent_dropout_rate)
          if self._recurrent_dropout_rate < 0.0:
              self._recurrent_dropout_rate = component.master.hyperparams.dropout_rate
  
          tf.logging.info('[%s] input_dropout_rate=%s recurrent_dropout_rate=%s',
                          component.name, self._input_dropout_rate,
                          self._recurrent_dropout_rate)
  
          layers, context_layers = self.create_hidden_layers(component,
                                                             self._hidden_layer_sizes)
          last_layer_dim = layers[-1].dim
          layers.append(
              dragnn.Layer(component, name='last_layer', dim=last_layer_dim))
          layers.append(
              dragnn.Layer(component, name='logits', dim=component.num_actions))
  
          # Provide initial layers and context layers, so the base class constructor
          # can safely use accessors like get_layer_size().
          super(BaseLSTMNetwork, self).__init__(
              component, init_layers=layers, init_context_layers=context_layers)
  
          # Allocate parameters for the softmax.
          self._params.append(
              tf.get_variable(
                  'weights_softmax', [last_layer_dim, component.num_actions],
                  initializer=tf.random_normal_initializer(stddev=1e-4)))
          self._params.append(
              tf.get_variable(
                  'bias_softmax', [component.num_actions],
                  initializer=tf.zeros_initializer()))
  
      def get_logits(self, network_tensors):
          """Returns the logits for prediction."""
          return network_tensors[self.get_layer_index('logits')]
  
      @abc.abstractmethod
      def create_hidden_layers(self, component, hidden_layer_sizes):
          """Creates hidden network layers.
  
          Args:
            component: Parent ComponentBuilderBase object.
            hidden_layer_sizes: List of requested hidden layer activation sizes.
  
          Returns:
            layers: List of layers created by this network.
            context_layers: List of context layers created by this network.
          """
          pass
  
      def _append_base_layers(self, hidden_layers):
          """Appends layers defined by the base class to the |hidden_layers|."""
          last_layer = hidden_layers[-1]
  
          logits = tf.nn.xw_plus_b(last_layer,
                                   self._component.get_variable('weights_softmax'),
                                   self._component.get_variable('bias_softmax'))
          return hidden_layers + [last_layer, logits]
  
      def _create_cell(self, num_units, during_training):
          """Creates a single LSTM cell, possibly with dropout.
  
          Requires that BaseLSTMNetwork.__init__() was called.
  
          Args:
            num_units: Number of hidden units in the cell.
            during_training: Whether to create a cell for training (vs inference).
  
          Returns:
            A RNNCell of the requested size, possibly with dropout.
          """
          # No dropout in inference mode.
          if not during_training:
              return tf.contrib.rnn.LayerNormBasicLSTMCell(
                  num_units, layer_norm=self._attrs['layer_norm'], reuse=True)
  
          # Otherwise, apply dropout to inputs and recurrences.
          cell = tf.contrib.rnn.LayerNormBasicLSTMCell(
              num_units,
              dropout_keep_prob=self._recurrent_dropout_rate,
              layer_norm=self._attrs['layer_norm'])
          cell = tf.contrib.rnn.DropoutWrapper(
              cell, input_keep_prob=self._input_dropout_rate)
          return cell
  
      def _create_train_cells(self):
          """Creates a list of LSTM cells for training."""
          return [
              self._create_cell(num_units, during_training=True)
              for num_units in self._hidden_layer_sizes
          ]
  
      def _create_inference_cells(self):
          """Creates a list of LSTM cells for inference."""
          return [
              self._create_cell(num_units, during_training=False)
              for num_units in self._hidden_layer_sizes
          ]
  
      def _capture_variables_as_params(self, function):
          """Captures variables created by a function in |self._params|."""
          self._params.extend(capture_variables(function, 'cell'))
  
      def _apply_with_captured_variables(self, function):
          """Applies a function using previously-captured variables."""
          return apply_with_captured_variables(function, 'cell', self._component)
  
  
  class LayerNormBasicLSTMNetwork(BaseLSTMNetwork):
      """Wrapper around tf.contrib.rnn.LayerNormBasicLSTMCell.
  
      Features:
        All inputs are concatenated.
  
      Subclass-specific layers:
        state_c_<n>: Cell states for the <n>'th LSTM layer (0-origin).
        state_h_<n>: Hidden states for the <n>'th LSTM layer (0-origin).
      """
  
      def __init__(self, component):
          """Sets up context and output layers, as well as a final softmax."""
          super(LayerNormBasicLSTMNetwork, self).__init__(component)
  
          # Wrap lists of training and inference sub-cells into multi-layer RNN cells.
          # Note that a |MultiRNNCell| state is a tuple of per-layer sub-states.
          self._train_cell = tf.contrib.rnn.MultiRNNCell(self._create_train_cells())
          self._inference_cell = tf.contrib.rnn.MultiRNNCell(
              self._create_inference_cells())
  
          def _cell_closure(scope):
              """Applies the LSTM cell to placeholder inputs and state."""
              placeholder_inputs = tf.placeholder(
                  dtype=tf.float32, shape=(1, self._concatenated_input_dim))
  
              placeholder_substates = []
              for num_units in self._hidden_layer_sizes:
                  placeholder_substate = tf.contrib.rnn.LSTMStateTuple(
                      tf.placeholder(dtype=tf.float32, shape=(1, num_units)),
                      tf.placeholder(dtype=tf.float32, shape=(1, num_units)))
                  placeholder_substates.append(placeholder_substate)
              placeholder_state = tuple(placeholder_substates)
  
              self._train_cell(
                  inputs=placeholder_inputs, state=placeholder_state, scope=scope)
  
          self._capture_variables_as_params(_cell_closure)
  
      def create_hidden_layers(self, component, hidden_layer_sizes):
          """See base class."""
          # Construct the layer meta info for the DRAGNN builder. Note that the order
          # of h and c are reversed compared to the vanilla DRAGNN LSTM cell, as
          # this is the standard in tf.contrib.rnn.
          #
          # NB: The h activations of the last LSTM must be the last layer, in order
          # for _append_base_layers() to work.
          layers = []
          for index, num_units in enumerate(hidden_layer_sizes):
              layers.append(
                  dragnn.Layer(component, name='state_c_%d' % index, dim=num_units))
              layers.append(
                  dragnn.Layer(component, name='state_h_%d' % index, dim=num_units))
          context_layers = list(layers)  # copy |layers|, don't alias it
          return layers, context_layers
  
      def create(self,
                 fixed_embeddings,
                 linked_embeddings,
                 context_tensor_arrays,
                 attention_tensor,
                 during_training,
                 stride=None):
          """See base class."""
          # NB: This cell pulls the lstm's h and c vectors from context_tensor_arrays
          # instead of through linked features.
          check.Eq(
              len(context_tensor_arrays), 2 * len(self._hidden_layer_sizes),
              'require two context tensors per hidden layer')
  
          # Rearrange the context tensors into a tuple of LSTM sub-states.
          length = context_tensor_arrays[0].size()
          substates = []
          for index, num_units in enumerate(self._hidden_layer_sizes):
              state_c = context_tensor_arrays[2 * index].read(length - 1)
              state_h = context_tensor_arrays[2 * index + 1].read(length - 1)
  
              # Fix shapes that for some reason are not set properly for an unknown
              # reason. TODO(googleuser): Why are the shapes not set?
              state_c.set_shape([tf.Dimension(None), num_units])
              state_h.set_shape([tf.Dimension(None), num_units])
              substates.append(tf.contrib.rnn.LSTMStateTuple(state_c, state_h))
          state = tuple(substates)
  
          input_tensor = dragnn.get_input_tensor(fixed_embeddings, linked_embeddings)
          cell = self._train_cell if during_training else self._inference_cell
  
          def _cell_closure(scope):
              """Applies the LSTM cell to the current inputs and state."""
              return cell(input_tensor, state, scope)
  
          unused_h, state = self._apply_with_captured_variables(_cell_closure)
  
          # Return tensors to be put into the tensor arrays / used to compute
          # objective.
          output_tensors = []
          for new_substate in state:
              new_c, new_h = new_substate
              output_tensors.append(new_c)
              output_tensors.append(new_h)
          return self._append_base_layers(output_tensors)
  
  
  class BulkBiLSTMNetwork(BaseLSTMNetwork):
      """Bulk wrapper around tf.contrib.rnn.stack_bidirectional_dynamic_rnn().
  
      Features:
        lengths: [stride, 1] sequence lengths per batch item.
        All other features are concatenated into input activations.
  
      Subclass-specific layers:
        outputs: [stride * num_steps, self._output_dim] bi-LSTM activations.
      """
  
      def __init__(self, component):
          """Initializes the bulk bi-LSTM.
  
          Parameters used:
            parallel_iterations (1): Parallelism of the underlying tf.while_loop().
              Defaults to 1 thread to encourage deterministic behavior, but can be
              increased to trade memory for speed.
  
          Args:
            component: parent ComponentBuilderBase object.
          """
          super(BulkBiLSTMNetwork, self).__init__(
              component, additional_attr_defaults={'parallel_iterations': 1})
  
          check.In('lengths', self._linked_feature_dims,
                   'Missing required linked feature')
          check.Eq(self._linked_feature_dims['lengths'], 1,
                   'Wrong dimension for "lengths" feature')
          self._input_dim = self._concatenated_input_dim - 1  # exclude 'lengths'
          self._output_dim = self.get_layer_size('outputs')
          tf.logging.info('[%s] Bulk bi-LSTM with input_dim=%d output_dim=%d',
                          component.name, self._input_dim, self._output_dim)
  
          # Create one training and inference cell per layer and direction.
          self._train_cells_forward = self._create_train_cells()
          self._train_cells_backward = self._create_train_cells()
          self._inference_cells_forward = self._create_inference_cells()
          self._inference_cells_backward = self._create_inference_cells()
  
          def _bilstm_closure(scope):
              """Applies the bi-LSTM to placeholder inputs and lengths."""
              # Use singleton |stride| and |steps| because their values don't affect the
              # weight variables.
              stride, steps = 1, 1
              placeholder_inputs = tf.placeholder(
                  dtype=tf.float32, shape=[stride, steps, self._input_dim])
              placeholder_lengths = tf.placeholder(dtype=tf.int64, shape=[stride])
  
              # Omit the initial states and sequence lengths for simplicity; they don't
              # affect the weight variables.
              tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
                  self._train_cells_forward,
                  self._train_cells_backward,
                  placeholder_inputs,
                  dtype=tf.float32,
                  sequence_length=placeholder_lengths,
                  scope=scope)
  
          self._capture_variables_as_params(_bilstm_closure)
  
          # Allocate parameters for the initial states.  Note that an LSTM state is a
          # tuple of two substates (c, h), so there are 4 variables per layer.
          for index, num_units in enumerate(self._hidden_layer_sizes):
              for direction in ['forward', 'backward']:
                  for substate in ['c', 'h']:
                      self._params.append(
                          tf.get_variable(
                              'initial_state_%s_%s_%d' % (direction, substate, index),
                              [1, num_units],  # leading 1 for later batch-wise tiling
                              dtype=tf.float32,
                              initializer=tf.constant_initializer(0.0)))
  
      def create_hidden_layers(self, component, hidden_layer_sizes):
          """See base class."""
          dim = 2 * hidden_layer_sizes[-1]
          return [dragnn.Layer(component, name='outputs', dim=dim)], []
  
      def create(self,
                 fixed_embeddings,
                 linked_embeddings,
                 context_tensor_arrays,
                 attention_tensor,
                 during_training,
                 stride=None):
          """Requires |stride|; otherwise see base class."""
          check.NotNone(stride,
                        'BulkBiLSTMNetwork requires "stride" and must be called '
                        'in the bulk feature extractor component.')
  
          # Flatten the lengths into a vector.
          lengths = dragnn.lookup_named_tensor('lengths', linked_embeddings)
          lengths_s = tf.squeeze(lengths.tensor, [1])
  
          # Collect all other inputs into a batched tensor.
          linked_embeddings = [
              named_tensor for named_tensor in linked_embeddings
              if named_tensor.name != 'lengths'
          ]
          inputs_sxnxd = dragnn.get_input_tensor_with_stride(
              fixed_embeddings, linked_embeddings, stride)
  
          # Since get_input_tensor_with_stride() concatenates the input embeddings, it
          # obscures the static activation dimension, which the RNN library requires.
          # Restore it using set_shape().  Note that set_shape() merges into the known
          # shape, so only specify the activation dimension.
          inputs_sxnxd.set_shape(
              [tf.Dimension(None), tf.Dimension(None), self._input_dim])
  
          initial_states_forward, initial_states_backward = (
              self._create_initial_states(stride))
  
          if during_training:
              cells_forward = self._train_cells_forward
              cells_backward = self._train_cells_backward
          else:
              cells_forward = self._inference_cells_forward
              cells_backward = self._inference_cells_backward
  
          def _bilstm_closure(scope):
              """Applies the bi-LSTM to the current inputs."""
              outputs_sxnxd, _, _ = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
                  cells_forward,
                  cells_backward,
                  inputs_sxnxd,
                  initial_states_fw=initial_states_forward,
                  initial_states_bw=initial_states_backward,
                  sequence_length=lengths_s,
                  parallel_iterations=self._attrs['parallel_iterations'],
                  scope=scope)
              return outputs_sxnxd
  
          # Layer outputs are not batched; flatten out the batch dimension.
          outputs_sxnxd = self._apply_with_captured_variables(_bilstm_closure)
          outputs_snxd = tf.reshape(outputs_sxnxd, [-1, self._output_dim])
          return self._append_base_layers([outputs_snxd])
  
      def _create_initial_states(self, stride):
          """Returns stacked and batched initial states for the bi-LSTM."""
          initial_states_forward = []
          initial_states_backward = []
          for index in range(len(self._hidden_layer_sizes)):
              # Retrieve the initial states for this layer.
              states_sxd = []
              for direction in ['forward', 'backward']:
                  for substate in ['c', 'h']:
                      state_1xd = self._component.get_variable('initial_state_%s_%s_%d' %
                                                               (direction, substate, index))
                      state_sxd = tf.tile(state_1xd, [stride, 1])  # tile across the batch
                      states_sxd.append(state_sxd)
  
              # Assemble and append forward and backward LSTM states.
              initial_states_forward.append(
                  tf.contrib.rnn.LSTMStateTuple(states_sxd[0], states_sxd[1]))
              initial_states_backward.append(
                  tf.contrib.rnn.LSTMStateTuple(states_sxd[2], states_sxd[3]))
          return initial_states_forward, initial_states_backward
  
apis: tensorflow.shape tensorflow.truncated_normal_initializer tensorflow.variable_scope tensorflow.concat tensorflow.control_dependencies
apis: tensorflow.shape tensorflow.variable_scope tensorflow.control_dependencies
apis: tensorflow.contrib.rnn.stack_bidirectional_dynamic_rnn tensorflow.placeholder tensorflow.constant_initializer tensorflow.logging.info tensorflow.reshape tensorflow.zeros_initializer tensorflow.contrib.rnn.DropoutWrapper tensorflow.variable_scope tensorflow.contrib.rnn.LayerNormBasicLSTMCell tensorflow.squeeze tensorflow.Dimension tensorflow.random_normal_initializer tensorflow.tile tensorflow.contrib.rnn.LSTMStateTuple