code: def sequence_labeler(
      hidden,
      targets,
      n_targets,
      config,
      pad_id,
      multilabel=False,
      train=False,
      reuse=None,
      lengths=None,
      use_crf=True,
      **kwargs,
  ):
      """
      An Attention based sequence labeler model.
  
      In the case of unidirectional base models such as GPT this model takes the output of the pre-trained model,
      applies an additional randomly initialised multihead attention block, with residuals on top.
      The extra attention is not future masked to allow the model to label sequences based on context in both directions.
      The representations fed into this model are necessarily future masked because a language modelling loss is the
      original objective of the featurizer.
  
      For bidirectional base models we apply the crf model directly to the output of the base model.
  
      :param hidden: The output of the featurizer. [batch_size, sequence_length, embed_dim]
      :param targets: The placeholder representing the sequence labeling targets. [batch_size, sequence_length]
      :param n_targets: A python int containing the number of classes that the model should be learning to predict over.
      :param config: A config object, containing all parameters for the featurizer.
      :param train: If this flag is true, dropout and losses are added to the graph.
      :param reuse: Should reuse be set within this scope.
      :param lengths: The number of non-padding tokens in the input.
      :param kwargs: Spare arguments.
      :return: dict containing:
          "logits": The un-normalised log probabilities of each class being in each location. For usable predictions,
              sampling from this distribution is not sufficient and a viterbi decoding method should be used.
          "losses": The negative log likelihood for the sequence targets.
          "predict_params": A dictionary of params to be fed to the viterbi decode function.
      """
      with tf.compat.v1.variable_scope("sequence-labeler", reuse=reuse):
          if targets is not None:
              targets = tf.cast(targets, dtype=tf.int32)
  
          nx = config.n_embed
  
          def seq_lab_internal(hidden):
              if config.base_model.is_bidirectional:
                  n = hidden
              else:
                  attn_fn = functools.partial(
                      attn,
                      scope="seq_label_attn",
                      n_state=nx,
                      n_head=config.seq_num_heads,
                      resid_pdrop=config.resid_p_drop,
                      attn_pdrop=config.attn_p_drop,
                      train=train,
                      scale=False,
                      mask=False,
                  )
                  n = norm(
                      attn_fn(hidden) + hidden, "seq_label_residual"
                  )
  
              flat_logits = tf.compat.v1.layers.dense(n, n_targets)
              logits = tf.reshape(
                  flat_logits,
                  tf.concat(
                      [tf.shape(input=hidden)[:2], [n_targets]], 0
                  ),
              )
              return logits
  
          with tf.compat.v1.variable_scope("seq_lab_attn"):
              if config.low_memory_mode and train:
                  seq_lab_internal = recompute_grad(
                      seq_lab_internal, use_entire_scope=True
                  )
              logits = seq_lab_internal(hidden)
              logits = tf.cast(
                  logits, tf.float32
              )  # always run the crf in float32
  
          loss = 0.0
  
          default_lengths = tf.shape(input=hidden)[1] * tf.ones(
              tf.shape(input=hidden)[0], dtype=tf.int32
          )
          if lengths is None:
              lengths = default_lengths
  
          class_weights = kwargs.get("class_weights")
  
          with tf.device("CPU:0" if train else logits.device):
              if multilabel:
                  transition_params = []
                  logits_individual = tf.unstack(
                      logits, n_targets, axis=-1
                  )
                  if targets is not None:
                      targets_individual = tf.unstack(
                          targets, n_targets, axis=-1
                      )
                  logits = []
                  for i in range(n_targets):
                      transition_params.append(
                          tf.cast(
                              tf.compat.v1.get_variable(
                                  "Transition_matrix_{}".format(i),
                                  shape=[2, 2],
                              ),
                              tf.float32,
                          )
                      )
                      logits.append(
                          tf.stack(
                              (
                                  logits_individual[pad_id],
                                  logits_individual[i],
                              ),
                              axis=-1,
                          )
                      )
                      if targets is not None and i != pad_id:
                          if class_weights is not None:
                              is_pos_cls = tf.cast(
                                  targets_individual[i],
                                  dtype=tf.float32,
                              )
                              class_weight = tf.expand_dims(
                                  class_weights[i] * is_pos_cls
                                  + class_weights[pad_id]
                                  * (1.0 - is_pos_cls),
                                  -1,
                              )
                              logits_i = class_reweighted_grad(
                                  logits[-1],
                                  class_weight,
                                  norm_grads=config.renorm_after_class_weights,
                              )
                          else:
                              logits_i = logits[i]
                          if use_crf:
                              loss -= crf_log_likelihood(
                                  logits_i,
                                  targets_individual[i],
                                  lengths,
                                  transition_params=transition_params[
                                      -1
                                  ],
                              )[0]
                          else:
                              weights = tf.math.divide_no_nan(
                                  tf.sequence_mask(
                                      lengths,
                                      maxlen=tf.shape(
                                          input=targets_individual[i]
                                      )[1],
                                      dtype=tf.float32,
                                  ),
                                  tf.expand_dims(
                                      tf.cast(lengths, tf.float32), -1
                                  ),
                              )
                              loss += tf.compat.v1.losses.sparse_softmax_cross_entropy(
                                  targets_individual[i],
                                  logits_i,
                                  weights=weights,
                              )
                  logits = tf.stack(logits, axis=-1)
              else:
                  if class_weights is not None and train:
                      class_weights = tf.reshape(
                          class_weights, [1, 1, -1]
                      )
                      one_hot_class_weights = (
                          class_weights
                          * tf.one_hot(targets, depth=n_targets)
                      )
                      per_token_weights = tf.reduce_sum(
                          input_tensor=one_hot_class_weights,
                          axis=-1,
                          keepdims=True,
                      )
                      logits = class_reweighted_grad(
                          logits,
                          per_token_weights,
                          norm_grads=config.renorm_after_class_weights,
                      )
  
                  transition_params = tf.cast(
                      tf.compat.v1.get_variable(
                          "Transition_matrix",
                          shape=[n_targets, n_targets],
                      ),
                      tf.float32,
                  )
                  if targets is not None:
                      if use_crf:
                          log_likelihood, _ = crf_log_likelihood(
                              logits,
                              targets,
                              lengths,
                              transition_params=transition_params,
                          )
                          loss = -log_likelihood
                      else:
                          weights = tf.math.divide_no_nan(
                              tf.sequence_mask(
                                  lengths,
                                  maxlen=tf.shape(input=targets)[1],
                                  dtype=tf.float32,
                              ),
                              tf.expand_dims(
                                  tf.cast(lengths, tf.float32), -1
                              ),
                          )
                          loss = tf.compat.v1.losses.sparse_softmax_cross_entropy(
                              targets, logits, weights=weights
                          )
  
          return {
              "logits": logits,
              "losses": loss,
              "predict_params": {
                  "transition_matrix": transition_params,
                  "sequence_length": lengths,
              },
          }
  
imports: import functools
  import tensorflow
  from tensorflow_addons.text.crf import crf_log_likelihood
  from finetune.base_models.gpt.featurizer import attn, dropout, norm
  from finetune.util.shapes import shape_list, merge_leading_dims
  from finetune.optimizers.recompute_grads import recompute_grad
  from finetune.errors import FinetuneError
  from finetune.nn.activations import act_fns
  from finetune.nn.nn_utils import norm
  from tensorflow.python.framework import function
label_desc: Use image node_id 12 for calling a global function with example usage: sequence_labeler(hidden, targets, n_targets, config, pad_id, multilabel, train, reuse, lengths, use_crf) and returns: dict, logits