code: def gpt2_featurizer(
      X, encoder, config, train=False, reuse=None, **kwargs
  ):
      initial_shape = tf.shape(input=X)
      X = tf.reshape(X, shape=tf.concat(([-1], initial_shape[-1:]), 0))
      X.set_shape([None, None])
  
      pos_values = get_pos_values(
          tf.shape(input=X)[1], encoder.vocab_size
      )
      X = tf.stack(
          (X, tf.tile(pos_values, [tf.shape(input=X)[0], 1])), 2
      )
  
      with tf.compat.v1.variable_scope("model/featurizer", reuse=reuse):
          embed_weights = tf.compat.v1.get_variable(
              name="we",
              shape=[
                  encoder.vocab_size + config.max_length,
                  config.n_embed,
              ],
              initializer=tf.compat.v1.random_normal_initializer(
                  stddev=config.weight_stddev
              ),
          )
          if config.train_embeddings:
              embed_weights = dropout(
                  embed_weights, config.embed_p_drop, train
              )
          else:
              embed_weights = tf.stop_gradient(embed_weights)
          h = embed(X, embed_weights)
  
          # Transformer
          pasts = [None] * config.n_layer
          for layer, past in enumerate(pasts):
              if (
                  (config.n_layer - layer) == config.num_layers_trained
                  and config.num_layers_trained != config.n_layer
              ):
                  h = tf.stop_gradient(h)
                  train_layer = False
              else:
                  train_layer = train
  
              with tf.compat.v1.variable_scope("h%d" % layer):
                  block_fn = functools.partial(
                      block, past=past, hparams=config, train=train
                  )
                  if config.low_memory_mode and train_layer:
                      block_fn = recompute_grad(
                          block_fn, use_entire_scope=True
                      )
                  h = block_fn(h)
  
          h = norm(h, "ln_f")
  
          # Use hidden state at classifier token as input to final proj. + softmax
          clf_h = tf.reshape(
              h, [-1, config.n_embed]
          )  # [batch * seq_len, embed]
          clf_token = encoder["_classify_"]
          pool_idx = tf.cast(
              tf.argmax(
                  input=tf.cast(
                      tf.equal(X[:, :, 0], clf_token), tf.float32
                  ),
                  axis=1,
              ),
              tf.int32,
          )
          batch, length, _ = shape_list(X)
          clf_h = tf.gather(
              clf_h,
              tf.range(batch, dtype=tf.int32) * length + pool_idx,
          )
          clf_h = tf.reshape(
              clf_h,
              shape=tf.concat(
                  (initial_shape[:-1], [config.n_embed]), 0
              ),
          )
          seq_feats = tf.reshape(
              h, shape=tf.concat((initial_shape, [config.n_embed]), 0)
          )
  
          lengths = lengths_from_eos_idx(
              eos_idx=pool_idx, max_length=shape_list(X)[0]
          )
  
          return {
              "embed_weights": embed_weights,
              "features": clf_h,
              "sequence_features": seq_feats,
              "eos_idx": pool_idx,
              "length": lengths,
          }
  
file_path: finetune/finetune/base_models/gpt2/featurizer.py
imports: import functools
  import tensorflow
  from finetune.util.shapes import shape_list, lengths_from_eos_idx
  from finetune.optimizers.recompute_grads import recompute_grad
  from finetune.nn.activations import gelu
  from finetune.base_models.gpt.featurizer import norm, dropout, get_pos_values
label_desc: Use image node_id 13 for calling a global function with example usage: gpt2_featurizer(X, encoder, config, train, reuse) and returns: dict