code: def process_request(x, seq):
      global tokenizer
  
      ctx, cont = x
  
      ctx_tokens = tokenizer.encode(
          "<|endoftext|>" + ftfy.fix_text(ctx, normalization="NFKC")
      )
      cont_tokens = tokenizer.encode(
          ftfy.fix_text(cont, normalization="NFKC")
      )
  
      all_tokens = ctx_tokens + cont_tokens
      all_tokens = np.array(all_tokens)[
          -seq:
      ]  # truncate sequence at seq length
  
      provided_ctx = len(all_tokens) - 1
      pad_amount = seq - provided_ctx
  
      return {
          "obs": np.pad(
              all_tokens[:-1], ((0, pad_amount),), constant_values=50256
          ),
          "target": np.pad(
              all_tokens[1:], ((0, pad_amount),), constant_values=50256
          ),
          "ctx_length": seq,
          "eval_mask": np.logical_and(
              np.arange(0, seq)
              > len(all_tokens) - len(cont_tokens) - 2,
              np.arange(0, seq) < len(all_tokens) - 1,
          ),
      }
  
file_path: mesh-transformer-jax/tasks/eval_harness.py
imports: from functools import partial
  import transformers
  from lm_eval.base import LM
  from tqdm import tqdm
  import numpy
  from tasks.util import sample_batch, shrink_seq
  import multiprocessing
  import ftfy
label_desc: Use image node_id 2 for calling a global function with example usage: process_request(x, seq) and returns: dict