class_docstr: Simultaneous Translation
  Text agent for Japanese
class_docstr_tok: ["Simultaneous","Translation","Text","agent","for","Japanese"]
code: def units_to_segment(self, units, states):
      # Merge sub words (units) to full word (segment).
      # For Japanese, we can directly send
      # the untokenized token to server except the BOS token
      # with following option
      # --sacrebleu-tokenizer MeCab
      # --eval-latency-unit char
      # --no-space
      token = units.value.pop()
  
      if (
          token == self.dict["tgt"].eos_word
          or len(states.segments.target) > self.max_len
      ):
          return DEFAULT_EOS
  
      if BOS_PREFIX == token:
          return None
      if token[0] == BOS_PREFIX:
          return token[1:]
      else:
          return token
  
file_path: fairseq/examples/simultaneous_translation/eval/agents/simul_t2t_enja.py
imports: import os
  from fairseq import checkpoint_utils, tasks
  import sentencepiece
  import torch
label_desc: Use image node_id 10 for calling the SimulTransTextAgentJA obj's underlying member method code with example usage: obj.units_to_segment(units, states) and returns: DEFAULT_EOS, None, token, token