class_docstr: Re-implementation of Tercom Tokenizer in Python 3.
  
  See src/ter/core/Normalizer.java in https://github.com/jhclark/tercom Note that Python doesn't support named Unicode
  blocks so the mapping for relevant blocks was taken from here: https://unicode-table.com/en/blocks/
  
  This implementation follows the implementation from
  https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_ter.py.
docstr: Apply a different tokenization techniques according.
  
  Args:
      sentence: An input sentence to pre-process and tokenize.
  
  Return:
      A tokenized and pre-processed sentence.
docstr_tok: ["Apply","a","different","tokenization","techniques","according",".","Args",":","sentence",":","An","input","sentence","to","pre-process","and","tokenize",".","Return",":","A","tokenized","and","pre-processed","sentence","."]
code: def __call__(self, sentence: str) -> str:
      """Apply a different tokenization techniques according.
  
      Args:
          sentence: An input sentence to pre-process and tokenize.
  
      Return:
          A tokenized and pre-processed sentence.
  
      """
      if not sentence:
          return ""
  
      if self.lowercase:
          sentence = sentence.lower()
  
      if self.normalize:
          sentence = self._normalize_general_and_western(sentence)
          if self.asian_support:
              sentence = self._normalize_asian(sentence)
  
      if self.no_punctuation:
          sentence = self._remove_punct(sentence)
          if self.asian_support:
              sentence = self._remove_asian_punct(sentence)
  
      # Strip extra whitespaces
      return " ".join(sentence.split())
  
imports: import re
  from functools import lru_cache
  from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Type, Union
  from torch import Tensor, tensor
  from torchmetrics.functional.text.helper import _flip_trace, _LevenshteinEditDistance, _trace_to_alignment, _validate_inputs
label_desc: Use image node_id 2 for calling the _TercomTokenizer obj's underlying member method code with example usage: obj.__call__(sentence) and returns: str, str