class_docstr: Re-implementation of Tercom Tokenizer in Python 3.
  
  See src/ter/core/Normalizer.java in https://github.com/jhclark/tercom Note that Python doesn't support named Unicode
  blocks so the mapping for relevant blocks was taken from here: https://unicode-table.com/en/blocks/
  
  This implementation follows the implementation from
  https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_ter.py.
docstr: Split Chinese chars and Japanese kanji down to character level.
docstr_tok: ["Split","Chinese","chars","and","Japanese","kanji","down","to","character","level","."]
code: def _normalize_asian(
      cls: Type["_TercomTokenizer"], sentence: str
  ) -> str:
      """Split Chinese chars and Japanese kanji down to character level."""
      # 4E00—9FFF CJK Unified Ideographs
      # 3400—4DBF CJK Unified Ideographs Extension A
      sentence = re.sub(
          r"([\u4e00-\u9fff\u3400-\u4dbf])", r" \1 ", sentence
      )
      # 31C0—31EF CJK Strokes
      # 2E80—2EFF CJK Radicals Supplement
      sentence = re.sub(
          r"([\u31c0-\u31ef\u2e80-\u2eff])", r" \1 ", sentence
      )
      # 3300—33FF CJK Compatibility
      # F900—FAFF CJK Compatibility Ideographs
      # FE30—FE4F CJK Compatibility Forms
      sentence = re.sub(
          r"([\u3300-\u33ff\uf900-\ufaff\ufe30-\ufe4f])",
          r" \1 ",
          sentence,
      )
      # 3200—32FF Enclosed CJK Letters and Months
      sentence = re.sub(r"([\u3200-\u3f22])", r" \1 ", sentence)
      # Split Hiragana, Katakana, and KatakanaPhoneticExtensions
      # only when adjacent to something else
      # 3040—309F Hiragana
      # 30A0—30FF Katakana
      # 31F0—31FF Katakana Phonetic Extensions
      sentence = re.sub(
          r"(^|^[\u3040-\u309f])([\u3040-\u309f]+)(?=$|^[\u3040-\u309f])",
          r"\1 \2 ",
          sentence,
      )
      sentence = re.sub(
          r"(^|^[\u30a0-\u30ff])([\u30a0-\u30ff]+)(?=$|^[\u30a0-\u30ff])",
          r"\1 \2 ",
          sentence,
      )
      sentence = re.sub(
          r"(^|^[\u31f0-\u31ff])([\u31f0-\u31ff]+)(?=$|^[\u31f0-\u31ff])",
          r"\1 \2 ",
          sentence,
      )
  
      sentence = re.sub(cls._ASIAN_PUNCTUATION, r" \1 ", sentence)
      return re.sub(cls._FULL_WIDTH_PUNCTUATION, r" \1 ", sentence)
  
imports: import re
  from functools import lru_cache
  from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Type, Union
  from torch import Tensor, tensor
  from torchmetrics.functional.text.helper import _flip_trace, _LevenshteinEditDistance, _trace_to_alignment, _validate_inputs
label_desc: Use image node_id 4 for calling the _TercomTokenizer obj's underlying member method code with example usage: obj._normalize_asian(cls, sentence) and returns: re