class_docstr: Runs basic tokenization with jumanpp morphological parser.
class_docstr_tok: ["Runs","basic","tokenization","with","jumanpp","morphological","parser","."]
docstr: Tokenizes a piece of text.
docstr_tok: ["Tokenizes","a","piece","of","text","."]
code: def tokenize(self, text, never_split=None, **kwargs):
      """Tokenizes a piece of text."""
      if self.normalize_text:
          text = unicodedata.normalize("NFKC", text)
  
      never_split = self.never_split + (
          never_split if never_split is not None else []
      )
      tokens = []
  
      for mrph in self.juman.analysis(text).mrph_list():
          token = mrph.midasi
  
          if self.do_lower_case and token not in never_split:
              token = token.lower()
  
          if self.trim_whitespace:
              if token.strip() == "":
                  continue
              else:
                  token = token.strip()
  
          tokens.append(token)
  
      return tokens
  
imports: import collections
  import copy
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 2 for calling the JumanppTokenizer obj's underlying member method code with example usage: obj.tokenize(text, never_split) and returns: tokens