class_docstr: Runs basic tokenization with jumanpp morphological parser.
class_docstr_tok: ["Runs","basic","tokenization","with","jumanpp","morphological","parser","."]
docstr: Constructs a JumanppTokenizer.
  
  Args:
      **do_lower_case**: (*optional*) boolean (default True)
          Whether to lowercase the input.
      **never_split**: (*optional*) list of str
          Kept for backward compatibility purposes. Now implemented directly at the base class level (see
          [`PreTrainedTokenizer.tokenize`]) List of tokens not to split.
      **normalize_text**: (*optional*) boolean (default True)
          Whether to apply unicode normalization to text before tokenization.
      **trim_whitespace**: (*optional*) boolean (default False)
          Whether to trim all whitespace, tab, newline from tokens.
docstr_tok: ["Constructs","a","JumanppTokenizer",".","Args",":","*","*","do_lower_case","*","*",":","(","*","optional","*",")","boolean","(","default","True",")","Whether","to","lowercase","the","input",".","*","*","never_split","*","*",":","(","*","optional","*",")","list","of","str","Kept","for","backward","compatibility","purposes",".","Now","implemented","directly","at","the","base","class","level","(","see","[","`","PreTrainedTokenizer.tokenize","`","]",")","List","of","tokens","not","to","split",".","*","*","normalize_text","*","*",":","(","*","optional","*",")","boolean","(","default","True",")","Whether","to","apply","unicode","normalization","to","text","before","tokenization",".","*","*","trim_whitespace","*","*",":","(","*","optional","*",")","boolean","(","default","False",")","Whether","to","trim","all","whitespace",",","tab",",","newline","from","tokens","."]
code: def __init__(
      self,
      do_lower_case=False,
      never_split=None,
      normalize_text=True,
      trim_whitespace=False,
  ):
      """
      Constructs a JumanppTokenizer.
  
      Args:
          **do_lower_case**: (*optional*) boolean (default True)
              Whether to lowercase the input.
          **never_split**: (*optional*) list of str
              Kept for backward compatibility purposes. Now implemented directly at the base class level (see
              [`PreTrainedTokenizer.tokenize`]) List of tokens not to split.
          **normalize_text**: (*optional*) boolean (default True)
              Whether to apply unicode normalization to text before tokenization.
          **trim_whitespace**: (*optional*) boolean (default False)
              Whether to trim all whitespace, tab, newline from tokens.
      """
  
      self.do_lower_case = do_lower_case
      self.never_split = never_split if never_split is not None else []
      self.normalize_text = normalize_text
      self.trim_whitespace = trim_whitespace
  
      try:
          import pyknp
      except ImportError:
          raise ImportError(
              "You need to install pyknp to use JumanppTokenizer. "
              "See https://github.com/ku-nlp/pyknp for installation."
          )
  
      self.juman = pyknp.Juman(jumanpp=True)
  
imports: import collections
  import copy
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 1 to create a new JumanppTokenizer object  with example: obj = JumanppTokenizer(do_lower_case, never_split, normalize_text, trim_whitespace)