class_docstr: Juman tokenizer
  
  Note:
      `spacy.Token._.fstring` is set. The Juman's output is stored into it during tokenizing.
class_docstr_tok: ["Juman","tokenizer","Note",":","`","spacy.Token._.fstring","`","is","set",".","The","Juman","'s","output","is","stored","into","it","during","tokenizing","."]
docstr: Args:
      juman_kwargs: passed to `pyknp.Juman.__init__`
      preprocessor: applied to text before tokenizing. `mojimoji.han_to_zen` is often used.
docstr_tok: ["Args",":","juman_kwargs",":","passed","to","`","pyknp.Juman.__init__","`","preprocessor",":","applied","to","text","before","tokenizing",".","`","mojimoji.han_to_zen","`","is","often","used","."]
code: def __init__(
      self,
      juman_kwargs: Optional[Dict[str, Any]] = None,
      preprocessor: Optional[
          Callable[[str], str]
      ] = han_to_zen_normalize,
  ):
      """
  
      Args:
          juman_kwargs: passed to `pyknp.Juman.__init__`
          preprocessor: applied to text before tokenizing. `mojimoji.han_to_zen` is often used.
      """
  
      juman_kwargs = juman_kwargs or {}
      default_command = get_juman_command()
      assert default_command
      juman_kwargs.setdefault("command", default_command)
  
      self.juman_kwargs = juman_kwargs
      self.preprocessor = preprocessor
      self.set_tokenizer()
  
imports: from dataclasses import dataclass
  from camphr.doc import Doc, UserDataProto
  import itertools
  import distutils.spawn
  from typing import Any, Callable, Dict, Iterator, List, Optional
  from typing_extensions import Literal
  from camphr.serde import SerializationMixin
label_desc: Use image node_id 3 to create a new Tokenizer object from inherited base classes: SerializationMixin with example: obj = Tokenizer(juman_kwargs, preprocessor)