class_docstr: Juman tokenizer
  
  Note:
      `spacy.Token._.fstring` is set. The Juman's output is stored into it during tokenizing.
class_docstr_tok: ["Juman","tokenizer","Note",":","`","spacy.Token._.fstring","`","is","set",".","The","Juman","'s","output","is","stored","into","it","during","tokenizing","."]
docstr: Tokenize text with Juman and format the outputs for further processing
docstr_tok: ["Tokenize","text","with","Juman","and","format","the","outputs","for","further","processing"]
code: def _detailed_tokens(self, juman_lines: str) -> List[ShortUnitWord]:
      """Tokenize text with Juman and format the outputs for further processing"""
      from pyknp import MList, Morpheme  # type: ignore
  
      ml: List[Morpheme] = MList(juman_lines).mrph_list()
      words: List[ShortUnitWord] = []
      for m in ml:
          surface: str = m.midasi  # type: ignore
          pos: str = m.hinsi + "," + m.bunrui  # type: ignore
          lemma: str = m.genkei or surface  # type: ignore
          words.append(ShortUnitWord(surface, lemma, pos, m.fstring, ""))  # type: ignore
      return words
  
imports: from dataclasses import dataclass
  from camphr.doc import Doc, UserDataProto
  import itertools
  import distutils.spawn
  from typing import Any, Callable, Dict, Iterator, List, Optional
  from typing_extensions import Literal
  from camphr.serde import SerializationMixin
label_desc: Use image node_id 8 for calling the Tokenizer obj's underlying member method code with example usage: obj._detailed_tokens(juman_lines) and returns: words