docstr: Tokenize text with Mecab and format the outputs for further processing
docstr_tok: ["Tokenize","text","with","Mecab","and","format","the","outputs","for","further","processing"]
code: def detailed_tokens(self, text: str) -> List[ShortUnitWord]:
      """Tokenize text with Mecab and format the outputs for further processing"""
      node: _MecabNode = self.tokenizer.parseToNode(text)  # type: ignore
      node = node.next
      if self.dictionary_type == "unidic":
          lemma_idx = 10
      elif self.dictionary_type == "juman":
          lemma_idx = 4
      else:
          lemma_idx = 6
      words: List[ShortUnitWord] = []
      while node.posid != 0:
          parts: List[str] = node.feature.split(",")
          pos = ",".join(parts[0:4])
          surface = node.surface
          base = parts[lemma_idx] if len(parts) > lemma_idx else surface
          nextnode = node.next
          # TODO: make non-destructive
          if nextnode.length != nextnode.rlength:
              # next node contains space, so attach it to this node.
              words.append(
                  ShortUnitWord(surface, base, pos, " ", node.feature)
              )
          elif nextnode.surface == "\u3000":
              # next node is full space, so attatch it to this node and skip the nextnode.
              words.append(
                  ShortUnitWord(
                      surface, base, pos, nextnode.surface, node.feature
                  )
              )
              nextnode = nextnode.next
          else:
              words.append(
                  ShortUnitWord(surface, base, pos, "", node.feature)
              )
          node = nextnode
      return words
  
file_path: camphr/packages/camphr/camphr/tokenizer/mecab/__init__.py
imports: from camphr.doc import Doc, UserDataProto
  from typing import List, NamedTuple, TYPE_CHECKING
  from camphr.serde import SerDe
  from typing_extensions import Literal, Protocol
label_desc: Use image node_id 5 for calling the Tokenizer obj's underlying member method code with example usage: obj.detailed_tokens(text) and returns: words