class_docstr: Tokenizes and translates text according to a character mapping object.
  Characters that map to None are considered token break characters. For all
  other characters the map is used to translate the character. This is useful
  for case and accent folding.
  
  This tokenizer loops character-by-character and so will likely be much
  slower than :class:`RegexTokenizer`.
  
  One way to get a character mapping object is to convert a Sphinx charset
  table file using :func:`whoosh.support.charset.charset_table_to_dict`.
  
  >>> from whoosh.support.charset import charset_table_to_dict
  >>> from whoosh.support.charset import default_charset
  >>> charmap = charset_table_to_dict(default_charset)
  >>> chtokenizer = CharsetTokenizer(charmap)
  >>> [t.text for t in chtokenizer(u'Stra\xdfe ABC')]
  [u'strase', u'abc']
  
  The Sphinx charset table format is described at
  http://www.sphinxsearch.com/docs/current.html#conf-charset-table.
docstr: :param value: The unicode string to tokenize.
  :param positions: Whether to record token positions in the token.
  :param chars: Whether to record character offsets in the token.
  :param start_pos: The position number of the first token. For example,
      if you set start_pos=2, the tokens will be numbered 2,3,4,...
      instead of 0,1,2,...
  :param start_char: The offset of the first character of the first
      token. For example, if you set start_char=2, the text "aaa bbb"
      will have chars (2,5),(6,9) instead (0,3),(4,7).
  :param tokenize: if True, the text should be tokenized.
docstr_tok: [":","param","value",":","The","unicode","string","to","tokenize",".",":","param","positions",":","Whether","to","record","token","positions","in","the","token",".",":","param","chars",":","Whether","to","record","character","offsets","in","the","token",".",":","param","start_pos",":","The","position","number","of","the","first","token",".","For","example",",","if","you","set","start_pos=2",",","the","tokens","will","be","numbered","2,3,4",",","...","instead","of","0,1,2",",","...",":","param","start_char",":","The","offset","of","the","first","character","of","the","first","token",".","For","example",",","if","you","set","start_char=2",",","the","text","``","aaa","bbb","''","will","have","chars","(","2,5",")",",","(","6,9",")","instead","(","0,3",")",",","(","4,7",")",".",":","param","tokenize",":","if","True",",","the","text","should","be","tokenized","."]
code: def __call__(
      self,
      value,
      positions=False,
      chars=False,
      keeporiginal=False,
      removestops=True,
      start_pos=0,
      start_char=0,
      tokenize=True,
      mode="",
      **kwargs,
  ):
      """
      :param value: The unicode string to tokenize.
      :param positions: Whether to record token positions in the token.
      :param chars: Whether to record character offsets in the token.
      :param start_pos: The position number of the first token. For example,
          if you set start_pos=2, the tokens will be numbered 2,3,4,...
          instead of 0,1,2,...
      :param start_char: The offset of the first character of the first
          token. For example, if you set start_char=2, the text "aaa bbb"
          will have chars (2,5),(6,9) instead (0,3),(4,7).
      :param tokenize: if True, the text should be tokenized.
      """
  
      assert isinstance(value, text_type), "%r is not unicode" % value
  
      t = Token(
          positions, chars, removestops=removestops, mode=mode, **kwargs
      )
      if not tokenize:
          t.original = t.text = value
          t.boost = 1.0
          if positions:
              t.pos = start_pos
          if chars:
              t.startchar = start_char
              t.endchar = start_char + len(value)
          yield t
      else:
          text = u("")
          charmap = self.charmap
          pos = start_pos
          startchar = currentchar = start_char
          for char in value:
              tchar = charmap[ord(char)]
              if tchar:
                  text += tchar
              else:
                  if currentchar > startchar:
                      t.text = text
                      t.boost = 1.0
                      if keeporiginal:
                          t.original = t.text
                      if positions:
                          t.pos = pos
                          pos += 1
                      if chars:
                          t.startchar = startchar
                          t.endchar = currentchar
                      yield t
                  startchar = currentchar + 1
                  text = u("")
  
              currentchar += 1
  
          if currentchar > startchar:
              t.text = value[startchar:currentchar]
              t.boost = 1.0
              if keeporiginal:
                  t.original = t.text
              if positions:
                  t.pos = pos
              if chars:
                  t.startchar = startchar
                  t.endchar = currentchar
              yield t
  
imports: from whoosh.compat import u, text_type
  from whoosh.analysis.acore import Composable, Token
  from whoosh.util.text import rcompile
label_desc: Use image node_id 3 for calling the CharsetTokenizer obj's underlying member method code with example usage: obj.__call__(value, positions, chars, keeporiginal, removestops, start_pos, start_char, tokenize, mode) without return types