class_docstr: Uses a regular expression to extract tokens from text.
  
  >>> rex = RegexTokenizer()
  >>> [token.text for token in rex(u("hi there 3.141 big-time under_score"))]
  ["hi", "there", "3.141", "big", "time", "under_score"]
docstr: :param value: The unicode string to tokenize.
  :param positions: Whether to record token positions in the token.
  :param chars: Whether to record character offsets in the token.
  :param start_pos: The position number of the first token. For example,
      if you set start_pos=2, the tokens will be numbered 2,3,4,...
      instead of 0,1,2,...
  :param start_char: The offset of the first character of the first
      token. For example, if you set start_char=2, the text "aaa bbb"
      will have chars (2,5),(6,9) instead (0,3),(4,7).
  :param tokenize: if True, the text should be tokenized.
docstr_tok: [":","param","value",":","The","unicode","string","to","tokenize",".",":","param","positions",":","Whether","to","record","token","positions","in","the","token",".",":","param","chars",":","Whether","to","record","character","offsets","in","the","token",".",":","param","start_pos",":","The","position","number","of","the","first","token",".","For","example",",","if","you","set","start_pos=2",",","the","tokens","will","be","numbered","2,3,4",",","...","instead","of","0,1,2",",","...",":","param","start_char",":","The","offset","of","the","first","character","of","the","first","token",".","For","example",",","if","you","set","start_char=2",",","the","text","``","aaa","bbb","''","will","have","chars","(","2,5",")",",","(","6,9",")","instead","(","0,3",")",",","(","4,7",")",".",":","param","tokenize",":","if","True",",","the","text","should","be","tokenized","."]
code: def __call__(
      self,
      value,
      positions=False,
      chars=False,
      keeporiginal=False,
      removestops=True,
      start_pos=0,
      start_char=0,
      tokenize=True,
      mode="",
      **kwargs,
  ):
      """
      :param value: The unicode string to tokenize.
      :param positions: Whether to record token positions in the token.
      :param chars: Whether to record character offsets in the token.
      :param start_pos: The position number of the first token. For example,
          if you set start_pos=2, the tokens will be numbered 2,3,4,...
          instead of 0,1,2,...
      :param start_char: The offset of the first character of the first
          token. For example, if you set start_char=2, the text "aaa bbb"
          will have chars (2,5),(6,9) instead (0,3),(4,7).
      :param tokenize: if True, the text should be tokenized.
      """
  
      assert isinstance(value, text_type), "%s is not unicode" % repr(
          value
      )
  
      t = Token(
          positions, chars, removestops=removestops, mode=mode, **kwargs
      )
      if not tokenize:
          t.original = t.text = value
          t.boost = 1.0
          if positions:
              t.pos = start_pos
          if chars:
              t.startchar = start_char
              t.endchar = start_char + len(value)
          yield t
      elif not self.gaps:
          # The default: expression matches are used as tokens
          for pos, match in enumerate(self.expression.finditer(value)):
              t.text = match.group(0)
              t.boost = 1.0
              if keeporiginal:
                  t.original = t.text
              t.stopped = False
              if positions:
                  t.pos = start_pos + pos
              if chars:
                  t.startchar = start_char + match.start()
                  t.endchar = start_char + match.end()
              yield t
      else:
          # When gaps=True, iterate through the matches and
          # yield the text between them.
          prevend = 0
          pos = start_pos
          for match in self.expression.finditer(value):
              start = prevend
              end = match.start()
              text = value[start:end]
              if text:
                  t.text = text
                  t.boost = 1.0
                  if keeporiginal:
                      t.original = t.text
                  t.stopped = False
                  if positions:
                      t.pos = pos
                      pos += 1
                  if chars:
                      t.startchar = start_char + start
                      t.endchar = start_char + end
  
                  yield t
  
              prevend = match.end()
  
          # If the last "gap" was before the end of the text,
          # yield the last bit of text as a final token.
          if prevend < len(value):
              t.text = value[prevend:]
              t.boost = 1.0
              if keeporiginal:
                  t.original = t.text
              t.stopped = False
              if positions:
                  t.pos = pos
              if chars:
                  t.startchar = prevend
                  t.endchar = len(value)
              yield t
  
imports: from whoosh.compat import u, text_type
  from whoosh.analysis.acore import Composable, Token
  from whoosh.util.text import rcompile
label_desc: Use image node_id 3 for calling the RegexTokenizer obj's underlying member method code with example usage: obj.__call__(value, positions, chars, keeporiginal, removestops, start_pos, start_char, tokenize, mode) without return types