class_docstr: Splits token text into N-grams.
  
  >>> rext = RegexTokenizer()
  >>> stream = rext("hello there")
  >>> ngf = NgramFilter(4)
  >>> [token.text for token in ngf(stream)]
  ["hell", "ello", "ther", "here"]
class_docstr_tok: ["Splits","token","text","into","N-grams",".",">",">",">","rext","=","RegexTokenizer","(",")",">",">",">","stream","=","rext","(","``","hello","there","''",")",">",">",">","ngf","=","NgramFilter","(","4",")",">",">",">","[","token.text","for","token","in","ngf","(","stream",")","]","[","``","hell","''",",","``","ello","''",",","``","ther","''",",","``","here","''","]"]
code: def __call__(self, tokens):
      assert hasattr(tokens, "__iter__")
      at = self.at
      for t in tokens:
          text = t.text
          if len(text) < self.min:
              continue
  
          chars = t.chars
          if chars:
              startchar = t.startchar
          # Token positions don't mean much for N-grams,
          # so we'll leave the token's original position
          # untouched.
  
          if t.mode == "query":
              size = min(self.max, len(t.text))
              if at == -1:
                  t.text = text[:size]
                  if chars:
                      t.endchar = startchar + size
                  yield t
              elif at == 1:
                  t.text = text[0 - size :]
                  if chars:
                      t.startchar = t.endchar - size
                  yield t
              else:
                  for start in xrange(0, len(text) - size + 1):
                      t.text = text[start : start + size]
                      if chars:
                          t.startchar = startchar + start
                          t.endchar = startchar + start + size
                      yield t
          else:
              if at == -1:
                  limit = min(self.max, len(text))
                  for size in xrange(self.min, limit + 1):
                      t.text = text[:size]
                      if chars:
                          t.endchar = startchar + size
                      yield t
  
              elif at == 1:
                  if chars:
                      original_startchar = t.startchar
                  start = max(0, len(text) - self.max)
                  for i in xrange(start, len(text) - self.min + 1):
                      t.text = text[i:]
                      if chars:
                          t.startchar = original_startchar + i
                      yield t
              else:
                  for start in xrange(0, len(text) - self.min + 1):
                      for size in xrange(self.min, self.max + 1):
                          end = start + size
                          if end > len(text):
                              continue
  
                          t.text = text[start:end]
  
                          if chars:
                              t.startchar = startchar + start
                              t.endchar = startchar + end
  
                          yield t
  
file_path: whoosh/src/whoosh/analysis/ngrams.py
imports: from whoosh.compat import text_type
  from whoosh.compat import xrange
  from whoosh.analysis.acore import Token
  from whoosh.analysis.filters import Filter, LowercaseFilter
  from whoosh.analysis.tokenizers import Tokenizer, RegexTokenizer
label_desc: Use image node_id 3 for calling the NgramFilter obj's underlying member method code with example usage: obj.__call__(tokens) without return types