 
 ReadDelay = 500 # ms between read() and push(). If 0, we'll push() non-stop.
 MaxNTokens = 500 # tokens send to client
+
+# Builds a list of wanted Tokens, assuming there are only unigrams in the text.
+#
+# We have a special code path when we know it's impossible for the result to
+# contain anything but unigrams. In that case, we assume the maximum number of
+# unique tokens to be <100k, so we just throw every token into one TokenBin
+# and then exclude every invalid token from the result.
+class UnigramTokenListBuilder
+  constructor: (@includeFilters, @excludeFilters) ->
+    @tokenBin = new TokenBin([])
+
+  addDocumentText: (text) ->
+    tokens = tokenize(text)
+    @tokenBin.addTokens(tokens)
+
+  getTokensByFrequency: ->
+    ret = @tokenBin.getTokensByFrequency()
+
+    for filter in @includeFilters
+      ret = (token for token in ret when filter.test(token.name))
+
+    for filter in @excludeFilters
+      ret = (token for token in ret when !filter.test(token.name))
+
+    ret
+
+# Builds a list of wanted Tokens, including ngrams.
+#
+# Since we allow ngrams, we assume there could be an arbitrarily-high number of
+# ngrams. To keep memory under control, we can't store unwanted tokens
+# temporarily, like we would in UnigramTokenListBuilder.
+class NgramTokenListBuilder
+  constructor: (@includeFilters, @excludeFilters) ->
+    @tokenBin = new TokenBin([])
+
+  addDocumentText: (text) ->
+    tokens = tokenize(text)
+    tokensString = tokens.join(' ')
+
+    toAdd = [] # list of all tokens, with repeats
+    toAddSet = {} # token -> null. Ensure when we union we don't count tokens twice
+
+    for filter, filterIndex in @includeFilters
+      moreToAdd = filter.findTokensFromUnigrams(tokensString)
+      if toAdd.length
+        # Remove duplicates: tokens we found in a previous filter
+        moreToAdd = (token for token in moreToAdd when token not of toAddSet)
+      if filterIndex < @includeFilters.length - 1
+        # Remember these tokens for the next filter
+        (toAddSet[token] = null) for token in moreToAdd
+      toAdd = toAdd.concat(moreToAdd)
+
+    @tokenBin.addTokens(toAdd)
+
+  getTokensByFrequency: ->
+    ret = @tokenBin.getTokensByFrequency()
+
+    for filter in @excludeFilters
+      ret = (token for token in ret when !filter.test(token.name))
+
+    ret
 
 # Outputs JSON objects corresponding to the current status.
 #

# Any of the above variables may be unset. If `progress` is unset, that means
# it is 1.0.
#
# Handle errors, too: they do happen.
module.exports = class TokenBinStream extends stream.Readable
  constructor: (@options) ->
    throw new Error('Must pass options.server, the Overview server base URL') if !@options.server

  #
  # Callers should call this when the client stops listening.
  destroy: ->
    @destroyed = true
    @stream?.abort() # It might be the case that @stream was never created

    @_clearReadTimeout() # No more reads, please
