anchor.diff:              | 0.0 -> None
               | cutOffValue -> Some(float32 <| cutOffValue, totalDocs.GetMaxScore())
           
  -        let highlighterOptions = 
  -            if notNull searchQuery.Highlights then 
  -                match searchQuery.Highlights.HighlightedFields with
  -                | x when x.Length = 1 -> 
  -                    match indexWriter.Settings.Fields.TryGetValue(x.First()) with
  -                    | (true, field) -> 
  -                        let htmlFormatter = 
  -                            new SimpleHTMLFormatter(searchQuery.Highlights.PreTag, searchQuery.Highlights.PostTag)
  -                        Some(field, new Highlighter(htmlFormatter, new QueryScorer(query)))
  -                    | _ -> None
  -                | _ -> None
  -            else None
  -        
  -        let inline getHighlighter (document : LuceneDocument, shardIndex, doc) = 
  -            if highlighterOptions.IsSome then 
  -                let (field, highlighter) = highlighterOptions.Value
  -                let text = document.Get(field.SchemaName)
  -                if text <> null then 
  -                    let tokenStream = 
  -                        TokenSources.GetAnyTokenStream
  -                            (indexSearchers.[shardIndex].IndexReader, doc, field.SchemaName, 
  -                             indexWriter.Settings.SearchAnalyzer)
  -                    let frags = 
  -                        highlighter.GetBestTextFragments
  -                            (tokenStream, text, false, searchQuery.Highlights.FragmentsToReturn)
  -                    frags
  -                    |> Array.filter (fun frag -> notNull (frag) && frag.GetScore() > float32 (0.0))
  -                    |> Array.map (fun frag -> frag.ToString())
  -                else Array.empty<string>
  -            else Array.empty<string>
  -        
  -        let processDocument (hit : ScoreDoc, document : LuceneDocument) = 
  -            let timeStamp = 
  -                let t = document.Get(indexWriter.GetSchemaName(TimeStampField.Name))
  -                if isNull t then 0L
  -                else int64 t
  -            
  -            let fields = getDocument (indexWriter, searchQuery, document)
  -            let resultDoc = new Model.Document()
  -            resultDoc.Id <- document.Get(indexWriter.GetSchemaName(IdField.Name))
  -            resultDoc.IndexName <- indexWriter.Settings.IndexName
  -            resultDoc.TimeStamp <- timeStamp
  -            resultDoc.Fields <- fields
  -            resultDoc.Score <- if searchQuery.ReturnScore then float (hit.Score)
  -                               else 0.0
  -            resultDoc.Highlights <- getHighlighter (document, hit.ShardIndex, hit.Doc)
  -            resultDoc
  -        
  -        let distinctByFilter (document : LuceneDocument) = 
  -            match distinctBy with
  -            | Some(field, hashSet) -> 
  -                let distinctByValue = document.Get(indexWriter.GetSchemaName(field.FieldName))
  -                if notNull distinctByValue && hashSet.Add(distinctByValue) then Some(document)
  -                else None
  -            | None -> Some(document)
  -        
  -        let cutOffFilter (hit : ScoreDoc) (document : LuceneDocument option) = 
  -            match cutOff with
  -            | Some(cutOffValue, maxScore) -> 
  -                if (hit.Score / maxScore * 100.0f >= cutOffValue) then document
  -                else None
  -            | None -> document
  -        
  -        // Start composing the seach results
  +        let highlighterOptions = state |> getHighlightOptions
  +        
  +        // Start composing the search results
           let results = 
               seq { 
                   for i = searchQuery.Skip to hits.Length - 1 do
                       let hit = hits.[i]
  -                    let document = indexSearchers.[hit.ShardIndex].IndexSearcher.Doc(hit.Doc)
  -                    let result = distinctByFilter document |> cutOffFilter hit
  +                    let document = state.GetDoc(hit.ShardIndex, hit.Doc)
  +                    
  +                    let result = 
  +                        state
  +                        |> distinctByFilter document distinctBy
  +                        |> cutOffFilter hit cutOff
                       match result with
  -                    | Some(doc) -> yield processDocument (hit, document)
  +                    | Some(doc) -> yield state |> processDocument (hit, document, highlighterOptions)
                       | None -> ()
  -                // Dispose the searchers
  -                for i in 0..indexSearchers.Length - 1 do
  -                    (indexSearchers.[i] :> IDisposable).Dispose()
  +                (state :> IDisposable).Dispose()
               }
  -        
           new SearchResults(RecordsReturned = recordsReturned, BestScore = totalDocs.GetMaxScore(), 
                             TotalAvailable = totalAvailable, Documents = results.ToArray())
  
positive.code: // the search time filters. 
  [<AutoOpen>]
  module Searcher = 
      /// Returns a document from the index
      let getDocument (indexWriter : IndexWriter, search : SearchQuery, document : LuceneDocument) = 
          let fields = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
          
          let getValue (field : FieldSchema) = 
              let value = document.Get(field.SchemaName)
              if notNull value then 
                  if value = Constants.StringDefaultValue && search.ReturnEmptyStringForNull then 
                      fields.Add(field.FieldName, String.Empty)
                  else fields.Add(field.FieldName, value)
          match search.Columns with
          // Return no other columns when nothing is passed
          | _ when search.Columns.Length = 0 -> ()
          // Return all columns when *
          | _ when search.Columns.First() = "*" -> 
              for field in indexWriter.Settings.Fields do
                  if [ IdField.Name; TimeStampField.Name; ModifyIndexField.Name; StateField.Name ] 
                     |> Seq.contains field.FieldName then ()
                  else getValue (field)
          // Return only the requested columns
          | _ -> 
              for fieldName in search.Columns do
                  match indexWriter.Settings.Fields.TryGetValue(fieldName) with
                  | (true, field) -> getValue (field)
                  | _ -> ()
          fields
      
      let search (indexWriter : IndexWriter, query : Query, searchQuery : SearchQuery) = 
          (!>) "Input Query:%s \nGenerated Query : %s" (searchQuery.QueryString) (query.ToString())
          let indexSearchers = indexWriter |> IndexWriter.getRealTimeSearchers
          // Each thread only works on a separate part of the array and as no parts are shared across
          // multiple threads the below variables are thread safe. The cost of using blocking collection vs. 
          // array per search is high
          let topDocsCollection : TopFieldDocs array = Array.zeroCreate indexSearchers.Length
          
          let sortOrder = 
              match searchQuery.OrderByDirection with
              | Constants.OrderByDirection.Ascending -> false
              | _ -> true
          
          let sort = 
              match searchQuery.OrderBy with
              | null -> Sort.RELEVANCE
              | _ -> 
                  match indexWriter.Settings.Fields.TryGetValue(searchQuery.OrderBy) with
                  | (true, field) -> 
                      if field |> FieldSchema.hasDocValues then 
                          new Sort(new SortField(field.SchemaName, field.FieldType.SortFieldType, sortOrder))
                      else Sort.RELEVANCE
                  | _ -> Sort.RELEVANCE
          
          let distinctBy = 
              if not <| String.IsNullOrWhiteSpace(searchQuery.DistinctBy) then 
                  match indexWriter.Settings.Fields.TryGetValue(searchQuery.DistinctBy) with
                  | true, field -> 
                      match field |> FieldSchema.isTokenized with
                      | false -> Some(field, new HashSet<string>(StringComparer.OrdinalIgnoreCase))
                      | true -> None
                  | _ -> None
              else None
          
          let count = 
              match searchQuery.Count with
  
negative.code:         let searchShard (x : ShardWriter) = 
              // This is to enable proper sorting
              let topFieldCollector = TopFieldCollector.Create(sort, count, null, true, true, true)
              state.IndexSearchers.[x.ShardNo].IndexSearcher.Search(query, topFieldCollector)
              topDocsCollection.[x.ShardNo] <- topFieldCollector.TopDocs()
          
          indexWriter.ShardWriters |> Array.Parallel.iter searchShard
  