anchor.diff:                  yield Choice1Of2 (regexIndex, token)
   
               | None ->
  -                // If there are any pending symbols, reject the first one and restart tokenizing.
  -                if not <| Queue.isEmpty tokenizerState.Pending then
  -                    let rejectedSymbol, pending = Queue.dequeue tokenizerState.Pending
  -                    yield Choice2Of2 [| rejectedSymbol |]
  -
  -                    // Move the remaining pending symbols into the buffer.
  +                // If the last final DFA state is the initial DFA state and there
  +                // are some symbols we could still parse, reject the first pending
  +                // symbol and restart tokenizing.
  +                if tokenizerState.LastFinalState = dfa.InitialState then
  +                    if not <| Queue.isEmpty tokenizerState.Pending then
  +                        let rejectedSymbol, pending = Queue.dequeue tokenizerState.Pending
  +                        yield Choice2Of2 [| rejectedSymbol |]
  +
  +                        // Move the remaining pending symbols into the buffer.
  +                        let tokenizerState =
  +                            { tokenizerState with
  +                                Pending = Queue.empty;
  +                                Buffer = pending; }
  +
  +                        // Start matching again at the initial DFA state.
  +                        let (), tokenizerState = TokenizerState.transition dfa.InitialState tokenizerState
  +                        let (), tokenizerState = TokenizerState.checkpoint tokenizerState
  +                        yield! tokenize symbolEnumerator tokenizerState
  +                else
  +                    // Emit the current token (set at the last-encountered final state).
  +                    let token, tokenizerState = TokenizerState.accept tokenizerState
  +                    let regexIndex = Map.find tokenizerState.LastFinalState dfa.FinalStates
  +                    yield Choice1Of2 (regexIndex, token)
  +
  +                    // Move the pending symbols into the buffer.
  +                    // They are placed at the front of the buffer to keep
  +                    // the symbols in the same order as they were in the symbol stream.
                       let tokenizerState =
                           { tokenizerState with
                               Pending = Queue.empty;
  -                            Buffer = pending; }
  -
  -                    // Start matching again at the initial DFA state.
  +                            Buffer =
  +                                // OPTIMIZE : Implement a Queue.fold function for this.
  +                                let buffer = Queue.toArray tokenizerState.Buffer
  +                                (tokenizerState.Pending, buffer)
  +                                ||> Array.fold (fun buffer el ->
  +                                    Queue.enqueue el buffer); }
  +                    
  +                    // Start tokenizing again at the initial DFA state.
                       let (), tokenizerState = TokenizerState.transition dfa.InitialState tokenizerState
  -                    let (), tokenizerState = TokenizerState.checkpoint tokenizerState
  +                    let (), tokenizerState = TokenizerState.markFinalState tokenizerState
                       yield! tokenize symbolEnumerator tokenizerState
   
           | Some symbol ->
  
positive.code:                     let tokenizerState =
                          { tokenizerState with
                              Pending = Queue.empty;
                              Buffer = 
                                  // OPTIMIZE : Implement a Queue.fold function for this.
                                  let pending = Queue.toArray tokenizerState.Pending
                                  (tokenizerState.Buffer, pending)
                                  ||> Array.fold (fun buffer el ->
                                      Queue.enqueue el buffer); }
                      
                      // Start tokenizing again at the initial DFA state.
                      let (), tokenizerState = TokenizerState.transition dfa.InitialState tokenizerState
                      let (), tokenizerState = TokenizerState.checkpoint tokenizerState
                      yield! tokenize symbolEnumerator tokenizerState
  
              | Some transitionTarget ->
                  // Add this symbol to the queue of pending symbols.
                  let (), tokenizerState = TokenizerState.addPending symbol tokenizerState
  
                  // If the current DFA state is a final (accepting) state,
                  // mark it as the last final state.
                  let (), tokenizerState =
                      if Map.containsKey tokenizerState.CurrentState dfa.FinalStates then
                          TokenizerState.checkpoint tokenizerState
                      else
                          (), tokenizerState
  
                  // Set the transition target as the current DFA state,
                  // then recurse to continue tokenizing.
                  let (), tokenizerState = TokenizerState.transition transitionTarget tokenizerState
                  yield! tokenize symbolEnumerator tokenizerState
      }
  
      // Run the tokenize function.
  
negative.code:             LastFinalState = initialState; }
  
      /// Marks the current DFA state as the last final DFA state visited
      /// by the tokenizer. This function should only be called when the
      /// current DFA state is a final (accepting) state.
      let markFinalState (tokenizerState : TokenizerState<'Symbol>) =
          (),
          { tokenizerState with
              LastFinalState = tokenizerState.CurrentState; }
  
      /// Accepts any symbols pending acceptance.
      let acceptPending (tokenizerState : TokenizerState<'Symbol>) =
          (),
          { tokenizerState with
              Accepted =
  