anchor.diff:     | EOL
      | Token of TokenInformation
   
  -let tokenize (content:string) =
  -    seq { let sourceTokenizer = SourceTokenizer([], "/tmp.fsx")
  -          let lines = content.Replace("\r\n","\n").Split('\r', '\n')
  -          let lexState = ref 0L
  -          for line in lines do 
  -              let lineTokenizer = sourceTokenizer.CreateLineTokenizer line
  -              let finLine = ref false
  -              while not !finLine do
  -                  let tok, newLexState = lineTokenizer.ScanToken(!lexState)
  -                  lexState := newLexState
  -                  match tok with 
  -                  | None -> 
  -                      yield (EOL, System.Environment.NewLine) // new line
  -                      finLine := true
  -                  | Some t -> 
  -                      yield (Token t, line.[t.LeftColumn..t.RightColumn]) }
  +let tokenize (content : string) =
  +    seq { 
  +        let sourceTokenizer = SourceTokenizer([], "/tmp.fsx")
  +        let lines = content.Replace("\r\n","\n").Split('\r', '\n')
  +        let lexState = ref 0L
  +        for (i, line) in lines |> Seq.zip [1..lines.Length] do 
  +            let lineTokenizer = sourceTokenizer.CreateLineTokenizer line
  +            let finLine = ref false
  +            while not !finLine do
  +                let tok, newLexState = lineTokenizer.ScanToken(!lexState)
  +                lexState := newLexState
  +                match tok with 
  +                | None ->
  +                    if i <> lines.Length then
  +                        // New line except at the last token 
  +                        yield (EOL, System.Environment.NewLine) 
  +                    finLine := true
  +                | Some t -> 
  +                    yield (Token t, line.[t.LeftColumn..t.RightColumn]) }
   
   type LineCommentStickiness = | StickyLeft | StickyRight | NotApplicable
   
  
positive.code:             loop moreOrigTokens newTokens 
  
          // Dangling text at the end 
          | [], ((newTok, newTokText) :: moreNewTokens) ->
              Debug.WriteLine("dangling new token '{0}'", newTokText)
              addText newTokText 
              loop [] moreNewTokens 
  
negative.code: 
      let rec loop origTokens newTokens = 
          match origTokens, newTokens with 
          | (Marked(Token origTok, _, _) :: moreOrigTokens),  _ 
                when origTok.CharClass = TokenCharKind.WhiteSpace && origTok.ColorClass <> TokenColorKind.InactiveCode 
                     && origTok.ColorClass <> TokenColorKind.PreprocessorKeyword ->
                Debug.WriteLine "dropping whitespace from orig tokens" 
                loop moreOrigTokens newTokens 
  
  
          | (Marked(EOL, _, _) :: moreOrigTokens),  _ ->
                Debug.WriteLine "dropping newline from orig tokens" 
                loop moreOrigTokens newTokens 
  
  