anchor.diff:  let Eof : Alphabet = 0xFFFFFFFEu
   let Epsilon : Alphabet = 0xFFFFFFFFu
   
  -
   let unicode = ref false
   
  -let unicodeCategories = 
  - dict 
  -  [| "Pe", System.Globalization.UnicodeCategory.ClosePunctuation; // (Pe)
  -    "Pc", System.Globalization.UnicodeCategory.ConnectorPunctuation; // (Pc)
  -    "Cc", System.Globalization.UnicodeCategory.Control; // (Cc)
  -    "Sc", System.Globalization.UnicodeCategory.CurrencySymbol; // (Sc)
  -    "Pd", System.Globalization.UnicodeCategory.DashPunctuation; // (Pd)
  -    "Nd", System.Globalization.UnicodeCategory.DecimalDigitNumber; // (Nd)
  -    "Me", System.Globalization.UnicodeCategory.EnclosingMark; // (Me)
  -    "Pf", System.Globalization.UnicodeCategory.FinalQuotePunctuation; // (Pf)
  -    "Cf", System.Globalization.UnicodeCategory.Format; // (Cf)
  -    "Pi", System.Globalization.UnicodeCategory.InitialQuotePunctuation; // (Pi)
  -    "Nl", System.Globalization.UnicodeCategory.LetterNumber; // (Nl)
  -    "Zl", System.Globalization.UnicodeCategory.LineSeparator; // (Zl)
  -    "Ll", System.Globalization.UnicodeCategory.LowercaseLetter; // (Ll)
  -    "Sm", System.Globalization.UnicodeCategory.MathSymbol; // (Sm)
  -    "Lm", System.Globalization.UnicodeCategory.ModifierLetter; // (Lm)
  -    "Sk", System.Globalization.UnicodeCategory.ModifierSymbol; // (Sk)
  -    "Mn", System.Globalization.UnicodeCategory.NonSpacingMark; // (Mn)
  -    "Ps", System.Globalization.UnicodeCategory.OpenPunctuation; // (Ps)
  -    "Lo", System.Globalization.UnicodeCategory.OtherLetter; // (Lo)
  -    "Cn", System.Globalization.UnicodeCategory.OtherNotAssigned; // (Cn)
  -    "No", System.Globalization.UnicodeCategory.OtherNumber; // (No)
  -    "Po", System.Globalization.UnicodeCategory.OtherPunctuation; // (Po)
  -    "So", System.Globalization.UnicodeCategory.OtherSymbol; // (So)
  -    "Zp", System.Globalization.UnicodeCategory.ParagraphSeparator; // (Zp)
  -    "Co", System.Globalization.UnicodeCategory.PrivateUse; // (Co)
  -    "Zs", System.Globalization.UnicodeCategory.SpaceSeparator; // (Zs)
  -    "Mc", System.Globalization.UnicodeCategory.SpacingCombiningMark; // (Mc)
  -    "Cs", System.Globalization.UnicodeCategory.Surrogate; // (Cs)
  -    "Lt", System.Globalization.UnicodeCategory.TitlecaseLetter; // (Lt)
  -    "Lu", System.Globalization.UnicodeCategory.UppercaseLetter; // (Lu)
  +let unicodeCategories =
  + dict
  +  [| "Pe", UnicodeCategory.ClosePunctuation; // (Pe)
  +    "Pc", UnicodeCategory.ConnectorPunctuation; // (Pc)
  +    "Cc", UnicodeCategory.Control; // (Cc)
  +    "Sc", UnicodeCategory.CurrencySymbol; // (Sc)
  +    "Pd", UnicodeCategory.DashPunctuation; // (Pd)
  +    "Nd", UnicodeCategory.DecimalDigitNumber; // (Nd)
  +    "Me", UnicodeCategory.EnclosingMark; // (Me)
  +    "Pf", UnicodeCategory.FinalQuotePunctuation; // (Pf)
  +    "Cf", UnicodeCategory.Format; // (Cf)
  +    "Pi", UnicodeCategory.InitialQuotePunctuation; // (Pi)
  +    "Nl", UnicodeCategory.LetterNumber; // (Nl)
  +    "Zl", UnicodeCategory.LineSeparator; // (Zl)
  +    "Ll", UnicodeCategory.LowercaseLetter; // (Ll)
  +    "Sm", UnicodeCategory.MathSymbol; // (Sm)
  +    "Lm", UnicodeCategory.ModifierLetter; // (Lm)
  +    "Sk", UnicodeCategory.ModifierSymbol; // (Sk)
  +    "Mn", UnicodeCategory.NonSpacingMark; // (Mn)
  +    "Ps", UnicodeCategory.OpenPunctuation; // (Ps)
  +    "Lo", UnicodeCategory.OtherLetter; // (Lo)
  +    "Cn", UnicodeCategory.OtherNotAssigned; // (Cn)
  +    "No", UnicodeCategory.OtherNumber; // (No)
  +    "Po", UnicodeCategory.OtherPunctuation; // (Po)
  +    "So", UnicodeCategory.OtherSymbol; // (So)
  +    "Zp", UnicodeCategory.ParagraphSeparator; // (Zp)
  +    "Co", UnicodeCategory.PrivateUse; // (Co)
  +    "Zs", UnicodeCategory.SpaceSeparator; // (Zs)
  +    "Mc", UnicodeCategory.SpacingCombiningMark; // (Mc)
  +    "Cs", UnicodeCategory.Surrogate; // (Cs)
  +    "Lt", UnicodeCategory.TitlecaseLetter; // (Lt)
  +    "Lu", UnicodeCategory.UppercaseLetter; // (Lu)
     |]
   
   let NumUnicodeCategories = unicodeCategories.Count
  -let _ = assert (NumUnicodeCategories = 30) // see table interpreter
  +assert (NumUnicodeCategories = 30) // see table interpreter
   let encodedUnicodeCategoryBase = 0xFFFFFF00u
   let EncodeUnicodeCategoryIndex(idx:int) = encodedUnicodeCategoryBase + uint32 idx
  -let EncodeUnicodeCategory(s:string) = 
  -    if not (!unicode) then 
  -         failwith "unicode category classes may only be used if --unicode is specified";
  -    if unicodeCategories.ContainsKey(s) then 
  +let EncodeUnicodeCategory s =
  +    if not (!unicode) then
  +        failwith "unicode category classes may only be used if --unicode is specified"
  +    if unicodeCategories.ContainsKey(s) then
           EncodeUnicodeCategoryIndex (int32 unicodeCategories.[s])
       else
           failwithf "invalid Unicode category: '%s'" s
  
positive.code:         if not <| doneSet.Contains(nfaSet) then
              let moves = ComputeMoves nfaSet
              for (KeyValue(inp,movesForInput)) in moves do
                  assert (inp <> Epsilon);
                  let moveSet = EClosure movesForInput;
                  if not moveSet.IsEmpty then 
                      //incr count
                      let dfaNode = GetDfaNode nfaSet
                      dfaNode.Transitions <- (inp, GetDfaNode moveSet) :: dfaNode.Transitions;
                      // Printf.printf "%d (%s) : %s --> %d (%s)\n" dfaNode.Id dfaNode.Name (match inp with EncodeChar c -> String.make 1 c | LEof -> "eof") moveSetDfaNode.Id moveSetDfaNode.Name;
                      workList.Push(moveSet);
                   
                  doneSet.Add(nfaSet) |> ignore
  
      //Printf.printfn "count = %d" !count;
      let ruleStartNode = GetDfaNode nfaSet0
      let ruleNodes = 
          dfaNodes
          |> Seq.map (fun kvp -> kvp.Value) 
          |> Seq.toList
          |> List.sortBy (fun s -> s.Id)
      ruleStartNode,ruleNodes
  
  let Compile spec = 
      List.foldBack
          (fun (name,args,clauses) (perRuleData,dfaNodes) -> 
              let nfa, actions, nfaNodeMap = LexerStateToNfa (Map.ofList spec.Macros) clauses
              let ruleStartNode, ruleNodes = NfaToDfa nfaNodeMap nfa
              //Printf.printfn "name = %s, ruleStartNode = %O" name ruleStartNode.Id;
              (ruleStartNode,actions) :: perRuleData, ruleNodes @ dfaNodes)
          spec.Rules
          ([],[])
  
negative.code:     let prev = LookupMultiMap trDict a
      trDict.[a] <- b::prev
  
  type NfaNodeMap() =
      let map = new Dictionary<int,NfaNode>(100)
      member x.Item with get(nid) = map.[nid]
      member x.Count = map.Count
  
      member x.NewNfaNode(trs,ac) =
          let nodeId = map.Count+1 // ID zero is reserved
          let trDict = new Dictionary<_,_>(List.length trs)
          for (a,b) in trs do
             AddToMultiMap trDict a b
  
          let node : NfaNode = {Id=nodeId; Name=string nodeId; Transitions=trDict; Accepted=ac}
          map.[nodeId] <-node
          node
  
  let LexerStateToNfa (macros: Map<string,_>) (clauses: Clause list) =
  
      /// Table allocating node ids
      let nfaNodeMap = new NfaNodeMap()
  
      /// Compile a regular expression into the NFA
      let rec CompileRegexp re dest =
          match re with
          | Alt res ->
              let trs = res |> List.map (fun re -> (Epsilon,CompileRegexp re dest))
              nfaNodeMap.NewNfaNode(trs,[])
          | Seq res ->
              List.foldBack (CompileRegexp) res dest
          | Inp (Alphabet c) ->
              nfaNodeMap.NewNfaNode([(c, dest)],[])
  
          | Star re ->
              let nfaNode = nfaNodeMap.NewNfaNode([(Epsilon, dest)],[])
              let sre = CompileRegexp re nfaNode
              AddToMultiMap nfaNode.Transitions Epsilon sre
              nfaNodeMap.NewNfaNode([(Epsilon,sre); (Epsilon,dest)],[])
          | Macro m ->
              if not (macros.ContainsKey(m)) then failwithf "The macro %s is not defined" m
              CompileRegexp (macros.[m]) dest
  
          // These cases unwind the difficult cases in the syntax that rely on knowing the
          // entire alphabet.
          //
          // Note we've delayed the expension of these until we've worked out all the 'special' Unicode characters
          // mentioned in the entire lexer spec, i.e. we wait until GetAlphabet returns a reliable and stable answer.
          | Inp (UnicodeCategory uc) ->
              let re = Alt([ yield Inp(Alphabet(EncodeUnicodeCategory uc))
                             // Also include any specific characters in this category
                             for c in GetSingleCharAlphabet() do
                                 if System.Char.GetUnicodeCategory(c) = unicodeCategories.[uc] then
                                      yield Inp(Alphabet(EncodeChar(c))) ])
              CompileRegexp re dest
  
          | Inp Any ->
              let re = Alt([ for n in GetAlphabet() do yield Inp(Alphabet(n)) ])
              CompileRegexp re dest
  
          | Inp (NotCharSet chars) ->
              let re = Alt [ // Include any characters from those in the alphabet besides those that are not immediately excluded
                             for c in GetSingleCharAlphabet() do
                                 let ec = EncodeChar c
                                 if not (chars.Contains(ec)) then
                                     yield Inp(Alphabet(ec))
  
                             // Include all unicode categories
                             // That is, negations _only_ exclude precisely the given set of characters. You can't
                             // exclude whole classes of characters as yet
                             if !unicode then
                                 let ucs = chars |> Set.map(DecodeChar >> System.Char.GetUnicodeCategory)
                                 for KeyValue(nm,uc) in unicodeCategories do
                                     //if ucs.Contains(uc) then
                                     //    printfn "warning: the unicode category '\\%s' ('%O') is automatically excluded by this character set negation. Consider adding this to the negation." nm uc
                                     //else
                                         yield Inp(Alphabet(EncodeUnicodeCategory nm))
                           ]
              CompileRegexp re dest
  
      let actions = new System.Collections.Generic.List<_>()
  
      /// Compile an acceptance of a regular expression into the NFA
      let sTrans macros nodeId (regexp,code) =
          let actionId = actions.Count
          actions.Add(code)
          let sAccept = nfaNodeMap.NewNfaNode([],[(nodeId,actionId)])
          CompileRegexp regexp sAccept
  
      let trs = clauses |> List.mapi (fun n x -> (Epsilon,sTrans macros n x))
      let nfaStartNode = nfaNodeMap.NewNfaNode(trs,[])
      nfaStartNode,(actions |> Seq.readonly), nfaNodeMap
  
  // TODO: consider a better representation here.
  type internal NfaNodeIdSetBuilder = HashSet<NodeId>
  
  type internal NfaNodeIdSet(nodes: NfaNodeIdSetBuilder) =
      // BEWARE: the next line is performance critical
      let s = nodes |> Seq.toArray
      do Array.sortInPlaceWith compare s // 19
  
      // These are all surprisingly slower (because they create two arrays):
      //let s = nodes |> Seq.toArray |> Array.sort
      //let s = nodes |> Seq.toArray |> Array.sortWith compare // 76
      //let s = nodes |> Seq.toArray |> (fun arr -> Array.sortInPlace arr; arr) // 76
  
      member x.Representation = s
      member x.Elements = s
      member x.Fold f z = Array.fold f z s
      interface System.IComparable with
          member x.CompareTo(y:obj) =
              let y = (y :?> NfaNodeIdSet)
              let xr = x.Representation
              let yr = y.Representation
              let c = compare xr.Length yr.Length
              if c <> 0 then c else
              let n = yr.Length
              let rec go i =
                  if i >= n then 0 else
                  let c = compare xr.[i] yr.[i]
                  if c <> 0 then c else
                  go (i+1)
              go 0
  
      override x.Equals(y:obj) =
          match y with
          | :? NfaNodeIdSet as y ->
              let xr = x.Representation
              let yr = y.Representation
              let n = yr.Length
              xr.Length = n &&
              (let rec go i = (i < n) && xr.[i] = yr.[i] && go (i+1)
               go 0)
          | _ -> false
  
  