anchor.diff:  
       let long (msg:string) = msg.Length > len
   
  -    let ham,spam = 
  -        dataset 
  +    let ham,spam =
  +        dataset
           |> Array.partition (fun (docType,_) -> docType = Ham)
  -    let spamAndLongCount = 
  -        spam 
  -        |> Array.filter (fun (_,sms) -> long sms)  
  +    let spamAndLongCount =
  +        spam
  +        |> Array.filter (fun (_,sms) -> long sms)
           |> Array.length
  -    let longCount = 
  -        dataset 
  -        |> Array.filter (fun (_,sms) -> long sms)  
  +    let longCount =
  +        dataset
  +        |> Array.filter (fun (_,sms) -> long sms)
           |> Array.length
   
       let pSpam = (float spam.Length) / (float dataset.Length)
  -    let pLongIfSpam =        
  -        float spamAndLongCount / float spam.Length 
  -    let pLong = 
  +    let pLongIfSpam =
  +        float spamAndLongCount / float spam.Length
  +    let pLong =
           float longCount /
           float (dataset.Length)
   
       let pSpamIfLong = pLongIfSpam * pSpam / pLong
       pSpamIfLong
   
  -for l in 10 .. 10 .. 130 do 
  +for l in 10 .. 10 .. 130 do
       printfn "P(Spam if Length > %i) = %.4f" l (lengthAnalysis l)
   
   
  
positive.code: // Super simple model: predict based on one single token, "txt"
  let txtClassifier = train training wordTokenizer (["txt"] |> set)
  
  validation 
  |> Seq.averageBy (fun (l,o) -> 
      if l = txtClassifier o then 1. else 0.)
  |> printfn "Based on 'txt', correctly classified: %.3f"
  
  // Even simpler model: always predict the most frequent class,
  // in that case, Ham
  validation 
  |> Seq.averageBy (fun (l,o) -> 
      if l = Ham then 1. else 0.)
  |> printfn "Aways Ham, correctly classified: %.3f"
  
  // Use every single token
  let tokenized = 
      training 
      |> Seq.map (fun (lbl,sms) -> lbl,wordTokenizer sms)
  
  let allTokens = 
      training
      |> Seq.map snd
      |> vocabulary wordTokenizer
  
  let fullClassifier = train training wordTokenizer allTokens
  validation 
  |> Seq.averageBy (fun (l,o) -> 
      if l = fullClassifier o then 1. else 0.)
  |> printfn "Based on all tokens, correctly classified: %.3f"
  
  // generic model evaluation function
  let evaluate (tokenizer:Tokenizer) (tokens:Token Set) = 
      let model = train training tokenizer tokens
      validation 
      |> Seq.averageBy (fun (l,o) -> 
          if l = model o then 1. else 0.)
      |> printfn "Correctly classified: %.3f"
  
  evaluate wordTokenizer allTokens
  
  // does casing matter?
  let casedTokenizer (text:string) = 
      text
      |> matchWords.Matches
      |> Seq.cast<Match>
      |> Seq.map (fun m -> m.Value)
      |> Set.ofSeq
  
  let casedTokens = 
      training 
      |> Seq.map snd 
      |> vocabulary casedTokenizer
  
  evaluate casedTokenizer casedTokens
  
  // most frequent tokens
  
  let top n (tokenizer:Tokenizer) (docs:string []) = 
      let tokenized = docs |> Array.map tokenizer
      let tokens = tokenized |> Set.unionMany
      tokens 
      |> Seq.sortBy (fun t -> - countIn tokenized t)
      |> Seq.take n
      |> Set.ofSeq
  
  let ham,spam = 
      let rawHam,rawSpam =
          training 
          |> Array.partition (fun (lbl,_) -> lbl=Ham)
      rawHam |> Array.map snd, 
      rawSpam |> Array.map snd
  
  let hamCount = ham |> vocabulary casedTokenizer |> Set.count
  let spamCount = spam |> vocabulary casedTokenizer |> Set.count
  
  let topHam = ham |> top (hamCount / 10) casedTokenizer 
  let topSpam = spam |> top (spamCount / 10) casedTokenizer 
  
  let topTokens = Set.union topHam topSpam
  
  
negative.code: // making tokens out of phone and sms numbers
  
  let phoneWords = Regex(@"0[7-9]\d{9}")
  let phone (text:string) =
      match (phoneWords.IsMatch text) with
      | true -> "__PHONE__"
      | false -> text
  