{-# LANGUAGE TupleSections #-}
-- | Description: Simple document analysis.
--
-- This module implements some simple document analysis algorithms using a "bag
-- of words" model.
module Document.Analysis where

import Data.Char
import Data.Digest.Adler32
import Data.List
import Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as T
import Data.Word
import Sparse.Matrix (Key (Key), Mat)
import qualified Sparse.Matrix as M

-- | Convert a single word into zero or more features.
--
-- This involves converting to lower case, splitting on punctuation, filtering
-- empty strings, etc.
feature :: Text -> [Text]
feature s =
    filter (not . T.null) . T.split (not . isLetter) . T.toLower $ s

-- | Hash a feature using Adler32 algorithm.
hashFeature :: Text -> Word
hashFeature = fromIntegral . adler32 . T.encodeUtf8

-- | Inefficiently, and in memory, convert a list of documents into a
-- document-term matrix.
documentMatrix :: [(Word, Text)] -> Mat Int
documentMatrix = foldl' worker M.empty
  where
    worker :: Mat Int -> (Word, Text) -> Mat Int
    worker mat (i, doc) =
        let fs = map (\w -> (Key i . hashFeature $ w, 1)) $ T.words doc
            mat' = foldl' (\m f -> m + M.fromList [f]) mat fs
        in mat'
