main :: IO ()
main = do
    gbSize <- countWds "gb"
    putStrLn $ "Did gb: " ++ show gbSize
    (wiktSize, wiktIndex) <- loadWds "wikt"
    putStrLn $ "Did wikt: " ++ show wiktSize
    (wikiSize, wikiIndex) <- loadWds "wiki"
    putStrLn $ "Did wiki: " ++ show wikiSize
    let myLookup :: Str -> Maybe (Str, Float)
        myLookup line =
            case (BST.lookup wd2 wiktIndex, BST.lookup wd2 wikiIndex) of
              (Just wiktCount, Just wikiCount) ->
                Just (wd, minimum [gbFrac, wiktFrac, wikiFrac])
                  where
                    wiktFrac = fromIntegral wiktCount / fromIntegral wiktSize
                    wikiFrac = fromIntegral wikiCount / fromIntegral wikiSize
              _ -> Nothing
          where
            (wd:countStr:_) = BSC.split '\t' line
            wd2 = DTE.encodeUtf8 . DT.toLower $ DTE.decodeUtf8 wd
            Just (count, _) = BSC.readInt countStr
            gbFrac = fromIntegral count / fromIntegral gbSize
        myMap :: (Str, Float) -> Str
        myMap (wd, count) = wd <> "\t" <>
            BSC.pack (show $ log (1 / count) / log 10) <> "\n"
    runResourceT $
        CC.sourceFile "/home/danl/p/l/melang/lang/de/base/gb-wds.txt"
        $$ CB.lines
        =$ CC.map myLookup
        =$ CC.filter isJust
        =$ CC.map (myMap . fromJust)
        =$ CC.stdout
main :: IO ()
main = do
    gbSize <- countWds "gb"
    putStrLn $ "Did gb: " ++ show gbSize
    (wiktSize, wiktIndex) <- loadWds "wikt"
    putStrLn $ "Did wikt: " ++ show wiktSize
    (wikiSize, wikiIndex) <- loadWds "wiki"
    putStrLn $ "Did wiki: " ++ show wikiSize
    let myLookup :: Str -> Maybe (Str, Float)
        myLookup line =
            case (BST.lookup wd2 wiktIndex, BST.lookup wd2 wikiIndex) of
              (Just wiktCount, Just wikiCount) ->
                Just (wd, minimum [gbFrac, wiktFrac, wikiFrac])
                  where
                    wiktFrac = fromIntegral wiktCount / fromIntegral wiktSize
                    wikiFrac = fromIntegral wikiCount / fromIntegral wikiSize
              _ -> Nothing
          where
            (wd:countStr:_) = BSC.split '\t' line
            wd2 = DTE.encodeUtf8 . DT.toLower $ DTE.decodeUtf8 wd
            Just (count, _) = BSC.readInt countStr
            gbFrac = fromIntegral count / fromIntegral gbSize
        myMap :: (Str, Float) -> Str
        myMap (wd, count) = wd <> "\t" <>
            BSC.pack (show $ log (1 / count) / log 10) <> "\n"
    runResourceT $
        CC.sourceFile "/home/danl/p/l/melang/lang/de/base/gb-wds.txt"
        $$ CB.lines
        =$ CC.map myLookup
        =$ CC.filter isJust
        =$ CC.map (myMap . fromJust)
        =$ CC.stdout
main = do
    gbSize <- countWds "gb"
    putStrLn $ "Did gb: " ++ show gbSize
    (wiktSize, wiktIndex) <- loadWds "wikt"
    putStrLn $ "Did wikt: " ++ show wiktSize
    (wikiSize, wikiIndex) <- loadWds "wiki"
    putStrLn $ "Did wiki: " ++ show wikiSize
    let myLookup :: Str -> Maybe (Str, Float)
        myLookup line =
            case (BST.lookup wd2 wiktIndex, BST.lookup wd2 wikiIndex) of
              (Just wiktCount, Just wikiCount) ->
                Just (wd, minimum [gbFrac, wiktFrac, wikiFrac])
                  where
                    wiktFrac = fromIntegral wiktCount / fromIntegral wiktSize
                    wikiFrac = fromIntegral wikiCount / fromIntegral wikiSize
              _ -> Nothing
          where
            (wd:countStr:_) = BSC.split '\t' line
            wd2 = DTE.encodeUtf8 . DT.toLower $ DTE.decodeUtf8 wd
            Just (count, _) = BSC.readInt countStr
            gbFrac = fromIntegral count / fromIntegral gbSize
        myMap :: (Str, Float) -> Str
        myMap (wd, count) = wd <> "\t" <>
            BSC.pack (show $ log (1 / count) / log 10) <> "\n"
    runResourceT $
        CC.sourceFile "/home/danl/p/l/melang/lang/de/base/gb-wds.txt"
        $$ CB.lines
        =$ CC.map myLookup
        =$ CC.filter isJust
        =$ CC.map (myMap . fromJust)
        =$ CC.stdout