# OPTIONS #

-- ------------------------------------------------------------

module Hayoo.IndexTypes
    ( module Hayoo.IndexTypes
    , Document
    , Documents
    , SmallDocuments
    , FunctionInfo(..)
    , PackageInfo(..)
    , Score

    , docTable2smallDocTable
    )
where

import           Control.Arrow
import           Control.DeepSeq

import           Data.Binary
  import qualified Data . ByteString . Char8                as C
import qualified Data.IntSet                    as IS
import qualified Data.IntMap                    as IM
import           Data.List                      ( foldl' )
import qualified Data.Map                       as M    ( lookup )
import           Data.Maybe

import           Hayoo.FunctionInfo
import           Hayoo.PackageInfo
import           Hayoo.PackageRank

import           Holumbus.Crawler
import           Holumbus.Crawler.IndexerCore

import qualified Holumbus.Data.PrefixTree       as PT

import           Holumbus.Index.Common          ( Document(..)
                                                , Occurrences
                                                , toList, fromList
                                                , custom, editDocIds, removeById, toMap, updateDocIds', updateDocuments
                                                )
import           Holumbus.Index.CompactDocuments
                                                ( Documents(..)
                                                , emptyDocuments
                                                )

import           Holumbus.Index.CompactSmallDocuments
                                                ( SmallDocuments(..)
                                                , docTable2smallDocTable
                                                )

import           Holumbus.Query.Result          ( Score )

-- import           Debug.Trace

-- ------------------------------------------------------------

  .1 : direct use of prefix tree with simple-9 encoded occurences 

    concerning efficiency this implementation is about the same as the 2 . one , 
    space and time are minimally better , the reason could be less code working with classes 

 import            Holumbus . Index . Inverted . PrefixMem 



   concerning efficiency this implementation is about the same as the 2. one,
   space and time are minimally better, the reason could be less code working with classes

import           Holumbus.Index.Inverted.PrefixMem

-}
-- ------------------------------------------------------------

  .2 : indirect use of prefix tree with simple-9 encoded occurences via InvertedCompressed 

    minimal overhead compared to .1 
    but less efficient in time ( 1598s / 1038s ) and space 
    total mem use ( 2612 MB / 2498 MB ) than .3 

 import qualified Holumbus . Index . Inverted . CompressedPrefixMem     as PM 

 type Inverted                    = PM.InvertedCompressed 

 emptyInverted                    : : Inverted 
 emptyInverted                    = PM.emptyInvertedCompressed 


   minimal overhead compared to .1
   but less efficient in time (1598s / 1038s) and space
   total mem use (2612MB / 2498MB) than .3

import qualified Holumbus.Index.Inverted.CompressedPrefixMem    as PM

type Inverted                   = PM.InvertedCompressed

emptyInverted                   :: Inverted
emptyInverted                   = PM.emptyInvertedCompressed
-}

-- ------------------------------------------------------------

  .3 : indirect prefix tree without compression of position sets 

    best of these 3 implementations 

    implementations with serializations become much more inefficient 
    in runtime and are not worth to be considered 


   best of these 3 implementations

   implementations with serializations become much more inefficient
   in runtime and are not worth to be considered
-}
 
import qualified Holumbus.Index.Inverted.CompressedPrefixMem    as PM

type Inverted                   = PM.Inverted0

emptyInverted                   :: Inverted
emptyInverted                   = PM.emptyInverted0

removeDocIdsInverted            :: Occurrences -> Inverted -> Inverted
removeDocIdsInverted            = PM.removeDocIdsInverted

type CompactInverted            = PM.InvertedOSerialized

inverted2compactInverted        :: Inverted -> CompactInverted
inverted2compactInverted        = fromList PM.emptyInvertedOSerialized . toList

-- ------------------------------------------------------------

type HayooState  di             = IndexerState       Inverted Documents di
type HayooConfig di             = IndexCrawlerConfig Inverted Documents di

type HayoorCrawlerState di      = CrawlerState (HayooState di)

emptyHayooState                 :: HayooState di
emptyHayooState                 = emptyIndexerState emptyInverted emptyDocuments

-- ------------------------------------------------------------

getPkgNameFct                   :: Document FunctionInfo -> String
getPkgNameFct                   = package . fromJust . custom

getPkgNamePkg                   :: Document PackageInfo -> String
getPkgNamePkg                   = p_name . fromJust . custom

-- ------------------------------------------------------------

removePackages'                 :: (Binary di, NFData di) =>
                                   (Document di -> String) -> String -> [String] -> Bool -> IO (HayooState di)
removePackages' pkgName ixName pkgList defragment
                                = do
                                  ix <- decodeFile ixName
                                  let ix1  = removePack' pkgName pkgList ix
                                  let ix2  = if defragment
                                             then defragmentIndex ix1
                                             else ix1
                                  rnf ix2 `seq` return ix2

-- ------------------------------------------------------------

removePack'                     :: (Binary di) =>
                                   (Document di -> String) -> [String] ->
                                   HayooState di -> HayooState di
removePack' pkgName ps IndexerState
              { ixs_index     = ix
              , ixs_documents = ds
              }                 = IndexerState
                                  { ixs_index     = ix'
                                  , ixs_documents = ds'
                                  }
    where
  collect all DocIds used in the given packages
    docIds                      = IM.foldWithKey checkDoc IM.empty . toMap $ ds
    checkDoc did doc xs
        | docPartOfPack         = IM.insert did IS.empty xs
        | otherwise             =                        xs
        where
        docPartOfPack           = (`elem` ps) . pkgName $ doc

  remove all DocIds from index
    ix'                         = removeDocIdsInverted docIds ix

                                                        -- restrict document table
    ds'                         = foldl' removeById ds $ IM.keys docIds

-- ------------------------------------------------------------

defragmentIndex                 :: (Binary di) =>
                                   HayooState di -> HayooState di
defragmentIndex IndexerState
              { ixs_index     = ix
              , ixs_documents = ds
              }                 = IndexerState
                                  { ixs_index     = ix'
                                  , ixs_documents = ds'
                                  }
    where
    ix'                         = updateDocIds' editId ix
    ds'                         = editDocIds editId ds
    idMap                       = IM.fromList . flip zip [1..] . IM.keys . toMap $ ds
    editId i                    = fromJust . IM.lookup i $ idMap

-- ------------------------------------------------------------

-- | package ranking is implemented by the following algorithm
--
  .1 the rank of a package not used by another package is 1.0
--
  .2 the rank of a package used by other packages is 1.0 + 0.5 * sum of the ranks of the
--    directly dependent packages. Example: a depends on b, b depends on c, d depends on c:
     rank a = 1.0 , rank b = 1.5 , rank c = 2.25 , rank d = 1.0
--
  .3 this leads to a ranking where rank base > 1000.0 and rank bytestring > 300 . To
     reduce the weight differences , the log to base 2 is taken instead of the direct value

packageRanking                  :: HayooPkgIndexerState -> HayooPkgIndexerState
packageRanking ixs@(IndexerState { ixs_documents = ds })
                                = ixs { ixs_documents = updateDocuments insertRank ds }
    where
    deflate                     = 0.5
    scale                       = (/10.0) . fromInteger . round . (*10) . (+1.0) . logBase 2
    rank                        = ranking deflate
                                  . dagFromList
                                  . map (\ p -> (getPackageName p, getPackageDependencies p))
                                  . map fromJust
                                  . filter isJust               -- all illegal package refs are filtered out (there are illegal refs)
                                  . map custom
                                  . IM.elems
                                  . toMap $ ds

    insertRank d                = d { custom = fmap insertRank' (custom d) }
        where
        insertRank' ci          = setPackageRank (scale . fromMaybe (1.0) . M.lookup (getPackageName ci) $ rank) ci


 traceNothing d 
     | isJust . custom $ d        = d 
     | otherwise                  = traceShow d $ d 

traceNothing d
    | isJust . custom $ d       = d
    | otherwise                 = traceShow d $ d
-}

-- ------------------------------------------------------------

type RankTable                  = PT.PrefixTree Score

lookupRankTable                 :: String -> RankTable -> Score
lookupRankTable p               = fromMaybe 1.0 . PT.lookup p

buildRankTable                  :: SmallDocuments PackageInfo -> RankTable
buildRankTable                  = toMap
                                  >>> IM.elems
                                  >>> map ( custom
                                            >>> fromJust
                                            >>> (p_name &&& p_rank)
                                          )
                                  >>> PT.fromList 

-- ------------------------------------------------------------

type HayooIndexerState                  = HayooState   FunctionInfo
type HayooIndexerConfig                 = HayooConfig  FunctionInfo

type HayooIndexerCrawlerState           = CrawlerState HayooIndexerState

-- ------------------------------------------------------------

type HayooPkgIndexerState               = HayooState   PackageInfo
type HayooPkgIndexerConfig              = HayooConfig  PackageInfo

type HayooPkgIndexerCrawlerState        = CrawlerState HayooPkgIndexerState

-- ------------------------------------------------------------


 ------------------------------------------------------------
 import           Debug.Trace
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 restrict document table
 ------------------------------------------------------------
 ------------------------------------------------------------
 | package ranking is implemented by the following algorithm


    directly dependent packages. Example: a depends on b, b depends on c, d depends on c:

 all illegal package refs are filtered out (there are illegal refs)
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
 ------------------------------------------------------------
# OPTIONS #


module Hayoo.IndexTypes
    ( module Hayoo.IndexTypes
    , Document
    , Documents
    , SmallDocuments
    , FunctionInfo(..)
    , PackageInfo(..)
    , Score

    , docTable2smallDocTable
    )
where

import           Control.Arrow
import           Control.DeepSeq

import           Data.Binary
  import qualified Data . ByteString . Char8                as C
import qualified Data.IntSet                    as IS
import qualified Data.IntMap                    as IM
import           Data.List                      ( foldl' )
import qualified Data.Map                       as M    ( lookup )
import           Data.Maybe

import           Hayoo.FunctionInfo
import           Hayoo.PackageInfo
import           Hayoo.PackageRank

import           Holumbus.Crawler
import           Holumbus.Crawler.IndexerCore

import qualified Holumbus.Data.PrefixTree       as PT

import           Holumbus.Index.Common          ( Document(..)
                                                , Occurrences
                                                , toList, fromList
                                                , custom, editDocIds, removeById, toMap, updateDocIds', updateDocuments
                                                )
import           Holumbus.Index.CompactDocuments
                                                ( Documents(..)
                                                , emptyDocuments
                                                )

import           Holumbus.Index.CompactSmallDocuments
                                                ( SmallDocuments(..)
                                                , docTable2smallDocTable
                                                )

import           Holumbus.Query.Result          ( Score )



  .1 : direct use of prefix tree with simple-9 encoded occurences 

    concerning efficiency this implementation is about the same as the 2 . one , 
    space and time are minimally better , the reason could be less code working with classes 

 import            Holumbus . Index . Inverted . PrefixMem 



   concerning efficiency this implementation is about the same as the 2. one,
   space and time are minimally better, the reason could be less code working with classes

import           Holumbus.Index.Inverted.PrefixMem

-}

  .2 : indirect use of prefix tree with simple-9 encoded occurences via InvertedCompressed 

    minimal overhead compared to .1 
    but less efficient in time ( 1598s / 1038s ) and space 
    total mem use ( 2612 MB / 2498 MB ) than .3 

 import qualified Holumbus . Index . Inverted . CompressedPrefixMem     as PM 

 type Inverted                    = PM.InvertedCompressed 

 emptyInverted                    : : Inverted 
 emptyInverted                    = PM.emptyInvertedCompressed 


   minimal overhead compared to .1
   but less efficient in time (1598s / 1038s) and space
   total mem use (2612MB / 2498MB) than .3

import qualified Holumbus.Index.Inverted.CompressedPrefixMem    as PM

type Inverted                   = PM.InvertedCompressed

emptyInverted                   :: Inverted
emptyInverted                   = PM.emptyInvertedCompressed
-}


  .3 : indirect prefix tree without compression of position sets 

    best of these 3 implementations 

    implementations with serializations become much more inefficient 
    in runtime and are not worth to be considered 


   best of these 3 implementations

   implementations with serializations become much more inefficient
   in runtime and are not worth to be considered
-}
 
import qualified Holumbus.Index.Inverted.CompressedPrefixMem    as PM

type Inverted                   = PM.Inverted0

emptyInverted                   :: Inverted
emptyInverted                   = PM.emptyInverted0

removeDocIdsInverted            :: Occurrences -> Inverted -> Inverted
removeDocIdsInverted            = PM.removeDocIdsInverted

type CompactInverted            = PM.InvertedOSerialized

inverted2compactInverted        :: Inverted -> CompactInverted
inverted2compactInverted        = fromList PM.emptyInvertedOSerialized . toList


type HayooState  di             = IndexerState       Inverted Documents di
type HayooConfig di             = IndexCrawlerConfig Inverted Documents di

type HayoorCrawlerState di      = CrawlerState (HayooState di)

emptyHayooState                 :: HayooState di
emptyHayooState                 = emptyIndexerState emptyInverted emptyDocuments


getPkgNameFct                   :: Document FunctionInfo -> String
getPkgNameFct                   = package . fromJust . custom

getPkgNamePkg                   :: Document PackageInfo -> String
getPkgNamePkg                   = p_name . fromJust . custom


removePackages'                 :: (Binary di, NFData di) =>
                                   (Document di -> String) -> String -> [String] -> Bool -> IO (HayooState di)
removePackages' pkgName ixName pkgList defragment
                                = do
                                  ix <- decodeFile ixName
                                  let ix1  = removePack' pkgName pkgList ix
                                  let ix2  = if defragment
                                             then defragmentIndex ix1
                                             else ix1
                                  rnf ix2 `seq` return ix2


removePack'                     :: (Binary di) =>
                                   (Document di -> String) -> [String] ->
                                   HayooState di -> HayooState di
removePack' pkgName ps IndexerState
              { ixs_index     = ix
              , ixs_documents = ds
              }                 = IndexerState
                                  { ixs_index     = ix'
                                  , ixs_documents = ds'
                                  }
    where
  collect all DocIds used in the given packages
    docIds                      = IM.foldWithKey checkDoc IM.empty . toMap $ ds
    checkDoc did doc xs
        | docPartOfPack         = IM.insert did IS.empty xs
        | otherwise             =                        xs
        where
        docPartOfPack           = (`elem` ps) . pkgName $ doc

  remove all DocIds from index
    ix'                         = removeDocIdsInverted docIds ix

    ds'                         = foldl' removeById ds $ IM.keys docIds


defragmentIndex                 :: (Binary di) =>
                                   HayooState di -> HayooState di
defragmentIndex IndexerState
              { ixs_index     = ix
              , ixs_documents = ds
              }                 = IndexerState
                                  { ixs_index     = ix'
                                  , ixs_documents = ds'
                                  }
    where
    ix'                         = updateDocIds' editId ix
    ds'                         = editDocIds editId ds
    idMap                       = IM.fromList . flip zip [1..] . IM.keys . toMap $ ds
    editId i                    = fromJust . IM.lookup i $ idMap


  .1 the rank of a package not used by another package is 1.0
  .2 the rank of a package used by other packages is 1.0 + 0.5 * sum of the ranks of the
     rank a = 1.0 , rank b = 1.5 , rank c = 2.25 , rank d = 1.0
  .3 this leads to a ranking where rank base > 1000.0 and rank bytestring > 300 . To
     reduce the weight differences , the log to base 2 is taken instead of the direct value

packageRanking                  :: HayooPkgIndexerState -> HayooPkgIndexerState
packageRanking ixs@(IndexerState { ixs_documents = ds })
                                = ixs { ixs_documents = updateDocuments insertRank ds }
    where
    deflate                     = 0.5
    scale                       = (/10.0) . fromInteger . round . (*10) . (+1.0) . logBase 2
    rank                        = ranking deflate
                                  . dagFromList
                                  . map (\ p -> (getPackageName p, getPackageDependencies p))
                                  . map fromJust
                                  . map custom
                                  . IM.elems
                                  . toMap $ ds

    insertRank d                = d { custom = fmap insertRank' (custom d) }
        where
        insertRank' ci          = setPackageRank (scale . fromMaybe (1.0) . M.lookup (getPackageName ci) $ rank) ci


 traceNothing d 
     | isJust . custom $ d        = d 
     | otherwise                  = traceShow d $ d 

traceNothing d
    | isJust . custom $ d       = d
    | otherwise                 = traceShow d $ d
-}


type RankTable                  = PT.PrefixTree Score

lookupRankTable                 :: String -> RankTable -> Score
lookupRankTable p               = fromMaybe 1.0 . PT.lookup p

buildRankTable                  :: SmallDocuments PackageInfo -> RankTable
buildRankTable                  = toMap
                                  >>> IM.elems
                                  >>> map ( custom
                                            >>> fromJust
                                            >>> (p_name &&& p_rank)
                                          )
                                  >>> PT.fromList 


type HayooIndexerState                  = HayooState   FunctionInfo
type HayooIndexerConfig                 = HayooConfig  FunctionInfo

type HayooIndexerCrawlerState           = CrawlerState HayooIndexerState


type HayooPkgIndexerState               = HayooState   PackageInfo
type HayooPkgIndexerConfig              = HayooConfig  PackageInfo

type HayooPkgIndexerCrawlerState        = CrawlerState HayooPkgIndexerState


