-- write all the URLs, docs and enough info to pretty print it to a result
-- and replace each with an identifier (index in the space) - big reduction in memory
writeItems :: StoreWrite -> (Conduit (Maybe Target, Item) IO (Maybe TargetId, Item) -> IO a) -> IO a
writeItems store act = do
    storeWriteType store Items $ storeWriteParts store $ act $
        void $ (\f -> mapAccumMC f 0) $ \pos (target, item) -> case target of
            Nothing -> return (pos, (Nothing, item))
            Just target -> do
                let bs = BS.concat $ LBS.toChunks $ GZip.compress $ UTF8.fromString $ unlines $ outputItem (TargetId pos, target)
                liftIO $ do
                    storeWriteBS store $ intToBS $ BS.length bs
                    storeWriteBS store bs
                let pos2 = pos + fromIntegral (intSize + BS.length bs)
                return (pos2, (Just $ TargetId pos, item))
writeItems :: StoreWrite -> (Conduit (Maybe Target, Item) IO (Maybe TargetId, Item) -> IO a) -> IO a
writeItems store act = do
    storeWriteType store Items $ storeWriteParts store $ act $
        void $ (\f -> mapAccumMC f 0) $ \pos (target, item) -> case target of
            Nothing -> return (pos, (Nothing, item))
            Just target -> do
                let bs = BS.concat $ LBS.toChunks $ GZip.compress $ UTF8.fromString $ unlines $ outputItem (TargetId pos, target)
                liftIO $ do
                    storeWriteBS store $ intToBS $ BS.length bs
                    storeWriteBS store bs
                let pos2 = pos + fromIntegral (intSize + BS.length bs)
                return (pos2, (Just $ TargetId pos, item))
writeItems store act = do
    storeWriteType store Items $ storeWriteParts store $ act $
        void $ (\f -> mapAccumMC f 0) $ \pos (target, item) -> case target of
            Nothing -> return (pos, (Nothing, item))
            Just target -> do
                let bs = BS.concat $ LBS.toChunks $ GZip.compress $ UTF8.fromString $ unlines $ outputItem (TargetId pos, target)
                liftIO $ do
                    storeWriteBS store $ intToBS $ BS.length bs
                    storeWriteBS store bs
                let pos2 = pos + fromIntegral (intSize + BS.length bs)
                return (pos2, (Just $ TargetId pos, item))