leaf_focus/report/item/document.py
def load(cls, processing_dir: Path, feed_dir: Path) -> Iterable["Document"]:
        """Load documents from a directory."""

        items = {}

        for csv_path in feed_dir.rglob("*.csv"):
            for pdf_feed in CrawlItem.load(csv_path):
                # feed
                pdf_fee_path = pdf_feed.path
                if pdf_fee_path not in items:
                    items[pdf_fee_path] = {}
                items[pdf_fee_path]["feed"] = pdf_feed

        for json_path in processing_dir.rglob("pdf-identify.json"):
            # identify
            pdf_identify = PdfIdentifyItem.read(json_path)
            pdf_identify_path = str(pdf_identify.pdf_file)
            if pdf_identify_path not in items:
                items[pdf_identify_path] = {}
            items[pdf_identify_path]["identify"] = pdf_identify
            items[pdf_identify_path]["processing"] = json_path.parent

            # info
            info_path = json_path.with_name("pdf-info.json")
            if not info_path.exists():
                continue
            pdf_info = PdfInfoItem.load_json(info_path)
            pdf_info_path = str(pdf_info.pdf_path)
            if pdf_info_path not in items:
                items[pdf_info_path] = {}
            items[pdf_info_path]["info"] = pdf_info

        for pdf_file, item in items.items():
            pdf_path = Path(pdf_file)
            pdf_feed = item.get("feed")  # type: CrawlItem
            pdf_identify = item.get("identify")  # type: PdfIdentifyItem
            pdf_info = item.get("info")  # type: PdfInfoItem
            pdf_processing = item.get("processing")  # type: Path

            if pdf_info and pdf_info.entries:
                pdf_created_date = pdf_info.entries.get("CreationDate")
                pdf_modified_date = pdf_info.entries.get("ModDate")
            else:
                pdf_created_date = None
                pdf_modified_date = None

            if pdf_feed:
                website_modified_date = pdf_feed.last_updated
                pdf_url = pdf_feed.url
                referrer_url = pdf_feed.referrer
                assembly = pdf_feed.category
            else:
                website_modified_date = None
                pdf_url = None
                referrer_url = None
                assembly = None

            doc = Document(
                pdf_path=pdf_path,
                pdf_hash_type=pdf_identify.hash_type,
                pdf_hash_value=pdf_identify.file_hash,
                pdf_created_date=pdf_created_date,
                pdf_modified_date=pdf_modified_date,
                pdf_downloaded_date=cls.downloaded_date(pdf_path),
                website_modified_date=website_modified_date,
                pdf_url=pdf_url,
                referrer_url=referrer_url,
                assembly=assembly,
            )
            pages = Page.load(pdf_processing, doc)
            doc.pages = list(pages)
            yield doc
Load documents from a directory.
Load documents from a directory.
Load documents from a directory .
def load(cls, processing_dir: Path, feed_dir: Path) -> Iterable["Document"]:
        items = {}
        for csv_path in feed_dir.rglob("*.csv"):
            for pdf_feed in CrawlItem.load(csv_path):
                pdf_fee_path = pdf_feed.path
                if pdf_fee_path not in items:
                    items[pdf_fee_path] = {}
                items[pdf_fee_path]["feed"] = pdf_feed
        for json_path in processing_dir.rglob("pdf-identify.json"):
            pdf_identify = PdfIdentifyItem.read(json_path)
            pdf_identify_path = str(pdf_identify.pdf_file)
            if pdf_identify_path not in items:
                items[pdf_identify_path] = {}
            items[pdf_identify_path]["identify"] = pdf_identify
            items[pdf_identify_path]["processing"] = json_path.parent
            info_path = json_path.with_name("pdf-info.json")
            if not info_path.exists():
                continue
            pdf_info = PdfInfoItem.load_json(info_path)
            pdf_info_path = str(pdf_info.pdf_path)
            if pdf_info_path not in items:
                items[pdf_info_path] = {}
            items[pdf_info_path]["info"] = pdf_info
        for pdf_file, item in items.items():
            pdf_path = Path(pdf_file)
            pdf_feed = item.get("feed")  
            pdf_identify = item.get("identify")  
            pdf_info = item.get("info")  
            pdf_processing = item.get("processing")  
            if pdf_info and pdf_info.entries:
                pdf_created_date = pdf_info.entries.get("CreationDate")
                pdf_modified_date = pdf_info.entries.get("ModDate")
            else:
                pdf_created_date = None
                pdf_modified_date = None
            if pdf_feed:
                website_modified_date = pdf_feed.last_updated
                pdf_url = pdf_feed.url
                referrer_url = pdf_feed.referrer
                assembly = pdf_feed.category
            else:
                website_modified_date = None
                pdf_url = None
                referrer_url = None
                assembly = None
            doc = Document(
                pdf_path=pdf_path,
                pdf_hash_type=pdf_identify.hash_type,
                pdf_hash_value=pdf_identify.file_hash,
                pdf_created_date=pdf_created_date,
                pdf_modified_date=pdf_modified_date,
                pdf_downloaded_date=cls.downloaded_date(pdf_path),
                website_modified_date=website_modified_date,
                pdf_url=pdf_url,
                referrer_url=referrer_url,
                assembly=assembly,
            )
            pages = Page.load(pdf_processing, doc)
            doc.pages = list(pages)
            yield doc
def load ( cls , processing_dir : Path , feed_dir : Path ) -> Iterable [ "Document" ] : items = { } for csv_path in feed_dir . rglob ( "*.csv" ) : for pdf_feed in CrawlItem . load ( csv_path ) : pdf_fee_path = pdf_feed . path if pdf_fee_path not in items : items [ pdf_fee_path ] = { } items [ pdf_fee_path ] [ "feed" ] = pdf_feed for json_path in processing_dir . rglob ( "pdf-identify.json" ) : pdf_identify = PdfIdentifyItem . read ( json_path ) pdf_identify_path = str ( pdf_identify . pdf_file ) if pdf_identify_path not in items : items [ pdf_identify_path ] = { } items [ pdf_identify_path ] [ "identify" ] = pdf_identify items [ pdf_identify_path ] [ "processing" ] = json_path . parent info_path = json_path . with_name ( "pdf-info.json" ) if not info_path . exists ( ) : continue pdf_info = PdfInfoItem . load_json ( info_path ) pdf_info_path = str ( pdf_info . pdf_path ) if pdf_info_path not in items : items [ pdf_info_path ] = { } items [ pdf_info_path ] [ "info" ] = pdf_info for pdf_file , item in items . items ( ) : pdf_path = Path ( pdf_file ) pdf_feed = item . get ( "feed" ) pdf_identify = item . get ( "identify" ) pdf_info = item . get ( "info" ) pdf_processing = item . get ( "processing" ) if pdf_info and pdf_info . entries : pdf_created_date = pdf_info . entries . get ( "CreationDate" ) pdf_modified_date = pdf_info . entries . get ( "ModDate" ) else : pdf_created_date = None pdf_modified_date = None if pdf_feed : website_modified_date = pdf_feed . last_updated pdf_url = pdf_feed . url referrer_url = pdf_feed . referrer assembly = pdf_feed . category else : website_modified_date = None pdf_url = None referrer_url = None assembly = None doc = Document ( pdf_path = pdf_path , pdf_hash_type = pdf_identify . hash_type , pdf_hash_value = pdf_identify . file_hash , pdf_created_date = pdf_created_date , pdf_modified_date = pdf_modified_date , pdf_downloaded_date = cls . downloaded_date ( pdf_path ) , website_modified_date = website_modified_date , pdf_url = pdf_url , referrer_url = referrer_url , assembly = assembly , ) pages = Page . load ( pdf_processing , doc ) doc . pages = list ( pages ) yield doc
Load documents from a directory.
Load documents from a directory .
"""Load documents from a directory.""" # feed # identify # info # type: CrawlItem # type: PdfIdentifyItem # type: PdfInfoItem # type: Path
cls
processing_dir
Path
feed_dir
Path