leaf_focus/pipeline/prefect_flow/construct.py
def build_full(
        self,
        base_dir: Path,
        pdf_info_exe: Path,
        pdf_text_exe: Path,
        pdf_image_exe: Path,
    ):
        """Build the full Prefect flow."""

        with Flow("leaf-focus") as flow:
            feed_dir = Parameter("feed_dir")
            threshold = Parameter("threshold")

            download_task = DownloadCrawlTask()
            download_items = download_task(feed_dir)

            pdf_identify_task = PdfIdentifyTask(base_dir)
            pdf_identify_items = pdf_identify_task.map(download_items)

            pdf_info_task = PdfInfoTask(base_dir, pdf_info_exe)
            pdf_info_task.map(pdf_identify_items)

            pdf_text_task = PdfTextTask(base_dir, pdf_text_exe)
            pdf_text_task.map(pdf_identify_items)

            pdf_images_task = PdfImagesTask(base_dir, pdf_image_exe)
            pdf_image_items = pdf_images_task.map(pdf_identify_items)

            ocr_prepare_task = OcrPrepareTask(base_dir)
            ocr_prepare_task.map(
                input_item=flatten(pdf_image_items),
                threshold=unmapped(threshold),
            )

            # NOTE: Cannot run OCR as part of the Prefect flow.
            #       This is because tensorflow does not
            #       integrate very well into the Prefect Flow.
            # ocr_recognise_task = OcrRecogniseTask(base_dir)
            # ocr_recognise_task.map(
            #     input_item=flatten(ocr_prepare_items),
            #     threshold=unmapped(threshold),
            #     ocr_wrapper=ocr_resource,
            # )

        return flow
Build the full Prefect flow.
Build the full Prefect flow.
Build the full Prefect flow .
def build_full(
        self,
        base_dir: Path,
        pdf_info_exe: Path,
        pdf_text_exe: Path,
        pdf_image_exe: Path,
    ):
        with Flow("leaf-focus") as flow:
            feed_dir = Parameter("feed_dir")
            threshold = Parameter("threshold")
            download_task = DownloadCrawlTask()
            download_items = download_task(feed_dir)
            pdf_identify_task = PdfIdentifyTask(base_dir)
            pdf_identify_items = pdf_identify_task.map(download_items)
            pdf_info_task = PdfInfoTask(base_dir, pdf_info_exe)
            pdf_info_task.map(pdf_identify_items)
            pdf_text_task = PdfTextTask(base_dir, pdf_text_exe)
            pdf_text_task.map(pdf_identify_items)
            pdf_images_task = PdfImagesTask(base_dir, pdf_image_exe)
            pdf_image_items = pdf_images_task.map(pdf_identify_items)
            ocr_prepare_task = OcrPrepareTask(base_dir)
            ocr_prepare_task.map(
                input_item=flatten(pdf_image_items),
                threshold=unmapped(threshold),
            )
        return flow
def build_full ( self , base_dir : Path , pdf_info_exe : Path , pdf_text_exe : Path , pdf_image_exe : Path , ) : with Flow ( "leaf-focus" ) as flow : feed_dir = Parameter ( "feed_dir" ) threshold = Parameter ( "threshold" ) download_task = DownloadCrawlTask ( ) download_items = download_task ( feed_dir ) pdf_identify_task = PdfIdentifyTask ( base_dir ) pdf_identify_items = pdf_identify_task . map ( download_items ) pdf_info_task = PdfInfoTask ( base_dir , pdf_info_exe ) pdf_info_task . map ( pdf_identify_items ) pdf_text_task = PdfTextTask ( base_dir , pdf_text_exe ) pdf_text_task . map ( pdf_identify_items ) pdf_images_task = PdfImagesTask ( base_dir , pdf_image_exe ) pdf_image_items = pdf_images_task . map ( pdf_identify_items ) ocr_prepare_task = OcrPrepareTask ( base_dir ) ocr_prepare_task . map ( input_item = flatten ( pdf_image_items ) , threshold = unmapped ( threshold ) , ) return flow
Build the full Prefect flow.
Build the full Prefect flow .
"""Build the full Prefect flow.""" # NOTE: Cannot run OCR as part of the Prefect flow. #       This is because tensorflow does not #       integrate very well into the Prefect Flow. # ocr_recognise_task = OcrRecogniseTask(base_dir) # ocr_recognise_task.map( #     input_item=flatten(ocr_prepare_items), #     threshold=unmapped(threshold), #     ocr_wrapper=ocr_resource, # )
self
base_dir
Path
pdf_info_exe
Path
pdf_text_exe
Path
pdf_image_exe
Path