fulltext/extract.py
def extract(identifier: str, id_type: str, version: str,
            owner: Optional[str] = None,
            token: Optional[str] = None) -> Dict[str, str]:
    """Perform text extraction for a single arXiv document."""
    logger.debug('Perform extraction for %s in bucket %s with version %s',
                 identifier, id_type, version)
    storage = store.Storage.current_session()

    # This assumes we have a metadata record on disk already.
    extraction = storage.retrieve(identifier, version, bucket=id_type,
                                  meta_only=True)

    pdf_path: Optional[str] = None
    try:
        pdf_content = _retrieve_pdf(identifier, id_type, token)
        pdf_path = _store_pdf_in_workdir(identifier, id_type, pdf_content)
        content = extractor.do_extraction(pdf_path)

    except Exception as e:
        logger.error('Failed to process %s: %s', identifier, e)
        storage.store(extraction.copy(status=Extraction.Status.FAILED,
                                      ended=datetime.now(UTC),
                                      exception=str(e)))
        raise e
    finally:
        if pdf_path is not None:
            os.remove(pdf_path)    # Cleanup.

    extraction = extraction.copy(status=Extraction.Status.SUCCEEDED,
                                 ended=datetime.now(UTC),
                                 content=content)
    storage.store(extraction, 'plain')
    extraction = extraction.copy(content=psv.normalize_text_psv(content))
    storage.store(extraction, 'psv')
    result = extraction.to_dict()
    result.pop('content')
    return result
Perform text extraction for a single arXiv document.
Perform text extraction for a single arXiv document.
Perform text extraction for a single arXiv document .
def extract(identifier: str, id_type: str, version: str,
            owner: Optional[str] = None,
            token: Optional[str] = None) -> Dict[str, str]:
    logger.debug('Perform extraction for %s in bucket %s with version %s',
                 identifier, id_type, version)
    storage = store.Storage.current_session()
    extraction = storage.retrieve(identifier, version, bucket=id_type,
                                  meta_only=True)
    pdf_path: Optional[str] = None
    try:
        pdf_content = _retrieve_pdf(identifier, id_type, token)
        pdf_path = _store_pdf_in_workdir(identifier, id_type, pdf_content)
        content = extractor.do_extraction(pdf_path)
    except Exception as e:
        logger.error('Failed to process %s: %s', identifier, e)
        storage.store(extraction.copy(status=Extraction.Status.FAILED,
                                      ended=datetime.now(UTC),
                                      exception=str(e)))
        raise e
    finally:
        if pdf_path is not None:
            os.remove(pdf_path)    
    extraction = extraction.copy(status=Extraction.Status.SUCCEEDED,
                                 ended=datetime.now(UTC),
                                 content=content)
    storage.store(extraction, 'plain')
    extraction = extraction.copy(content=psv.normalize_text_psv(content))
    storage.store(extraction, 'psv')
    result = extraction.to_dict()
    result.pop('content')
    return result
def extract ( identifier : str , id_type : str , version : str , owner : Optional [ str ] = None , token : Optional [ str ] = None ) -> Dict [ str , str ] : logger . debug ( 'Perform extraction for %s in bucket %s with version %s' , identifier , id_type , version ) storage = store . Storage . current_session ( ) extraction = storage . retrieve ( identifier , version , bucket = id_type , meta_only = True ) pdf_path : Optional [ str ] = None try : pdf_content = _retrieve_pdf ( identifier , id_type , token ) pdf_path = _store_pdf_in_workdir ( identifier , id_type , pdf_content ) content = extractor . do_extraction ( pdf_path ) except Exception as e : logger . error ( 'Failed to process %s: %s' , identifier , e ) storage . store ( extraction . copy ( status = Extraction . Status . FAILED , ended = datetime . now ( UTC ) , exception = str ( e ) ) ) raise e finally : if pdf_path is not None : os . remove ( pdf_path ) extraction = extraction . copy ( status = Extraction . Status . SUCCEEDED , ended = datetime . now ( UTC ) , content = content ) storage . store ( extraction , 'plain' ) extraction = extraction . copy ( content = psv . normalize_text_psv ( content ) ) storage . store ( extraction , 'psv' ) result = extraction . to_dict ( ) result . pop ( 'content' ) return result
Perform text extraction for a single arXiv document.
Perform text extraction for a single arXiv document .
"""Perform text extraction for a single arXiv document.""" # This assumes we have a metadata record on disk already. # Cleanup.
identifier
str
id_type
str
version
str
owner
Optional[str]
token
Optional[str]