fulltext/process/psv.py
def split_on_references(lines: List[str], max_refs_fraction: float = 0.5) \
        -> Tuple[List[str], List[str]]:
    """
    Mark the start of the references.

    Does this by looking for the last occurrence of the word "Reference" or
    "Bibliography".
    """
    regex_refsection = re.compile(
        r'^[^a-zA-Z]*(Reference[s]?|Bibliography)[\W]*$', flags=re.IGNORECASE
    )

    psv: List[str] = []
    ref: List[str] = []
    line_num = 0
    last_refs = 0

    for line in lines:
        line_num += 1
        if regex_refsection.match(line):
            last_refs = line_num

    if line_num:
        refs_fraction = 1.0 - last_refs / line_num
        if refs_fraction > max_refs_fraction:
            print(
                "Not removing references as they comprise {}% of lines".format(
                    int(100*refs_fraction)
                )
            )
            last_refs = line_num + 1

    line_num = 0

    for line_num, line in enumerate(lines):
        if last_refs > 0 and line_num >= last_refs-1:
            ref.append(line)
        else:
            psv.append(line)
    return psv, ref

    Mark the start of the references.

    Does this by looking for the last occurrence of the word "Reference" or
    "Bibliography".
    
Mark the start of the references.
Does this by looking for the last occurrence of the word "Reference" or
"Bibliography".
Mark the start of the references . Does this by looking for the last occurrence of the word " Reference " or " Bibliography " .
def split_on_references(lines: List[str], max_refs_fraction: float = 0.5) \
        -> Tuple[List[str], List[str]]:
    regex_refsection = re.compile(
        r'^[^a-zA-Z]*(Reference[s]?|Bibliography)[\W]*$', flags=re.IGNORECASE
    )
    psv: List[str] = []
    ref: List[str] = []
    line_num = 0
    last_refs = 0
    for line in lines:
        line_num += 1
        if regex_refsection.match(line):
            last_refs = line_num
    if line_num:
        refs_fraction = 1.0 - last_refs / line_num
        if refs_fraction > max_refs_fraction:
            print(
                "Not removing references as they comprise {}% of lines".format(
                    int(100*refs_fraction)
                )
            )
            last_refs = line_num + 1
    line_num = 0
    for line_num, line in enumerate(lines):
        if last_refs > 0 and line_num >= last_refs-1:
            ref.append(line)
        else:
            psv.append(line)
    return psv, ref
def split_on_references ( lines : List [ str ] , max_refs_fraction : float = 0.5 ) -> Tuple [ List [ str ] , List [ str ] ] : regex_refsection = re . compile ( r'^[^a-zA-Z]*(Reference[s]?|Bibliography)[\W]*$' , flags = re . IGNORECASE ) psv : List [ str ] = [ ] ref : List [ str ] = [ ] line_num = 0 last_refs = 0 for line in lines : line_num += 1 if regex_refsection . match ( line ) : last_refs = line_num if line_num : refs_fraction = 1.0 - last_refs / line_num if refs_fraction > max_refs_fraction : print ( "Not removing references as they comprise {}% of lines" . format ( int ( 100 * refs_fraction ) ) ) last_refs = line_num + 1 line_num = 0 for line_num , line in enumerate ( lines ) : if last_refs > 0 and line_num >= last_refs - 1 : ref . append ( line ) else : psv . append ( line ) return psv , ref
Mark the start of the references.
Mark the start of the references .
"""
    Mark the start of the references.

    Does this by looking for the last occurrence of the word "Reference" or
    "Bibliography".
    """
lines
List[str]
max_refs_fraction
float