doc2json/jats2json/pmc_utils/all_tag_utils.py
def parse_all_paragraphs_in_section(
    sec_tag,
    par_to_text: Callable = None,
    replace_formula=True
) -> List[Dict]:
    """Internal function. Assumes section has no nested tags
    `par_to_text` is an optional function that converts the `par` tag into a string.  by default, calls `par_tag.text`.
    """
    outputs = []
    sp = BeautifulSoup('', 'lxml')
    for par_tag in sec_tag.find_all("p", recursive=True):
        cite_spans = []
        fig_spans = []
        table_spans = []
        # suppl_spans = []
        sup_spans = []
        sub_spans = []
        eq_spans = []

        if par_tag.find('display-formula'):
            raise NotImplementedError('Display formula!')

        if par_tag.find('formula'):
            raise NotImplementedError('Formula!')

        formula_dict = parse_formulas(par_tag, sp, replace_formula)

        par_text = par_to_text(par_tag) if par_to_text else par_tag.text
        par_text = re.sub(
            r"[^\S\n\t]", " ", par_text
        )  # replaces whitespace but not newline or tab
        par_text = re.sub(
            r"  ", " ", par_text
        )  # replaces two spaces w/ one

        # Tokenize the text into normal text and special placeholder tokens.
        pattern = r"(#!start#)|(#!sep#)|(#!end#)|(@!start@)|(@!end@)|(&!start&)|(&!end&)"
        tokens = [tok for tok in re.split(pattern, par_text) if tok]

        # To handle nested structures, use a shift-reduce algorithm to consume the text. Placeholder tags are merged away, and related spans are registered.
        stack = []
        full_text = []
        pos = 0
        disable_count = False
        for token in tokens:
            if token in START_TOKENS:
                stack.append(token)
                stack.append(pos)
                stack.append(token.replace('start', 'sep'))
            elif token in SEP_TOKENS:
                assert stack
                stack.append(token)
                disable_count = True
            elif token in END_TOKENS:
                assert stack
                disable_count = False
                args = _reduce_args(stack, token)
                start_pos = args[0][0]
                text = "".join(args[1])
                assert len(args) == 2 or len(args) == 4
                if len(args) == 2:
                    ref_id, ref_type = None, None
                elif len(args) == 4:
                    ref_id = args[2] and args[2][0]
                    ref_type = args[3] and args[3][0]
                stack.append(text)
                _add_spans(
                    token,
                    start_pos,
                    text,
                    ref_id,
                    ref_type,
                    cite_spans,
                    fig_spans,
                    table_spans,
                    sup_spans,
                    sub_spans,
                )
            else:  # just normal text
                stack.append(token)
                if not disable_count:  # metadata appearing after a separator
                    full_text.append(token)
                    pos += len(token)

        full_text = "".join(full_text)
        assert pos == len(full_text)

        title = sec_tag.find("title")
        title = title.text if title else ""

        # get all equation spans
        eq_spans = []
        for span in itertools.chain(
                re.finditer(r'(INLINEFORM\d+)', full_text),
                re.finditer(r'(DISPLAYFORM\d+)', full_text)
        ):
            try:
                matching_formula = formula_dict[span.group()]
                eq_spans.append({
                    "start": span.start(),
                    "end": span.start() + len(span.group()),
                    "text": matching_formula[0],
                    "latex": matching_formula[1],
                    "mathml": matching_formula[2],
                    "ref_id": span.group()
                })
            except KeyError:
                continue

        outputs.append(
            {
                "text": full_text,
                'cite_spans': cite_spans,
                'fig_spans': fig_spans,
                'table_spans': table_spans,
                # 'suppl_spans': suppl_spans,
                'sup_spans': sup_spans,
                'sub_spans': sub_spans,
                'eq_spans': eq_spans,
                "section": title,
            }
        )
    return outputs
Internal function. Assumes section has no nested tags
    `par_to_text` is an optional function that converts the `par` tag into a string.  by default, calls `par_tag.text`.
    
Internal function. Assumes section has no nested tags
`par_to_text` is an optional function that converts the `par` tag into a string.
Internal function . Assumes section has no nested tags ` par_to_text ` is an optional function that converts the ` par ` tag into a string .
def parse_all_paragraphs_in_section(
    sec_tag,
    par_to_text: Callable = None,
    replace_formula=True
) -> List[Dict]:
    outputs = []
    sp = BeautifulSoup('', 'lxml')
    for par_tag in sec_tag.find_all("p", recursive=True):
        cite_spans = []
        fig_spans = []
        table_spans = []
        sup_spans = []
        sub_spans = []
        eq_spans = []
        if par_tag.find('display-formula'):
            raise NotImplementedError('Display formula!')
        if par_tag.find('formula'):
            raise NotImplementedError('Formula!')
        formula_dict = parse_formulas(par_tag, sp, replace_formula)
        par_text = par_to_text(par_tag) if par_to_text else par_tag.text
        par_text = re.sub(
            r"[^\S\n\t]", " ", par_text
        )  
        par_text = re.sub(
            r"  ", " ", par_text
        )  
        pattern = r"(#!start#)|(#!sep#)|(#!end#)|(@!start@)|(@!end@)|(&!start&)|(&!end&)"
        tokens = [tok for tok in re.split(pattern, par_text) if tok]
        stack = []
        full_text = []
        pos = 0
        disable_count = False
        for token in tokens:
            if token in START_TOKENS:
                stack.append(token)
                stack.append(pos)
                stack.append(token.replace('start', 'sep'))
            elif token in SEP_TOKENS:
                assert stack
                stack.append(token)
                disable_count = True
            elif token in END_TOKENS:
                assert stack
                disable_count = False
                args = _reduce_args(stack, token)
                start_pos = args[0][0]
                text = "".join(args[1])
                assert len(args) == 2 or len(args) == 4
                if len(args) == 2:
                    ref_id, ref_type = None, None
                elif len(args) == 4:
                    ref_id = args[2] and args[2][0]
                    ref_type = args[3] and args[3][0]
                stack.append(text)
                _add_spans(
                    token,
                    start_pos,
                    text,
                    ref_id,
                    ref_type,
                    cite_spans,
                    fig_spans,
                    table_spans,
                    sup_spans,
                    sub_spans,
                )
            else:  
                stack.append(token)
                if not disable_count:  
                    full_text.append(token)
                    pos += len(token)
        full_text = "".join(full_text)
        assert pos == len(full_text)
        title = sec_tag.find("title")
        title = title.text if title else ""
        eq_spans = []
        for span in itertools.chain(
                re.finditer(r'(INLINEFORM\d+)', full_text),
                re.finditer(r'(DISPLAYFORM\d+)', full_text)
        ):
            try:
                matching_formula = formula_dict[span.group()]
                eq_spans.append({
                    "start": span.start(),
                    "end": span.start() + len(span.group()),
                    "text": matching_formula[0],
                    "latex": matching_formula[1],
                    "mathml": matching_formula[2],
                    "ref_id": span.group()
                })
            except KeyError:
                continue
        outputs.append(
            {
                "text": full_text,
                'cite_spans': cite_spans,
                'fig_spans': fig_spans,
                'table_spans': table_spans,
                'sup_spans': sup_spans,
                'sub_spans': sub_spans,
                'eq_spans': eq_spans,
                "section": title,
            }
        )
    return outputs
def parse_all_paragraphs_in_section ( sec_tag , par_to_text : Callable = None , replace_formula = True ) -> List [ Dict ] : outputs = [ ] sp = BeautifulSoup ( '' , 'lxml' ) for par_tag in sec_tag . find_all ( "p" , recursive = True ) : cite_spans = [ ] fig_spans = [ ] table_spans = [ ] sup_spans = [ ] sub_spans = [ ] eq_spans = [ ] if par_tag . find ( 'display-formula' ) : raise NotImplementedError ( 'Display formula!' ) if par_tag . find ( 'formula' ) : raise NotImplementedError ( 'Formula!' ) formula_dict = parse_formulas ( par_tag , sp , replace_formula ) par_text = par_to_text ( par_tag ) if par_to_text else par_tag . text par_text = re . sub ( r"[^\S\n\t]" , " " , par_text ) par_text = re . sub ( r"  " , " " , par_text ) pattern = r"(#!start#)|(#!sep#)|(#!end#)|(@!start@)|(@!end@)|(&!start&)|(&!end&)" tokens = [ tok for tok in re . split ( pattern , par_text ) if tok ] stack = [ ] full_text = [ ] pos = 0 disable_count = False for token in tokens : if token in START_TOKENS : stack . append ( token ) stack . append ( pos ) stack . append ( token . replace ( 'start' , 'sep' ) ) elif token in SEP_TOKENS : assert stack stack . append ( token ) disable_count = True elif token in END_TOKENS : assert stack disable_count = False args = _reduce_args ( stack , token ) start_pos = args [ 0 ] [ 0 ] text = "" . join ( args [ 1 ] ) assert len ( args ) == 2 or len ( args ) == 4 if len ( args ) == 2 : ref_id , ref_type = None , None elif len ( args ) == 4 : ref_id = args [ 2 ] and args [ 2 ] [ 0 ] ref_type = args [ 3 ] and args [ 3 ] [ 0 ] stack . append ( text ) _add_spans ( token , start_pos , text , ref_id , ref_type , cite_spans , fig_spans , table_spans , sup_spans , sub_spans , ) else : stack . append ( token ) if not disable_count : full_text . append ( token ) pos += len ( token ) full_text = "" . join ( full_text ) assert pos == len ( full_text ) title = sec_tag . find ( "title" ) title = title . text if title else "" eq_spans = [ ] for span in itertools . chain ( re . finditer ( r'(INLINEFORM\d+)' , full_text ) , re . finditer ( r'(DISPLAYFORM\d+)' , full_text ) ) : try : matching_formula = formula_dict [ span . group ( ) ] eq_spans . append ( { "start" : span . start ( ) , "end" : span . start ( ) + len ( span . group ( ) ) , "text" : matching_formula [ 0 ] , "latex" : matching_formula [ 1 ] , "mathml" : matching_formula [ 2 ] , "ref_id" : span . group ( ) } ) except KeyError : continue outputs . append ( { "text" : full_text , 'cite_spans' : cite_spans , 'fig_spans' : fig_spans , 'table_spans' : table_spans , 'sup_spans' : sup_spans , 'sub_spans' : sub_spans , 'eq_spans' : eq_spans , "section" : title , } ) return outputs
Internal function.
Internal function .
"""Internal function. Assumes section has no nested tags
    `par_to_text` is an optional function that converts the `par` tag into a string.  by default, calls `par_tag.text`.
    """ # suppl_spans = [] # replaces whitespace but not newline or tab # replaces two spaces w/ one # Tokenize the text into normal text and special placeholder tokens. # To handle nested structures, use a shift-reduce algorithm to consume the text. Placeholder tags are merged away, and related spans are registered. # just normal text # metadata appearing after a separator # get all equation spans # 'suppl_spans': suppl_spans,
sec_tag
par_to_text
Callable
replace_formula