doc2json/tex2json/xml_to_json.py
def process_tables_from_tex(sp: BeautifulSoup, ref_map: Dict) -> Dict:
    """
    Generate table dict and replace with id tokens
    :param sp:
    :param ref_map:
    :return:
    """
    # process floats first because they are on the outside
    for flt in sp.find_all('float'):
        try:
            if flt.name and flt.get('name') == 'table':
                if flt.get('id'):
                    # normalize table id
                    ref_id = flt.get('id').replace('uid', 'TABREF')
                    # remove equation tex
                    if flt.caption:
                        caption_el = replace_ref_tokens(sp, flt.caption, ref_map)
                        for eq in caption_el.find_all('texmath'):
                            eq.decompose()
                        caption_text = caption_el.text.strip()
                    elif flt.head:
                        head_el = replace_ref_tokens(sp, flt.head, ref_map)
                        for eq in head_el.find_all('texmath'):
                            eq.decompose()
                        caption_text = head_el.text.strip()
                    elif flt.p:
                        caption_parts = []
                        for tab_p in flt.find_all('p'):
                            p_el = replace_ref_tokens(sp, tab_p, ref_map)
                            for eq in p_el.find_all('texmath'):
                                eq.decompose()
                            caption_parts.append(p_el.text.strip())
                        caption_text = ' '.join(caption_parts)
                    else:
                        tab_el = replace_ref_tokens(sp, flt, ref_map)
                        caption_text = tab_el.text.strip()
                    if caption_text:
                        caption_text = re.sub(r'\s+', ' ', caption_text)
                        caption_text = re.sub(r'\s', ' ', caption_text)
                    # form tabmap entry
                    ref_map[ref_id]['text'] = caption_text
                flt.decompose()
        except AttributeError:
            print('Attribute error with table float: ', flt.name)
            continue

    for tab in sp.find_all('table'):
        try:
            # skip inline tables
            if tab.get('rend') == 'inline':
                continue
            # process them
            if tab.name and tab.get('id'):
                # normalize table id
                ref_id = tab.get('id').replace('uid', 'TABREF')
                # remove equation tex from caption and clean and resolve refs
                if tab.caption:
                    caption_el = replace_ref_tokens(sp, tab.caption, ref_map)
                    for eq in caption_el.find_all('texmath'):
                        eq.decompose()
                    caption_text = caption_el.text.strip()
                elif tab.head:
                    head_el = replace_ref_tokens(sp, tab.head, ref_map)
                    for eq in head_el.find_all('texmath'):
                        eq.decompose()
                    caption_text = head_el.text.strip()
                elif tab.p:
                    caption_parts = []
                    for tab_p in tab.find_all('p'):
                        p_el = replace_ref_tokens(sp, tab_p, ref_map)
                        for eq in p_el.find_all('texmath'):
                            eq.decompose()
                        caption_parts.append(p_el.text.strip())
                    caption_text = ' '.join(caption_parts)
                else:
                    tab_el = replace_ref_tokens(sp, tab, ref_map)
                    caption_text = tab_el.text.strip()
                if caption_text:
                    caption_text = re.sub(r'\s+', ' ', caption_text)
                    caption_text = re.sub(r'\s', ' ', caption_text)
                # form tabmap entry
                ref_map[ref_id]['text'] = caption_text
        except AttributeError:
            print('Attribute error with table: ', tab.name)
            continue
        tab.decompose()

    return ref_map

    Generate table dict and replace with id tokens
    :param sp:
    :param ref_map:
    :return:
    
Generate table dict and replace with id tokens
Generate table dict and replace with id tokens
def process_tables_from_tex(sp: BeautifulSoup, ref_map: Dict) -> Dict:
    for flt in sp.find_all('float'):
        try:
            if flt.name and flt.get('name') == 'table':
                if flt.get('id'):
                    ref_id = flt.get('id').replace('uid', 'TABREF')
                    if flt.caption:
                        caption_el = replace_ref_tokens(sp, flt.caption, ref_map)
                        for eq in caption_el.find_all('texmath'):
                            eq.decompose()
                        caption_text = caption_el.text.strip()
                    elif flt.head:
                        head_el = replace_ref_tokens(sp, flt.head, ref_map)
                        for eq in head_el.find_all('texmath'):
                            eq.decompose()
                        caption_text = head_el.text.strip()
                    elif flt.p:
                        caption_parts = []
                        for tab_p in flt.find_all('p'):
                            p_el = replace_ref_tokens(sp, tab_p, ref_map)
                            for eq in p_el.find_all('texmath'):
                                eq.decompose()
                            caption_parts.append(p_el.text.strip())
                        caption_text = ' '.join(caption_parts)
                    else:
                        tab_el = replace_ref_tokens(sp, flt, ref_map)
                        caption_text = tab_el.text.strip()
                    if caption_text:
                        caption_text = re.sub(r'\s+', ' ', caption_text)
                        caption_text = re.sub(r'\s', ' ', caption_text)
                    ref_map[ref_id]['text'] = caption_text
                flt.decompose()
        except AttributeError:
            print('Attribute error with table float: ', flt.name)
            continue
    for tab in sp.find_all('table'):
        try:
            if tab.get('rend') == 'inline':
                continue
            if tab.name and tab.get('id'):
                ref_id = tab.get('id').replace('uid', 'TABREF')
                 from caption and clean and resolve refs
                if tab.caption:
                    caption_el = replace_ref_tokens(sp, tab.caption, ref_map)
                    for eq in caption_el.find_all('texmath'):
                        eq.decompose()
                    caption_text = caption_el.text.strip()
                elif tab.head:
                    head_el = replace_ref_tokens(sp, tab.head, ref_map)
                    for eq in head_el.find_all('texmath'):
                        eq.decompose()
                    caption_text = head_el.text.strip()
                elif tab.p:
                    caption_parts = []
                    for tab_p in tab.find_all('p'):
                        p_el = replace_ref_tokens(sp, tab_p, ref_map)
                        for eq in p_el.find_all('texmath'):
                            eq.decompose()
                        caption_parts.append(p_el.text.strip())
                    caption_text = ' '.join(caption_parts)
                else:
                    tab_el = replace_ref_tokens(sp, tab, ref_map)
                    caption_text = tab_el.text.strip()
                if caption_text:
                    caption_text = re.sub(r'\s+', ' ', caption_text)
                    caption_text = re.sub(r'\s', ' ', caption_text)
                ref_map[ref_id]['text'] = caption_text
        except AttributeError:
            print('Attribute error with table: ', tab.name)
            continue
        tab.decompose()
    return ref_map
def process_tables_from_tex ( sp : BeautifulSoup , ref_map : Dict ) -> Dict : for flt in sp . find_all ( 'float' ) : try : if flt . name and flt . get ( 'name' ) == 'table' : if flt . get ( 'id' ) : ref_id = flt . get ( 'id' ) . replace ( 'uid' , 'TABREF' ) if flt . caption : caption_el = replace_ref_tokens ( sp , flt . caption , ref_map ) for eq in caption_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_text = caption_el . text . strip ( ) elif flt . head : head_el = replace_ref_tokens ( sp , flt . head , ref_map ) for eq in head_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_text = head_el . text . strip ( ) elif flt . p : caption_parts = [ ] for tab_p in flt . find_all ( 'p' ) : p_el = replace_ref_tokens ( sp , tab_p , ref_map ) for eq in p_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_parts . append ( p_el . text . strip ( ) ) caption_text = ' ' . join ( caption_parts ) else : tab_el = replace_ref_tokens ( sp , flt , ref_map ) caption_text = tab_el . text . strip ( ) if caption_text : caption_text = re . sub ( r'\s+' , ' ' , caption_text ) caption_text = re . sub ( r'\s' , ' ' , caption_text ) ref_map [ ref_id ] [ 'text' ] = caption_text flt . decompose ( ) except AttributeError : print ( 'Attribute error with table float: ' , flt . name ) continue for tab in sp . find_all ( 'table' ) : try : if tab . get ( 'rend' ) == 'inline' : continue if tab . name and tab . get ( 'id' ) : ref_id = tab . get ( 'id' ) . replace ( 'uid' , 'TABREF' ) if tab . caption : caption_el = replace_ref_tokens ( sp , tab . caption , ref_map ) for eq in caption_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_text = caption_el . text . strip ( ) elif tab . head : head_el = replace_ref_tokens ( sp , tab . head , ref_map ) for eq in head_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_text = head_el . text . strip ( ) elif tab . p : caption_parts = [ ] for tab_p in tab . find_all ( 'p' ) : p_el = replace_ref_tokens ( sp , tab_p , ref_map ) for eq in p_el . find_all ( 'texmath' ) : eq . decompose ( ) caption_parts . append ( p_el . text . strip ( ) ) caption_text = ' ' . join ( caption_parts ) else : tab_el = replace_ref_tokens ( sp , tab , ref_map ) caption_text = tab_el . text . strip ( ) if caption_text : caption_text = re . sub ( r'\s+' , ' ' , caption_text ) caption_text = re . sub ( r'\s' , ' ' , caption_text ) ref_map [ ref_id ] [ 'text' ] = caption_text except AttributeError : print ( 'Attribute error with table: ' , tab . name ) continue tab . decompose ( ) return ref_map
Generate table dict and replace with id tokens
Generate table dict and replace with id tokens
"""
    Generate table dict and replace with id tokens
    :param sp:
    :param ref_map:
    :return:
    """ # process floats first because they are on the outside # normalize table id # remove equation tex # form tabmap entry # skip inline tables # process them # normalize table id # remove equation tex from caption and clean and resolve refs # form tabmap entry
None
sp
BeautifulSoup
None
ref_map
Dict
None