doc2json/utils/latex_util.py
def normalize(path, out_dir, write_logs=True):
    """
    Normalize an arXiv file
    Adapted from https://github.com/IllDepence/unarXive
        with modifications

    Identifies the primary *.tex file, the bibliography file,
    and expands other tex files and the bibliography into the
    main tex file
    """
    def log(msg):
        if write_logs:
            with open(os.path.join(out_dir, 'log.txt'), 'a') as f:
                f.write('{}\n'.format(msg))

    # break path
    _, fn = os.path.split(path.strip('/'))

    # identify main tex file
    main_tex_path = None
    ignored_names = []

    # check .tex files first
    for tfn in os.listdir(path):

        if not TEX_EXT_PATT.match(os.path.splitext(tfn)[1]):
            ignored_names.append(tfn)
            continue

        try:
            cntnt = read_file(os.path.join(path, tfn))
        except:
            continue

        if re.search(MAIN_TEX_PATT, cntnt) is not None:
            main_tex_path = tfn

    # try other files
    if main_tex_path is None:
        for tfn in ignored_names:
            if NON_TEXT_PATT.match(os.path.splitext(tfn)[1]):
                continue
            try:
                cntnt = read_file(os.path.join(path, tfn))
                if re.search(MAIN_TEX_PATT, cntnt) is not None:
                    main_tex_path = tfn
            except:
                continue

    # give up
    if main_tex_path is None:
        log(('couldn\'t find main tex file in dump archive {}'
             '').format(fn))

    # flatten to single tex file and save
    with tempfile.TemporaryDirectory() as tmp_dir_path:
        temp_tex_fn = os.path.join(tmp_dir_path, f'{fn}.tex')

        # find bbl file
        main_tex_fn = os.path.join(path, main_tex_path)
        bbl_files = glob.glob(os.path.join(path, '*.bbl'))

        if bbl_files:
            latexpand_args = ['latexpand',
                              '--expand-bbl',
                              os.path.split(bbl_files[0])[1],
                              main_tex_path,
                              '--output',
                              temp_tex_fn]
        else:
            latexpand_args = ['latexpand',
                              main_tex_path,
                              '--output',
                              temp_tex_fn]

        # run latexpand
        with open(os.path.join(out_dir, 'log_latexpand.txt'), 'a+') as err:
            subprocess.run(latexpand_args, stderr=err, cwd=path)

        # re-read and write to ensure utf-8 b/c latexpand doesn't
        # behave
        new_tex_fn = os.path.join(out_dir, f'{fn}.tex')
        cntnt = read_file(temp_tex_fn)
        if PRE_FIX_NATBIB:
            cntnt = NATBIB_PATT.sub(r'\\cite{\3}', cntnt)
        if PRE_FIX_BIBOPT:
            cntnt = BIBOPT_PATT.sub(r'\\bibitem', cntnt)
        if PRE_FILTER_MATH:
            cntnt = remove_math(cntnt)
        with open(new_tex_fn, mode='w', encoding='utf-8') as f:
            f.write(cntnt)

    Normalize an arXiv file
    Adapted from https://github.com/IllDepence/unarXive
        with modifications

    Identifies the primary *.tex file, the bibliography file,
    and expands other tex files and the bibliography into the
    main tex file
    


Identifies the primary *.tex file, the bibliography file,
and expands other tex files and the bibliography into the
main tex file
Identifies the primary * . tex file the bibliography file and expands other tex files and the bibliography into the main tex file
def normalize(path, out_dir, write_logs=True):
    def log(msg):
        if write_logs:
            with open(os.path.join(out_dir, 'log.txt'), 'a') as f:
                f.write('{}\n'.format(msg))
    _, fn = os.path.split(path.strip('/'))
    main_tex_path = None
    ignored_names = []
    for tfn in os.listdir(path):
        if not TEX_EXT_PATT.match(os.path.splitext(tfn)[1]):
            ignored_names.append(tfn)
            continue
        try:
            cntnt = read_file(os.path.join(path, tfn))
        except:
            continue
        if re.search(MAIN_TEX_PATT, cntnt) is not None:
            main_tex_path = tfn
    if main_tex_path is None:
        for tfn in ignored_names:
            if NON_TEXT_PATT.match(os.path.splitext(tfn)[1]):
                continue
            try:
                cntnt = read_file(os.path.join(path, tfn))
                if re.search(MAIN_TEX_PATT, cntnt) is not None:
                    main_tex_path = tfn
            except:
                continue
    if main_tex_path is None:
        log(('couldn\'t find main tex file in dump archive {}'
             '').format(fn))
    with tempfile.TemporaryDirectory() as tmp_dir_path:
        temp_tex_fn = os.path.join(tmp_dir_path, f'{fn}.tex')
        main_tex_fn = os.path.join(path, main_tex_path)
        bbl_files = glob.glob(os.path.join(path, '*.bbl'))
        if bbl_files:
            latexpand_args = ['latexpand',
                              '--expand-bbl',
                              os.path.split(bbl_files[0])[1],
                              main_tex_path,
                              '--output',
                              temp_tex_fn]
        else:
            latexpand_args = ['latexpand',
                              main_tex_path,
                              '--output',
                              temp_tex_fn]
        with open(os.path.join(out_dir, 'log_latexpand.txt'), 'a+') as err:
            subprocess.run(latexpand_args, stderr=err, cwd=path)
        new_tex_fn = os.path.join(out_dir, f'{fn}.tex')
        cntnt = read_file(temp_tex_fn)
        if PRE_FIX_NATBIB:
            cntnt = NATBIB_PATT.sub(r'\\cite{\3}', cntnt)
        if PRE_FIX_BIBOPT:
            cntnt = BIBOPT_PATT.sub(r'\\bibitem', cntnt)
        if PRE_FILTER_MATH:
            cntnt = remove_math(cntnt)
        with open(new_tex_fn, mode='w', encoding='utf-8') as f:
            f.write(cntnt)
def normalize ( path , out_dir , write_logs = True ) : def log ( msg ) : if write_logs : with open ( os . path . join ( out_dir , 'log.txt' ) , 'a' ) as f : f . write ( '{}\n' . format ( msg ) ) _ , fn = os . path . split ( path . strip ( '/' ) ) main_tex_path = None ignored_names = [ ] for tfn in os . listdir ( path ) : if not TEX_EXT_PATT . match ( os . path . splitext ( tfn ) [ 1 ] ) : ignored_names . append ( tfn ) continue try : cntnt = read_file ( os . path . join ( path , tfn ) ) except : continue if re . search ( MAIN_TEX_PATT , cntnt ) is not None : main_tex_path = tfn if main_tex_path is None : for tfn in ignored_names : if NON_TEXT_PATT . match ( os . path . splitext ( tfn ) [ 1 ] ) : continue try : cntnt = read_file ( os . path . join ( path , tfn ) ) if re . search ( MAIN_TEX_PATT , cntnt ) is not None : main_tex_path = tfn except : continue if main_tex_path is None : log ( ( 'couldn\'t find main tex file in dump archive {}' '' ) . format ( fn ) ) with tempfile . TemporaryDirectory ( ) as tmp_dir_path : temp_tex_fn = os . path . join ( tmp_dir_path , f'{fn}.tex' ) main_tex_fn = os . path . join ( path , main_tex_path ) bbl_files = glob . glob ( os . path . join ( path , '*.bbl' ) ) if bbl_files : latexpand_args = [ 'latexpand' , '--expand-bbl' , os . path . split ( bbl_files [ 0 ] ) [ 1 ] , main_tex_path , '--output' , temp_tex_fn ] else : latexpand_args = [ 'latexpand' , main_tex_path , '--output' , temp_tex_fn ] with open ( os . path . join ( out_dir , 'log_latexpand.txt' ) , 'a+' ) as err : subprocess . run ( latexpand_args , stderr = err , cwd = path ) new_tex_fn = os . path . join ( out_dir , f'{fn}.tex' ) cntnt = read_file ( temp_tex_fn ) if PRE_FIX_NATBIB : cntnt = NATBIB_PATT . sub ( r'\\cite{\3}' , cntnt ) if PRE_FIX_BIBOPT : cntnt = BIBOPT_PATT . sub ( r'\\bibitem' , cntnt ) if PRE_FILTER_MATH : cntnt = remove_math ( cntnt ) with open ( new_tex_fn , mode = 'w' , encoding = 'utf-8' ) as f : f . write ( cntnt )
Normalize an arXiv file
Adapted from https://github.com/IllDepence/unarXive
with modifications
Normalize an arXiv file Adapted from https : // github . com / IllDepence / unarXive with modifications
"""
    Normalize an arXiv file
    Adapted from https://github.com/IllDepence/unarXive
        with modifications

    Identifies the primary *.tex file, the bibliography file,
    and expands other tex files and the bibliography into the
    main tex file
    """ # break path # identify main tex file # check .tex files first # try other files # give up # flatten to single tex file and save # find bbl file # run latexpand # re-read and write to ensure utf-8 b/c latexpand doesn't # behave
path
out_dir
write_logs