mtp/machamp/dataset_readers/reader_utils.py
def seqs2data(conllu_file, do_lowercase):
    """
    Reads a conllu-like file. We do not base the comment identification on
    the starting character being a '#' , as in some of the datasets we used
    the words where in column 0, and could start with a `#'. Instead we start
    at the back, and see how many columns (tabs) the file has. Then we judge
    any sentences at the start which do not have this amount of columns (tabs)
    as comments. Returns both the read column data as well as the full data.
    """
    sent = []
    for line in open(conllu_file, mode="r", encoding="utf-8"):
        if do_lowercase:
            line = line.lower()
        # because people use paste command, which includes empty tabs
        if len(line) < 2 or line.replace('\t', '') == '':
            if len(sent) == 0:
                continue
            num_cols = len(sent[-1])
            beg_idx = 0
            for i in range(len(sent)):
                back_idx = len(sent) - 1 - i
                if len(sent[back_idx]) == num_cols:
                    beg_idx = len(sent) - 1 - i
            yield sent[beg_idx:], sent
            sent = []
        else:
            sent.append([token for token in line[:-1].split('\t')])

    # adds the last sentence when there is no empty line
    if len(sent) != 0 and sent != ['']:
        num_cols = len(sent[-1])
        beg_idx = 0
        for i in range(len(sent)):
            back_idx = len(sent) - 1 - i
            if len(sent[back_idx]) == num_cols:
                beg_idx = len(sent) - 1 - i
        yield sent[beg_idx:], sent

    Reads a conllu-like file. We do not base the comment identification on
    the starting character being a '#' , as in some of the datasets we used
    the words where in column 0, and could start with a `#'. Instead we start
    at the back, and see how many columns (tabs) the file has. Then we judge
    any sentences at the start which do not have this amount of columns (tabs)
    as comments. Returns both the read column data as well as the full data.
    
Reads a conllu-like file. We do not base the comment identification on
the starting character being a '#' , as in some of the datasets we used
the words where in column 0, and could start with a `#'. Instead we start
at the back, and see how many columns (tabs) the file has. Then we judge
any sentences at the start which do not have this amount of columns (tabs)
as comments. Returns both the read column data as well as the full data.
Reads a conllu - like file . We do not base the comment identification on the starting character being a ' # ' as in some of the datasets we used the words where in column 0 and could start with a ` # ' . Instead we start at the back and see how many columns ( tabs ) the file has . Then we judge any sentences at the start which do not have this amount of columns ( tabs ) as comments . Returns both the read column data as well as the full data .
def seqs2data(conllu_file, do_lowercase):
    sent = []
    for line in open(conllu_file, mode="r", encoding="utf-8"):
        if do_lowercase:
            line = line.lower()
        if len(line) < 2 or line.replace('\t', '') == '':
            if len(sent) == 0:
                continue
            num_cols = len(sent[-1])
            beg_idx = 0
            for i in range(len(sent)):
                back_idx = len(sent) - 1 - i
                if len(sent[back_idx]) == num_cols:
                    beg_idx = len(sent) - 1 - i
            yield sent[beg_idx:], sent
            sent = []
        else:
            sent.append([token for token in line[:-1].split('\t')])
    if len(sent) != 0 and sent != ['']:
        num_cols = len(sent[-1])
        beg_idx = 0
        for i in range(len(sent)):
            back_idx = len(sent) - 1 - i
            if len(sent[back_idx]) == num_cols:
                beg_idx = len(sent) - 1 - i
        yield sent[beg_idx:], sent
def seqs2data ( conllu_file , do_lowercase ) : sent = [ ] for line in open ( conllu_file , mode = "r" , encoding = "utf-8" ) : if do_lowercase : line = line . lower ( ) if len ( line ) < 2 or line . replace ( '\t' , '' ) == '' : if len ( sent ) == 0 : continue num_cols = len ( sent [ - 1 ] ) beg_idx = 0 for i in range ( len ( sent ) ) : back_idx = len ( sent ) - 1 - i if len ( sent [ back_idx ] ) == num_cols : beg_idx = len ( sent ) - 1 - i yield sent [ beg_idx : ] , sent sent = [ ] else : sent . append ( [ token for token in line [ : - 1 ] . split ( '\t' ) ] ) if len ( sent ) != 0 and sent != [ '' ] : num_cols = len ( sent [ - 1 ] ) beg_idx = 0 for i in range ( len ( sent ) ) : back_idx = len ( sent ) - 1 - i if len ( sent [ back_idx ] ) == num_cols : beg_idx = len ( sent ) - 1 - i yield sent [ beg_idx : ] , sent
Reads a conllu-like file.
Reads a conllu - like file .
"""
    Reads a conllu-like file. We do not base the comment identification on
    the starting character being a '#' , as in some of the datasets we used
    the words where in column 0, and could start with a `#'. Instead we start
    at the back, and see how many columns (tabs) the file has. Then we judge
    any sentences at the start which do not have this amount of columns (tabs)
    as comments. Returns both the read column data as well as the full data.
    """ # because people use paste command, which includes empty tabs # adds the last sentence when there is no empty line
conllu_file
do_lowercase