def get_full_mapping(src_filename, trg_filename, align_filename, mapping_filename, reverse_src2trg=False, lowercase=True):
    ' Get full mapping give align.\n\n    Args:\n        src_filename:\n        trg_filename:\n        align_filename:\n        mapping_filename:\n        reverse_src2trg:\n        lowercase:\n\n    Returns:\n\n    '
    print('src: {}, trg: {}, align: {}, mapping: {}, reverse: {}'.format(src_filename, trg_filename, align_filename, mapping_filename, reverse_src2trg))
    src2trg_mapping = defaultdict((lambda : defaultdict(int)))
    processed_line = 0
    with open(src_filename) as fs, open(trg_filename) as ft, open(align_filename) as fa:
        for (ls, lt, la) in zip(fs, ft, fa):
            if lowercase:
                ls = ls.lower()
                lt = lt.lower()
            processed_line += 1
            ls_words = ls.split()
            lt_words = lt.split()
            la_aligns = la.split()
            src_pos_counter = Counter()
            trg_pos_counter = Counter()
            valid_src_pos = set()
            valid_trg_pos = set()
            for align in la_aligns:
                (src_pos, trg_pos) = align.split('-')
                src_pos = int(src_pos)
                trg_pos = int(trg_pos)
                if _is_token_alnum(ls_words[src_pos]):
                    src_pos_counter[src_pos] += 1
                if _is_token_alnum(lt_words[trg_pos]):
                    trg_pos_counter[trg_pos] += 1
            for (pos, c) in src_pos_counter.items():
                if (c == 1):
                    valid_src_pos.add(pos)
            for (pos, c) in trg_pos_counter.items():
                if (c == 1):
                    valid_trg_pos.add(pos)
            for align in la_aligns:
                (src_pos, trg_pos) = align.split('-')
                src_pos = int(src_pos)
                trg_pos = int(trg_pos)
                if (_is_token_alnum(ls_words[src_pos]) and _is_token_alnum(lt_words[trg_pos]) and (src_pos in valid_src_pos) and (trg_pos in valid_trg_pos)):
                    if reverse_src2trg:
                        src2trg_mapping[lt_words[trg_pos]][ls_words[src_pos]] += 1
                    else:
                        src2trg_mapping[ls_words[src_pos]][lt_words[trg_pos]] += 1
            if ((processed_line % 1000000) == 0):
                print('{} done.'.format(processed_line))
    with open(mapping_filename, 'w') as fw:
        print('dump to {} ...'.format(mapping_filename))
        json.dump(src2trg_mapping, fw)
    return src2trg_mapping
Get full mapping give align.

Args:
    src_filename:
    trg_filename:
    align_filename:
    mapping_filename:
    reverse_src2trg:
    lowercase:

Returns:
def get_full_mapping(src_filename, trg_filename, align_filename, mapping_filename, reverse_src2trg=False, lowercase=True):
    ' Get full mapping give align.\n\n    Args:\n        src_filename:\n        trg_filename:\n        align_filename:\n        mapping_filename:\n        reverse_src2trg:\n        lowercase:\n\n    Returns:\n\n    '
    print('src: {}, trg: {}, align: {}, mapping: {}, reverse: {}'.format(src_filename, trg_filename, align_filename, mapping_filename, reverse_src2trg))
    src2trg_mapping = defaultdict((lambda : defaultdict(int)))
    processed_line = 0
    with open(src_filename) as fs, open(trg_filename) as ft, open(align_filename) as fa:
        for (ls, lt, la) in zip(fs, ft, fa):
            if lowercase:
                ls = ls.lower()
                lt = lt.lower()
            processed_line += 1
            ls_words = ls.split()
            lt_words = lt.split()
            la_aligns = la.split()
            src_pos_counter = Counter()
            trg_pos_counter = Counter()
            valid_src_pos = set()
            valid_trg_pos = set()
            for align in la_aligns:
                (src_pos, trg_pos) = align.split('-')
                src_pos = int(src_pos)
                trg_pos = int(trg_pos)
                if _is_token_alnum(ls_words[src_pos]):
                    src_pos_counter[src_pos] += 1
                if _is_token_alnum(lt_words[trg_pos]):
                    trg_pos_counter[trg_pos] += 1
            for (pos, c) in src_pos_counter.items():
                if (c == 1):
                    valid_src_pos.add(pos)
            for (pos, c) in trg_pos_counter.items():
                if (c == 1):
                    valid_trg_pos.add(pos)
            for align in la_aligns:
                (src_pos, trg_pos) = align.split('-')
                src_pos = int(src_pos)
                trg_pos = int(trg_pos)
                if (_is_token_alnum(ls_words[src_pos]) and _is_token_alnum(lt_words[trg_pos]) and (src_pos in valid_src_pos) and (trg_pos in valid_trg_pos)):
                    if reverse_src2trg:
                        src2trg_mapping[lt_words[trg_pos]][ls_words[src_pos]] += 1
                    else:
                        src2trg_mapping[ls_words[src_pos]][lt_words[trg_pos]] += 1
            if ((processed_line % 1000000) == 0):
                print('{} done.'.format(processed_line))
    with open(mapping_filename, 'w') as fw:
        print('dump to {} ...'.format(mapping_filename))
        json.dump(src2trg_mapping, fw)
    return src2trg_mapping