examples/multilingual/data_scripts/download_wmt19_and_before.py

from typing import NamedTuple, List
from urllib.parse import urlparse
import os, sys
import subprocess
from subprocess import check_call, check_output
import glob
import wget
import re
import multiprocessing as mp
from functools import partial
import pathlib
from collections import OrderedDict 

WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None)

if WORKDIR_ROOT is None or  not WORKDIR_ROOT.strip():
    print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."')
    sys.exit(-1)

# scripts and data locations
CWD = os.getcwd()
UTILS = f"{CWD}/utils"

MOSES = f"{UTILS}/mosesdecoder"
SGM_TOOL = f'{MOSES}/scripts/ems/support/input-from-sgm.perl'

TMX2CORPUS = f"{UTILS}/tmx2corpus"
TMX_TOOL = f'python {TMX2CORPUS}/tmx2corpus.py'

to_data_path = f'{WORKDIR_ROOT}/wmt'
download_to = f'{to_data_path}/downloads'
manually_downloads = f'{to_data_path}/downloads'
extract_to = f'{to_data_path}/extracted'
#DESTDIR=${WORKDIR_ROOT}/ML50/raw/
raw_data = f'{WORKDIR_ROOT}/ML50/raw'
####

class DLDataset(NamedTuple):
    name: str
    train_urls: List[str]
    valid_urls: List[str]
    test_urls: List[str]        
    train_files_patterns: List[str] = []
    valid_files_patterns: List[str] = []
    test_files_patterns: List[str] = []



def bar_custom(current, total, width=80):
    print("Downloading: %d%% [%d / %d] Ks" % (current / total * 100, current / 1000, total / 1000), end='\r')

def get_downloaded_file(dl_folder, url):
    if isinstance(url, tuple):
        url, f = url
    else:
        url_f = urlparse(url)
        # f = os.path.split(url_f.path)[-1]
        f = '_'.join(url_f.path.split('/')[1:])
    return url, f"{dl_folder}/{f}"

def download_parts_and_combine(dl_folder, urls, filename):
    parts = []
    for url_record in urls:
        url, part_file = get_downloaded_file(dl_folder, url_record)     
        if os.path.exists(part_file):
            print(f'{part_file} has already been downloaded so skip')
        else: 
            part_file = wget.download(url, part_file, bar=bar_custom)  
        parts.append(part_file)

    def get_combine_cmd(parts):           
        #default as tar.gz.??
        return f'cat {" ".join(parts)} > {filename}'

    combine_cmd = get_combine_cmd(parts)
    call(combine_cmd, debug=True)
    return filename

def download_a_url(dl_folder, url):
    url, filename = get_downloaded_file(dl_folder, url)        
    if os.path.exists(filename):
        print(f'{filename} has already been downloaded so skip')
        return filename

    print(f'downloading {url} to {filename}')
    if isinstance(url, list) or isinstance(url, tuple):
        download_parts_and_combine(dl_folder, url, filename)
    else:
        wget.download(url, filename, bar=bar_custom)
    print(f'dowloaded: {filename}')
    return filename

def download_files(dl_folder, urls, completed_urls={}):
    for url_record in urls:
        url, _ = get_downloaded_file(dl_folder, url_record) 
        filename = download_a_url(dl_folder, url_record) 
        completed_urls[str(url)] = filename
    return completed_urls

def check_need_manual_downalod(dl_folder, to_manually_download_urls):
    to_be_manually_dowloaded = []
    manually_completed_urls = {}
    for url_record, instruction in to_manually_download_urls:
        url, filename = get_downloaded_file(dl_folder, url_record)
        if not os.path.exists(filename):
            print(f'{url} need to be download manually, please download it manually following {instruction}; and copy it to {filename}')
            to_be_manually_dowloaded.append((url, filename))
        else:
            manually_completed_urls[url] = filename
    # if len(to_be_manually_dowloaded) > 0:
    #     raise ValueError('Missing files that need to be downloaded manually; stop the process now.')
    return to_be_manually_dowloaded
        
def download_dataset(to_folder, dl_dataset, completed_urls={}):
    download_files(to_folder, dl_dataset.train_urls, completed_urls)
    download_files(to_folder, dl_dataset.valid_urls, completed_urls)
    download_files(to_folder, dl_dataset.test_urls, completed_urls)
    print('completed downloading')
    return completed_urls

def call(cmd, debug=False):
    if debug:
        print(cmd)
    check_call(cmd, shell=True)

    
def get_extract_name(file_path):
    path = os.path.split(file_path)
    return path[-1] + '_extract' #.split('.')[0]

def extract_file(downloaded_file, extract_folder, get_extract_name=get_extract_name, debug=False):
    extract_name = get_extract_name(downloaded_file)
    extract_to = f'{extract_folder}/{extract_name}'
    os.makedirs(extract_to, exist_ok=True)
    if os.path.exists(f'{extract_to}/DONE'):
        print(f'{downloaded_file} has already been extracted to {extract_to} so skip')
        return extract_to
    def get_extract_cmd(filename):
        if filename.endswith('.tgz') or filename.endswith('tar.gz'):
            return f'tar xzfv {filename} -C {extract_to}'
        elif filename.endswith('.gz.tar'): 
            return f'tar xfv {filename} -C {extract_to}; (cd {extract_to}; gzip -d *.gz; [ $? -eq 0 ]  || gzip -d */*.gz)'  
        elif filename.endswith('.tar'):
            return f'tar xfv {filename} -C {extract_to}'        
        elif filename.endswith('.gz'):
            return f'cp {filename} {extract_to}; (cd {extract_to}; gzip -d *.gz)'
        elif filename.endswith('.zip'):
            return f'unzip {filename} -d {extract_to}'        
    extract_cmd = get_extract_cmd(downloaded_file) 
    print(f'extracting {downloaded_file}')
    if isinstance(extract_cmd, list):
        for c in  extract_cmd:
            call(c, debug=debug)
    else:
        call(extract_cmd, debug=debug)
    call(f'echo DONE > {extract_to}/DONE')
    return extract_to


def extract_all_files(
    completed_urls, extract_folder,
    get_extract_name=get_extract_name,
    completed_extraction={},
    debug=False):
    extracted_folders = OrderedDict()
    for url, downloaded_file in set(completed_urls.items()):
        if downloaded_file in completed_extraction:
            print(f'{downloaded_file} is already extracted; so skip')
            continue
        folder = extract_file(downloaded_file, extract_folder, get_extract_name, debug)
        extracted_folders[url] = folder
    return extracted_folders


def my_glob(folder):
    for p in [f'{folder}/*', f'{folder}/*/*', f'{folder}/*/*/*']:
        for f in glob.glob(p):
            yield f


def sgm2raw(sgm, debug):
    to_file = sgm[0:len(sgm) - len('.sgm')]
    if os.path.exists(to_file):
        debug and print(f'{sgm} already converted to {to_file}; so skip')
        return to_file
    cmd = f'{SGM_TOOL} < {sgm} > {to_file}'
    call(cmd, debug)
    return to_file

def tmx2raw(tmx, debug):
    to_file = tmx[0:len(tmx) - len('.tmx')]
    to_folder = os.path.join(*os.path.split(tmx)[:-1])
    if os.path.exists(f'{to_folder}/bitext.en'):
        debug and print(f'{tmx} already extracted to {to_file}; so skip')
        return to_file
    cmd = f'(cd {to_folder}; {TMX_TOOL} {tmx})'
    call(cmd, debug)
    return to_file

CZENG16_REGEX = re.compile(r'.*?data.plaintext-format/0[0-9]train$')
WMT19_WIKITITLES_REGEX = re.compile(r'.*?wikititles-v1.(\w\w)-en.tsv.gz')
TSV_REGEX = re.compile(r'.*?(\w\w)-(\w\w).tsv$')



def cut_wikitles(wiki_file, debug):
    # different languages have different file names: 
    if wiki_file.endswith('wiki/fi-en/titles.fi-en'):
        to_file1 = f'{wiki_file}.fi'
        to_file2 = f'{wiki_file}.en'
        BACKSLASH = '\\'
        cmd1 = f"cat {wiki_file} | sed 's/|||/{BACKSLASH}t/g' |cut -f1 |awk '{{$1=$1}};1' > {to_file1}"
        cmd2 = f"cat {wiki_file} | sed 's/|||/{BACKSLASH}t/g' |cut -f2 |awk '{{$1=$1}};1' > {to_file2}"  
#     elif WMT19_WIKITITLES_REGEX.match(wiki_file):
#         src = WMT19_WIKITITLES_REGEX.match(wiki_file).groups()[0]
#         to_file1 = f'{wiki_file}.{src}'
#         to_file2 = f'{wiki_file}.en'
#         cmd1 = f"cat {wiki_file} | cut -f1 |awk '{{$1=$1}};1' > {to_file1}"
#         cmd2 = f"cat {wiki_file} | cut -f2 |awk '{{$1=$1}};1' > {to_file2}"
    else:
        return None
    if os.path.exists(to_file1) and os.path.exists(to_file2):
        debug and print(f'{wiki_file} already processed to {to_file1} and {to_file2}; so skip')
        return wiki_file    

    call(cmd1, debug=debug)
    call(cmd2, debug=debug)
    return wiki_file

def cut_tsv(file, debug):
    m = TSV_REGEX.match(file)
    if m is None:
        raise ValueError(f'{file} is not matching tsv pattern')
    src = m.groups()[0]
    tgt = m.groups()[1]

    to_file1 = f'{file}.{src}'
    to_file2 = f'{file}.{tgt}' 
    cmd1 = f"cat {file} | cut -f1 |awk '{{$1=$1}};1' > {to_file1}"
    cmd2 = f"cat {file} | cut -f2 |awk '{{$1=$1}};1' > {to_file2}"         
    if os.path.exists(to_file1) and os.path.exists(to_file2):
        debug and print(f'{file} already processed to {to_file1} and {to_file2}; so skip')
        return file    

    call(cmd1, debug=debug)
    call(cmd2, debug=debug)
    return file    

    
def convert_file_if_needed(file, debug):
    if file.endswith('.sgm'):
        return sgm2raw(file, debug)
    elif file.endswith('.tmx'):
        return tmx2raw(file, debug)
    elif file.endswith('wiki/fi-en/titles.fi-en'):
        return cut_wikitles(file, debug)
#     elif WMT19_WIKITITLES_REGEX.match(file):
#         return cut_wikitles(file, debug)
    elif file.endswith('.tsv'):
        return cut_tsv(file, debug)
    elif CZENG16_REGEX.match(file):
        return convert2czeng17(file, debug)
    else:
        return file


def convert_files_if_needed(extracted_foldrs, my_glob=my_glob, debug=False):
    return {
        url: list(sorted(set(convert_file_if_needed(f, debug)) for f in sorted(set(my_glob(folder)))))
        for url, folder in extracted_foldrs.items()
    }
        
def match_patt(file_path, file_pattern, src, tgt, lang):    
    return file_pattern.format(src=src, tgt=tgt, lang=lang) in file_path

def match_patts(file_path, file_patterns, src, tgt, lang):
    for file_pattern in file_patterns:
        params = { k: v for k, v in [('src', src), ('tgt', tgt), ('lang', lang)] if k in file_pattern}
        matching = file_pattern.format(**params)   

        if isinstance(file_pattern, tuple):
            pattern, directions = file_pattern
            if f'{src}-{tgt}' in directions and matching in file_path:
                return True
        else:
            if matching in file_path:
                return True
    return False

def extracted_glob(extracted_folder, file_patterns, src, tgt, lang):
    def get_matching_pattern(file_pattern):
        params = {
            k: v 
            for k, v in [('src', src), ('tgt', tgt), ('lang', lang)] 
            if '{' + k + '}' in file_pattern
        }
        file_pattern = re.sub(r'{src:(.*?)}', r'\1' if lang == src else '', file_pattern)
        file_pattern = re.sub(r'{tgt:(.*?)}', r'\1' if lang == tgt else '', file_pattern)
        file_pattern = file_pattern.format(**params)
        return file_pattern
    for file_pattern in file_patterns:
        if isinstance(file_pattern, tuple):
            file_pattern, lang_pairs = file_pattern
            if f'{src}-{tgt}' not in lang_pairs:
                continue
#         print('working on pattern: ', file_pattern, lang_pairs )
        matching_pattern = get_matching_pattern(file_pattern)
        if matching_pattern is None:
            continue
        glob_patterns = f'{extracted_folder}/{matching_pattern}'
#         print('glob_patterns: ', glob_patterns)
        for f in glob.glob(glob_patterns):
            yield f       

# for debug usage
def all_extracted_files(split, src, tgt, extracted_folders, split_urls):
    def get_url(url):
        if isinstance(url, tuple):
            url, downloaded_file = url        
        return url
    return [
        f
        for url in split_urls
        for f in my_glob(extracted_folders[str(get_url(url))])        
    ]

def concat_files(split, src, tgt, extracted_folders, split_urls, path_patterns, to_folder, debug=False):
#     if debug:
#         print('extracted files to be filtered by patterns: ', 
#               '\n\t'.join(sorted(all_extracted_files(split, src, tgt, extracted_folders, split_urls))))
    for lang in [src, tgt]:
        to_file = f'{to_folder}/{split}.{src}-{tgt}.{lang}'
        s_src, s_tgt, s_lang = src.split('_')[0], tgt.split('_')[0], lang.split('_')[0]
        files = []
        for url in split_urls:
            if isinstance(url, tuple):
                url, downloaded_file = url
            if str(url) not in extracted_folders:
                print(f'warning: {url} not in extracted files')
            for extracted_file in set(
                extracted_glob(
                    extracted_folders[str(url)], path_patterns, 
                    s_src, s_tgt, s_lang)):
                files.append(extracted_file)
        if len(files) == 0:
            print('warning: ', f'No files found for split {to_file}')
            continue
        files = sorted(set(files))
        print(f'concating {len(files)} files into {to_file}')
        cmd = ['cat'] + [f'"{f}"' for f in files] + [f'>{to_file}']
        cmd = " ".join(cmd)
        call(cmd, debug=debug)

UTILS = os.path.join(pathlib.Path(__file__).parent, 'utils')
LID_MODEL = f'{download_to}/lid.176.bin'
LID_MULTI = f'{UTILS}/fasttext_multi_filter.py'

def lid_filter(split, src, tgt, from_folder, to_folder, debug=False):
    if not os.path.exists(LID_MODEL):
        call(f'wget -nc https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -O {LID_MODEL}')
    from_prefix = f'{from_folder}/{split}.{src}-{tgt}'
    to_prefix = f'{to_folder}/{split}.{src}-{tgt}'
    if os.path.exists(f'{from_prefix}.{src}') and os.path.exists(f'{from_prefix}.{tgt}'):
        s_src, s_tgt = src.split('_')[0], tgt.split('_')[0]  
        cmd = (
            f'python {LID_MULTI} --model {LID_MODEL} --inputs {from_prefix}.{src} {from_prefix}.{tgt} '
            f'--langs {s_src} {s_tgt} --outputs {to_prefix}.{src} {to_prefix}.{tgt}'
        )
        print(f'filtering {from_prefix}')
        call(cmd, debug=debug)

def concat_into_splits(dl_dataset, src, tgt, extracted_folders, to_folder, debug):
    to_folder_tmp = f"{to_folder}_tmp"
    os.makedirs(to_folder_tmp, exist_ok=True)
    concat_files('train', src, tgt,
                 extracted_folders,
                 split_urls=dl_dataset.train_urls,
                 path_patterns=dl_dataset.train_files_patterns,
                 to_folder=to_folder_tmp, debug=debug)
    lid_filter('train', src, tgt, to_folder_tmp, to_folder, debug)

    concat_files('valid', src, tgt,
                 extracted_folders, 
                 split_urls=dl_dataset.valid_urls, 
                 path_patterns=dl_dataset.valid_files_patterns, 
                 to_folder=to_folder, debug=debug)
    concat_files('test', src, tgt, 
                 extracted_folders, 
                 split_urls=dl_dataset.test_urls, 
                 path_patterns=dl_dataset.test_files_patterns, 
                 to_folder=to_folder, debug=debug)
            

def download_multi(dl_folder, extract_folder, urls, num_processes=8, debug=False):
    pool = mp.Pool(processes=num_processes)
    download_f = partial(download_a_url, dl_folder)
    downloaded_files = pool.imap_unordered(download_f, urls)
    pool.close()
    pool.join()

BLEU_REGEX = re.compile("^BLEU\\S* = (\\S+) ")
def run_eval_bleu(cmd):
    output = check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode("utf-8").strip()
    print(output)
    bleu = -1.0
    for line in output.strip().split('\n'):
        m = BLEU_REGEX.search(line)
        if m is not None:
            bleu = m.groups()[0]
            bleu = float(bleu)
            break
    return bleu

def check_wmt_test_bleu(raw_folder, wmt_lang_pairs):
    not_matchings = []
    for wmt, src_tgts in wmt_lang_pairs:
        for src_tgt in src_tgts:
            print(f'checking test bleus for: {src_tgt} at {wmt}')
            src, tgt = src_tgt.split('-')
            ssrc, stgt = src[:2], tgt[:2]
            if os.path.exists(f'{raw_folder}/test.{tgt}-{src}.{src}'):
                # reversed direction may have different test set
                test_src = f'{raw_folder}/test.{tgt}-{src}.{src}'
            else:
                test_src = f'{raw_folder}/test.{src}-{tgt}.{src}'
            cmd1 = f'cat {test_src} | sacrebleu -t "{wmt}" -l {stgt}-{ssrc}; [ $? -eq 0 ] || echo ""'
            test_tgt = f'{raw_folder}/test.{src}-{tgt}.{tgt}'       
            cmd2 = f'cat {test_tgt} | sacrebleu -t "{wmt}" -l {ssrc}-{stgt}; [ $? -eq 0 ] || echo ""'
            bleu1 = run_eval_bleu(cmd1) 
            if bleu1 != 100.0:
                not_matchings.append(f'{wmt}:{src_tgt} source side not matching: {test_src}')
            bleu2 = run_eval_bleu(cmd2) 
            if bleu2 != 100.0:
                not_matchings.append(f'{wmt}:{src_tgt} target side not matching: {test_tgt}')
    return not_matchings         
 
def download_and_extract(
    to_folder, lang_pairs, dl_dataset, 
    to_manually_download_urls, 
    completed_urls={}, completed_extraction={},
    debug=False):

    dl_folder = f'{to_folder}/downloads'
    extract_folder = f'{to_folder}/extracted'
    raw_folder =  f'{to_folder}/raw'
    lid_filtered = f'{to_folder}/lid_filtered'

    os.makedirs(extract_folder, exist_ok=True)
    os.makedirs(raw_folder, exist_ok=True)
    os.makedirs(lid_filtered, exist_ok=True)

    
    to_be_manually_dowloaded = check_need_manual_downalod(dl_folder, to_manually_download_urls)

    completed_urls = download_dataset(
        dl_folder, dl_dataset, completed_urls)
    if debug:
        print('completed urls: ', completed_urls)
    

    extracted_folders = extract_all_files(
        completed_urls,
        extract_folder=extract_folder, 
        completed_extraction=completed_extraction,
        debug=debug)
    if debug:
        print('download files have been extracted to folders: ', extracted_folders)

    converted_files = convert_files_if_needed(extracted_folders, debug=False)
    for src_tgt in lang_pairs:
        print(f'working on {dl_dataset.name}: {src_tgt}')
        src, tgt = src_tgt.split('-')
        concat_into_splits(dl_dataset, 
                            src=src, tgt=tgt,
                            extracted_folders=extracted_folders, 
                            to_folder=raw_folder, debug=debug)                            
    print('completed data into: ', raw_folder)

def download_czang16(download_to, username=None):
    wgets = [
        f'wget --user={username} --password=czeng -P {download_to} http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar'
        for i in range(10)]
    cmds = []
    for i, cmd in enumerate(wgets):
        filename = f'{download_to}/data-plaintext-format.{i}.tar'
        if os.path.exists(filename):
            print(f'{filename} has already been downloaded; so skip')
            continue
        cmds.append(cmd)
    if cmds and username is None:
        raise ValueError('No czeng username is given; please register at http://ufal.mff.cuni.cz/czeng/czeng16 to obtain username to download')        
    for cmd in cmds:
        call(cmd)
    print('done with downloading czeng1.6')

def download_czeng17_script(download_to, extract_folder, debug=False):
    url = 'http://ufal.mff.cuni.cz/czeng/download.php?f=convert_czeng16_to_17.pl.zip'
    filename = f'{download_to}/convert_czeng16_to_17.pl.zip'
    extract_to = f'{extract_folder}/{get_extract_name(filename)}'
    script_path = f'{extract_to}/convert_czeng16_to_17.pl'
    
    if not os.path.exists(script_path):
        wget.download(url, filename, bar=bar_custom)  
        extract_to = extract_file(f'{download_to}/convert_czeng16_to_17.pl.zip', extract_folder, get_extract_name=get_extract_name, debug=debug)    
    return script_path

czeng17_script_path = ""
def convert2czeng17(file, debug):
    en_file = f'{file}.en'
    cs_file = f'{file}.cs'
    
    if not os.path.exists(en_file) or not os.path.exists(cs_file):
        cs_cmd = f'cat {file} | perl {czeng17_script_path} | cut -f3 > {cs_file}'
        en_cmd = f'cat {file} | perl {czeng17_script_path} | cut -f4 > {en_file}'
        call(cs_cmd, debug)
        call(en_cmd, debug)
    else:
        print(f'already extracted: {en_file} and {cs_file}')
    return file

def extract_czeng17(extract_folder, debug=False):
    url = 'http://ufal.mff.cuni.cz/czeng/download.php?f=convert_czeng16_to_17.pl.zip'
    filename = f'{download_to}/convert_czeng16_to_17.pl.zip'
    extract_to = f'{extract_folder}/{get_extract_name(filename)}'
    script_path = f'{extract_to}/convert_czeng16_to_17.pl'
    
    if not os.path.exists(script_path):
        wget.download(url, filename, bar=bar_custom)  
        extract_to = extract_file(f'{download_to}/convert_czeng16_to_17.pl.zip', extract_folder, get_extract_name=get_extract_name, debug=debug)    
    return script_path

#########
# definitions of wmt data sources
# for es-en
# Punctuation in the official test sets will be encoded with ASCII characters (not complex Unicode characters) as much as possible. You may want to normalize your system's output before submission. You are able able to use a rawer version of the test sets that does not have this normalization.
# script to normalize punctuation: http://www.statmt.org/wmt11/normalize-punctuation.perl
wmt13_es_en = DLDataset(
    name='wmt13_es-en',
    train_urls=[
        'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz',
        'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz',
        'http://www.statmt.org/wmt13/training-parallel-un.tgz',
        'http://www.statmt.org/wmt13/training-parallel-nc-v8.tgz',
    ],
    valid_urls=[
        ('http://www.statmt.org/wmt13/dev.tgz', 'wmt13_dev.tgz')
    ],
    test_urls=[
        ('http://www.statmt.org/wmt13/test.tgz', 'wmt13_test.tgz')
    ],
    train_files_patterns=[
        ('*/europarl-v7.{src}-{tgt}.{lang}', ['es-en']), 
        ('*commoncrawl.{src}-{tgt}.{lang}', ['es-en']),
        ('*/news-commentary-v8.{src}-{tgt}.{lang}', ['es-en']),
        ('un/*undoc.2000.{src}-{tgt}.{lang}', ['es-en']), 
    ] ,
    valid_files_patterns=[
    ('dev/newstest2012.{lang}', ['es-en'])
    ],
    test_files_patterns=[
    ('test/newstest*.{lang}', ['es-en'])
    ],
)

wmt14_de_fr_en = DLDataset(
    name='wmt14_de_fr_en',
    train_urls=[
        'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz',
        'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz',
        'http://www.statmt.org/wmt13/training-parallel-un.tgz',
        'http://www.statmt.org/wmt14/training-parallel-nc-v9.tgz',
        ('http://www.statmt.org/wmt10/training-giga-fren.tar', 'training-giga-fren.gz.tar'), #it is actuall a gz.tar 
    ],
    valid_urls=[
        ('http://www.statmt.org/wmt14/dev.tgz', 'wmt14_dev.tgz'),
    ],
    test_urls=[
        ('http://www.statmt.org/wmt14/test-full.tgz', 'wmt14_test_full.tgz'), # cleaned test sets
    ],
    train_files_patterns=[
        ('*/europarl-v7.{src}-{tgt}.{lang}', ['fr-en', 'de-en']), 
        ('*commoncrawl.{src}-{tgt}.{lang}', ['fr-en', 'de-en']),
        ('*/*news-commentary-v9.{src}-{tgt}.{lang}', ['fr-en', 'de-en']),
        ('un/undoc.2000.{src}-{tgt}.{lang}', ['fr-en']),    
        ('*giga-{src}{tgt}*{lang}', ['fr-en'])
    ],
    valid_files_patterns=[
    ('dev/newstest2013.{lang}', ['fr-en', 'de-en'])
    ],
    test_files_patterns=[ 
    ('test-full/newstest*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['en-de', 'de-en', 'fr-en', 'en-fr']),                      
    ],
)

# pip install git+https://github.com/amake/tmx2corpus.git
wmt16_ro_en = DLDataset(
    name='wmt16_ro-en',
    train_urls=[
        ('http://data.statmt.org/wmt16/translation-task/training-parallel-ep-v8.tgz', 'wmt16_training-parallel-ep-v8.tgz'),
        ('http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz', 'en-ro.tmx.gz'),
    ],
    valid_urls=[
        ('http://data.statmt.org/wmt16/translation-task/dev-romanian-updated.tgz', 'wmt16_dev.tgz')
    ],
    test_urls=[
        ('http://data.statmt.org/wmt16/translation-task/test.tgz', 'wmt16_test.tgz')
    ],
    train_files_patterns=[
        ('*/*europarl-v8.{src}-{tgt}.{lang}', ['ro-en']), 
        ('bitext.{lang}', ['ro-en']) #setimes from tmux
        ] ,
    valid_files_patterns=[
    ('dev/newsdev2016*{src}{tgt}*.{lang}', ['ro-en', 'ro-en'])
    ],
    test_files_patterns=[
    ('test/newstest*{src}{tgt}*.{lang}', ['ro-en', 'en-ro'])
    ],
)

cwmt_wmt_instruction = 'cwmt download instruction at: http://nlp.nju.edu.cn/cwmt-wmt'
wmt17_fi_lv_tr_zh_en_manual_downloads = [
    # fake urls to have unique keys for the data
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASIA2015.zip', 'CASIA2015.zip'), cwmt_wmt_instruction),
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2011.zip', 'CASICT2011.zip'), cwmt_wmt_instruction),
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2015.zip', 'CASICT2015.zip'), cwmt_wmt_instruction),
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2015.zip', 'Datum2015.zip'), cwmt_wmt_instruction),
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2017.zip', 'Datum2017.zip'), cwmt_wmt_instruction),
    ( ('http://nlp.nju.edu.cn/cwmt-wmt/NEU2017.zip', 'NEU2017.zip'), cwmt_wmt_instruction),    
]
wmt17_fi_lv_tr_zh_en = DLDataset(
    name='wmt17_fi_lv_tr_zh_en',
    train_urls=[
        ('http://data.statmt.org/wmt17/translation-task/training-parallel-ep-v8.tgz', 'wmt17_training-parallel-ep-v8.tgz'),
        'http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz',
        'http://www.statmt.org/wmt15/wiki-titles.tgz',
        ('http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-tr.tmx.gz', 'en-tr.tmx.gz'),
        ('http://data.statmt.org/wmt17/translation-task/rapid2016.tgz', 'wmt17_rapid2016.tgz'),
        'http://data.statmt.org/wmt17/translation-task/leta.v1.tgz',
        'http://data.statmt.org/wmt17/translation-task/dcep.lv-en.v1.tgz',
        'http://data.statmt.org/wmt17/translation-task/books.lv-en.v1.tgz',
        (('https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.00',
        'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.01',), 'UNv1.0.en-zh.tar.gz'),
        #manually download files:
        ('http://nlp.nju.edu.cn/cwmt-wmt/CASIA2015.zip', 'CASIA2015.zip'),  
        ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2011.zip', 'CASICT2011.zip'),  
        ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2015.zip', 'CASICT2015.zip'),  
        ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2015.zip', 'Datum2015.zip'), 
        ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2017.zip', 'Datum2017.zip'),  
        ('http://nlp.nju.edu.cn/cwmt-wmt/NEU2017.zip', 'NEU2017.zip'),          
    ],
    valid_urls=[
        ('http://data.statmt.org/wmt17/translation-task/dev.tgz', 'wmt17_dev.tgz'),
    ],
    test_urls=[
        #NEW: Improved translations for zh test sets
        ('http://data.statmt.org/wmt17/translation-task/test-update-1.tgz', 'wmt17_test_zh_en.tgz'),    
        ('http://data.statmt.org/wmt17/translation-task/test.tgz', 'wmt17_test_others.tgz')
    ],
    train_files_patterns=[
        ('casict*/cas*{src:ch}{tgt:en}.txt', ['zh-en', 'zh-en'] ),
        ('casia*/cas*{src:ch}{tgt:en}.txt', ['zh-en', 'zh-en'] ),
        ('dataum*/Book*{src:cn}{tgt:en}.txt', ['zh-en', 'zh-en']),
        ('neu*/NEU*{src:cn}{tgt:en}.txt', ['zh-en', 'zh-en'] ),
        ('*/*UNv1.0.en-zh.{src:zh}{tgt:en}', ['zh-en']),
        ('training/*news-commentary-v12.{src}-{tgt}.{lang}', ['zh-en', ]),
        
        ('*/*europarl-v8.{src}-{tgt}.{lang}', ['fi-en', 'lv-en']),
        ('wiki/fi-en/titles.{src}-{tgt}.{lang}', ['fi-en', ]),  
        ('rapid2016.{tgt}-{src}.{lang}', ['fi-en', 'lv-en']),
        ('*/leta.{lang}', ['lv-en']),
        ('*/dcep.{lang}', ['lv-en']),
        ('*/farewell.{lang}', ['lv-en']),       
        ('bitext.{lang}', ['tr-en']),
    ] ,
    valid_files_patterns=[
    ('dev/newsdev2017*{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    [
        'fi-en', 'lv-en', 'tr-en', 'zh-en',
        'en-fi', 'en-lv', 'en-tr', 'en-zh'
    ]),                      
    ('dev/newstest2016*{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    [
        'fi-en',  'tr-en',  
        'en-fi',  'en-tr',  
    ]),  
    ],
    test_files_patterns=[
    ('test/newstest2017-{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    [
        'fi-en', 'lv-en', 'tr-en', 
        'en-fi', 'en-lv', 'en-tr',  
    ]),
    ('newstest2017-{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    [
        'zh-en',
        'en-zh'
    ]),
    ],
)

czeng_instruction = 'download instruction at: http://ufal.mff.cuni.cz/czeng/czeng16'
#alternative: use the prepared data but detokenize it?
wmt18_cs_et_en_manual_downloads = [
#for cs, need to register and download; Register and download CzEng 1.6.  
#Better results can be obtained by using a subset of sentences, released under a new version name CzEng 1.7.
    # ((f'http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar', 
    #     f'data-plaintext-format.{i}.tar'), czeng_instruction)
    # for i in range(10)
]

wmt18_cs_et_en = DLDataset(
    name='wmt18_cs_et_en',
    train_urls=[
        'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz',
        'http://data.statmt.org/wmt18/translation-task/training-parallel-ep-v8.tgz',
        'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-cs.zipporah0-dedup-clean.tgz',
        'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-et.zipporah0-dedup-clean.tgz',
        'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz',
        'http://data.statmt.org/wmt18/translation-task/training-parallel-nc-v13.tgz',
        ('http://data.statmt.org/wmt18/translation-task/rapid2016.tgz', 'wmt18_rapid2016.tgz'),
        # (tuple(
        #     (f'http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar', 
        #     f'data-plaintext-format.{i}.tar')
        #     for i in range(10)
        # ), 
        # 'czeng16_data_plaintext.gz.tar'), 
    ],
    valid_urls=[
        ('http://data.statmt.org/wmt18/translation-task/dev.tgz', 'wmt18_dev.tgz'),
    ],
    test_urls=[
        ('http://data.statmt.org/wmt18/translation-task/test.tgz', 'wmt18_test.tgz'),
    ],
    train_files_patterns=[
        # ('*/*europarl-v7.{src}-{tgt}.{lang}', ['cs-en']),
        ('*/*europarl-v8.{src}-{tgt}.{lang}', ['et-en']),
        # ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['cs-en', 'et-en']),
        ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['et-en']),
        # ('*commoncrawl.{src}-{tgt}.{lang}', ['cs-en']),
        # ('*/news-commentary-v13.{src}-{tgt}.{lang}', ['cs-en']),
        # ('data.plaintext-format/*train.{lang}', ['cs-en']),
        ('rapid2016.{tgt}-{src}.{lang}', ['et-en']),
    ] ,
    valid_files_patterns=[
    ('dev/newsdev2018*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['et-en']),
    # ('dev/newstest2017*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['cs-en'])        
    ],
    test_files_patterns=[
    ('test/newstest2018-{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    # ['cs-en', 'et-en']),
    ['et-en']),
    ]
)

ru_en_yandex_instruction = 'Yandex Corpus download instruction at: https://translate.yandex.ru/corpus?lang=en'
wmt19_ru_gu_kk_lt_manual_downloads = [
    (('https://translate.yandex.ru/corpus?lang=en', 'wmt19_1mcorpus.zip'), ru_en_yandex_instruction)
]
wmt19_ru_gu_kk_lt = DLDataset(
    name='wmt19_ru_gu_kk_lt',
    train_urls=[
        'http://www.statmt.org/europarl/v9/training/europarl-v9.lt-en.tsv.gz',
        'https://s3.amazonaws.com/web-language-models/paracrawl/release3/en-lt.bicleaner07.tmx.gz',
        'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz',
        'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz',
        'http://data.statmt.org/news-commentary/v14/training/news-commentary-v14-wmt19.en-kk.tsv.gz',
        'http://data.statmt.org/news-commentary/v14/training/news-commentary-v14.en-ru.tsv.gz',
        'http://data.statmt.org/wikititles/v1/wikititles-v1.kk-en.tsv.gz',
        'http://data.statmt.org/wikititles/v1/wikititles-v1.ru-en.tsv.gz',
        'http://data.statmt.org/wikititles/v1/wikititles-v1.kk-en.tsv.gz',
        'http://data.statmt.org/wikititles/v1/wikititles-v1.lt-en.tsv.gz',
        'http://data.statmt.org/wikititles/v1/wikititles-v1.gu-en.tsv.gz',
        (('https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.00',
        'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.01',
        'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.02',), 
        'wmt19_UNv1.0.en-ru.tar.gz'),
        'https://tilde-model.s3-eu-west-1.amazonaws.com/rapid2016.en-lt.tmx.zip',
        ('https://translate.yandex.ru/corpus?lang=en', 'wmt19_1mcorpus.zip'),
    ],
    valid_urls=[
        ('http://data.statmt.org/wmt19/translation-task/dev.tgz', 'wmt19_dev.tgz'),
    ],
    test_urls=[
        ('http://data.statmt.org/wmt19/translation-task/test.tgz', 'wmt19_test.tgz'),
    ],
    train_files_patterns=[
        ('*europarl-v9.{src}-{tgt}.tsv.{lang}', ['lt-en']),
        #paracrawl
        ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['ru-en']),
        ('bitext.{lang}', ['lt-en',]),
        ('*commoncrawl.{src}-{tgt}.{lang}', ['ru-en',]),
        ('*news-commentary-v14-wmt19.{tgt}-{src}.tsv.{lang}', ['kk-en', ]),
        ('*news-commentary-v14.{tgt}-{src}.tsv.{lang}', ['ru-en']),
        #yandex
        ('corpus.{tgt}_{src}.1m.{lang}', ['ru-en']),
        ('wikititles_v1_wikititles-v1.{src}-{tgt}.tsv.{lang}', ['ru-en', 'kk-en', 'lt-en', 'gu-en']),
        ('*/UNv1.0.{tgt}-{src}.{lang}', ['ru-en']),
        #rapid
        ('bitext.{lang}', ['lt-en'])
    ],
    valid_files_patterns=[
    ('dev/newsdev2019*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['gu-en', 'kk-en', 'lt-en']),
    ('dev/newstest2018*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['ru-en']),       
    ],
    test_files_patterns=[
    ('sgm/newstest2019-{src}{tgt}-{src:src}{tgt:ref}.{lang}', 
    ['ru-en', 'gu-en', 'kk-en', 'lt-en', 'en-ru', 'en-gu', 'en-kk', 'en-lt']),
    ]    
)


#########

if __name__ == "__main__":
    # speed up the downloads with multiple processing
    dl_folder = f'{to_data_path}/downloads'
    extract_folder = f'{to_data_path}/extracted'

    urls = [
        url
        for dataset in [wmt13_es_en, wmt14_de_fr_en, wmt16_ro_en, wmt18_cs_et_en, wmt19_ru_gu_kk_lt]
        for urls in [dataset.train_urls, dataset.valid_urls, dataset.test_urls]
        for url in urls
    ]
    urls = set(urls)
    download_multi(dl_folder, extract_folder, urls, num_processes=8, debug=True)

    # check manually downlaods
    to_manually_download_urls = (
        wmt17_fi_lv_tr_zh_en_manual_downloads + wmt18_cs_et_en_manual_downloads + wmt19_ru_gu_kk_lt_manual_downloads
    )
    to_be_manually_dowloaded = check_need_manual_downalod(dl_folder, to_manually_download_urls)
    if len(to_be_manually_dowloaded) > 0:
        print('Missing files that need to be downloaded manually; stop the process now.')
        exit(-1)
    
    completed_urls = {}
    completed_extraction = {}
    def work_on_wmt(directions, wmt_data):
        download_and_extract(
            to_data_path, 
            directions, 
            wmt_data, 
            to_manually_download_urls=to_manually_download_urls,
            completed_urls=completed_urls, completed_extraction=completed_extraction, debug=True)
                
    work_on_wmt(
        ['es_XX-en_XX'], 
        wmt13_es_en,)
    work_on_wmt(
        [
            'fr_XX-en_XX',  'en_XX-fr_XX',
            # 'en_XX-de_DE', 'de_DE-en_XX',
        ], 
        wmt14_de_fr_en,)
    work_on_wmt(
        ['ro_RO-en_XX', 'en_XX-ro_XX'], 
        wmt16_ro_en,)
    work_on_wmt(
        [
            # 'zh_CN-en_XX', 
            'lv_LV-en_XX', 'fi_FI-en_XX', 'tr_TR-en_XX',
            #in case the reversed directions have different train/valid/test data
            # 'en_XX-zh_CN', 
            'en_XX-lv_LV', 'en_XX-fi_FI', 'en_XX-tr_TR',
        ], 
        wmt17_fi_lv_tr_zh_en, )
    # czeng17_script_path = download_czeng17_script(download_to, extract_to, debug=False)
    # cz_username =  None
    work_on_wmt(
        [
            # 'cs_CZ-en_XX', 
            'et_EE-en_XX'], 
        wmt18_cs_et_en,)
    work_on_wmt(
        [
            # 'ru_RU-en_XX', 'en_XX-ru_RU', 
            'gu_IN-en_XX', 'kk_KZ-en_XX', 'lt_LT-en_XX',
            #in case the reversed directions have different train/valid/test data
            'en_XX-gu_IN', 'en_XX-kk_KZ', 'en_XX-lt_LT'
        ], 
        wmt19_ru_gu_kk_lt,)

    not_matching = check_wmt_test_bleu(
        f'{to_data_path}/raw', 
        [
            ('wmt13', ['es_XX-en_XX']),
            ('wmt14/full', ['fr_XX-en_XX',]),
            ('wmt16', ['ro_RO-en_XX',]),
            # ('wmt17/improved', ['zh_CN-en_XX']),
            ('wmt17', [ 'lv_LV-en_XX', 'fi_FI-en_XX', 'tr_TR-en_XX']),
            ('wmt18', ['cs_CZ-en_XX', 'et_EE-en_XX']),
            ('wmt19', ['gu_IN-en_XX', 'kk_KZ-en_XX', 'lt_LT-en_XX']), 
            #'ru_RU-en_XX', 
        ]
        )    
    if len(not_matching) > 0:
        print('the following datasets do not have matching test datasets:\n\t', '\n\t'.join(not_matching))