def METHOD_NAME(spm_model_path, train_path, valid_path, test_path, dest_dir, remove_empty=False, output_format='piece', workers=20):
    with tempfile.TemporaryDirectory() as tmp:
        # Tokenize with SentencePiece
        for split, path in ('train', train_path), ('valid', valid_path), ('test', test_path):
            if path is None:
                continue
            if path == '-':
                path = sys.stdin.fileno()
            with open(path, encoding='utf-8', errors='surrogateescape') as fin:
                with open(f'{tmp}/{split}', mode='w', encoding='utf-8', errors='surrogateescape') as fout:
                    encoder = MultiprocessingEncoder(model=spm_model_path, remove_empty=remove_empty, output_format=output_format)
                    pool = Pool(workers, initializer=encoder.initializer)
                    encoded_lines = pool.imap(encoder.encode, fin, 10000)
                    for i, line in enumerate(encoded_lines, start=1):
                        if line is not None:
                            print(line, file=fout)
                        if i % 10000 == 0:
                            print("tokenized {} lines".format(i), file=sys.stderr)
        # Generate dictionary
        sp = spm.SentencePieceProcessor(model_file=spm_model_path)
        if output_format == 'piece':
            vocab = [sp.id_to_piece(i) for i in range(3, sp.vocab_size())]
        else:
            vocab = map(str, range(sp.vocab_size()))
        with open(f'{tmp}/dict.txt', mode='w', encoding='utf-8', errors='surrogateescape') as f:
            for word in vocab:
                print(word, 1, file=f)
        # Binarize
        command = [
            'python3', '-m', 'fairseq_cli.preprocess',
            '--only-source',
            '--thresholdsrc', '0',
            '--destdir', dest_dir,
            '--srcdict', f'{tmp}/dict.txt',
            '--workers', '20',
        ]
        for split, path in ('train', train_path), ('valid', valid_path), ('test', test_path):
            if path is not None:
                command += [f'--{split}pref', f'{tmp}/{split}']
        subprocess.run(command)
        # Copy SentencePiece model
        shutil.copyfile(spm_model_path, f'{dest_dir}/sentencepiece.bpe.model')