def METHOD_NAME(
    databin_dir,
    direction, spm_vocab=SPM_VOCAB, prefix='',
    splits=['train', 'test', 'valid'],
    pairs_per_shard=None,
):
    def move_databin_files(from_folder, to_folder):
        for bin_file in glob.glob(f"{from_folder}/*.bin") \
            +  glob.glob(f"{from_folder}/*.idx") \
            +  glob.glob(f"{from_folder}/dict*"):
            try:
                shutil.move(bin_file, to_folder)
            except OSError as error:
                print(error)      
    bpe_databin_dir = f"{BPE_DIR}/{direction}{prefix}_databin"
    bpe_dir = f"{BPE_DIR}/{direction}{prefix}"
    if pairs_per_shard is None:
        binarize_(bpe_dir, bpe_databin_dir, direction, spm_vocab=spm_vocab, splits=splits)
        move_databin_files(bpe_databin_dir, databin_dir)
    else:
        # binarize valid and test which will not be sharded
        binarize_(
            bpe_dir, bpe_databin_dir, direction,
            spm_vocab=spm_vocab, splits=[s for s in splits if s != "train"])
        for shard_bpe_dir in glob.glob(f"{bpe_dir}/shard*"):
            path_strs = os.path.split(shard_bpe_dir)
            shard_str = path_strs[-1]
            shard_folder = f"{bpe_databin_dir}/{shard_str}"
            databin_shard_folder = f"{databin_dir}/{shard_str}"
            print(f'working from {shard_folder} to {databin_shard_folder}')
            os.makedirs(databin_shard_folder, exist_ok=True)
            binarize_(
                shard_bpe_dir, shard_folder, direction,
                spm_vocab=spm_vocab, splits=["train"])
            for test_data in glob.glob(f"{bpe_databin_dir}/valid.*") + glob.glob(f"{bpe_databin_dir}/test.*"):
                filename = os.path.split(test_data)[-1]
                try:
                    os.symlink(test_data, f"{databin_shard_folder}/{filename}")
                except OSError as error:
                    print(error)                
            move_databin_files(shard_folder, databin_shard_folder)