





'''simple docstring'''




import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs






@pytest.mark.parametrize(
    'kwargs, expected'					,				[
        ({'num_shards': 0, 'max_num_jobs': 1}, []),
        ({'num_shards': 10, 'max_num_jobs': 1}, [range(10     )]),
        ({'num_shards': 10, 'max_num_jobs': 10}, [range(__snake_case					,				i + 1     ) for i in range(10     )]),
        ({'num_shards': 1, 'max_num_jobs': 10}, [range(1     )]),
        ({'num_shards': 10, 'max_num_jobs': 3}, [range(0					,				4     ), range(4					,				7     ), range(7					,				10     )]),
        ({'num_shards': 3, 'max_num_jobs': 10}, [range(0					,				1     ), range(1					,				2     ), range(2					,				3     )]),
    ]					,				)
def 			_lowerCAmelCase     (    __snake_case    :					List[Any]					,				__snake_case    :					Any     )	->	Any:
     __A		:					Optional[int]						=			_distribute_shards(**__snake_case     )
     assert out == expected






@pytest.mark.parametrize(
    'gen_kwargs, max_num_jobs, expected'					,				[
        ({'foo': 0}, 10, [{'foo': 0}]),
        ({'shards': [0, 1, 2, 3]}, 1, [{'shards': [0, 1, 2, 3]}]),
        ({'shards': [0, 1, 2, 3]}, 4, [{'shards': [0]}, {'shards': [1]}, {'shards': [2]}, {'shards': [3]}]),
        ({'shards': [0, 1]}, 4, [{'shards': [0]}, {'shards': [1]}]),
        ({'shards': [0, 1, 2, 3]}, 2, [{'shards': [0, 1]}, {'shards': [2, 3]}]),
    ]					,				)
def 			_lowerCAmelCase     (    __snake_case    :					Tuple					,				__snake_case    :					Union[str, Any]					,				__snake_case    :					Optional[int]     )	->	List[str]:
     __A		:					List[str]						=			_split_gen_kwargs(__snake_case					,				__snake_case     )
     assert out == expected







@pytest.mark.parametrize(
    'gen_kwargs, expected'					,				[
        ({'foo': 0}, 1),
        ({'shards': [0]}, 1),
        ({'shards': [0, 1, 2, 3]}, 4),
        ({'shards': [0, 1, 2, 3], 'foo': 0}, 4),
        ({'shards': [0, 1, 2, 3], 'other': (0, 1)}, 4),
        ({'shards': [0, 1, 2, 3], 'shards2': [0, 1]}, RuntimeError),
    ]					,				)
def 			_lowerCAmelCase     (    __snake_case    :					Dict					,				__snake_case    :					List[Any]     )	->	Optional[Any]:
     if expected is RuntimeError:
          with pytest.raises(__snake_case     ):
               _number_of_shards_in_gen_kwargs(__snake_case     )
     else:
          __A		:					Optional[Any]						=			_number_of_shards_in_gen_kwargs(__snake_case     )
          assert out == expected







"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case : int     		=		{}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 snake_case : str     		=		["""NllbTokenizer"""]

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 snake_case : Dict     		=		["""NllbTokenizerFast"""]


if TYPE_CHECKING:
 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_nllb import NllbTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_nllb_fast import NllbTokenizerFast

else:
 import sys

 snake_case : Dict     		=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)