





import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)

class      lowerCAmelCase							(      __UpperCamelCase    ):
 def __init__(   self						:   Tuple    ,  UpperCAmelCase						:   str=0.0_1    ,  UpperCAmelCase						:   Any=1000			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Dict      				=	p_stop
     lowerCamelCase__					:   Dict      				=	max_length







 def __iter__(   self						:   List[Any]			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Optional[Any]      				=	0
     lowerCamelCase__					:   str      				=	False
     while not stop and count < self.max_length:
         yield count
         count += 1
         lowerCamelCase__					:   Union[str, Any]      				=	random.random() < self.p_stop


class      lowerCAmelCase							(      unittest.TestCase    ):
 def   A_			(   self						:   Dict    ,  UpperCAmelCase						:   List[str]    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   Dict=False    ,  UpperCAmelCase						:   Optional[Any]=True			)    ->			Any:
     lowerCamelCase__					:   Union[str, Any]      				=	[
         BatchSamplerShard(UpperCAmelCase    ,  2    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)
         for i in range(2			)
     ]
     lowerCamelCase__					:   Any      				=	[list(UpperCAmelCase			) for batch_sampler_shard in batch_sampler_shards]
     if not split_batches:
         self.assertListEqual([len(UpperCAmelCase			) for shard in batch_sampler_shards]    ,  [len(UpperCAmelCase			) for e in expected]			)
     self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   Optional[Any]			)    ->			Dict:
     # Check the shards when the dataset is a round multiple of total batch size.
     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(24			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   int      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(24			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     # Expected shouldn't change
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     # Check the shards when the dataset is a round multiple of batch size but not total batch size.
     lowerCamelCase__					:   Tuple      				=	BatchSampler(range(21			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   int      				=	BatchSampler(range(21			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
     # num_processes batch.
     lowerCamelCase__					:   Optional[int]      				=	BatchSampler(range(22			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   str      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(22			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
     # num_processes batch.
     lowerCamelCase__					:   str      				=	BatchSampler(range(20			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   int      				=	BatchSampler(range(20			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Tuple      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     # Check the shards when the dataset is very small.
     lowerCamelCase__					:   Dict      				=	BatchSampler(range(2			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	[[[0, 1, 0]], [[1, 0, 1]]]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   Any      				=	BatchSampler(range(2			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   str      				=	[[], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   str			)    ->			Union[str, Any]:
     # Check the shards when the dataset is a round multiple of batch size.
     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(24			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(24			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     # Expected shouldn't change
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size.
     lowerCamelCase__					:   str      				=	BatchSampler(range(22			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Dict      				=	BatchSampler(range(22			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size or num_processes.
     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(21			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[int]      				=	BatchSampler(range(21			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     # Check the shards when the dataset is very small.
     lowerCamelCase__					:   Union[str, Any]      				=	BatchSampler(range(2			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	[[[0, 1]], [[0, 1]]]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(2			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	[[], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]			)    ->			Tuple:
     # Check the shards when the dataset is a round multiple of total batch size.
     lowerCamelCase__					:   List[str]      				=	BatchSampler(range(24			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   int      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(24			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     # Expected shouldn't change
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is a round multiple of batch size but not total batch size.
     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(21			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Tuple      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Any      				=	BatchSampler(range(21			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
     # num_processes batch.
     lowerCamelCase__					:   Any      				=	BatchSampler(range(22			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   str      				=	BatchSampler(range(22			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
     # num_processes batch.
     lowerCamelCase__					:   str      				=	BatchSampler(range(20			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Tuple      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Union[str, Any]      				=	BatchSampler(range(20			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	[
         [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
         [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is very small.
     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(2			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[[[0, 1]], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   int      				=	BatchSampler(range(2			)    ,  batch_size=3    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	[[], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]			)    ->			List[str]:
     # Check the shards when the dataset is a round multiple of batch size.
     lowerCamelCase__					:   Dict      				=	BatchSampler(range(24			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   int      				=	BatchSampler(range(24			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     # Expected shouldn't change
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size.
     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(22			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(22			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is not a round multiple of batch size or num_processes.
     lowerCamelCase__					:   Optional[int]      				=	BatchSampler(range(21			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	BatchSampler(range(21			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   int      				=	[
         [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
         [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
     ]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     # Check the shards when the dataset is very small.
     lowerCamelCase__					:   Tuple      				=	BatchSampler(range(2			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Tuple      				=	[[[0, 1]], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[Any]      				=	BatchSampler(range(2			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	[[], []]
     self.check_batch_sampler_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  even_batches=UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]			)    ->			List[Any]:
     lowerCamelCase__					:   Tuple      				=	[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
     lowerCamelCase__					:   str      				=	[BatchSamplerShard(UpperCAmelCase    ,  2    ,  UpperCAmelCase    ,  even_batches=UpperCAmelCase			) for i in range(2			)]

     self.assertEqual(len(batch_sampler_shards[0]			)    ,  3			)
     self.assertEqual(len(batch_sampler_shards[1]			)    ,  2			)

     self.assertListEqual(list(batch_sampler_shards[0]			)    ,  [[0, 1, 2], [5, 6, 7, 8], [12, 13]]			)
     self.assertListEqual(list(batch_sampler_shards[1]			)    ,  [[3, 4], [9, 10, 11]]			)
 def   A_			(   self						:   int    ,  UpperCAmelCase						:   Dict    ,  UpperCAmelCase						:   List[str]    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   List[str]=False    ,  UpperCAmelCase						:   Union[str, Any]=2    ,  UpperCAmelCase						:   int=False			)    ->			Tuple:
     random.seed(UpperCAmelCase			)
     lowerCamelCase__					:   Tuple      				=	list(UpperCAmelCase			)

     lowerCamelCase__					:   Tuple      				=	[
         IterableDatasetShard(
             UpperCAmelCase    ,  batch_size=UpperCAmelCase    ,  drop_last=UpperCAmelCase    ,  num_processes=UpperCAmelCase    ,  process_index=UpperCAmelCase    ,  split_batches=UpperCAmelCase    ,  )
         for i in range(UpperCAmelCase			)
     ]
     lowerCamelCase__					:   Optional[int]      				=	[]
     for iterable_dataset_shard in iterable_dataset_shards:
         # Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
         random.seed(UpperCAmelCase			)
         iterable_dataset_lists.append(list(UpperCAmelCase			)			)

     lowerCamelCase__					:   Optional[int]      				=	batch_size // num_processes if split_batches else batch_size
     # All iterable dataset shard should have the same length, a round multiple of shard_batch_size
     lowerCamelCase__					:   int      				=	iterable_dataset_lists[0]
     for l in iterable_dataset_lists[1:]:
         self.assertEqual(len(UpperCAmelCase			)    ,  len(UpperCAmelCase			)			)
         self.assertTrue(len(UpperCAmelCase			) % shard_batch_size == 0			)

     lowerCamelCase__					:   str      				=	[]
     for idx in range(0    ,  len(UpperCAmelCase			)    ,  UpperCAmelCase			):
         for l in iterable_dataset_lists:
             observed += l[idx : idx + shard_batch_size]

     if not drop_last:
         while len(UpperCAmelCase			) < len(UpperCAmelCase			):
             reference += reference
     self.assertListEqual(UpperCAmelCase    ,  reference[: len(UpperCAmelCase			)]			)
 def   A_			(   self						:   str			)    ->			Optional[int]:
     lowerCamelCase__					:   Tuple      				=	42
     lowerCamelCase__					:   str      				=	RandomIterableDataset()

     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)

     # Edge case with a very small dataset
     lowerCamelCase__					:   str      				=	RandomIterableDataset(max_length=2			)

     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
     self.check_iterable_dataset_shards(UpperCAmelCase    ,  UpperCAmelCase    ,  batch_size=4    ,  drop_last=UpperCAmelCase    ,  split_batches=UpperCAmelCase			)
 def   A_			(   self						:   Any			)    ->			int:
     lowerCamelCase__					:   Union[str, Any]      				=	BatchSampler(range(16			)    ,  batch_size=4    ,  drop_last=UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	SkipBatchSampler(UpperCAmelCase    ,  2			)
     self.assertListEqual(list(UpperCAmelCase			)    ,  [[8, 9, 10, 11], [12, 13, 14, 15]]			)
 def   A_			(   self						:   Dict			)    ->			Optional[Any]:
     lowerCamelCase__					:   List[str]      				=	SkipDataLoader(list(range(16			)			)    ,  batch_size=4    ,  skip_batches=2			)
     self.assertListEqual([t.tolist() for t in dataloader]    ,  [[8, 9, 10, 11], [12, 13, 14, 15]]			)
 def   A_			(   self						:   List[str]			)    ->			Tuple:
     lowerCamelCase__					:   Any      				=	DataLoader(list(range(16			)			)    ,  batch_size=4			)
     lowerCamelCase__					:   Union[str, Any]      				=	skip_first_batches(UpperCAmelCase    ,  num_batches=2			)
     self.assertListEqual([t.tolist() for t in new_dataloader]    ,  [[8, 9, 10, 11], [12, 13, 14, 15]]			)
 def   A_			(   self						:   Optional[int]			)    ->			int:
     lowerCamelCase__					:   Any      				=	DataLoaderShard(list(range(16			)			)    ,  batch_size=4			)
     for idx, _ in enumerate(UpperCAmelCase			):
         self.assertEqual(dataloader.end_of_dataloader    ,  idx == 3			)

     # Test it also works on the second iteration
     for idx, _ in enumerate(UpperCAmelCase			):
         self.assertEqual(dataloader.end_of_dataloader    ,  idx == 3			)







 def   A_			(   self						:   Tuple			)    ->			Union[str, Any]:
     Accelerator()
     lowerCamelCase__					:   Any      				=	DataLoaderDispatcher(range(16			)    ,  batch_size=4			)
     for idx, _ in enumerate(UpperCAmelCase			):
         self.assertEqual(dataloader.end_of_dataloader    ,  idx == 3			)

     # Test it also works on the second iteration
     for idx, _ in enumerate(UpperCAmelCase			):
         self.assertEqual(dataloader.end_of_dataloader    ,  idx == 3			)












import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available

class      lowerCAmelCase							(      __UpperCamelCase    ):
 UpperCAmelCase__									=	["""image_processor""", """tokenizer"""]
 UpperCAmelCase__									=	"""OwlViTImageProcessor"""
 UpperCAmelCase__									=	("""CLIPTokenizer""", """CLIPTokenizerFast""")
 def __init__(   self						:   Optional[int]    ,  UpperCAmelCase						:   str=None    ,  UpperCAmelCase						:   Any=None    ,  **UpperCAmelCase						:   Optional[Any]			)    ->			Dict:
     lowerCamelCase__					:   Any      				=	None
     if "feature_extractor" in kwargs:
         warnings.warn(
             'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
             ' instead.'    ,  UpperCAmelCase    ,  )
         lowerCamelCase__					:   Optional[Any]      				=	kwargs.pop('feature_extractor'			)

     lowerCamelCase__					:   Dict      				=	image_processor if image_processor is not None else feature_extractor
     if image_processor is None:
         raise ValueError('You need to specify an `image_processor`.'			)
     if tokenizer is None:
         raise ValueError('You need to specify a `tokenizer`.'			)

     super().__init__(UpperCAmelCase    ,  UpperCAmelCase			)
 def __call__(   self						:   List[Any]    ,  UpperCAmelCase						:   List[Any]=None    ,  UpperCAmelCase						:   Tuple=None    ,  UpperCAmelCase						:   int=None    ,  UpperCAmelCase						:   Optional[int]="max_length"    ,  UpperCAmelCase						:   Dict="np"    ,  **UpperCAmelCase						:   Union[str, Any]			)    ->			Union[str, Any]:

     if text is None and query_images is None and images is None:
         raise ValueError(
             'You have to specify at least one text or query image or image. All three cannot be none.'			)

     if text is not None:
         if isinstance(UpperCAmelCase    ,  UpperCAmelCase			) or (isinstance(UpperCAmelCase    ,  UpperCAmelCase			) and not isinstance(text[0]    ,  UpperCAmelCase			)):
             lowerCamelCase__					:   Any      				=	[self.tokenizer(UpperCAmelCase    ,  padding=UpperCAmelCase    ,  return_tensors=UpperCAmelCase    ,  **UpperCAmelCase			)]

         elif isinstance(UpperCAmelCase    ,  UpperCAmelCase			) and isinstance(text[0]    ,  UpperCAmelCase			):
             lowerCamelCase__					:   List[str]      				=	[]

             # Maximum number of queries across batch
             lowerCamelCase__					:   List[Any]      				=	max([len(UpperCAmelCase			) for t in text]			)

             # Pad all batch samples to max number of text queries
             for t in text:
                 if len(UpperCAmelCase			) != max_num_queries:
                     lowerCamelCase__					:   int      				=	t + [' '] * (max_num_queries - len(UpperCAmelCase			))

                 lowerCamelCase__					:   str      				=	self.tokenizer(UpperCAmelCase    ,  padding=UpperCAmelCase    ,  return_tensors=UpperCAmelCase    ,  **UpperCAmelCase			)
                 encodings.append(UpperCAmelCase			)
         else:
             raise TypeError('Input text should be a string, a list of strings or a nested list of strings'			)

         if return_tensors == "np":
             lowerCamelCase__					:   Optional[Any]      				=	np.concatenate([encoding['input_ids'] for encoding in encodings]    ,  axis=0			)
             lowerCamelCase__					:   int      				=	np.concatenate([encoding['attention_mask'] for encoding in encodings]    ,  axis=0			)

         elif return_tensors == "jax" and is_flax_available():
             import jax.numpy as jnp

             lowerCamelCase__					:   Optional[int]      				=	jnp.concatenate([encoding['input_ids'] for encoding in encodings]    ,  axis=0			)
             lowerCamelCase__					:   Any      				=	jnp.concatenate([encoding['attention_mask'] for encoding in encodings]    ,  axis=0			)

         elif return_tensors == "pt" and is_torch_available():
             import torch

             lowerCamelCase__					:   Union[str, Any]      				=	torch.cat([encoding['input_ids'] for encoding in encodings]    ,  dim=0			)
             lowerCamelCase__					:   Dict      				=	torch.cat([encoding['attention_mask'] for encoding in encodings]    ,  dim=0			)

         elif return_tensors == "tf" and is_tf_available():
             import tensorflow as tf

             lowerCamelCase__					:   Any      				=	tf.stack([encoding['input_ids'] for encoding in encodings]    ,  axis=0			)
             lowerCamelCase__					:   Optional[int]      				=	tf.stack([encoding['attention_mask'] for encoding in encodings]    ,  axis=0			)

         else:
             raise ValueError('Target return tensor type could not be returned'			)

         lowerCamelCase__					:   str      				=	BatchEncoding()
         lowerCamelCase__					:   int      				=	input_ids
         lowerCamelCase__					:   Any      				=	attention_mask

     if query_images is not None:
         lowerCamelCase__					:   Union[str, Any]      				=	BatchEncoding()
         lowerCamelCase__					:   Optional[Any]      				=	self.image_processor(
             UpperCAmelCase    ,  return_tensors=UpperCAmelCase    ,  **UpperCAmelCase			).pixel_values
         lowerCamelCase__					:   List[Any]      				=	query_pixel_values

     if images is not None:
         lowerCamelCase__					:   List[Any]      				=	self.image_processor(UpperCAmelCase    ,  return_tensors=UpperCAmelCase    ,  **UpperCAmelCase			)

     if text is not None and images is not None:
         lowerCamelCase__					:   Tuple      				=	image_features.pixel_values
         return encoding
     elif query_images is not None and images is not None:
         lowerCamelCase__					:   Dict      				=	image_features.pixel_values
         return encoding
     elif text is not None or query_images is not None:
         return encoding
     else:
         return BatchEncoding(data=dict(**UpperCAmelCase			)    ,  tensor_type=UpperCAmelCase			)
 def   A_			(   self						:   str    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   List[Any]			)    ->			Any:
     return self.image_processor.post_process(*UpperCAmelCase    ,  **UpperCAmelCase			)
 def   A_			(   self						:   int    ,  *UpperCAmelCase						:   List[Any]    ,  **UpperCAmelCase						:   Tuple			)    ->			str:
     return self.image_processor.post_process_object_detection(*UpperCAmelCase    ,  **UpperCAmelCase			)
 def   A_			(   self						:   Tuple    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   Optional[int]			)    ->			Optional[Any]:
     return self.image_processor.post_process_image_guided_detection(*UpperCAmelCase    ,  **UpperCAmelCase			)
 def   A_			(   self						:   Optional[Any]    ,  *UpperCAmelCase						:   Tuple    ,  **UpperCAmelCase						:   List[Any]			)    ->			Tuple:
     return self.tokenizer.batch_decode(*UpperCAmelCase    ,  **UpperCAmelCase			)
 def   A_			(   self						:   Union[str, Any]    ,  *UpperCAmelCase						:   List[Any]    ,  **UpperCAmelCase						:   List[Any]			)    ->			List[str]:
     return self.tokenizer.decode(*UpperCAmelCase    ,  **UpperCAmelCase			)
 @property
 def   A_			(   self						:   List[str]			)    ->			Optional[Any]:
     warnings.warn(
         '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'    ,  UpperCAmelCase    ,  )
     return self.image_processor_class







 @property
 def   A_			(   self						:   int			)    ->			int:
     warnings.warn(
         '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'    ,  UpperCAmelCase    ,  )
     return self.image_processor





