





import os
from argparse import ArgumentParser
from typing import List

import torch.utils.data

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node


_lowerCamelCase 					=      4
_lowerCamelCase 					=      3




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							pass






def 					SCREAMING_SNAKE_CASE__				(							UpperCamelCase__:				List[str]     ):
		for shard in shards:
				for i in range(UpperCamelCase__     ):
						yield {"i": i, "shard": shard}






def 					SCREAMING_SNAKE_CASE__				(							):
		SCREAMING_SNAKE_CASE__								=   int(os.environ["""RANK"""]     )
		SCREAMING_SNAKE_CASE__								=   int(os.environ["""WORLD_SIZE"""]     )

		SCREAMING_SNAKE_CASE__								=   ArgumentParser()
		parser.add_argument("""--streaming"""       , type=UpperCamelCase__     )
		parser.add_argument("""--local_rank"""       , type=UpperCamelCase__     )
		parser.add_argument("""--num_workers"""       , type=UpperCamelCase__       , default=0     )
		SCREAMING_SNAKE_CASE__								=   parser.parse_args()
		SCREAMING_SNAKE_CASE__								=   args.streaming
		SCREAMING_SNAKE_CASE__								=   args.num_workers

		SCREAMING_SNAKE_CASE__								=   {"""shards""": [f'''shard_{shard_idx}''' for shard_idx in range(UpperCamelCase__     )]}
		SCREAMING_SNAKE_CASE__								=   IterableDataset.from_generator(UpperCamelCase__       , gen_kwargs=UpperCamelCase__     )
		if not streaming:
				SCREAMING_SNAKE_CASE__								=   Dataset.from_list(list(UpperCamelCase__     )     )

		SCREAMING_SNAKE_CASE__								=   split_dataset_by_node(UpperCamelCase__       , rank=UpperCamelCase__       , world_size=UpperCamelCase__     )
		SCREAMING_SNAKE_CASE__								=   torch.utils.data.DataLoader(UpperCamelCase__       , num_workers=UpperCamelCase__     )

		SCREAMING_SNAKE_CASE__								=   NUM_SHARDS * NUM_ITEMS_PER_SHARD
		SCREAMING_SNAKE_CASE__								=   full_size // world_size
		expected_local_size += int(rank < (full_size % world_size)     )

		SCREAMING_SNAKE_CASE__								=   sum(1 for _ in dataloader     )
		if local_size != expected_local_size:
				raise FailedTestError(f'''local_size {local_size} != expected_local_size {expected_local_size}'''     )


if __name__ == "__main__":
	main()






import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible




class 	UpperCamelCase_							(  unittest.TestCase     ):


							def   _snake_case	(				self     :Tuple		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.bin""",
									    """safety_checker/model.safetensors""",
									    """vae/diffusion_pytorch_model.bin""",
									    """vae/diffusion_pytorch_model.safetensors""",
									    """text_encoder/pytorch_model.bin""",
									    """text_encoder/model.safetensors""",
									    """unet/diffusion_pytorch_model.bin""",
									    """unet/diffusion_pytorch_model.safetensors""",
									]
									self.assertTrue(is_safetensors_compatible(__A		)		)


							def   _snake_case	(				self     :List[str]		) -> int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """unet/diffusion_pytorch_model.bin""",
									    """unet/diffusion_pytorch_model.safetensors""",
									]
									self.assertTrue(is_safetensors_compatible(__A		)		)


							def   _snake_case	(				self     :Tuple		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.bin""",
									    """safety_checker/model.safetensors""",
									    """vae/diffusion_pytorch_model.bin""",
									    """vae/diffusion_pytorch_model.safetensors""",
									    """text_encoder/pytorch_model.bin""",
									    """text_encoder/model.safetensors""",
									    """unet/diffusion_pytorch_model.bin""",
									    # Removed: 'unet/diffusion_pytorch_model.safetensors',
									]
									self.assertFalse(is_safetensors_compatible(__A		)		)


							def   _snake_case	(				self     :Union[str, Any]		) -> Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """text_encoder/pytorch_model.bin""",
									    """text_encoder/model.safetensors""",
									]
									self.assertTrue(is_safetensors_compatible(__A		)		)


							def   _snake_case	(				self     :Optional[Any]		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.bin""",
									    """safety_checker/model.safetensors""",
									    """vae/diffusion_pytorch_model.bin""",
									    """vae/diffusion_pytorch_model.safetensors""",
									    """text_encoder/pytorch_model.bin""",
									    # Removed: 'text_encoder/model.safetensors',
									    """unet/diffusion_pytorch_model.bin""",
									    """unet/diffusion_pytorch_model.safetensors""",
									]
									self.assertFalse(is_safetensors_compatible(__A		)		)


							def   _snake_case	(				self     :Tuple		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.fp16.bin""",
									    """safety_checker/model.fp16.safetensors""",
									    """vae/diffusion_pytorch_model.fp16.bin""",
									    """vae/diffusion_pytorch_model.fp16.safetensors""",
									    """text_encoder/pytorch_model.fp16.bin""",
									    """text_encoder/model.fp16.safetensors""",
									    """unet/diffusion_pytorch_model.fp16.bin""",
									    """unet/diffusion_pytorch_model.fp16.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertTrue(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :Any		) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """unet/diffusion_pytorch_model.fp16.bin""",
									    """unet/diffusion_pytorch_model.fp16.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertTrue(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :str		) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """unet/diffusion_pytorch_model.bin""",
									    """unet/diffusion_pytorch_model.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertTrue(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :List[Any]		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.fp16.bin""",
									    """safety_checker/model.fp16.safetensors""",
									    """vae/diffusion_pytorch_model.fp16.bin""",
									    """vae/diffusion_pytorch_model.fp16.safetensors""",
									    """text_encoder/pytorch_model.fp16.bin""",
									    """text_encoder/model.fp16.safetensors""",
									    """unet/diffusion_pytorch_model.fp16.bin""",
									    # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertFalse(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :str		) -> List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """text_encoder/pytorch_model.fp16.bin""",
									    """text_encoder/model.fp16.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertTrue(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :Optional[int]		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """text_encoder/pytorch_model.bin""",
									    """text_encoder/model.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertTrue(is_safetensors_compatible(__A				,					variant=__A		)		)


							def   _snake_case	(				self     :Union[str, Any]		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [
									    """safety_checker/pytorch_model.fp16.bin""",
									    """safety_checker/model.fp16.safetensors""",
									    """vae/diffusion_pytorch_model.fp16.bin""",
									    """vae/diffusion_pytorch_model.fp16.safetensors""",
									    """text_encoder/pytorch_model.fp16.bin""",
									    # 'text_encoder/model.fp16.safetensors',
									    """unet/diffusion_pytorch_model.fp16.bin""",
									    """unet/diffusion_pytorch_model.fp16.safetensors""",
									]
									SCREAMING_SNAKE_CASE__								=   """fp16"""
									self.assertFalse(is_safetensors_compatible(__A				,					variant=__A		)		)