
"""simple docstring"""

from math import ceil






def 							A						(    snake_case				:str    , snake_case				:Tuple		)				->     str:
 __UpperCamelCase      =	list(range(0    , snake_case		)		)

 __UpperCamelCase      =	[item for sublist in list(device_map.values()		) for item in sublist]

 # Duplicate check
 __UpperCamelCase      =	[]
 for i in device_map_blocks:
  if device_map_blocks.count(snake_case		) > 1 and i not in duplicate_blocks:
   duplicate_blocks.append(snake_case		)
    # Missing blocks
 __UpperCamelCase      =	[i for i in blocks if i not in device_map_blocks]
 __UpperCamelCase      =	[i for i in device_map_blocks if i not in blocks]

 if len(snake_case		) != 0:
  raise ValueError(
      'Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device.'
      ' These attention blocks were specified more than once: ' + str(snake_case		)		)
 if len(snake_case		) != 0:
  raise ValueError(
      'There are attention blocks for this model that are not specified in the device_map. Add these attention '
      'blocks to a device on the device_map: ' + str(snake_case		)		)
 if len(snake_case		) != 0:
  raise ValueError(
      'The device_map contains more attention blocks than this model has. Remove these from the device_map:'
      + str(snake_case		)		)






def 							A						(    snake_case				:int    , snake_case				:str		)				->     Any:
 __UpperCamelCase      =	list(range(snake_case		)		)
 __UpperCamelCase      =	int(ceil(n_layers / len(snake_case		)		)		)
 __UpperCamelCase      =	[layers[i : i + n_blocks] for i in range(0    , snake_case    , snake_case		)]

 return dict(zip(snake_case    , snake_case		)		)





"""simple docstring"""

from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters






def 							A						(    )				->     Any:
 __UpperCamelCase      =	{
     'repo_name': ['test_repo1', 'test_repo2', 'test_repo3'],
     'path': ['test_1.py', 'test_2.py', 'unit_test.py'],
     'content': ['a ' * 2_0, 'a ' * 3_0, 'b ' * 7],
 }
 __UpperCamelCase      =	Dataset.from_dict(snake_case		)
 return dataset





class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	get_dataset()
      __UpperCamelCase      =	make_duplicate_clusters(__UpperCAmelCase     ,					0.8_5				)
      self.assertEqual(len(duplicate_clusters[0]				)     ,					2				)







     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	get_dataset()
      __UpperCamelCase  ,     __UpperCamelCase      =	deduplicate_dataset(__UpperCAmelCase				)
      self.assertEqual(len(__UpperCAmelCase				)     ,					2				)
      print(__UpperCAmelCase				)
      self.assertEqual(duplicate_clusters[0][0]['copies']     ,					2				)
      self.assertEqual(duplicate_clusters[0][0]['is_extreme']     ,					__UpperCAmelCase				)



