


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__							:						Dict							     =   """▁"""

lowerCamelCase__							:						List[str]							     =   get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
@require_tokenizers
class 		__magic_name__						(snake_case_   ,unittest.TestCase				):





     '''simple docstring'''



     __lowercase				:							Optional[Any]	   =       BigBirdTokenizer
     __lowercase				:							Dict	   =       BigBirdTokenizerFast
     __lowercase				:							List[Any]	   =       True
     __lowercase				:							List[Any]	   =       True

     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       super().setUp()

       snake_case__						= self.tokenizer_class(_a     ,		keep_accents=_a   )
       tokenizer.save_pretrained(self.tmpdirname   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Union[str, Any]   ):
       snake_case__						= '''<s>'''
       snake_case__						= 1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a   )     ,		_a   )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a   )     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       snake_case__						= list(self.get_tokenizer().get_vocab().keys()   )

       self.assertEqual(vocab_keys[0]     ,		'''<unk>'''   )
       self.assertEqual(vocab_keys[1]     ,		'''<s>'''   )
       self.assertEqual(vocab_keys[-1]     ,		'''[MASK]'''   )
       self.assertEqual(len(_a   )     ,		10_04   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Union[str, Any]   ):
       self.assertEqual(self.get_tokenizer().vocab_size     ,		10_00   )

     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       if not self.test_rust_tokenizer:
         return

       snake_case__						= self.get_tokenizer()
       snake_case__						= self.get_rust_tokenizer()

       snake_case__						= '''I was born in 92000, and this is falsé.'''

       snake_case__						= tokenizer.tokenize(_a   )
       snake_case__						= rust_tokenizer.tokenize(_a   )
       self.assertListEqual(_a     ,		_a   )

       snake_case__						= tokenizer.encode(_a     ,		add_special_tokens=_a   )
       snake_case__						= rust_tokenizer.encode(_a     ,		add_special_tokens=_a   )
       self.assertListEqual(_a     ,		_a   )

       snake_case__						= self.get_rust_tokenizer()
       snake_case__						= tokenizer.encode(_a   )
       snake_case__						= rust_tokenizer.encode(_a   )
       self.assertListEqual(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       snake_case__						= BigBirdTokenizer(_a     ,		keep_accents=_a   )

       snake_case__						= tokenizer.tokenize('''This is a test'''   )
       self.assertListEqual(_a     ,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']   )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_a   )     ,		[2_85, 46, 10, 1_70, 3_82]     ,		)

       snake_case__						= tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
       self.assertListEqual(
           _a     ,		[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]     ,		)
       snake_case__						= tokenizer.convert_tokens_to_ids(_a   )
       self.assertListEqual(
           _a     ,		[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]     ,		)

       snake_case__						= tokenizer.convert_ids_to_tokens(_a   )
       self.assertListEqual(
           _a     ,		[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ]     ,		)

     @cached_property
     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''   )

     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       snake_case__						= '''Hello World!'''
       snake_case__						= [65, 1_85_36, 22_60, 1_01, 66]

       self.assertListEqual(_a     ,		self.big_tokenizer.encode(_a   )   )

     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       snake_case__						= (
           '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
           ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
       )
       # fmt: off
       snake_case__						= [65, 8_71, 4_19, 3_58, 9_46, 9_91, 25_21, 4_52, 3_58, 13_57, 3_87, 77_51, 35_36, 1_12, 9_85, 4_56, 1_26, 8_65, 9_38, 54_00, 57_34, 4_58, 13_68, 4_67, 7_86, 24_62, 52_46, 11_59, 6_33, 8_65, 45_19, 4_57, 5_82, 8_52, 25_57, 4_27, 9_16, 5_08, 4_05, 3_43_24, 4_97, 3_91, 4_08, 1_13_42, 12_44, 3_85, 1_00, 9_38, 9_85, 4_56, 5_74, 3_62, 1_25_97, 32_00, 31_29, 11_72, 66]  # noqa: E231
       # fmt: on
       self.assertListEqual(_a     ,		self.big_tokenizer.encode(_a   )   )

     @require_torch
     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       import torch

       from transformers import BigBirdConfig, BigBirdModel

       # Build sequence
       snake_case__						= list(self.big_tokenizer.get_vocab().keys()   )[:10]
       snake_case__						= ''' '''.join(_a   )
       snake_case__						= self.big_tokenizer.encode_plus(_a     ,		return_tensors='''pt'''     ,		return_token_type_ids=_a   )
       snake_case__						= self.big_tokenizer.batch_encode_plus(
           [sequence + ''' ''' + sequence]     ,		return_tensors='''pt'''     ,		return_token_type_ids=_a   )

       snake_case__						= BigBirdConfig(attention_type='''original_full'''   )
       snake_case__						= BigBirdModel(_a   )

       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
         model(**_a   )
         model(**_a   )

     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       snake_case__						= BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''   )
       snake_case__						= tokenizer.decode(tokenizer('''Paris is the [MASK].'''   ).input_ids   )

       self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]'''   )







     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       # fmt: off
       snake_case__						= {'''input_ids''': [[65, 3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14, 66], [65, 4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_a     ,		model_name='''google/bigbird-roberta-base'''     ,		revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''     ,		)





import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs







@pytest.mark.parametrize(
    '''kwargs, expected'''							,   [
        ({'''num_shards''': 0, '''max_num_jobs''': 1}, []),
        ({'''num_shards''': 10, '''max_num_jobs''': 1}, [range(10				)]),
        ({'''num_shards''': 10, '''max_num_jobs''': 10}, [range(__lowerCAmelCase							,   i + 1				) for i in range(10				)]),
        ({'''num_shards''': 1, '''max_num_jobs''': 10}, [range(1				)]),
        ({'''num_shards''': 10, '''max_num_jobs''': 3}, [range(0							,   4				), range(4							,   7				), range(7							,   10				)]),
        ({'''num_shards''': 3, '''max_num_jobs''': 10}, [range(0							,   1				), range(1							,   2				), range(2							,   3				)]),
    ]							,   )
def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase				)    ->				Optional[int]:
  snake_case__						= _distribute_shards(**__lowerCAmelCase				)
  assert out == expected







@pytest.mark.parametrize(
    '''gen_kwargs, max_num_jobs, expected'''							,   [
        ({'''foo''': 0}, 10, [{'''foo''': 0}]),
        ({'''shards''': [0, 1, 2, 3]}, 1, [{'''shards''': [0, 1, 2, 3]}]),
        ({'''shards''': [0, 1, 2, 3]}, 4, [{'''shards''': [0]}, {'''shards''': [1]}, {'''shards''': [2]}, {'''shards''': [3]}]),
        ({'''shards''': [0, 1]}, 4, [{'''shards''': [0]}, {'''shards''': [1]}]),
        ({'''shards''': [0, 1, 2, 3]}, 2, [{'''shards''': [0, 1]}, {'''shards''': [2, 3]}]),
    ]							,   )
def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)    ->				Dict:
  snake_case__						= _split_gen_kwargs(__lowerCAmelCase							,   __lowerCAmelCase				)
  assert out == expected







@pytest.mark.parametrize(
    '''gen_kwargs, expected'''							,   [
        ({'''foo''': 0}, 1),
        ({'''shards''': [0]}, 1),
        ({'''shards''': [0, 1, 2, 3]}, 4),
        ({'''shards''': [0, 1, 2, 3], '''foo''': 0}, 4),
        ({'''shards''': [0, 1, 2, 3], '''other''': (0, 1)}, 4),
        ({'''shards''': [0, 1, 2, 3], '''shards2''': [0, 1]}, RuntimeError),
    ]							,   )
def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase				)    ->				List[Any]:
  if expected is RuntimeError:
    with pytest.raises(__lowerCAmelCase				):
      _number_of_shards_in_gen_kwargs(__lowerCAmelCase				)
  else:
    snake_case__						= _number_of_shards_in_gen_kwargs(__lowerCAmelCase				)
    assert out == expected

