




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class        __lowerCAmelCase					(			lowerCamelCase_     ,							unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			BertTokenizer
			snake_case_  						=			BertTokenizerFast
			snake_case_  						=			True
			snake_case_  						=			True
			snake_case_  						=			filter_non_english


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							super().setUp()

							__lowerCamelCase									=       [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							__lowerCamelCase									=       os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['vocab_file']				)
							with open(self.vocab_file     ,		'w'     ,		encoding='utf-8'				) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)


			def  lowercase_       ( self     ,		lowerCamelCase__				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       'UNwant\u00E9d,running'
							__lowerCamelCase									=       'unwanted, running'
							return input_text, output_text


			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.tokenizer_class(self.vocab_file				)

							__lowerCamelCase									=       tokenizer.tokenize('UNwant\u00E9d,running'				)
							self.assertListEqual(snake_case__     ,		['un', '##want', '##ed', ',', 'runn', '##ing']				)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				)     ,		[9, 6, 7, 12, 10, 11]				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							if not self.test_rust_tokenizer:
											return

							__lowerCamelCase									=       self.get_tokenizer()
							__lowerCamelCase									=       self.get_rust_tokenizer()

							__lowerCamelCase									=       'UNwant\u00E9d,running'

							__lowerCamelCase									=       tokenizer.tokenize(snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.tokenize(snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)

							__lowerCamelCase									=       tokenizer.encode(snake_case__     ,		add_special_tokens=snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.encode(snake_case__     ,		add_special_tokens=snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)

							__lowerCamelCase									=       self.get_rust_tokenizer()
							__lowerCamelCase									=       tokenizer.encode(snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.encode(snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)

							# With lower casing
							__lowerCamelCase									=       self.get_tokenizer(do_lower_case=snake_case__				)
							__lowerCamelCase									=       self.get_rust_tokenizer(do_lower_case=snake_case__				)

							__lowerCamelCase									=       'UNwant\u00E9d,running'

							__lowerCamelCase									=       tokenizer.tokenize(snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.tokenize(snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)

							__lowerCamelCase									=       tokenizer.encode(snake_case__     ,		add_special_tokens=snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.encode(snake_case__     ,		add_special_tokens=snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)

							__lowerCamelCase									=       self.get_rust_tokenizer()
							__lowerCamelCase									=       tokenizer.encode(snake_case__				)
							__lowerCamelCase									=       rust_tokenizer.encode(snake_case__				)
							self.assertListEqual(snake_case__     ,		snake_case__				)


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				)     ,		['ah', '\u535A', '\u63A8', 'zz']				)


			def  lowercase_       ( self				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)     ,		['hello', '!', 'how', 'are', 'you', '?']				)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)     ,		['hello']				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__     ,		strip_accents=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)     ,		['hällo', '!', 'how', 'are', 'you', '?']				)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)     ,		['h\u00E9llo']				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__     ,		strip_accents=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)     ,		['hallo', '!', 'how', 'are', 'you', '?']				)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)     ,		['hello']				)


			def  lowercase_       ( self				)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)     ,		['hallo', '!', 'how', 'are', 'you', '?']				)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)     ,		['hello']				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)     ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__     ,		strip_accents=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)     ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__     ,		strip_accents=snake_case__				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)     ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)


			def  lowercase_       ( self				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer(do_lower_case=snake_case__     ,		never_split=['[UNK]']				)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				)     ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)


			def  lowercase_       ( self				)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       BasicTokenizer()
							__lowerCamelCase									=       'a\n\'ll !!to?\'d of, can\'t.'
							__lowerCamelCase									=       ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
							self.assertListEqual(tokenizer.tokenize(snake_case__				)     ,		snake_case__				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							__lowerCamelCase									=       {}
							for i, token in enumerate(snake_case__				):
											__lowerCamelCase									=       i
							__lowerCamelCase									=       WordpieceTokenizer(vocab=snake_case__     ,		unk_token='[UNK]'				)

							self.assertListEqual(tokenizer.tokenize(''				)     ,		[]				)

							self.assertListEqual(tokenizer.tokenize('unwanted running'				)     ,		['un', '##want', '##ed', 'runn', '##ing']				)

							self.assertListEqual(tokenizer.tokenize('unwantedX running'				)     ,		['[UNK]', 'runn', '##ing']				)


			def  lowercase_       ( self				)       -> Optional[int]:



							'''simple docstring'''
							self.assertTrue(_is_whitespace(' '				)				)
							self.assertTrue(_is_whitespace('\t'				)				)
							self.assertTrue(_is_whitespace('\r'				)				)
							self.assertTrue(_is_whitespace('\n'				)				)
							self.assertTrue(_is_whitespace('\u00A0'				)				)

							self.assertFalse(_is_whitespace('A'				)				)
							self.assertFalse(_is_whitespace('-'				)				)


			def  lowercase_       ( self				)       -> List[Any]:



							'''simple docstring'''
							self.assertTrue(_is_control('\u0005'				)				)

							self.assertFalse(_is_control('A'				)				)
							self.assertFalse(_is_control(' '				)				)
							self.assertFalse(_is_control('\t'				)				)
							self.assertFalse(_is_control('\r'				)				)


			def  lowercase_       ( self				)       -> List[Any]:



							'''simple docstring'''
							self.assertTrue(_is_punctuation('-'				)				)
							self.assertTrue(_is_punctuation('$'				)				)
							self.assertTrue(_is_punctuation('`'				)				)
							self.assertTrue(_is_punctuation('.'				)				)

							self.assertFalse(_is_punctuation('A'				)				)
							self.assertFalse(_is_punctuation(' '				)				)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       self.get_tokenizer()
							__lowerCamelCase									=       self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(snake_case__				) for t in ['Test', '\xad', 'test']]     ,		[['[UNK]'], [], ['[UNK]']]				)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(snake_case__				) for t in ['Test', '\xad', 'test']]     ,		[['[UNK]'], [], ['[UNK]']]				)


			@slow
			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.tokenizer_class.from_pretrained('bert-base-uncased'				)

							__lowerCamelCase									=       tokenizer.encode('sequence builders'     ,		add_special_tokens=snake_case__				)
							__lowerCamelCase									=       tokenizer.encode('multi-sequence build'     ,		add_special_tokens=snake_case__				)

							__lowerCamelCase									=       tokenizer.build_inputs_with_special_tokens(snake_case__				)
							__lowerCamelCase									=       tokenizer.build_inputs_with_special_tokens(snake_case__     ,		snake_case__				)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
															__lowerCamelCase									=       self.rust_tokenizer_class.from_pretrained(snake_case__     ,		**snake_case__				)

															__lowerCamelCase									=       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
															__lowerCamelCase									=       tokenizer_r.encode_plus(
															    snake_case__     ,		return_attention_mask=snake_case__     ,		return_token_type_ids=snake_case__     ,		return_offsets_mapping=snake_case__     ,		add_special_tokens=snake_case__     ,		)

															__lowerCamelCase									=       tokenizer_r.do_lower_case if hasattr(snake_case__     ,		'do_lower_case'				) else False
															__lowerCamelCase									=       (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), 'A'),
															        ((1, 2), ','),
															        ((3, 5), 'na'),
															        ((5, 6), '##ï'),
															        ((6, 8), '##ve'),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), 'Allen'),
															        ((21, 23), '##NL'),
															        ((23, 24), '##P'),
															        ((25, 33), 'sentence'),
															        ((33, 34), '.'),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), 'a'),
															        ((1, 2), ','),
															        ((3, 8), 'naive'),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), 'allen'),
															        ((21, 23), '##nl'),
															        ((23, 24), '##p'),
															        ((25, 33), 'sentence'),
															        ((33, 34), '.'),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]     ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
															self.assertEqual([e[0] for e in expected_results]     ,		tokens['offset_mapping']				)






			def  lowercase_       ( self				)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       ['的', '人', '有']
							__lowerCamelCase									=       ''.join(snake_case__				)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
															__lowerCamelCase									=       True
															__lowerCamelCase									=       self.tokenizer_class.from_pretrained(snake_case__     ,		**snake_case__				)
															__lowerCamelCase									=       self.rust_tokenizer_class.from_pretrained(snake_case__     ,		**snake_case__				)

															__lowerCamelCase									=       tokenizer_p.encode(snake_case__     ,		add_special_tokens=snake_case__				)
															__lowerCamelCase									=       tokenizer_r.encode(snake_case__     ,		add_special_tokens=snake_case__				)

															__lowerCamelCase									=       tokenizer_r.convert_ids_to_tokens(snake_case__				)
															__lowerCamelCase									=       tokenizer_p.convert_ids_to_tokens(snake_case__				)

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(snake_case__     ,		snake_case__				)
															self.assertListEqual(snake_case__     ,		snake_case__				)

															__lowerCamelCase									=       False
															__lowerCamelCase									=       self.rust_tokenizer_class.from_pretrained(snake_case__     ,		**snake_case__				)
															__lowerCamelCase									=       self.tokenizer_class.from_pretrained(snake_case__     ,		**snake_case__				)

															__lowerCamelCase									=       tokenizer_r.encode(snake_case__     ,		add_special_tokens=snake_case__				)
															__lowerCamelCase									=       tokenizer_p.encode(snake_case__     ,		add_special_tokens=snake_case__				)

															__lowerCamelCase									=       tokenizer_r.convert_ids_to_tokens(snake_case__				)
															__lowerCamelCase									=       tokenizer_p.convert_ids_to_tokens(snake_case__				)

															# it is expected that only the first Chinese character is not preceded by "##".
															__lowerCamelCase									=       [
															    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__				)
															]
															self.assertListEqual(snake_case__     ,		snake_case__				)
															self.assertListEqual(snake_case__     ,		snake_case__				)








import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    'files'  ,							[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     int  )       ->  int:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tmp_path_factory.mktemp('dset_infos_dir'  )
			if "full:README.md" in files:
						with open(dataset_infos_dir / 'README.md'  ,							'w'  ) as f:
									f.write('---\ndataset_info:\n  dataset_size: 42\n---'  )
			if "empty:README.md" in files:
						with open(dataset_infos_dir / 'README.md'  ,							'w'  ) as f:
									f.write(''  )
    # we want to support dataset_infos.json for backward compatibility
			if "dataset_infos.json" in files:
						with open(dataset_infos_dir / 'dataset_infos.json'  ,							'w'  ) as f:
									f.write('{"default": {"dataset_size": 42}}'  )
			SCREAMING_SNAKE_CASE_   : List[Any]					  =      DatasetInfosDict.from_directory(lowerCamelCase_  )
			assert dataset_infos
			assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    'dataset_info'  ,							[
        DatasetInfo(),
        DatasetInfo(
            description='foo'  ,							features=Features({'a': Value('int32'  )}  )  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train'}]  ,							download_size=42  ,							),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     DatasetInfo  )       ->  Any:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Dict					  =      str(lowerCamelCase_  )
			dataset_info.write_to_directory(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : List[str]					  =      DatasetInfo.from_directory(lowerCamelCase_  )
			assert dataset_info == reloaded
			assert os.path.exists(os.path.join(lowerCamelCase_  ,							'dataset_info.json'  )  )





def        __UpperCAmelCase						(							)       ->  List[Any]:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : str					  =      DatasetInfo(
			    description='foo'  ,							citation='bar'  ,							homepage='https://foo.bar'  ,							license='CC0'  ,							features=Features({'a': Value('int32'  )}  )  ,							post_processed={}  ,							supervised_keys=()  ,							task_templates=[]  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train', 'num_examples': 42}]  ,							download_checksums={}  ,							download_size=13_37  ,							post_processing_size=4_42  ,							dataset_size=12_34  ,							size_in_bytes=13_37 + 4_42 + 12_34  ,							)
			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      dataset_info._to_yaml_dict()
			assert sorted(lowerCamelCase_  ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML  )
			for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
						assert key in dataset_info_yaml_dict
						assert isinstance(dataset_info_yaml_dict[key]  ,							(list, dict, int, str)  )
			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      yaml.safe_dump(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : int					  =      yaml.safe_load(lowerCamelCase_  )
			assert dataset_info_yaml_dict == reloaded





def        __UpperCAmelCase						(							)       ->  Dict:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      DatasetInfo()
			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      dataset_info._to_yaml_dict()
			assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    'dataset_infos_dict'  ,							[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()}  ),
        DatasetInfosDict({'my_config_name': DatasetInfo()}  ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'  ,							features=Features({'a': Value('int32'  )}  )  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train'}]  ,							download_size=42  ,							)
            }  ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42  ),
                'v2': DatasetInfo(dataset_size=13_37  ),
            }  ),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     Optional[Any]  ,							lowerCamelCase_ :     DatasetInfosDict  )       ->  Tuple:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Any					  =      str(lowerCamelCase_  )
			dataset_infos_dict.write_to_directory(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : List[Any]					  =      DatasetInfosDict.from_directory(lowerCamelCase_  )

			# the config_name of the dataset_infos_dict take over the attribute
			for config_name, dataset_info in dataset_infos_dict.items():
						SCREAMING_SNAKE_CASE_   : Any					  =      config_name
						# the yaml representation doesn't include fields like description or citation
						# so we just test that we can recover what we can from the yaml
						SCREAMING_SNAKE_CASE_   : Any					  =      DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()  )
			assert dataset_infos_dict == reloaded

			if dataset_infos_dict:
						assert os.path.exists(os.path.join(lowerCamelCase_  ,							'README.md'  )  )



