




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 				lowerCAmelCase_       (					lowerCamelCase_						,     unittest.TestCase							):
				__a     :       Any						=     BertTokenizer
				__a     :       Tuple						=     BertTokenizerFast
				__a     :       Union[str, Any]						=     True
				__a     :       int						=     True
				__a     :       Union[str, Any]						=     filter_non_english






				def 		snake_case	(			self ):
							super().setUp()

							SCREAMING_SNAKE_CASE_   : Any					  =      [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['vocab_file'] )
							with open(self.vocab_file      ,'w'      ,encoding='utf-8' ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )






				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      'UNwant\u00E9d,running'
							SCREAMING_SNAKE_CASE_   : List[str]					  =      'unwanted, running'
							return input_text, output_text






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      self.tokenizer_class(self.vocab_file )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.tokenize('UNwant\u00E9d,running' )
							self.assertListEqual(snake_case__      ,['un', '##want', '##ed', ',', 'runn', '##ing'] )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )      ,[9, 6, 7, 12, 10, 11] )






				def 		snake_case	(			self ):
							if not self.test_rust_tokenizer:
										return

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_tokenizer()
							SCREAMING_SNAKE_CASE_   : int					  =      self.get_rust_tokenizer()

							SCREAMING_SNAKE_CASE_   : str					  =      'UNwant\u00E9d,running'

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.tokenize(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      rust_tokenizer.tokenize(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : int					  =      rust_tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Any					  =      rust_tokenizer.encode(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							# With lower casing
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.get_tokenizer(do_lower_case=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_rust_tokenizer(do_lower_case=snake_case__ )

							SCREAMING_SNAKE_CASE_   : int					  =      'UNwant\u00E9d,running'

							SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer.tokenize(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      rust_tokenizer.tokenize(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : str					  =      rust_tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer.encode(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      rust_tokenizer.encode(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )      ,['ah', '\u535A', '\u63A8', 'zz'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )      ,['hello', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hällo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['h\u00E9llo'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hallo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hallo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )      ,['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      BasicTokenizer(do_lower_case=snake_case__      ,never_split=['[UNK]'] )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )      ,['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      BasicTokenizer()
							SCREAMING_SNAKE_CASE_   : Any					  =      'a\n\'ll !!to?\'d of, can\'t.'
							SCREAMING_SNAKE_CASE_   : Tuple					  =      ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
							self.assertListEqual(tokenizer.tokenize(snake_case__ )      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							SCREAMING_SNAKE_CASE_   : List[str]					  =      {}
							for i, token in enumerate(snake_case__ ):
										SCREAMING_SNAKE_CASE_   : Tuple					  =      i
							SCREAMING_SNAKE_CASE_   : List[str]					  =      WordpieceTokenizer(vocab=snake_case__      ,unk_token='[UNK]' )

							self.assertListEqual(tokenizer.tokenize('' )      ,[] )

							self.assertListEqual(tokenizer.tokenize('unwanted running' )      ,['un', '##want', '##ed', 'runn', '##ing'] )

							self.assertListEqual(tokenizer.tokenize('unwantedX running' )      ,['[UNK]', 'runn', '##ing'] )






				def 		snake_case	(			self ):
							self.assertTrue(_is_whitespace(' ' ) )
							self.assertTrue(_is_whitespace('\t' ) )
							self.assertTrue(_is_whitespace('\r' ) )
							self.assertTrue(_is_whitespace('\n' ) )
							self.assertTrue(_is_whitespace('\u00A0' ) )

							self.assertFalse(_is_whitespace('A' ) )
							self.assertFalse(_is_whitespace('-' ) )






				def 		snake_case	(			self ):
							self.assertTrue(_is_control('\u0005' ) )

							self.assertFalse(_is_control('A' ) )
							self.assertFalse(_is_control(' ' ) )
							self.assertFalse(_is_control('\t' ) )
							self.assertFalse(_is_control('\r' ) )






				def 		snake_case	(			self ):
							self.assertTrue(_is_punctuation('-' ) )
							self.assertTrue(_is_punctuation('$' ) )
							self.assertTrue(_is_punctuation('`' ) )
							self.assertTrue(_is_punctuation('.' ) )

							self.assertFalse(_is_punctuation('A' ) )
							self.assertFalse(_is_punctuation(' ' ) )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_tokenizer()
							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(snake_case__ ) for t in ['Test', '\xad', 'test']]      ,[['[UNK]'], [], ['[UNK]']] )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(snake_case__ ) for t in ['Test', '\xad', 'test']]      ,[['[UNK]'], [], ['[UNK]']] )






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      self.tokenizer_class.from_pretrained('bert-base-uncased' )

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.encode('sequence builders'      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer.encode('multi-sequence build'      ,add_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : int					  =      tokenizer.build_inputs_with_special_tokens(snake_case__ )
							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.build_inputs_with_special_tokens(snake_case__      ,snake_case__ )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]






				def 		snake_case	(			self ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													SCREAMING_SNAKE_CASE_   : int					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : List[Any]					  =      F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer_r.encode_plus(
													    snake_case__      ,return_attention_mask=snake_case__      ,return_token_type_ids=snake_case__      ,return_offsets_mapping=snake_case__      ,add_special_tokens=snake_case__      ,)

													SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer_r.do_lower_case if hasattr(snake_case__      ,'do_lower_case' ) else False
													SCREAMING_SNAKE_CASE_   : Any					  =      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'A'),
													        ((1, 2), ','),
													        ((3, 5), 'na'),
													        ((5, 6), '##ï'),
													        ((6, 8), '##ve'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'Allen'),
													        ((21, 23), '##NL'),
													        ((23, 24), '##P'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'a'),
													        ((1, 2), ','),
													        ((3, 8), 'naive'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'allen'),
													        ((21, 23), '##nl'),
													        ((23, 24), '##p'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]      ,tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
													self.assertEqual([e[0] for e in expected_results]      ,tokens['offset_mapping'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Dict					  =      ['的', '人', '有']
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      ''.join(snake_case__ )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      True
													SCREAMING_SNAKE_CASE_   : Dict					  =      self.tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )
													SCREAMING_SNAKE_CASE_   : str					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer_p.encode(snake_case__      ,add_special_tokens=snake_case__ )
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_r.encode(snake_case__      ,add_special_tokens=snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_r.convert_ids_to_tokens(snake_case__ )
													SCREAMING_SNAKE_CASE_   : Tuple					  =      tokenizer_p.convert_ids_to_tokens(snake_case__ )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(snake_case__      ,snake_case__ )
													self.assertListEqual(snake_case__      ,snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      False
													SCREAMING_SNAKE_CASE_   : Dict					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer_r.encode(snake_case__      ,add_special_tokens=snake_case__ )
													SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer_p.encode(snake_case__      ,add_special_tokens=snake_case__ )

													SCREAMING_SNAKE_CASE_   : int					  =      tokenizer_r.convert_ids_to_tokens(snake_case__ )
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_p.convert_ids_to_tokens(snake_case__ )

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      [
													    F'##{token}' if idx != 0 else token for idx, token in enumerate(snake_case__ )
													]
													self.assertListEqual(snake_case__      ,snake_case__ )
													self.assertListEqual(snake_case__      ,snake_case__ )









import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    'files'  ,							[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     int  )       ->  int:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tmp_path_factory.mktemp('dset_infos_dir'  )
			if "full:README.md" in files:
						with open(dataset_infos_dir / 'README.md'  ,							'w'  ) as f:
									f.write('---\ndataset_info:\n  dataset_size: 42\n---'  )
			if "empty:README.md" in files:
						with open(dataset_infos_dir / 'README.md'  ,							'w'  ) as f:
									f.write(''  )
    # we want to support dataset_infos.json for backward compatibility
			if "dataset_infos.json" in files:
						with open(dataset_infos_dir / 'dataset_infos.json'  ,							'w'  ) as f:
									f.write('{"default": {"dataset_size": 42}}'  )
			SCREAMING_SNAKE_CASE_   : List[Any]					  =      DatasetInfosDict.from_directory(lowerCamelCase_  )
			assert dataset_infos
			assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    'dataset_info'  ,							[
        DatasetInfo(),
        DatasetInfo(
            description='foo'  ,							features=Features({'a': Value('int32'  )}  )  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train'}]  ,							download_size=42  ,							),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     DatasetInfo  )       ->  Any:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Dict					  =      str(lowerCamelCase_  )
			dataset_info.write_to_directory(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : List[str]					  =      DatasetInfo.from_directory(lowerCamelCase_  )
			assert dataset_info == reloaded
			assert os.path.exists(os.path.join(lowerCamelCase_  ,							'dataset_info.json'  )  )





def        __UpperCAmelCase						(							)       ->  List[Any]:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : str					  =      DatasetInfo(
			    description='foo'  ,							citation='bar'  ,							homepage='https://foo.bar'  ,							license='CC0'  ,							features=Features({'a': Value('int32'  )}  )  ,							post_processed={}  ,							supervised_keys=()  ,							task_templates=[]  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train', 'num_examples': 42}]  ,							download_checksums={}  ,							download_size=13_37  ,							post_processing_size=4_42  ,							dataset_size=12_34  ,							size_in_bytes=13_37 + 4_42 + 12_34  ,							)
			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      dataset_info._to_yaml_dict()
			assert sorted(lowerCamelCase_  ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML  )
			for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
						assert key in dataset_info_yaml_dict
						assert isinstance(dataset_info_yaml_dict[key]  ,							(list, dict, int, str)  )
			SCREAMING_SNAKE_CASE_   : Optional[int]					  =      yaml.safe_dump(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : int					  =      yaml.safe_load(lowerCamelCase_  )
			assert dataset_info_yaml_dict == reloaded





def        __UpperCAmelCase						(							)       ->  Dict:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      DatasetInfo()
			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      dataset_info._to_yaml_dict()
			assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    'dataset_infos_dict'  ,							[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()}  ),
        DatasetInfosDict({'my_config_name': DatasetInfo()}  ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'  ,							features=Features({'a': Value('int32'  )}  )  ,							builder_name='builder'  ,							config_name='config'  ,							version='1.0.0'  ,							splits=[{'name': 'train'}]  ,							download_size=42  ,							)
            }  ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42  ),
                'v2': DatasetInfo(dataset_size=13_37  ),
            }  ),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     Optional[Any]  ,							lowerCamelCase_ :     DatasetInfosDict  )       ->  Tuple:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Any					  =      str(lowerCamelCase_  )
			dataset_infos_dict.write_to_directory(lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : List[Any]					  =      DatasetInfosDict.from_directory(lowerCamelCase_  )

			# the config_name of the dataset_infos_dict take over the attribute
			for config_name, dataset_info in dataset_infos_dict.items():
						SCREAMING_SNAKE_CASE_   : Any					  =      config_name
						# the yaml representation doesn't include fields like description or citation
						# so we just test that we can recover what we can from the yaml
						SCREAMING_SNAKE_CASE_   : Any					  =      DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()  )
			assert dataset_infos_dict == reloaded

			if dataset_infos_dict:
						assert os.path.exists(os.path.join(lowerCamelCase_  ,							'README.md'  )  )



