



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 			_a (					lowerCAmelCase			,							unittest.TestCase):
			"""simple docstring"""
			UpperCamelCase__			      =	BertTokenizer
			UpperCamelCase__			      =	BertTokenizerFast
			UpperCamelCase__			      =	True
			UpperCamelCase__			      =	True
			UpperCamelCase__			      =	filter_non_english



			def  lowercase__						(		self     :    Tuple   )->Tuple:
							super().setUp()

							_UpperCAmelCase					=    [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   VOCAB_FILES_NAMES['''vocab_file''']   )
							with open(self.vocab_file				,   '''w'''				,   encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )



			def  lowercase__						(		self     :    Optional[int]				,   __UpperCamelCase     :    Optional[Any]   )->Optional[int]:
							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''
							_UpperCAmelCase					=    '''unwanted, running'''
							return input_text, output_text



			def  lowercase__						(		self     :    Union[str, Any]   )->Optional[int]:
							_UpperCAmelCase					=    self.tokenizer_class(self.vocab_file   )

							_UpperCAmelCase					=    tokenizer.tokenize('''UNwant\u00E9d,running'''   )
							self.assertListEqual(__UpperCamelCase				,   ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase   )				,   [9, 6, 7, 1_2, 1_0, 1_1]   )



			def  lowercase__						(		self     :    List[str]   )->int:
							if not self.test_rust_tokenizer:
											return

							_UpperCAmelCase					=    self.get_tokenizer()
							_UpperCAmelCase					=    self.get_rust_tokenizer()

							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''

							_UpperCAmelCase					=    tokenizer.tokenize(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.tokenize(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    self.get_rust_tokenizer()
							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							# With lower casing
							_UpperCAmelCase					=    self.get_tokenizer(do_lower_case=__UpperCamelCase   )
							_UpperCAmelCase					=    self.get_rust_tokenizer(do_lower_case=__UpperCamelCase   )

							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''

							_UpperCAmelCase					=    tokenizer.tokenize(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.tokenize(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    self.get_rust_tokenizer()
							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    Tuple   )->Optional[Any]:
							_UpperCAmelCase					=    BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )				,   ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )



			def  lowercase__						(		self     :    Any   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,   ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    Optional[int]   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''h\u00E9llo''']   )



			def  lowercase__						(		self     :    Optional[int]   )->Dict:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    List[Any]   )->int:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    str   )->Tuple:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    Union[str, Any]   )->Dict:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    int   )->Optional[Any]:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    Union[str, Any]   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   never_split=['''[UNK]''']   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )				,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )



			def  lowercase__						(		self     :    Optional[Any]   )->str:
							_UpperCAmelCase					=    BasicTokenizer()
							_UpperCAmelCase					=    '''a\n\'ll !!to?\'d of, can\'t.'''
							_UpperCAmelCase					=    ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
							self.assertListEqual(tokenizer.tokenize(__UpperCamelCase   )				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    int   )->str:
							_UpperCAmelCase					=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							_UpperCAmelCase					=    {}
							for i, token in enumerate(__UpperCamelCase   ):
											_UpperCAmelCase					=    i
							_UpperCAmelCase					=    WordpieceTokenizer(vocab=__UpperCamelCase				,   unk_token='''[UNK]'''   )

							self.assertListEqual(tokenizer.tokenize(''''''   )				,   []   )

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )				,   ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )				,   ['''[UNK]''', '''runn''', '''##ing''']   )



			def  lowercase__						(		self     :    List[str]   )->str:
							self.assertTrue(_is_whitespace(''' '''   )   )
							self.assertTrue(_is_whitespace('''\t'''   )   )
							self.assertTrue(_is_whitespace('''\r'''   )   )
							self.assertTrue(_is_whitespace('''\n'''   )   )
							self.assertTrue(_is_whitespace('''\u00A0'''   )   )

							self.assertFalse(_is_whitespace('''A'''   )   )
							self.assertFalse(_is_whitespace('''-'''   )   )



			def  lowercase__						(		self     :    Optional[int]   )->Dict:
							self.assertTrue(_is_control('''\u0005'''   )   )

							self.assertFalse(_is_control('''A'''   )   )
							self.assertFalse(_is_control(''' '''   )   )
							self.assertFalse(_is_control('''\t'''   )   )
							self.assertFalse(_is_control('''\r'''   )   )



			def  lowercase__						(		self     :    Dict   )->int:
							self.assertTrue(_is_punctuation('''-'''   )   )
							self.assertTrue(_is_punctuation('''$'''   )   )
							self.assertTrue(_is_punctuation('''`'''   )   )
							self.assertTrue(_is_punctuation('''.'''   )   )

							self.assertFalse(_is_punctuation('''A'''   )   )
							self.assertFalse(_is_punctuation(''' '''   )   )



			def  lowercase__						(		self     :    Dict   )->Optional[int]:
							_UpperCAmelCase					=    self.get_tokenizer()
							_UpperCAmelCase					=    self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__UpperCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,   [['''[UNK]'''], [], ['''[UNK]''']]   )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__UpperCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,   [['''[UNK]'''], [], ['''[UNK]''']]   )



			@slow
			def  lowercase__						(		self     :    Union[str, Any]   )->int:
							_UpperCAmelCase					=    self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

							_UpperCAmelCase					=    tokenizer.encode('''sequence builders'''				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    tokenizer.encode('''multi-sequence build'''				,   add_special_tokens=__UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase   )
							_UpperCAmelCase					=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				,   __UpperCamelCase   )

							assert encoded_sentence == [1_0_1] + text + [1_0_2]
							assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



			def  lowercase__						(		self     :    List[Any]   )->int:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
															_UpperCAmelCase					=    tokenizer_r.encode_plus(
															    __UpperCamelCase				,   return_attention_mask=__UpperCamelCase				,   return_token_type_ids=__UpperCamelCase				,   return_offsets_mapping=__UpperCamelCase				,   add_special_tokens=__UpperCamelCase				,   )

															_UpperCAmelCase					=    tokenizer_r.do_lower_case if hasattr(__UpperCamelCase				,   '''do_lower_case'''   ) else False
															_UpperCAmelCase					=    (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''A'''),
															        ((1, 2), ''','''),
															        ((3, 5), '''na'''),
															        ((5, 6), '''##ï'''),
															        ((6, 8), '''##ve'''),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), '''Allen'''),
															        ((2_1, 2_3), '''##NL'''),
															        ((2_3, 2_4), '''##P'''),
															        ((2_5, 3_3), '''sentence'''),
															        ((3_3, 3_4), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''a'''),
															        ((1, 2), ''','''),
															        ((3, 8), '''naive'''),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), '''allen'''),
															        ((2_1, 2_3), '''##nl'''),
															        ((2_3, 2_4), '''##p'''),
															        ((2_5, 3_3), '''sentence'''),
															        ((3_3, 3_4), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]				,   tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
															self.assertEqual([e[0] for e in expected_results]				,   tokens['''offset_mapping''']   )



			def  lowercase__						(		self     :    List[str]   )->Tuple:
							_UpperCAmelCase					=    ['''的''', '''人''', '''有''']
							_UpperCAmelCase					=    ''''''.join(__UpperCamelCase   )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															_UpperCAmelCase					=    True
															_UpperCAmelCase					=    self.tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_p.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_r.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.convert_ids_to_tokens(__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.convert_ids_to_tokens(__UpperCamelCase   )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

															_UpperCAmelCase					=    False
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )
															_UpperCAmelCase					=    self.tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.convert_ids_to_tokens(__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.convert_ids_to_tokens(__UpperCamelCase   )

															# it is expected that only the first Chinese character is not preceded by "##".
															_UpperCAmelCase					=    [
															    F'##{token}' if idx != 0 else token for idx, token in enumerate(__UpperCamelCase   )
															]
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )








"""simple docstring"""


from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml
class 			_a (					yaml.SafeLoader):
			"""simple docstring"""



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Any   )->List[Any]:
							_UpperCAmelCase					=    [self.constructed_objects[key_node] for key_node, _ in node.value]
							_UpperCAmelCase					=    [tuple(__UpperCamelCase   ) if isinstance(__UpperCamelCase				,   __UpperCamelCase   ) else key for key in keys]
							_UpperCAmelCase					=    Counter(__UpperCamelCase   )
							_UpperCAmelCase					=    [key for key in counter if counter[key] > 1]
							if duplicate_keys:
											raise TypeError(F'Got duplicate yaml keys: {duplicate_keys}'   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[Any]				,   __UpperCamelCase     :    str=False   )->Dict:
							_UpperCAmelCase					=    super().construct_mapping(__UpperCamelCase				,   deep=__UpperCamelCase   )
							self._check_no_duplicates_on_constructed_node(__UpperCamelCase   )
							return mapping







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						str				):






				'''simple docstring'''



				_UpperCAmelCase					=    list(readme_content.splitlines()				)
				if full_content and full_content[0] == "---" and "---" in full_content[1:]:
								_UpperCAmelCase					=    full_content[1:].index('''---'''				) + 1
								_UpperCAmelCase					=    '''\n'''.join(full_content[1:sep_idx]				)
								return yamlblock, "\n".join(full_content[sep_idx + 1 :]				)

				return None, "\n".join(_SCREAMING_SNAKE_CASE				)
class 			_a (					lowerCAmelCase):
			"""simple docstring"""
			# class attributes
			UpperCamelCase__			      =	{"""train_eval_index"""}  # train-eval-index in the YAML metadata



			@classmethod
			def  lowercase__						(		cls     :    List[Any]				,   __UpperCamelCase     :    Path   )->"DatasetMetadata":
							with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(readme_file.read()   )
							if yaml_string is not None:
											return cls.from_yaml_string(__UpperCamelCase   )
							else:
											return cls()



			def  lowercase__						(		self     :    Tuple				,   __UpperCamelCase     :    Path   )->List[Any]:
							if path.exists():
											with open(__UpperCamelCase				,   encoding='''utf-8'''   ) as readme_file:
															_UpperCAmelCase					=    readme_file.read()
							else:
											_UpperCAmelCase					=    None
							_UpperCAmelCase					=    self._to_readme(__UpperCamelCase   )
							with open(__UpperCamelCase				,   '''w'''				,   encoding='''utf-8'''   ) as readme_file:
											readme_file.write(__UpperCamelCase   )



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    Optional[str] = None   )->str:
							if readme_content is not None:
											_UpperCAmelCase		,						_UpperCAmelCase					=    _split_yaml_from_readme(__UpperCamelCase   )
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n''' + content
							else:
											_UpperCAmelCase					=    '''---\n''' + self.to_yaml_string() + '''---\n'''
							return full_content



			@classmethod
			def  lowercase__						(		cls     :    str				,   __UpperCamelCase     :    str   )->"DatasetMetadata":
							_UpperCAmelCase					=    yaml.load(__UpperCamelCase				,   Loader=_NoDuplicateSafeLoader   ) or {}

							# Convert the YAML keys to DatasetMetadata fields
							_UpperCAmelCase					=    {
							    (key.replace('''-'''				,   '''_'''   ) if key.replace('''-'''				,   '''_'''   ) in cls._FIELDS_WITH_DASHES else key): value
							    for key, value in metadata_dict.items()
							}
							return cls(**__UpperCamelCase   )



			def  lowercase__						(		self     :    str   )->str:
							return yaml.safe_dump(
							    {
							        (key.replace('''_'''				,   '''-'''   ) if key in self._FIELDS_WITH_DASHES else key): value
							        for key, value in self.items()
							    }				,   sort_keys=__UpperCamelCase				,   allow_unicode=__UpperCamelCase				,   encoding='''utf-8'''				,   ).decode('''utf-8'''   )



__A   :							str										=   {
    "image-classification": [],
    "translation": [],
    "image-segmentation": [],
    "fill-mask": [],
    "automatic-speech-recognition": [],
    "token-classification": [],
    "sentence-similarity": [],
    "audio-classification": [],
    "question-answering": [],
    "summarization": [],
    "zero-shot-classification": [],
    "table-to-text": [],
    "feature-extraction": [],
    "other": [],
    "multiple-choice": [],
    "text-classification": [],
    "text-to-image": [],
    "text2text-generation": [],
    "zero-shot-image-classification": [],
    "tabular-classification": [],
    "tabular-regression": [],
    "image-to-image": [],
    "tabular-to-text": [],
    "unconditional-image-generation": [],
    "text-retrieval": [],
    "text-to-speech": [],
    "object-detection": [],
    "audio-to-audio": [],
    "text-generation": [],
    "conversational": [],
    "table-question-answering": [],
    "visual-question-answering": [],
    "image-to-text": [],
    "reinforcement-learning": [],
    "voice-activity-detection": [],
    "time-series-forecasting": [],
    "document-question-answering": [],
}


if __name__ == "__main__":
					from argparse import ArgumentParser

					__A   :							str										=   ArgumentParser(usage="Validate the yaml metadata block of a README.md file.")
					ap.add_argument("readme_filepath")
					__A   :							Union[str, Any]										=   ap.parse_args()

					__A   :							Dict										=   Path(args.readme_filepath)
					__A   :							Tuple										=   DatasetMetadata.from_readme(readme_filepath)
					print(dataset_metadata)
					dataset_metadata.to_readme(readme_filepath)



