



"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case   					=				get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
     from transformers.models.mbart.modeling_mbart import shift_tokens_right

snake_case   					=				2_5_0_0_0_4
snake_case   					=				2_5_0_0_2_0



@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	Tuple           =  MBartaaTokenizer
   A__		:	Dict           =  MBartaaTokenizerFast
   A__		:	int           =  True
   A__		:	str           =  True



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     super().setUp()

     # We have a SentencePiece fixture for testing
     _snake_case   	=      MBartaaTokenizer(__lowerCamelCase							,       src_lang='''en_XX'''							,       tgt_lang='''ro_RO'''							,       keep_accents=__lowerCamelCase				)
     tokenizer.save_pretrained(self.tmpdirname				)



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     _snake_case   	=      '''<s>'''
     _snake_case   	=      0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase				)							,       __lowerCamelCase				)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase				)							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      list(self.get_tokenizer().get_vocab().keys()				)

     self.assertEqual(vocab_keys[0]							,       '''<s>'''				)
     self.assertEqual(vocab_keys[1]							,       '''<pad>'''				)
     self.assertEqual(vocab_keys[-1]							,       '''<mask>'''				)
     self.assertEqual(len(__lowerCamelCase				)							,       1_0_5_4				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     self.assertEqual(self.get_tokenizer().vocab_size							,       1_0_5_4				)



   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      MBartaaTokenizer(__lowerCamelCase							,       src_lang='''en_XX'''							,       tgt_lang='''ro_RO'''							,       keep_accents=__lowerCamelCase				)

     _snake_case   	=      tokenizer.tokenize('''This is a test'''				)
     self.assertListEqual(__lowerCamelCase							,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(__lowerCamelCase				)							,       [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,       )

     _snake_case   	=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
     self.assertListEqual(
         __lowerCamelCase							,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']							,       )
     _snake_case   	=      tokenizer.convert_tokens_to_ids(__lowerCamelCase				)
     self.assertListEqual(
         __lowerCamelCase							,       [
             value + tokenizer.fairseq_offset
             for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
         ]							,       )

     _snake_case   	=      tokenizer.convert_ids_to_tokens(__lowerCamelCase				)
     self.assertListEqual(
         __lowerCamelCase							,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']							,       )



   @slow
   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     # fmt: off
     _snake_case   	=      {'''input_ids''': [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__lowerCamelCase							,       model_name='''facebook/mbart-large-50'''							,       revision='''d3913889c59cd5c9e456b269c376325eabad57e2'''							,       )





   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     if not self.test_slow_tokenizer:
       # as we don't have a slow version, we can't compare the outputs between slow and fast versions
       return

     _snake_case   	=      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart50''', {})
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
         _snake_case   	=      self.rust_tokenizer_class.from_pretrained(__lowerCamelCase							,       **__lowerCamelCase				)
         _snake_case   	=      self.tokenizer_class.from_pretrained(__lowerCamelCase							,       **__lowerCamelCase				)

         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      tokenizer_r.save_pretrained(__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.save_pretrained(__lowerCamelCase				)

         # Checks it save with the same files + the tokenizer.json file for the fast one
         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)
         _snake_case   	=      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f				)
         self.assertSequenceEqual(__lowerCamelCase							,       __lowerCamelCase				)

         # Checks everything loads correctly in the same way
         _snake_case   	=      tokenizer_r.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.from_pretrained(__lowerCamelCase				)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(__lowerCamelCase							,       __lowerCamelCase				)				)
           # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
           # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

         shutil.rmtree(__lowerCamelCase				)

         # Save tokenizer rust, legacy_format=True
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      tokenizer_r.save_pretrained(__lowerCamelCase							,       legacy_format=__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.save_pretrained(__lowerCamelCase				)

         # Checks it save with the same files
         self.assertSequenceEqual(__lowerCamelCase							,       __lowerCamelCase				)

         # Checks everything loads correctly in the same way
         _snake_case   	=      tokenizer_r.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.from_pretrained(__lowerCamelCase				)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(__lowerCamelCase							,       __lowerCamelCase				)				)

         shutil.rmtree(__lowerCamelCase				)

         # Save tokenizer rust, legacy_format=False
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      tokenizer_r.save_pretrained(__lowerCamelCase							,       legacy_format=__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.save_pretrained(__lowerCamelCase				)

         # Checks it saved the tokenizer.json file
         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)

         # Checks everything loads correctly in the same way
         _snake_case   	=      tokenizer_r.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      tokenizer_p.from_pretrained(__lowerCamelCase				)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(__lowerCamelCase							,       __lowerCamelCase				)				)

         shutil.rmtree(__lowerCamelCase				)



@require_torch
@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase    (	unittest.TestCase  ):
   A__		:	Dict           =  '''facebook/mbart-large-50-one-to-many-mmt'''
   A__		:	Any           =  [
       ''' UN Chief Says There Is No Military Solution in Syria''',
       ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
   ]
   A__		:	Dict           =  [
       '''Şeful ONU declară că nu există o soluţie militară în Siria''',
       '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
       ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
       ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
   ]
   A__		:	Optional[Any]           =  [EN_CODE, 8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2]



   @classmethod
   def    __UpperCAmelCase    (       cls    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      MBartaaTokenizer.from_pretrained(
         cls.checkpoint_name							,       src_lang='''en_XX'''							,       tgt_lang='''ro_RO'''				)
     _snake_case   	=      1
     return cls



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']							,       2_5_0_0_0_1				)
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']							,       2_5_0_0_0_4				)
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']							,       2_5_0_0_2_0				)
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''mr_IN''']							,       2_5_0_0_3_8				)



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     _snake_case   	=      self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
     self.assertListEqual(self.expected_src_tokens							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     self.assertIn(__lowerCamelCase							,       self.tokenizer.all_special_ids				)
     _snake_case   	=      [RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
     _snake_case   	=      self.tokenizer.decode(__lowerCamelCase							,       skip_special_tokens=__lowerCamelCase				)
     _snake_case   	=      self.tokenizer.decode(generated_ids[1:]							,       skip_special_tokens=__lowerCamelCase				)
     self.assertEqual(__lowerCamelCase							,       __lowerCamelCase				)
     self.assertNotIn(self.tokenizer.eos_token							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      ['''this is gunna be a long sentence ''' * 2_0]
     assert isinstance(src_text[0]							,       __lowerCamelCase				)
     _snake_case   	=      1_0
     _snake_case   	=      self.tokenizer(__lowerCamelCase							,       max_length=__lowerCamelCase							,       truncation=__lowerCamelCase				).input_ids[0]
     self.assertEqual(ids[0]							,       __lowerCamelCase				)
     self.assertEqual(ids[-1]							,       2				)
     self.assertEqual(len(__lowerCamelCase				)							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']				)							,       [2_5_0_0_5_3, 2_5_0_0_0_1]				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      tempfile.mkdtemp()
     _snake_case   	=      self.tokenizer.fairseq_tokens_to_ids
     self.tokenizer.save_pretrained(__lowerCamelCase				)
     _snake_case   	=      MBartaaTokenizer.from_pretrained(__lowerCamelCase				)
     self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,       __lowerCamelCase				)



   @require_torch
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      self.tokenizer(self.src_text							,       text_target=self.tgt_text							,       padding=__lowerCamelCase							,       return_tensors='''pt'''				)
     _snake_case   	=      shift_tokens_right(batch['''labels''']							,       self.tokenizer.pad_token_id				)

     # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
     assert batch.input_ids[1][0] == EN_CODE
     assert batch.input_ids[1][-1] == 2
     assert batch.labels[1][0] == RO_CODE
     assert batch.labels[1][-1] == 2
     assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]



   @require_torch
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      self.tokenizer(
         self.src_text							,       text_target=self.tgt_text							,       padding=__lowerCamelCase							,       truncation=__lowerCamelCase							,       max_length=len(self.expected_src_tokens				)							,       return_tensors='''pt'''							,       )
     _snake_case   	=      shift_tokens_right(batch['''labels''']							,       self.tokenizer.pad_token_id				)

     self.assertIsInstance(__lowerCamelCase							,       __lowerCamelCase				)

     self.assertEqual((2, 1_4)							,       batch.input_ids.shape				)
     self.assertEqual((2, 1_4)							,       batch.attention_mask.shape				)
     _snake_case   	=      batch.input_ids.tolist()[0]
     self.assertListEqual(self.expected_src_tokens							,       __lowerCamelCase				)
     self.assertEqual(2							,       batch.decoder_input_ids[0, 0]				)  # decoder_start_token_id
     # Test that special tokens are reset
     self.assertEqual(self.tokenizer.prefix_tokens							,       [EN_CODE]				)
     self.assertEqual(self.tokenizer.suffix_tokens							,       [self.tokenizer.eos_token_id]				)



   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      self.tokenizer(self.src_text							,       padding=__lowerCamelCase							,       truncation=__lowerCamelCase							,       max_length=3							,       return_tensors='''pt'''				)
     _snake_case   	=      self.tokenizer(
         text_target=self.tgt_text							,       padding=__lowerCamelCase							,       truncation=__lowerCamelCase							,       max_length=1_0							,       return_tensors='''pt'''				)
     _snake_case   	=      targets['''input_ids''']
     _snake_case   	=      shift_tokens_right(__lowerCamelCase							,       self.tokenizer.pad_token_id				)

     self.assertEqual(batch.input_ids.shape[1]							,       3				)
     self.assertEqual(batch.decoder_input_ids.shape[1]							,       1_0				)





   @require_torch
   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      self.tokenizer._build_translation_inputs(
         '''A test'''							,       return_tensors='''pt'''							,       src_lang='''en_XX'''							,       tgt_lang='''ar_AR'''				)

     self.assertEqual(
         nested_simplify(__lowerCamelCase				)							,       {
             # en_XX, A, test, EOS
             '''input_ids''': [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
             '''attention_mask''': [[1, 1, 1, 1]],
             # ar_AR
             '''forced_bos_token_id''': 2_5_0_0_0_1,
         }							,       )










"""simple docstring"""






from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml



class     UpperCAmelCase    (	yaml.SafeLoader  ):



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      [self.constructed_objects[key_node] for key_node, _ in node.value]
     _snake_case   	=      [tuple(__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else key for key in keys]
     _snake_case   	=      Counter(__lowerCamelCase				)
     _snake_case   	=      [key for key in counter if counter[key] > 1]
     if duplicate_keys:
       raise TypeError(f"""Got duplicate yaml keys: {duplicate_keys}"""				)





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	Optional[Any]=False				):



     """simple docstring"""
     _snake_case   	=      super().construct_mapping(__lowerCamelCase							,       deep=__lowerCamelCase				)
     self._check_no_duplicates_on_constructed_node(__lowerCamelCase				)
     return mapping




def 			snake_case		(		lowerCAmelCase_	)						->				Tuple[Optional[str], str]:
  _snake_case   	=      list(readme_content.splitlines()	)
  if full_content and full_content[0] == "---" and "---" in full_content[1:]:
    _snake_case   	=      full_content[1:].index('''---'''	) + 1
    _snake_case   	=      '''\n'''.join(full_content[1:sep_idx]	)
    return yamlblock, "\n".join(full_content[sep_idx + 1 :]	)

  return None, "\n".join(lowerCAmelCase_	)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   # class attributes
   A__		:	int           =  {'''train_eval_index'''}  # train-eval-index in the YAML metadata



   @classmethod
   def    __UpperCAmelCase    (       cls    :	str							,       __lowerCamelCase    :	Path				):



     """simple docstring"""
     with open(__lowerCamelCase							,       encoding='''utf-8'''				) as readme_file:
       _snake_case	,     _snake_case   	=      _split_yaml_from_readme(readme_file.read()				)
     if yaml_string is not None:
       return cls.from_yaml_string(__lowerCamelCase				)
     else:
       return cls()



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	Path				):



     """simple docstring"""
     if path.exists():
       with open(__lowerCamelCase							,       encoding='''utf-8'''				) as readme_file:
         _snake_case   	=      readme_file.read()
     else:
       _snake_case   	=      None
     _snake_case   	=      self._to_readme(__lowerCamelCase				)
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as readme_file:
       readme_file.write(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if readme_content is not None:
       _snake_case	,     _snake_case   	=      _split_yaml_from_readme(__lowerCamelCase				)
       _snake_case   	=      '''---\n''' + self.to_yaml_string() + '''---\n''' + content
     else:
       _snake_case   	=      '''---\n''' + self.to_yaml_string() + '''---\n'''
     return full_content



   @classmethod
   def    __UpperCAmelCase    (       cls    :	Tuple							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      yaml.load(__lowerCamelCase							,       Loader=_NoDuplicateSafeLoader				) or {}

     # Convert the YAML keys to DatasetMetadata fields
     _snake_case   	=      {
         (key.replace('''-'''							,       '''_'''				) if key.replace('''-'''							,       '''_'''				) in cls._FIELDS_WITH_DASHES else key): value
         for key, value in metadata_dict.items()
     }
     return cls(**__lowerCamelCase				)





   def    __UpperCAmelCase    (       self    :	Dict				):



     """simple docstring"""
     return yaml.safe_dump(
         {
             (key.replace('''_'''							,       '''-'''				) if key in self._FIELDS_WITH_DASHES else key): value
             for key, value in self.items()
         }							,       sort_keys=__lowerCamelCase							,       allow_unicode=__lowerCamelCase							,       encoding='''utf-8'''							,       ).decode('''utf-8'''				)

snake_case   					=				{
    '''image-classification''': [],
    '''translation''': [],
    '''image-segmentation''': [],
    '''fill-mask''': [],
    '''automatic-speech-recognition''': [],
    '''token-classification''': [],
    '''sentence-similarity''': [],
    '''audio-classification''': [],
    '''question-answering''': [],
    '''summarization''': [],
    '''zero-shot-classification''': [],
    '''table-to-text''': [],
    '''feature-extraction''': [],
    '''other''': [],
    '''multiple-choice''': [],
    '''text-classification''': [],
    '''text-to-image''': [],
    '''text2text-generation''': [],
    '''zero-shot-image-classification''': [],
    '''tabular-classification''': [],
    '''tabular-regression''': [],
    '''image-to-image''': [],
    '''tabular-to-text''': [],
    '''unconditional-image-generation''': [],
    '''text-retrieval''': [],
    '''text-to-speech''': [],
    '''object-detection''': [],
    '''audio-to-audio''': [],
    '''text-generation''': [],
    '''conversational''': [],
    '''table-question-answering''': [],
    '''visual-question-answering''': [],
    '''image-to-text''': [],
    '''reinforcement-learning''': [],
    '''voice-activity-detection''': [],
    '''time-series-forecasting''': [],
    '''document-question-answering''': [],
}


if __name__ == "__main__":
     from argparse import ArgumentParser

     snake_case   					=				ArgumentParser(usage='''Validate the yaml metadata block of a README.md file.''')
     ap.add_argument('''readme_filepath''')
     snake_case   					=				ap.parse_args()

     snake_case   					=				Path(args.readme_filepath)
     snake_case   					=				DatasetMetadata.from_readme(readme_filepath)
     print(dataset_metadata)
     dataset_metadata.to_readme(readme_filepath)





