





import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
       from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
       from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
       from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


lowercase    : Optional[Any]    	=  DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

lowercase    : List[Any]    	=  """main"""
# Default branch name
lowercase    : Optional[Any]    	=  """f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"""
# One particular commit (not the top of `main`)
lowercase    : Union[str, Any]    	=  """aaaaaaa"""
# This commit does not exist, so we should 404.

lowercase    : Union[str, Any]    	=  """d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"""
# Sha-1 of config.json on the top of `main`, for checking purposes
lowercase    : Tuple    	=  """4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"""


@contextlib.contextmanager
def     SCREAMING_SNAKE_CASE			(      ):
       print('''Welcome!'''					)
       yield
       print('''Bye!'''					)


@contextlib.contextmanager
def     SCREAMING_SNAKE_CASE			(      ):
       print('''Bonjour!'''					)
       yield
       print('''Au revoir!'''					)

class  __A(						unittest.TestCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            assert transformers.__spec__ is not None
            assert importlib.util.find_spec('''transformers'''      ) is not None






class  __A(						unittest.TestCase ):

     @unittest.mock.patch('''sys.stdout''',       new_callable=io.StringIO      )
     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            with ContextManagers([]      ):
                   print('''Transformers are awesome!'''      )
            # The print statement adds a new line at the end of the output
            self.assertEqual(mock_stdout.getvalue(),       '''Transformers are awesome!\n'''      )

     @unittest.mock.patch('''sys.stdout''',       new_callable=io.StringIO      )
     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            with ContextManagers([context_en()]      ):
                   print('''Transformers are awesome!'''      )
            # The output should be wrapped with an English welcome and goodbye
            self.assertEqual(mock_stdout.getvalue(),       '''Welcome!\nTransformers are awesome!\nBye!\n'''      )

     @unittest.mock.patch('''sys.stdout''',       new_callable=io.StringIO      )
     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            with ContextManagers([context_fr(), context_en()]      ):
                   print('''Transformers are awesome!'''      )
            # The output should be wrapped with an English and French welcome and goodbye
            self.assertEqual(mock_stdout.getvalue(),       '''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''      )

     @require_torch
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(find_labels(_A      ),       ['''labels''']      )
            self.assertEqual(find_labels(_A      ),       ['''labels''', '''next_sentence_label''']      )
            self.assertEqual(find_labels(_A      ),       ['''start_positions''', '''end_positions''']      )

            class  __A(						a__ ):
                 pass






            self.assertEqual(find_labels(_A      ),       ['''labels''']      )

     @require_tf
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(find_labels(_A      ),       ['''labels''']      )
            self.assertEqual(find_labels(_A      ),       ['''labels''', '''next_sentence_label''']      )
            self.assertEqual(find_labels(_A      ),       ['''start_positions''', '''end_positions''']      )

            class  __A(						a__ ):
                 pass






            self.assertEqual(find_labels(_A      ),       ['''labels''']      )



     @require_flax
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(find_labels(_A      ),       []      )
            self.assertEqual(find_labels(_A      ),       []      )
            self.assertEqual(find_labels(_A      ),       []      )

            class  __A(						a__ ):
                 pass






            self.assertEqual(find_labels(_A      ),       []      )













import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase    : str    	=  logging.get_logger(__name__)

lowercase    : Tuple    	=  {"""vocab_file""": """vocab.json"""}

lowercase    : int    	=  {
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

lowercase    : int    	=  {"""mgp-str""": 27}

class  __A(						__UpperCAmelCase ):
     __A	  =       VOCAB_FILES_NAMES
     __A	  =       PRETRAINED_VOCAB_FILES_MAP
     __A	  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

     def __init__(      self,       A,       A="[GO]",       A="[GO]",       A="[s]",       A="[GO]",       **A      ):





            """simple docstring"""

            super().__init__(
                unk_token=A,       bos_token=A,       eos_token=A,       pad_token=A,       **A,       )

            with open(A,       encoding='''utf-8'''      ) as vocab_handle:
                   _UpperCamelCase              =						json.load(A      )
            _UpperCamelCase              =						{v: k for k, v in self.vocab.items()}

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return len(self.vocab      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return dict(self.vocab,       **self.added_tokens_encoder      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						[]
            for s in text:
                   char_tokens.extend(A      )
            return char_tokens

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.vocab.get(A,       self.vocab.get(self.unk_token      )      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.decoder.get(A      )



     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            if not os.path.isdir(A      ):
                   logger.error('''Vocabulary path ({}) should be a directory'''.format(A      )      )
                   return
            _UpperCamelCase              =						os.path.join(
                A,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

            with open(A,       '''w''',       encoding='''utf-8'''      ) as f:
                   f.write(json.dumps(self.vocab,       indent=2,       sort_keys=A,       ensure_ascii=A      ) + '''\n'''      )

            return (vocab_file,)






