





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast




class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        # A mock response for an HTTP head request to emulate server down
        A__		   =		mock.Mock()
        A__		   =		500
        A__		   =		{}
        A__		   =		HTTPError
        A__		   =		{}

        # Download this model to make sure it's in the cache.
        A__		   =		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request''',return_value=__lowerCamelCase					) as mock_head:
               A__		   =		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)
               # This check we did call the fake head request
               mock_head.assert_called()



 @require_tokenizers
 def 							UpperCamelCase     (				self					):
        # A mock response for an HTTP head request to emulate server down
        A__		   =		mock.Mock()
        A__		   =		500
        A__		   =		{}
        A__		   =		HTTPError
        A__		   =		{}

        # Download this model to make sure it's in the cache.
        A__		   =		GPTaTokenizerFast.from_pretrained('''gpt2'''					)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request''',return_value=__lowerCamelCase					) as mock_head:
               A__		   =		GPTaTokenizerFast.from_pretrained('''gpt2'''					)
               # This check we did call the fake head request
               mock_head.assert_called()



 def 							UpperCamelCase     (				self					):
        # This test is for deprecated behavior and can be removed in v5
        try:
               A__		   =		tempfile.mktemp()
               with open(__lowerCamelCase,'''wb'''					) as f:
                      http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',__lowerCamelCase					)

               A__		   =		AlbertTokenizer.from_pretrained(__lowerCamelCase					)
        finally:
               os.remove(__lowerCamelCase					)

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('''tokenizer.json'''					):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
        try:
               with open('''tokenizer.json''','''wb'''					) as f:
                      http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''',__lowerCamelCase					)
               A__		   =		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''					)
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size,1000					)
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
               os.remove('''tokenizer.json'''					)




 def 							UpperCamelCase     (				self					):
        # This test is for deprecated behavior and can be removed in v5
        A__		   =		AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''					)




@is_staging_test
class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):
 __SCREAMING_SNAKE_CASE							=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']



 @classmethod
 def 							UpperCamelCase     (				cls					):
        A__		   =		TOKEN
        HfFolder.save_token(__lowerCamelCase					)



 @classmethod
 def 							UpperCamelCase     (				cls					):
        try:
               delete_repo(token=cls._token,repo_id='''test-tokenizer'''					)
        except HTTPError:
               pass

        try:
               delete_repo(token=cls._token,repo_id='''valid_org/test-tokenizer-org'''					)
        except HTTPError:
               pass

        try:
               delete_repo(token=cls._token,repo_id='''test-dynamic-tokenizer'''					)
        except HTTPError:
               pass



 def 							UpperCamelCase     (				self					):
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		os.path.join(__lowerCamelCase,'''vocab.txt'''					)
               with open(__lowerCamelCase,'''w''',encoding='''utf-8'''					) as vocab_writer:
                      vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
               A__		   =		BertTokenizer(__lowerCamelCase					)

        tokenizer.push_to_hub('''test-tokenizer''',use_auth_token=self._token					)
        A__		   =		BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"					)
        self.assertDictEqual(new_tokenizer.vocab,tokenizer.vocab					)

        # Reset repo
        delete_repo(token=self._token,repo_id='''test-tokenizer'''					)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__lowerCamelCase,repo_id='''test-tokenizer''',push_to_hub=__lowerCamelCase,use_auth_token=self._token					)

        A__		   =		BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"					)
        self.assertDictEqual(new_tokenizer.vocab,tokenizer.vocab					)



 def 							UpperCamelCase     (				self					):
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		os.path.join(__lowerCamelCase,'''vocab.txt'''					)
               with open(__lowerCamelCase,'''w''',encoding='''utf-8'''					) as vocab_writer:
                      vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
               A__		   =		BertTokenizer(__lowerCamelCase					)

        tokenizer.push_to_hub('''valid_org/test-tokenizer-org''',use_auth_token=self._token					)
        A__		   =		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''					)
        self.assertDictEqual(new_tokenizer.vocab,tokenizer.vocab					)

        # Reset repo
        delete_repo(token=self._token,repo_id='''valid_org/test-tokenizer-org'''					)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   __lowerCamelCase,repo_id='''valid_org/test-tokenizer-org''',push_to_hub=__lowerCamelCase,use_auth_token=self._token					)

        A__		   =		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''					)
        self.assertDictEqual(new_tokenizer.vocab,tokenizer.vocab					)




 @require_tokenizers
 def 							UpperCamelCase     (				self					):
        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		os.path.join(__lowerCamelCase,'''vocab.txt'''					)
               with open(__lowerCamelCase,'''w''',encoding='''utf-8'''					) as vocab_writer:
                      vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
               A__		   =		CustomTokenizer(__lowerCamelCase					)

        # No fast custom tokenizer
        tokenizer.push_to_hub('''test-dynamic-tokenizer''',use_auth_token=self._token					)

        A__		   =		AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer",trust_remote_code=__lowerCamelCase					)
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,'''CustomTokenizer'''					)

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		os.path.join(__lowerCamelCase,'''vocab.txt'''					)
               with open(__lowerCamelCase,'''w''',encoding='''utf-8'''					) as vocab_writer:
                      vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)

               A__		   =		BertTokenizerFast.from_pretrained(__lowerCamelCase					)
               bert_tokenizer.save_pretrained(__lowerCamelCase					)
               A__		   =		CustomTokenizerFast.from_pretrained(__lowerCamelCase					)

        tokenizer.push_to_hub('''test-dynamic-tokenizer''',use_auth_token=self._token					)

        A__		   =		AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer",trust_remote_code=__lowerCamelCase					)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,'''CustomTokenizerFast'''					)
        A__		   =		AutoTokenizer.from_pretrained(
            f"{USER}/test-dynamic-tokenizer",use_fast=__lowerCamelCase,trust_remote_code=__lowerCamelCase					)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,'''CustomTokenizer'''					)




class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''Hello 友達'''					)
        self.assertEqual(trie.data,{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}					)
        trie.add('''Hello'''					)
        trie.data
        self.assertEqual(trie.data,{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''					),['''[CLS] This is a extra_id_100''']					)
        trie.add('''[CLS]'''					)
        trie.add('''extra_id_1'''					)
        trie.add('''extra_id_100'''					)
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''					),['''[CLS]''', ''' This is a ''', '''extra_id_100''']					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''A'''					)
        self.assertEqual(trie.split('''ABC'''					),['''A''', '''BC''']					)
        self.assertEqual(trie.split('''BCA'''					),['''BC''', '''A''']					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''TOKEN]'''					)
        trie.add('''[SPECIAL_TOKEN]'''					)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''					),['''This is something ''', '''[SPECIAL_TOKEN]''']					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''A'''					)
        trie.add('''P'''					)
        trie.add('''[SPECIAL_TOKEN]'''					)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''					),['''This is something ''', '''[SPECIAL_TOKEN]''']					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''AB'''					)
        trie.add('''B'''					)
        trie.add('''C'''					)
        self.assertEqual(trie.split('''ABC'''					),['''AB''', '''C''']					)



 def 							UpperCamelCase     (				self					):
        A__		   =		Trie()
        trie.add('''ABC'''					)
        trie.add('''B'''					)
        trie.add('''CD'''					)
        self.assertEqual(trie.split('''ABCD'''					),['''ABC''', '''D''']					)




 def 							UpperCamelCase     (				self					):
        # Even if the offsets are wrong, we necessarily output correct string
        # parts.
        A__		   =		Trie()
        A__		   =		trie.cut_text('''ABC''',[0, 0, 2, 1, 2, 3]					)
        self.assertEqual(__lowerCamelCase,['''AB''', '''C''']					)












import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()


def       UpperCamelCase__(			UpperCamelCase__     :			Any					,	UpperCamelCase__     :			Optional[Any]					,	UpperCamelCase__     :			Dict					,	UpperCamelCase__     :			int    )->Union[str, Any]:
       # Initialise PyTorch model
       A__		   =		BigBirdConfig.from_json_file(UpperCamelCase__    )
       print(f"Building PyTorch model from configuration: {config}"    )

       if is_trivia_qa:
              A__		   =		BigBirdForQuestionAnswering(UpperCamelCase__    )
       else:
              A__		   =		BigBirdForPreTraining(UpperCamelCase__    )

       # Load weights from tf checkpoint
       load_tf_weights_in_big_bird(UpperCamelCase__					,	UpperCamelCase__					,	is_trivia_qa=UpperCamelCase__    )

       # Save pytorch-model
       print(f"Save PyTorch model to {pytorch_dump_path}"    )
       model.save_pretrained(UpperCamelCase__    )


if __name__ == "__main__":
       a__: Optional[int]							   =						argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
       )
       parser.add_argument(
           '--big_bird_config_file',
           default=None,
           type=str,
           required=True,
           help=(
               'The config json file corresponding to the pre-trained BERT model. \n'
               'This specifies the model architecture.'
           ),
       )
       parser.add_argument(
           '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--is_trivia_qa', action='store_true', help='Whether to convert a model with a trivia_qa head.'
       )
       a__: Tuple							   =						parser.parse_args()
       convert_tf_checkpoint_to_pytorch(
           args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
       )





