





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast



class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			mock.Mock()
        SCREAMING_SNAKE_CASE__											=			5_00
        SCREAMING_SNAKE_CASE__											=			{}
        SCREAMING_SNAKE_CASE__											=			HTTPError
        SCREAMING_SNAKE_CASE__											=			{}

        # Download this model to make sure it's in the cache.
        SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''     )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''   ,				return_value=A_     ) as mock_head:
            SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''     )
            # This check we did call the fake head request
            mock_head.assert_called()



    @require_tokenizers
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			mock.Mock()
        SCREAMING_SNAKE_CASE__											=			5_00
        SCREAMING_SNAKE_CASE__											=			{}
        SCREAMING_SNAKE_CASE__											=			HTTPError
        SCREAMING_SNAKE_CASE__											=			{}

        # Download this model to make sure it's in the cache.
        SCREAMING_SNAKE_CASE__											=			GPTaTokenizerFast.from_pretrained('''gpt2'''     )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''   ,				return_value=A_     ) as mock_head:
            SCREAMING_SNAKE_CASE__											=			GPTaTokenizerFast.from_pretrained('''gpt2'''     )
            # This check we did call the fake head request
            mock_head.assert_called()



    def  lowercase_					(       self     ):





        '''simple docstring'''

        try:
            SCREAMING_SNAKE_CASE__											=			tempfile.mktemp()
            with open(A_   ,				'''wb'''     ) as f:
                http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   ,				A_     )

            SCREAMING_SNAKE_CASE__											=			AlbertTokenizer.from_pretrained(A_     )
        finally:
            os.remove(A_     )

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('''tokenizer.json'''     ):
            # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
            return
        try:
            with open('''tokenizer.json'''   ,				'''wb'''     ) as f:
                http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''     )
            # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
            self.assertEqual(tokenizer.vocab_size   ,				10_00     )
            # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
            os.remove('''tokenizer.json'''     )




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''     )





@is_staging_test
class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Optional[int]				=	["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]



    @classmethod
    def  lowercase_					(       cls     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			TOKEN
        HfFolder.save_token(A_     )



    @classmethod
    def  lowercase_					(       cls     ):





        '''simple docstring'''

        try:
            delete_repo(token=cls._token   ,				repo_id='''test-tokenizer'''     )
        except HTTPError:
            pass

        try:
            delete_repo(token=cls._token   ,				repo_id='''valid_org/test-tokenizer-org'''     )
        except HTTPError:
            pass

        try:
            delete_repo(token=cls._token   ,				repo_id='''test-dynamic-tokenizer'''     )
        except HTTPError:
            pass



    def  lowercase_					(       self     ):





        '''simple docstring'''

        with tempfile.TemporaryDirectory() as tmp_dir:
            SCREAMING_SNAKE_CASE__											=			os.path.join(A_   ,				'''vocab.txt'''     )
            with open(A_   ,				'''w'''   ,				encoding='''utf-8'''     ) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]     )     )
            SCREAMING_SNAKE_CASE__											=			BertTokenizer(A_     )

        tokenizer.push_to_hub('''test-tokenizer'''   ,				use_auth_token=self._token     )
        SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
        self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab     )

        # Reset repo
        delete_repo(token=self._token   ,				repo_id='''test-tokenizer'''     )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(A_   ,				repo_id='''test-tokenizer'''   ,				push_to_hub=A_   ,				use_auth_token=self._token     )

        SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
        self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        with tempfile.TemporaryDirectory() as tmp_dir:
            SCREAMING_SNAKE_CASE__											=			os.path.join(A_   ,				'''vocab.txt'''     )
            with open(A_   ,				'''w'''   ,				encoding='''utf-8'''     ) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]     )     )
            SCREAMING_SNAKE_CASE__											=			BertTokenizer(A_     )

        tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''   ,				use_auth_token=self._token     )
        SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''     )
        self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab     )

        # Reset repo
        delete_repo(token=self._token   ,				repo_id='''valid_org/test-tokenizer-org'''     )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(
                A_   ,				repo_id='''valid_org/test-tokenizer-org'''   ,				push_to_hub=A_   ,				use_auth_token=self._token     )

        SCREAMING_SNAKE_CASE__											=			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''     )
        self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab     )




    @require_tokenizers
    def  lowercase_					(       self     ):





        '''simple docstring'''

        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
            SCREAMING_SNAKE_CASE__											=			os.path.join(A_   ,				'''vocab.txt'''     )
            with open(A_   ,				'''w'''   ,				encoding='''utf-8'''     ) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]     )     )
            SCREAMING_SNAKE_CASE__											=			CustomTokenizer(A_     )

        # No fast custom tokenizer
        tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,				use_auth_token=self._token     )

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''   ,				trust_remote_code=A_     )
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,				'''CustomTokenizer'''     )

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
            SCREAMING_SNAKE_CASE__											=			os.path.join(A_   ,				'''vocab.txt'''     )
            with open(A_   ,				'''w'''   ,				encoding='''utf-8'''     ) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]     )     )

            SCREAMING_SNAKE_CASE__											=			BertTokenizerFast.from_pretrained(A_     )
            bert_tokenizer.save_pretrained(A_     )
            SCREAMING_SNAKE_CASE__											=			CustomTokenizerFast.from_pretrained(A_     )

        tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,				use_auth_token=self._token     )

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''   ,				trust_remote_code=A_     )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,				'''CustomTokenizerFast'''     )
        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained(
            f'''{USER}/test-dynamic-tokenizer'''   ,				use_fast=A_   ,				trust_remote_code=A_     )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,				'''CustomTokenizer'''     )





class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''Hello 友達'''     )
        self.assertEqual(trie.data   ,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}     )
        trie.add('''Hello'''     )
        trie.data
        self.assertEqual(trie.data   ,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''     )   ,				['''[CLS] This is a extra_id_100''']     )
        trie.add('''[CLS]'''     )
        trie.add('''extra_id_1'''     )
        trie.add('''extra_id_100'''     )
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''     )   ,				['''[CLS]''', ''' This is a ''', '''extra_id_100''']     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''A'''     )
        self.assertEqual(trie.split('''ABC'''     )   ,				['''A''', '''BC''']     )
        self.assertEqual(trie.split('''BCA'''     )   ,				['''BC''', '''A''']     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''TOKEN]'''     )
        trie.add('''[SPECIAL_TOKEN]'''     )
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''     )   ,				['''This is something ''', '''[SPECIAL_TOKEN]''']     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''A'''     )
        trie.add('''P'''     )
        trie.add('''[SPECIAL_TOKEN]'''     )
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''     )   ,				['''This is something ''', '''[SPECIAL_TOKEN]''']     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''AB'''     )
        trie.add('''B'''     )
        trie.add('''C'''     )
        self.assertEqual(trie.split('''ABC'''     )   ,				['''AB''', '''C''']     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        trie.add('''ABC'''     )
        trie.add('''B'''     )
        trie.add('''CD'''     )
        self.assertEqual(trie.split('''ABCD'''     )   ,				['''ABC''', '''D''']     )




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Trie()
        SCREAMING_SNAKE_CASE__											=			trie.cut_text('''ABC'''   ,				[0, 0, 2, 1, 2, 3]     )
        self.assertEqual(A_   ,				['''AB''', '''C''']     )



import logging
from dataclasses import dataclass, field
from typing import Optional

from seqaseq_trainer import arg_to_scheduler

from transformers import TrainingArguments


__snake_case				:      Union[str, Any]   		=			logging.getLogger(__name__)



@dataclass
class        A__(a_							):
						"""simple docstring"""






						_A    :				Optional[float]       					=			field(
						    default=0.0,  metadata={'''help''': '''The label smoothing epsilon to apply (if not zero).'''}							)
						_A    :				bool       					=			field(default=a_,  metadata={'''help''': '''Whether to SortishSamler or not.'''}							)
						_A    :				bool       					=			field(
						    default=a_,  metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}							)
						_A    :				bool       					=			field(default=a_,  metadata={'''help''': '''whether to use adafactor'''}							)
						_A    :				Optional[float]       					=			field(
						    default=a_,  metadata={'''help''': '''Encoder layer dropout probability. Goes into model.config.'''}							)
						_A    :				Optional[float]       					=			field(
						    default=a_,  metadata={'''help''': '''Decoder layer dropout probability. Goes into model.config.'''}							)
						_A    :				Optional[float]       					=			field(default=a_,  metadata={'''help''': '''Dropout probability. Goes into model.config.'''}							)
						_A    :				Optional[float]       					=			field(
						    default=a_,  metadata={'''help''': '''Attention dropout probability. Goes into model.config.'''}							)
						_A    :				Optional[str]       					=			field(
						    default='''linear''',  metadata={'''help''': f'''Which lr scheduler to use. Selected in {sorted(arg_to_scheduler.keys()							)}'''},  )


