


import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 	__lowerCamelCase			(_a				,      unittest.TestCase						):
       _lowercase											=     LEDTokenizer
       _lowercase											=     LEDTokenizerFast
       _lowercase											=     True







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            super().setUp()
            __UpperCamelCase =			[
                'l',
                'o',
                'w',
                'e',
                'r',
                's',
                't',
                'i',
                'd',
                'n',
                '\u0120',
                '\u0120l',
                '\u0120n',
                '\u0120lo',
                '\u0120low',
                'er',
                '\u0120lowest',
                '\u0120newer',
                '\u0120wider',
                '<unk>',
            ]
            __UpperCamelCase =			dict(zip(A_,range(len(A_					)					)					)					)
            __UpperCamelCase =			['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
            __UpperCamelCase =			{'unk_token': '<unk>'}

            __UpperCamelCase =			os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['vocab_file']					)
            __UpperCamelCase =			os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['merges_file']					)
            with open(self.vocab_file,'w',encoding='utf-8'					) as fp:
                 fp.write(json.dumps(A_					) + '\n'					)
            with open(self.merges_file,'w',encoding='utf-8'					) as fp:
                 fp.write('\n'.join(A_					)					)







       def 		snake_case_	(			self: Union[str, Any],**A_: Union[str, Any]					):






            '''simple docstring'''




            kwargs.update(self.special_tokens_map					)
            return self.tokenizer_class.from_pretrained(self.tmpdirname,**A_					)







       def 		snake_case_	(			self: List[Any],**A_: int					):






            '''simple docstring'''




            kwargs.update(self.special_tokens_map					)
            return self.rust_tokenizer_class.from_pretrained(self.tmpdirname,**A_					)







       def 		snake_case_	(			self: int,A_: Tuple					):






            '''simple docstring'''




            return "lower newer", "lower newer"







       @cached_property
       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            return LEDTokenizer.from_pretrained('allenai/led-base-16384'					)







       @cached_property
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'					)







       @require_torch
       def 		snake_case_	(			self: int					):






            '''simple docstring'''




            __UpperCamelCase =			['A long paragraph for summarization.', 'Another paragraph for summarization.']
            __UpperCamelCase =			[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			tokenizer(A_,max_length=len(A_					),padding=A_,return_tensors='pt'					)
                 self.assertIsInstance(A_,A_					)

                 self.assertEqual((2, 9),batch.input_ids.shape					)
                 self.assertEqual((2, 9),batch.attention_mask.shape					)
                 __UpperCamelCase =			batch.input_ids.tolist()[0]
                 self.assertListEqual(A_,A_					)







       @require_torch
       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            __UpperCamelCase =			['A long paragraph for summarization.', 'Another paragraph for summarization.']
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			tokenizer(A_,padding=A_,return_tensors='pt'					)
                 self.assertIn('input_ids',A_					)
                 self.assertIn('attention_mask',A_					)
                 self.assertNotIn('labels',A_					)
                 self.assertNotIn('decoder_attention_mask',A_					)







       @require_torch
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            __UpperCamelCase =			[
                'Summary of the text.',
                'Another summary.',
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			tokenizer(text_target=A_,max_length=32,padding='max_length',return_tensors='pt'					)
                 self.assertEqual(32,targets['input_ids'].shape[1]					)







       @require_torch
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			tokenizer(
                     ['I am a small frog' * 1024, 'I am a small frog'],padding=A_,truncation=A_,return_tensors='pt'					)
                 self.assertIsInstance(A_,A_					)
                 self.assertEqual(batch.input_ids.shape,(2, 5122)					)







       @require_torch
       def 		snake_case_	(			self: Optional[int]					):






            '''simple docstring'''




            __UpperCamelCase =			['A long paragraph for summarization.']
            __UpperCamelCase =			[
                'Summary of the text.',
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			tokenizer(A_,return_tensors='pt'					)
                 __UpperCamelCase =			tokenizer(text_target=A_,return_tensors='pt'					)
                 __UpperCamelCase =			inputs['input_ids']
                 __UpperCamelCase =			targets['input_ids']
                 self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()					)
                 self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()					)
                 self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()					)
                 self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()					)







       @require_torch
       def 		snake_case_	(			self: List[Any]					):






            '''simple docstring'''




            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                 __UpperCamelCase =			['Summary of the text.', 'Another summary.']
                 __UpperCamelCase =			[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                 __UpperCamelCase =			tokenizer(A_,padding=A_					)
                 __UpperCamelCase =			[[0] * len(A_					) for x in encoded_output['input_ids']]
                 __UpperCamelCase =			tokenizer.pad(A_					)
                 self.assertSequenceEqual(outputs['global_attention_mask'],A_					)







       def 		snake_case_	(			self: Dict					):






            '''simple docstring'''




            pass







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                      __UpperCamelCase =			self.rust_tokenizer_class.from_pretrained(A_,**A_					)
                      __UpperCamelCase =			self.tokenizer_class.from_pretrained(A_,**A_					)
                      __UpperCamelCase =			'A, <mask> AllenNLP sentence.'
                      __UpperCamelCase =			tokenizer_r.encode_plus(A_,add_special_tokens=A_,return_token_type_ids=A_					)
                      __UpperCamelCase =			tokenizer_p.encode_plus(A_,add_special_tokens=A_,return_token_type_ids=A_					)
                      self.assertEqual(sum(tokens_r['token_type_ids']					),sum(tokens_p['token_type_ids']					)					)
                      self.assertEqual(
                          sum(tokens_r['attention_mask']					) / len(tokens_r['attention_mask']					),sum(tokens_p['attention_mask']					) / len(tokens_p['attention_mask']					),)

                      __UpperCamelCase =			tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']					)
                      __UpperCamelCase =			tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']					)
                      self.assertSequenceEqual(tokens_p['input_ids'],[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)
                      self.assertSequenceEqual(tokens_r['input_ids'],[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)

                      self.assertSequenceEqual(
                          A_,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)
                      self.assertSequenceEqual(
                          A_,['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__snake_case   							=						logging.get_logger(__name__)

__snake_case   							=						{
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/config.json''',
    '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/config.json''',
    '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/config.json''',
    '''bert-base-multilingual-uncased''': '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json''',
    '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json''',
    '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/config.json''',
    '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/config.json''',
    '''bert-large-uncased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-base-cased-finetuned-mrpc''': '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json''',
    '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json''',
    '''bert-base-german-dbmdz-uncased''': '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese''': '''https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-cased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-uncased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json'''
    ),
    '''wietsedv/bert-base-dutch-cased''': '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json''',
    # See all BERT models at https://huggingface.co/models?filter=bert
}



class 	__lowerCamelCase			(_a						):
       _lowercase											=     """bert"""







       def __init__(			self: Any,A_: Dict=3_0522,A_: Optional[Any]=768,A_: Union[str, Any]=12,A_: List[Any]=12,A_: Optional[int]=3072,A_: Union[str, Any]="gelu",A_: List[str]=0.1,A_: Dict=0.1,A_: Optional[int]=512,A_: Optional[Any]=2,A_: Union[str, Any]=0.0_2,A_: List[Any]=1E-12,A_: Optional[int]=0,A_: List[Any]="absolute",A_: str=True,A_: Union[str, Any]=None,**A_: int,):






            '''simple docstring'''




            super().__init__(pad_token_id=A_,**A_					)

            __UpperCamelCase =			vocab_size
            __UpperCamelCase =			hidden_size
            __UpperCamelCase =			num_hidden_layers
            __UpperCamelCase =			num_attention_heads
            __UpperCamelCase =			hidden_act
            __UpperCamelCase =			intermediate_size
            __UpperCamelCase =			hidden_dropout_prob
            __UpperCamelCase =			attention_probs_dropout_prob
            __UpperCamelCase =			max_position_embeddings
            __UpperCamelCase =			type_vocab_size
            __UpperCamelCase =			initializer_range
            __UpperCamelCase =			layer_norm_eps
            __UpperCamelCase =			position_embedding_type
            __UpperCamelCase =			use_cache
            __UpperCamelCase =			classifier_dropout



class 	__lowerCamelCase			(_a						):







       @property
       def 		snake_case_	(			self: Optional[int]					):






            '''simple docstring'''




            if self.task == "multiple-choice":
                 __UpperCamelCase =			{0: 'batch', 1: 'choice', 2: 'sequence'}
            else:
                 __UpperCamelCase =			{0: 'batch', 1: 'sequence'}
            return OrderedDict(
                [
                    ('input_ids', dynamic_axis),
                    ('attention_mask', dynamic_axis),
                    ('token_type_ids', dynamic_axis),
                ]					)


