
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__lowerCAmelCase					   =       logging.get_logger(__name__)

__lowerCAmelCase					   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase					   =       {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase					   =       {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

__lowerCAmelCase					   =       {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}





class     lowerCamelCase      (       __lowerCamelCase					):
	UpperCamelCase_  :					Optional[Any]    		=	VOCAB_FILES_NAMES
	UpperCamelCase_  :					Optional[Any]    		=	PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase_  :					List[Any]    		=	PRETRAINED_INIT_CONFIGURATION
	UpperCamelCase_  :					Tuple    		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase_  :					Any    		=	RealmTokenizer







	def __init__(     self      :List[Any]   ,     lowercase      :List[Any]=None   ,     lowercase      :List[Any]=None   ,     lowercase      :Optional[int]=True   ,     lowercase      :List[Any]="[UNK]"   ,     lowercase      :Dict="[SEP]"   ,     lowercase      :Optional[Any]="[PAD]"   ,     lowercase      :Tuple="[CLS]"   ,     lowercase      :Optional[int]="[MASK]"   ,     lowercase      :int=True   ,     lowercase      :Any=None   ,     **lowercase      :List[Any]   ,     )		->      Optional[Any]:







				"""simple docstring"""






				super().__init__(
				    lowercase   ,     tokenizer_file=lowercase   ,     do_lower_case=lowercase   ,     unk_token=lowercase   ,     sep_token=lowercase   ,     pad_token=lowercase   ,     cls_token=lowercase   ,     mask_token=lowercase   ,     tokenize_chinese_chars=lowercase   ,     strip_accents=lowercase   ,     **lowercase   ,     )

				SCREAMING_SNAKE_CASE     						= json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
				if (
				    normalizer_state.get('''lowercase'''   ,     lowercase			) != do_lower_case
				    or normalizer_state.get('''strip_accents'''   ,     lowercase			) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''   ,     lowercase			) != tokenize_chinese_chars
				):
							SCREAMING_SNAKE_CASE     						= getattr(lowercase   ,     normalizer_state.pop('''type'''			)			)
							SCREAMING_SNAKE_CASE     						= do_lower_case
							SCREAMING_SNAKE_CASE     						= strip_accents
							SCREAMING_SNAKE_CASE     						= tokenize_chinese_chars
							SCREAMING_SNAKE_CASE     						= normalizer_class(**lowercase			)

				SCREAMING_SNAKE_CASE     						= do_lower_case







	def  snake_case__     (     self      :Optional[Any]   ,     lowercase      :Tuple   ,     **lowercase      :List[str]			)		->      List[Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= PaddingStrategy.MAX_LENGTH

				SCREAMING_SNAKE_CASE     						= text
				SCREAMING_SNAKE_CASE     						= kwargs.pop('''text_pair'''   ,     lowercase			)
				SCREAMING_SNAKE_CASE     						= kwargs.pop('''return_tensors'''   ,     lowercase			)

				SCREAMING_SNAKE_CASE     						= {
				    '''input_ids''': [],
				    '''attention_mask''': [],
				    '''token_type_ids''': [],
				}

				for idx, candidate_text in enumerate(lowercase			):
							if batch_text_pair is not None:
										SCREAMING_SNAKE_CASE     						= batch_text_pair[idx]
							else:
										SCREAMING_SNAKE_CASE     						= None

							SCREAMING_SNAKE_CASE     						= super().__call__(lowercase   ,     lowercase   ,     return_tensors=lowercase   ,     **lowercase			)

							SCREAMING_SNAKE_CASE     						= encoded_candidates.get('''input_ids'''			)
							SCREAMING_SNAKE_CASE     						= encoded_candidates.get('''attention_mask'''			)
							SCREAMING_SNAKE_CASE     						= encoded_candidates.get('''token_type_ids'''			)

							if encoded_input_ids is not None:
										output_data["input_ids"].append(lowercase			)
							if encoded_attention_mask is not None:
										output_data["attention_mask"].append(lowercase			)
							if encoded_token_type_ids is not None:
										output_data["token_type_ids"].append(lowercase			)

				SCREAMING_SNAKE_CASE     						= {key: item for key, item in output_data.items() if len(lowercase			) != 0}

				return BatchEncoding(lowercase   ,     tensor_type=lowercase			)







	def  snake_case__     (     self      :Dict   ,     lowercase      :int   ,     lowercase      :Any=None			)		->      List[Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output







	def  snake_case__     (     self      :Tuple   ,     lowercase      :List[int]   ,     lowercase      :Optional[List[int]] = None			)		->      List[int]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= [self.sep_token_id]
				SCREAMING_SNAKE_CASE     						= [self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep			) * [0]
				return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







	def  snake_case__     (     self      :str   ,     lowercase      :str   ,     lowercase      :Optional[str] = None			)		->      Tuple[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= self._tokenizer.model.save(lowercase   ,     name=lowercase			)
				return tuple(lowercase			)

import tempfile
import unittest

from transformers import AutoModelForSeqaSeqLM, AutoTokenizer
from transformers.testing_utils import (
    is_torch_available,
    require_optimum,
    require_torch,
    slow,
)


if is_torch_available():
							import torch





@require_torch
@require_optimum
@slow
class     lowerCamelCase      (       unittest.TestCase					):







	def  snake_case__     (     self      :List[Any]			)		->      Optional[Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= '''hf-internal-testing/tiny-random-t5'''
				SCREAMING_SNAKE_CASE     						= AutoTokenizer.from_pretrained(lowercase			)
				SCREAMING_SNAKE_CASE     						= AutoModelForSeqaSeqLM.from_pretrained(lowercase			)

				SCREAMING_SNAKE_CASE     						= tokenizer('''This is me'''   ,     return_tensors='''pt'''			)

				SCREAMING_SNAKE_CASE     						= model.to_bettertransformer()

				self.assertTrue(any('''BetterTransformer''' in mod.__class__.__name__ for _, mod in model.named_modules()			)			)

				SCREAMING_SNAKE_CASE     						= model.generate(**lowercase			)

				SCREAMING_SNAKE_CASE     						= model.reverse_bettertransformer()

				self.assertFalse(any('''BetterTransformer''' in mod.__class__.__name__ for _, mod in model.named_modules()			)			)

				with tempfile.TemporaryDirectory() as tmpdirname:
							model.save_pretrained(lowercase			)

							SCREAMING_SNAKE_CASE     						= AutoModelForSeqaSeqLM.from_pretrained(lowercase			)

							self.assertFalse(
							    any('''BetterTransformer''' in mod.__class__.__name__ for _, mod in model_reloaded.named_modules()			)			)

							SCREAMING_SNAKE_CASE     						= model_reloaded.generate(**lowercase			)
							self.assertTrue(torch.allclose(lowercase   ,     lowercase			)			)







	def  snake_case__     (     self      :Union[str, Any]			)		->      Optional[int]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= '''hf-internal-testing/tiny-random-t5'''
				SCREAMING_SNAKE_CASE     						= AutoModelForSeqaSeqLM.from_pretrained(lowercase			)

				SCREAMING_SNAKE_CASE     						= model.to_bettertransformer()

				with tempfile.TemporaryDirectory() as tmpdirname:
							with self.assertRaises(lowercase			):
										model.save_pretrained(lowercase			)

							SCREAMING_SNAKE_CASE     						= model.reverse_bettertransformer()
							model.save_pretrained(lowercase			)