

import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
    import jax.numpy as jnp

    from transformers.models.albert.modeling_flax_albert import (
        FlaxAlbertForMaskedLM,
        FlaxAlbertForMultipleChoice,
        FlaxAlbertForPreTraining,
        FlaxAlbertForQuestionAnswering,
        FlaxAlbertForSequenceClassification,
        FlaxAlbertForTokenClassification,
        FlaxAlbertModel,
    )





class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   def __init__(       self					:			Optional[Any]     , __lowerCamelCase					:			Tuple     , __lowerCamelCase					:			List[str]=13     , __lowerCamelCase					:			List[str]=7     , __lowerCamelCase					:			Union[str, Any]=True     , __lowerCamelCase					:			Union[str, Any]=True     , __lowerCamelCase					:			Dict=True     , __lowerCamelCase					:			Optional[int]=True     , __lowerCamelCase					:			Optional[int]=99     , __lowerCamelCase					:			Tuple=32     , __lowerCamelCase					:			Any=5     , __lowerCamelCase					:			int=4     , __lowerCamelCase					:			Any=37     , __lowerCamelCase					:			Any="gelu"     , __lowerCamelCase					:			int=0.1     , __lowerCamelCase					:			Optional[int]=0.1     , __lowerCamelCase					:			Optional[Any]=512     , __lowerCamelCase					:			Union[str, Any]=16     , __lowerCamelCase					:			Any=2     , __lowerCamelCase					:			List[Any]=0.02     , __lowerCamelCase					:			int=4     , ):
       SCREAMING_SNAKE_CASE										=   parent
       SCREAMING_SNAKE_CASE										=   batch_size
       SCREAMING_SNAKE_CASE										=   seq_length
       SCREAMING_SNAKE_CASE										=   is_training
       SCREAMING_SNAKE_CASE										=   use_attention_mask
       SCREAMING_SNAKE_CASE										=   use_token_type_ids
       SCREAMING_SNAKE_CASE										=   use_labels
       SCREAMING_SNAKE_CASE										=   vocab_size
       SCREAMING_SNAKE_CASE										=   hidden_size
       SCREAMING_SNAKE_CASE										=   num_hidden_layers
       SCREAMING_SNAKE_CASE										=   num_attention_heads
       SCREAMING_SNAKE_CASE										=   intermediate_size
       SCREAMING_SNAKE_CASE										=   hidden_act
       SCREAMING_SNAKE_CASE										=   hidden_dropout_prob
       SCREAMING_SNAKE_CASE										=   attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE										=   max_position_embeddings
       SCREAMING_SNAKE_CASE										=   type_vocab_size
       SCREAMING_SNAKE_CASE										=   type_sequence_label_size
       SCREAMING_SNAKE_CASE										=   initializer_range
       SCREAMING_SNAKE_CASE										=   num_choices





   def      _snake_case		(       self					:			str			):
       SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size, self.seq_length]     , self.vocab_size			)

       SCREAMING_SNAKE_CASE										=   None
       if self.use_attention_mask:
           SCREAMING_SNAKE_CASE										=   random_attention_mask([self.batch_size, self.seq_length]			)

       SCREAMING_SNAKE_CASE										=   None
       if self.use_token_type_ids:
           SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size, self.seq_length]     , self.type_vocab_size			)

       SCREAMING_SNAKE_CASE										=   AlbertConfig(
           vocab_size=self.vocab_size     , hidden_size=self.hidden_size     , num_hidden_layers=self.num_hidden_layers     , num_attention_heads=self.num_attention_heads     , intermediate_size=self.intermediate_size     , hidden_act=self.hidden_act     , hidden_dropout_prob=self.hidden_dropout_prob     , attention_probs_dropout_prob=self.attention_probs_dropout_prob     , max_position_embeddings=self.max_position_embeddings     , type_vocab_size=self.type_vocab_size     , is_decoder=__lowerCamelCase     , initializer_range=self.initializer_range     , )

       return config, input_ids, token_type_ids, attention_mask





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   self.prepare_config_and_inputs()
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   config_and_inputs
       SCREAMING_SNAKE_CASE										=   {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
       return config, inputs_dict





@require_flax
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      (
       (
           FlaxAlbertModel,
           FlaxAlbertForPreTraining,
           FlaxAlbertForMaskedLM,
           FlaxAlbertForMultipleChoice,
           FlaxAlbertForQuestionAnswering,
           FlaxAlbertForSequenceClassification,
           FlaxAlbertForTokenClassification,
           FlaxAlbertForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )





   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   FlaxAlbertModelTester(self			)





   @slow
   def      _snake_case		(       self					:			Any			):
       for model_class_name in self.all_model_classes:
           SCREAMING_SNAKE_CASE										=   model_class_name.from_pretrained("albert-base-v2"			)
           SCREAMING_SNAKE_CASE										=   model(np.ones((1, 1)			)			)
           self.assertIsNotNone(__lowerCamelCase			)





@require_flax
class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   @slow
   def      _snake_case		(       self					:			str			):
       SCREAMING_SNAKE_CASE										=   FlaxAlbertModel.from_pretrained("albert-base-v2"			)
       SCREAMING_SNAKE_CASE										=   np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]			)
       SCREAMING_SNAKE_CASE										=   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]			)
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase			)[0]
       SCREAMING_SNAKE_CASE										=   (1, 11, 768)
       self.assertEqual(output.shape     , __lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   np.array(
           [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]			)

       self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]     , __lowerCamelCase     , atol=1e-4			)			)


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

__A			:					Any						=			logging.get_logger(__name__)


__A			:					Any						=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__A			:					Optional[Any]						=			{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

__A			:					Union[str, Any]						=			{'facebook/blenderbot-3B': 1_2_8}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	__a     (      ):
    SCREAMING_SNAKE_CASE										=   (
        list(range(ord("!"       )			,  ord("~"       ) + 1       )       ) + list(range(ord("¡"       )			,  ord("¬"       ) + 1       )       ) + list(range(ord("®"       )			,  ord("ÿ"       ) + 1       )       )
    )
    SCREAMING_SNAKE_CASE										=   bs[:]
    SCREAMING_SNAKE_CASE										=   0
    for b in range(2**8       ):
        if b not in bs:
            bs.append(A__       )
            cs.append(2**8 + n       )
            n += 1
    SCREAMING_SNAKE_CASE										=   [chr(A__       ) for n in cs]
    return dict(zip(A__			,  A__       )       )





def 	__a     (      A__			:  List[Any]       ):
    SCREAMING_SNAKE_CASE										=   set()
    SCREAMING_SNAKE_CASE										=   word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)       )
        SCREAMING_SNAKE_CASE										=   char
    return pairs





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      ["input_ids", "attention_mask"]





   def __init__(       self					:			Union[str, Any]     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Optional[Any]     , __lowerCamelCase					:			Any="replace"     , __lowerCamelCase					:			Optional[Any]="<s>"     , __lowerCamelCase					:			Optional[Any]="</s>"     , __lowerCamelCase					:			Any="</s>"     , __lowerCamelCase					:			Union[str, Any]="<s>"     , __lowerCamelCase					:			List[str]="<unk>"     , __lowerCamelCase					:			Optional[Any]="<pad>"     , __lowerCamelCase					:			Dict="<mask>"     , __lowerCamelCase					:			Any=False     , **__lowerCamelCase					:			Optional[Any]     , ):
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else bos_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else eos_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else sep_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else cls_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else unk_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else mask_token

       super().__init__(
           errors=__lowerCamelCase     , bos_token=__lowerCamelCase     , eos_token=__lowerCamelCase     , unk_token=__lowerCamelCase     , sep_token=__lowerCamelCase     , cls_token=__lowerCamelCase     , pad_token=__lowerCamelCase     , mask_token=__lowerCamelCase     , add_prefix_space=__lowerCamelCase     , **__lowerCamelCase     , )

       with open(__lowerCamelCase     , encoding="utf-8"			) as vocab_handle:
           SCREAMING_SNAKE_CASE										=   json.load(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   {v: k for k, v in self.encoder.items()}
       SCREAMING_SNAKE_CASE										=   errors  # how to handle errors in decoding
       SCREAMING_SNAKE_CASE										=   bytes_to_unicode()
       SCREAMING_SNAKE_CASE										=   {v: k for k, v in self.byte_encoder.items()}
       with open(__lowerCamelCase     , encoding="utf-8"			) as merges_handle:
           SCREAMING_SNAKE_CASE										=   merges_handle.read().split("\n"			)[1:-1]
       SCREAMING_SNAKE_CASE										=   [tuple(merge.split()			) for merge in bpe_merges]
       SCREAMING_SNAKE_CASE										=   dict(zip(__lowerCamelCase     , range(len(__lowerCamelCase			)			)			)			)
       SCREAMING_SNAKE_CASE										=   {}
       SCREAMING_SNAKE_CASE										=   add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       SCREAMING_SNAKE_CASE										=   re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)





   @property
   # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
   def      _snake_case		(       self					:			str			):
       return len(self.encoder			)





   def      _snake_case		(       self					:			Union[str, Any]			):
       return dict(self.encoder     , **self.added_tokens_encoder			)





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			List[Any]			):
       if token in self.cache:
           return self.cache[token]
       SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)

       if not pairs:
           return token

       while True:
           SCREAMING_SNAKE_CASE										=   min(__lowerCamelCase     , key=lambda __lowerCamelCase			: self.bpe_ranks.get(__lowerCamelCase     , float("inf"			)			)			)
           if bigram not in self.bpe_ranks:
               break
           SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   bigram
           SCREAMING_SNAKE_CASE										=   []
           SCREAMING_SNAKE_CASE										=   0
           while i < len(__lowerCamelCase			):
               try:
                   SCREAMING_SNAKE_CASE										=   word.index(__lowerCamelCase     , __lowerCamelCase			)
               except ValueError:
                   new_word.extend(word[i:]			)
                   break
               else:
                   new_word.extend(word[i:j]			)
                   SCREAMING_SNAKE_CASE										=   j

               if word[i] == first and i < len(__lowerCamelCase			) - 1 and word[i + 1] == second:
                   new_word.append(first + second			)
                   i += 2
               else:
                   new_word.append(word[i]			)
                   i += 1
           SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   new_word
           if len(__lowerCamelCase			) == 1:
               break
           else:
               SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   " ".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   word
       return word





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   []
       for token in re.findall(self.pat     , __lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   "".join(
               self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase			).split(" "			)			)
       return bpe_tokens





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			Dict			):
       return self.encoder.get(__lowerCamelCase     , self.encoder.get(self.unk_token			)			)





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			Optional[int]			):
       return self.decoder.get(__lowerCamelCase			)





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   "".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"     , errors=self.errors			)
       return text





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       if not os.path.isdir(__lowerCamelCase			):
           logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
           return
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as f:
           f.write(json.dumps(self.encoder     , indent=2     , sort_keys=__lowerCamelCase     , ensure_ascii=__lowerCamelCase			) + "\n"			)

       SCREAMING_SNAKE_CASE										=   0
       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as writer:
           writer.write("#version: 0.2\n"			)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     , key=lambda __lowerCamelCase			: kv[1]			):
               if index != token_index:
                   logger.warning(
                       f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                       " Please check that the tokenizer is not corrupted!"			)
                   SCREAMING_SNAKE_CASE										=   token_index
               writer.write(" ".join(__lowerCamelCase			) + "\n"			)
               index += 1

       return vocab_file, merge_file





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None     , __lowerCamelCase					:			bool = False			):
       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__lowerCamelCase     , token_ids_a=__lowerCamelCase     , already_has_special_tokens=__lowerCamelCase			)

       if token_ids_a is None:
           return [1] + ([0] * len(__lowerCamelCase			)) + [1]
       return [1] + ([0] * len(__lowerCamelCase			)) + [1, 1] + ([0] * len(__lowerCamelCase			)) + [1]





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Any=False     , **__lowerCamelCase					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   kwargs.pop("add_prefix_space"     , self.add_prefix_space			)
       if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase			) > 0 and not text[0].isspace()):
           SCREAMING_SNAKE_CASE										=   " " + text
       return (text, kwargs)





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       return token_ids_a + [self.eos_token_id]





   def      _snake_case		(       self					:			int     , __lowerCamelCase					:			"Conversation"			):
       SCREAMING_SNAKE_CASE										=   []
       for is_user, text in conversation.iter_texts():
           if is_user:
               # We need to space prefix as it's being done within blenderbot
               inputs.append(" " + text			)
           else:
               # Generated responses should contain them already.
               inputs.append(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "  ".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   self.encode(__lowerCamelCase			)
       if len(__lowerCamelCase			) > self.model_max_length:
           SCREAMING_SNAKE_CASE										=   input_ids[-self.model_max_length :]
           logger.warning(f"Trimmed input from conversation as it was longer than {self.model_max_length} tokens."			)
       return input_ids