






'''simple docstring'''





import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 				lowercase       (							_lowerCamelCase ,       unittest.TestCase      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							GPTaTokenizer
    lowerCAmelCase__      				=							GPTaTokenizerFast
    lowerCAmelCase__      				=							True
    lowerCAmelCase__      				=							{"add_prefix_space": True}
    lowerCAmelCase__      				=							False

    def    a__				(					self     :     Tuple     )							->							str:

     '''simple docstring'''




     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     lowerCamelCase__  =     [
         "l",
         "o",
         "w",
         "e",
         "r",
         "s",
         "t",
         "i",
         "d",
         "n",
         "\u0120",
         "\u0120l",
         "\u0120n",
         "\u0120lo",
         "\u0120low",
         "er",
         "\u0120lowest",
         "\u0120newer",
         "\u0120wider",
         "<unk>",
         "<|endoftext|>",
     ]
     lowerCamelCase__  =     dict(zip(__lowerCamelCase				,     range(len(__lowerCamelCase     )     )     )     )
     lowerCamelCase__  =     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
     lowerCamelCase__  =     {"unk_token": "<unk>"}

     lowerCamelCase__  =     os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["vocab_file"]     )
     lowerCamelCase__  =     os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["merges_file"]     )
     with open(self.vocab_file				,     "w"				,     encoding="utf-8"     ) as fp:
      fp.write(json.dumps(__lowerCamelCase     ) + "\n"     )
     with open(self.merges_file				,     "w"				,     encoding="utf-8"     ) as fp:
      fp.write("\n".join(__lowerCamelCase     )     )

    def    a__				(					self     :     List[Any]				,     **__lowerCamelCase     :     Optional[int]     )							->							Dict:

     '''simple docstring'''




     kwargs.update(self.special_tokens_map     )
     return GPTaTokenizer.from_pretrained(self.tmpdirname				,     **__lowerCamelCase     )

    def    a__				(					self     :     Any				,     **__lowerCamelCase     :     int     )							->							List[Any]:

     '''simple docstring'''




     kwargs.update(self.special_tokens_map     )
     return GPTaTokenizerFast.from_pretrained(self.tmpdirname				,     **__lowerCamelCase     )

    def    a__				(					self     :     List[str]				,     __lowerCamelCase     :     Any     )							->							Optional[int]:

     '''simple docstring'''




     lowerCamelCase__  =     "lower newer"
     lowerCamelCase__  =     "lower newer"
     return input_text, output_text

    def    a__				(					self     :     int     )							->							List[str]:

     '''simple docstring'''




     lowerCamelCase__  =     GPTaTokenizer(self.vocab_file				,     self.merges_file				,     **self.special_tokens_map     )
     lowerCamelCase__  =     "lower newer"
     lowerCamelCase__  =     ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
     lowerCamelCase__  =     tokenizer.tokenize(__lowerCamelCase				,     add_prefix_space=__lowerCamelCase     )
     self.assertListEqual(__lowerCamelCase				,     __lowerCamelCase     )

     lowerCamelCase__  =     tokens + [tokenizer.unk_token]
     lowerCamelCase__  =     [14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase     )				,     __lowerCamelCase     )

    def    a__				(					self     :     List[str]     )							->							Any:

     '''simple docstring'''




     if not self.test_rust_tokenizer:
      return

     lowerCamelCase__  =     self.get_tokenizer()
     lowerCamelCase__  =     self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase     )

     lowerCamelCase__  =     "lower newer"

     # Testing tokenization
     lowerCamelCase__  =     tokenizer.tokenize(__lowerCamelCase				,     add_prefix_space=__lowerCamelCase     )
     lowerCamelCase__  =     rust_tokenizer.tokenize(__lowerCamelCase     )
     self.assertListEqual(__lowerCamelCase				,     __lowerCamelCase     )

     # Testing conversion to ids without special tokens
     lowerCamelCase__  =     tokenizer.encode(__lowerCamelCase				,     add_special_tokens=__lowerCamelCase				,     add_prefix_space=__lowerCamelCase     )
     lowerCamelCase__  =     rust_tokenizer.encode(__lowerCamelCase				,     add_special_tokens=__lowerCamelCase     )
     self.assertListEqual(__lowerCamelCase				,     __lowerCamelCase     )

     # Testing conversion to ids with special tokens
     lowerCamelCase__  =     self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase     )
     lowerCamelCase__  =     tokenizer.encode(__lowerCamelCase				,     add_prefix_space=__lowerCamelCase     )
     lowerCamelCase__  =     rust_tokenizer.encode(__lowerCamelCase     )
     self.assertListEqual(__lowerCamelCase				,     __lowerCamelCase     )

     # Testing the unknown token
     lowerCamelCase__  =     tokens + [rust_tokenizer.unk_token]
     lowerCamelCase__  =     [14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCamelCase     )				,     __lowerCamelCase     )

    def    a__				(					self     :     Optional[Any]				,     *__lowerCamelCase     :     List[Any]				,     **__lowerCamelCase     :     Tuple     )							->							str:

     '''simple docstring'''




     pass

    def    a__				(					self     :     str				,     __lowerCamelCase     :     Optional[int]=15     )							->							Dict:

     '''simple docstring'''




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
       lowerCamelCase__  =     self.rust_tokenizer_class.from_pretrained(__lowerCamelCase				,     **__lowerCamelCase     )

       # Simple input
       lowerCamelCase__  =     "This is a simple input"
       lowerCamelCase__  =     ["This is a simple input 1", "This is a simple input 2"]
       lowerCamelCase__  =     ("This is a simple input", "This is a pair")
       lowerCamelCase__  =     [
           ("This is a simple input 1", "This is a simple input 2"),
           ("This is a simple pair 1", "This is a simple pair 2"),
       ]

       # Simple input tests
       self.assertRaises(__lowerCamelCase				,     tokenizer_r.encode				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"     )

       # Simple input
       self.assertRaises(__lowerCamelCase				,     tokenizer_r.encode_plus				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"     )

       # Simple input
       self.assertRaises(
           __lowerCamelCase				,     tokenizer_r.batch_encode_plus				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"				,     )

       # Pair input
       self.assertRaises(__lowerCamelCase				,     tokenizer_r.encode				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"     )

       # Pair input
       self.assertRaises(__lowerCamelCase				,     tokenizer_r.encode_plus				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"     )

       # Pair input
       self.assertRaises(
           __lowerCamelCase				,     tokenizer_r.batch_encode_plus				,     __lowerCamelCase				,     max_length=__lowerCamelCase				,     padding="max_length"				,     )

    def    a__				(					self     :     Dict     )							->							Any:

     '''simple docstring'''




     lowerCamelCase__  =     GPTaTokenizer.from_pretrained(self.tmpdirname				,     pad_token="<pad>"     )

     # Simple input
     lowerCamelCase__  =     "This is a simple input"
     lowerCamelCase__  =     ["This is a simple input looooooooong", "This is a simple input"]
     lowerCamelCase__  =     ("This is a simple input", "This is a pair")
     lowerCamelCase__  =     [
         ("This is a simple input loooooong", "This is a simple input"),
         ("This is a simple pair loooooong", "This is a simple pair"),
     ]

     lowerCamelCase__  =     tokenizer.pad_token_id

     lowerCamelCase__  =     tokenizer(__lowerCamelCase				,     padding="max_length"				,     max_length=30				,     return_tensors="np"     )
     lowerCamelCase__  =     tokenizer(__lowerCamelCase				,     padding=__lowerCamelCase				,     truncate=__lowerCamelCase				,     return_tensors="np"     )
     lowerCamelCase__  =     tokenizer(*__lowerCamelCase				,     padding="max_length"				,     max_length=60				,     return_tensors="np"     )
     lowerCamelCase__  =     tokenizer(__lowerCamelCase				,     padding=__lowerCamelCase				,     truncate=__lowerCamelCase				,     return_tensors="np"     )

     # s
     # test single string max_length padding
     self.assertEqual(out_s["input_ids"].shape[-1]				,     30     )
     self.assertTrue(pad_token_id in out_s["input_ids"]     )
     self.assertTrue(0 in out_s["attention_mask"]     )

     # s2
     # test automatic padding
     self.assertEqual(out_sa["input_ids"].shape[-1]				,     33     )
     # long slice doesn't have padding
     self.assertFalse(pad_token_id in out_sa["input_ids"][0]     )
     self.assertFalse(0 in out_sa["attention_mask"][0]     )
     # short slice does have padding
     self.assertTrue(pad_token_id in out_sa["input_ids"][1]     )
     self.assertTrue(0 in out_sa["attention_mask"][1]     )

     # p
     # test single pair max_length padding
     self.assertEqual(out_p["input_ids"].shape[-1]				,     60     )
     self.assertTrue(pad_token_id in out_p["input_ids"]     )
     self.assertTrue(0 in out_p["attention_mask"]     )

     # p2
     # test automatic padding pair
     self.assertEqual(out_pa["input_ids"].shape[-1]				,     52     )
     # long slice pair doesn't have padding
     self.assertFalse(pad_token_id in out_pa["input_ids"][0]     )
     self.assertFalse(0 in out_pa["attention_mask"][0]     )
     # short slice pair does have padding
     self.assertTrue(pad_token_id in out_pa["input_ids"][1]     )
     self.assertTrue(0 in out_pa["attention_mask"][1]     )

    def    a__				(					self     :     List[Any]     )							->							int:

     '''simple docstring'''




     lowerCamelCase__  =     "$$$"
     lowerCamelCase__  =     GPTaTokenizer.from_pretrained(self.tmpdirname				,     bos_token=__lowerCamelCase				,     add_bos_token=__lowerCamelCase     )

     lowerCamelCase__  =     "This is a simple input"
     lowerCamelCase__  =     ["This is a simple input 1", "This is a simple input 2"]

     lowerCamelCase__  =     tokenizer.bos_token_id

     lowerCamelCase__  =     tokenizer(__lowerCamelCase     )
     lowerCamelCase__  =     tokenizer(__lowerCamelCase     )

     self.assertEqual(out_s.input_ids[0]				,     __lowerCamelCase     )
     self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids     )     )

     lowerCamelCase__  =     tokenizer.decode(out_s.input_ids     )
     lowerCamelCase__  =     tokenizer.batch_decode(out_sa.input_ids     )

     self.assertEqual(decode_s.split()[0]				,     __lowerCamelCase     )
     self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa     )     )

    def    a__				(					self     :     Optional[Any]     )							->							Optional[int]:

     '''simple docstring'''




     pass




    def    a__				(					self     :     int     )							->							Union[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     [self.get_tokenizer(do_lower_case=__lowerCamelCase				,     add_bos_token=__lowerCamelCase     )]
     for tokenizer in tokenizers:
      with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
       lowerCamelCase__  =     "Encode this."
       lowerCamelCase__  =     "This one too please."
       lowerCamelCase__  =     tokenizer.encode(__lowerCamelCase				,     add_special_tokens=__lowerCamelCase     )
       encoded_sequence += tokenizer.encode(__lowerCamelCase				,     add_special_tokens=__lowerCamelCase     )
       lowerCamelCase__  =     tokenizer.encode_plus(
           __lowerCamelCase				,     __lowerCamelCase				,     add_special_tokens=__lowerCamelCase				,     return_special_tokens_mask=__lowerCamelCase				,     )
       lowerCamelCase__  =     encoded_sequence_dict["input_ids"]
       lowerCamelCase__  =     encoded_sequence_dict["special_tokens_mask"]
       self.assertEqual(len(__lowerCamelCase     )				,     len(__lowerCamelCase     )     )

       lowerCamelCase__  =     [
           (x if not special_tokens_mask[i] else None) for i, x in enumerate(__lowerCamelCase     )
       ]
       lowerCamelCase__  =     [x for x in filtered_sequence if x is not None]
       self.assertEqual(__lowerCamelCase				,     __lowerCamelCase     )
@require_tokenizers
class 				lowercase       (							unittest.TestCase      ):


    '''simple docstring'''

    def    a__				(					self     :     Any     )							->							Tuple:

     '''simple docstring'''




     lowerCamelCase__  =     AutoTokenizer.from_pretrained("facebook/opt-350m"				,     from_slow=__lowerCamelCase     )
     lowerCamelCase__  =     "A photo of a cat"

     lowerCamelCase__  =     tokenizer.encode(
         __lowerCamelCase				,     )
     self.assertEqual(__lowerCamelCase				,     [2, 250, 1345, 9, 10, 4758]     )
     tokenizer.save_pretrained("test_opt"     )

     lowerCamelCase__  =     AutoTokenizer.from_pretrained("./test_opt"     )
     lowerCamelCase__  =     tokenizer.encode(
         __lowerCamelCase				,     )
     self.assertEqual(__lowerCamelCase				,     [2, 250, 1345, 9, 10, 4758]     )

    def    a__				(					self     :     int     )							->							Optional[int]:

     '''simple docstring'''




     lowerCamelCase__  =     AutoTokenizer.from_pretrained("facebook/opt-350m"				,     use_slow=__lowerCamelCase     )
     lowerCamelCase__  =     "A photo of a cat"

     lowerCamelCase__  =     tokenizer.encode(
         __lowerCamelCase				,     )
     # Same as above
     self.assertEqual(__lowerCamelCase				,     [2, 250, 1345, 9, 10, 4758]     )




    @unittest.skip("This test is failing because of a bug in the fast tokenizer"     )
    def    a__				(					self     :     List[str]     )							->							Tuple:

     '''simple docstring'''




     lowerCamelCase__  =     AutoTokenizer.from_pretrained("facebook/opt-350m"				,     from_slow=__lowerCamelCase     )

     lowerCamelCase__  =     "bos"
     lowerCamelCase__  =     tokenizer.get_vocab()["bos"]

     lowerCamelCase__  =     "A photo of a cat"
     lowerCamelCase__  =     tokenizer.encode(
         __lowerCamelCase				,     )
     # We changed the bos token
     self.assertEqual(__lowerCamelCase				,     [31957, 250, 1345, 9, 10, 4758]     )
     tokenizer.save_pretrained("./tok"     )
     lowerCamelCase__  =     AutoTokenizer.from_pretrained("./tok"     )
     self.assertTrue(tokenizer.is_fast     )
     lowerCamelCase__  =     tokenizer.encode(
         __lowerCamelCase				,     )
     self.assertEqual(__lowerCamelCase				,     [31957, 250, 1345, 9, 10, 4758]     )









'''simple docstring'''





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


__A							:							Union[str, Any]  							=		logging.get_logger(__name__)

__A							:							Tuple  							=		{
    """EleutherAI/gpt-neo-1.3B""": """https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json""",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}
class 				lowercase       (							_lowerCamelCase      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							"gpt_neo"
    lowerCAmelCase__      				=							["past_key_values"]
    lowerCAmelCase__      				=							{"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}

    def __init__(					self     :     str				,     __lowerCamelCase     :     Optional[int]=50257				,     __lowerCamelCase     :     Optional[int]=2048				,     __lowerCamelCase     :     Optional[int]=2048				,     __lowerCamelCase     :     Any=24				,     __lowerCamelCase     :     List[Any]=[[["global", "local"], 12]]				,     __lowerCamelCase     :     Any=16				,     __lowerCamelCase     :     Any=None				,     __lowerCamelCase     :     Optional[Any]=256				,     __lowerCamelCase     :     List[Any]="gelu_new"				,     __lowerCamelCase     :     Any=0.0				,     __lowerCamelCase     :     List[Any]=0.0				,     __lowerCamelCase     :     Union[str, Any]=0.0				,     __lowerCamelCase     :     Any=0.1				,     __lowerCamelCase     :     Dict=1E-5				,     __lowerCamelCase     :     List[Any]=0.0_2				,     __lowerCamelCase     :     List[Any]=True				,     __lowerCamelCase     :     Optional[int]=50256				,     __lowerCamelCase     :     Tuple=50256				,     **__lowerCamelCase     :     Optional[int]				,     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     vocab_size
     lowerCamelCase__  =     max_position_embeddings
     lowerCamelCase__  =     hidden_size
     lowerCamelCase__  =     num_layers
     lowerCamelCase__  =     num_heads
     lowerCamelCase__  =     intermediate_size
     lowerCamelCase__  =     window_size
     lowerCamelCase__  =     activation_function
     lowerCamelCase__  =     resid_dropout
     lowerCamelCase__  =     embed_dropout
     lowerCamelCase__  =     attention_dropout
     lowerCamelCase__  =     classifier_dropout
     lowerCamelCase__  =     layer_norm_epsilon
     lowerCamelCase__  =     initializer_range
     lowerCamelCase__  =     use_cache

     lowerCamelCase__  =     bos_token_id
     lowerCamelCase__  =     eos_token_id

     lowerCamelCase__  =     attention_types
     lowerCamelCase__  =     self.expand_attention_types_params(__lowerCamelCase     )

     if len(self.attention_layers     ) != self.num_layers:
      raise ValueError(
          "Configuration for convolutional module is incorrect. "
          "It is required that `len(config.attention_layers)` == `config.num_layers` "
          f'''but is `len(config.attention_layers) = {len(self.attention_layers     )}`, '''
          f'''`config.num_layers = {self.num_layers}`. '''
          "`config.attention_layers` is prepared using `config.attention_types`. "
          "Please verify the value of `config.attention_types` argument."     )

     super().__init__(bos_token_id=__lowerCamelCase				,     eos_token_id=__lowerCamelCase				,     **__lowerCamelCase     )




    @staticmethod
    def    a__				(					__lowerCamelCase     :     int     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     []
     for item in attention_types:
      for _ in range(item[1]     ):
       attentions.extend(item[0]     )
     return attentions






def    lowerCamelCase_     (    lowercase__				,     lowercase__				,     lowercase__				,     lowercase__):
 import torch

 lowerCamelCase__  =     input.size()
 lowerCamelCase__  =     len(lowercase__)
 lowerCamelCase__  =     shape[dimension]

 lowerCamelCase__  =     torch.arange(0				,     lowercase__				,     lowercase__)
 lowerCamelCase__  =     torch.div(sizedim - size				,     lowercase__				,     rounding_mode="floor") + 1
 lowerCamelCase__  =     torch.arange(lowercase__) + low_indices[:min_length][:, None]

 lowerCamelCase__  =     [slice(lowercase__)] * rank
 lowerCamelCase__  =     indices
 lowerCamelCase__  =     input[s]

 lowerCamelCase__  =     list(range(0				,     rank + 1))
 perm.append(perm.pop(dimension + 1))

 return sliced.permute(lowercase__)






def    lowerCamelCase_     (    lowercase__				,     lowercase__):
 import torch

 lowerCamelCase__  =     torch.arange(1				,     lowercase__)
 lowerCamelCase__  =     torch.remainder(lowercase__				,     lowercase__)
 lowerCamelCase__  =     remainders == 0
 lowerCamelCase__  =     candidates[divisor_indices]
 lowerCamelCase__  =     torch.max(lowercase__)
 return largest_divisor, torch.div(lowercase__				,     lowercase__				,     rounding_mode="floor")
class 				lowercase       (							_lowerCamelCase      ):


    '''simple docstring'''

    @property
    def    a__				(					self     :     Any     )							->							Mapping[str, Mapping[int, str]]:

     '''simple docstring'''




     lowerCamelCase__  =     OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}     )
     if self.use_past:
      self.fill_with_past_key_values_(__lowerCamelCase				,     direction="inputs"     )
      lowerCamelCase__  =     {0: "batch", 1: "past_sequence + sequence"}
     else:
      lowerCamelCase__  =     {0: "batch", 1: "sequence"}

     return common_inputs

    @property
    def    a__				(					self     :     Any     )							->							int:

     '''simple docstring'''




     return self._config.num_heads

    def    a__				(					self     :     Optional[Any]				,     __lowerCamelCase     :     PreTrainedTokenizer				,     __lowerCamelCase     :     int = -1				,     __lowerCamelCase     :     int = -1				,     __lowerCamelCase     :     bool = False				,     __lowerCamelCase     :     Optional[TensorType] = None				,     )							->							Mapping[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     super(__lowerCamelCase				,     self     ).generate_dummy_inputs(
         __lowerCamelCase				,     batch_size=__lowerCamelCase				,     seq_length=__lowerCamelCase				,     is_pair=__lowerCamelCase				,     framework=__lowerCamelCase     )

     # We need to order the input in the way they appears in the forward()
     lowerCamelCase__  =     OrderedDict({"input_ids": common_inputs["input_ids"]}     )

     # Need to add the past_keys
     if self.use_past:
      if not is_torch_available():
       raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."     )
      else:
       import torch

       lowerCamelCase__  ,	lowerCamelCase__  =     common_inputs["input_ids"].shape
       # Not using the same length for past_key_values
       lowerCamelCase__  =     seqlen + 2
       lowerCamelCase__  =     (
           batch,
           self.num_attention_heads,
           past_key_values_length,
           self._config.hidden_size // self.num_attention_heads,
       )
       lowerCamelCase__  =     [
           (torch.zeros(__lowerCamelCase     ), torch.zeros(__lowerCamelCase     )) for _ in range(self.num_layers     )
       ]

     lowerCamelCase__  =     common_inputs["attention_mask"]
     if self.use_past:
      lowerCamelCase__  =     ordered_inputs["attention_mask"].dtype
      lowerCamelCase__  =     torch.cat(
          [ordered_inputs["attention_mask"], torch.ones(__lowerCamelCase				,     __lowerCamelCase				,     dtype=__lowerCamelCase     )]				,     dim=1     )

     return ordered_inputs




    @property
    def    a__				(					self     :     Optional[Any]     )							->							int:

     '''simple docstring'''




     return 13

