



"""simple docstring"""




import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_lowerCAmelCase							:   Any				=				random.Random()

if is_torch_available():
 import torch

def 		__snake_case		( SCREAMING_SNAKE_CASE__      :		Tuple		,    SCREAMING_SNAKE_CASE__      :		Tuple=1.0		,    SCREAMING_SNAKE_CASE__      :		Union[str, Any]=None		,    SCREAMING_SNAKE_CASE__      :		Union[str, Any]=None							)    ->  Optional[Any]:





     '''simple docstring'''

     if rng is None:
          _UpperCAmelCase     :      Dict														=       global_rng

     _UpperCAmelCase     :      Union[str, Any]														=       []
     for batch_idx in range(shape[0]							):
          values.append([]							)
          for _ in range(shape[1]							):
               values[-1].append(rng.random() * scale							)

     return values






class       UpperCAmelCase_    (							unittest.TestCase		):



      def __init__(	self	:      List[Any]	,      A	:      int	,      A	:      Tuple=7	,      A	:      Any=4_0_0	,      A	:      str=2_0_0_0	,      A	:      List[Any]=1	,      A	:      int=0.0	,      A	:      Optional[Any]=1_6_0_0_0	,      A	:      List[str]=True	,      A	:      Union[str, Any]=True	,      ):
           _UpperCAmelCase     :      Dict														=       parent
           _UpperCAmelCase     :      str														=       batch_size
           _UpperCAmelCase     :      str														=       min_seq_length
           _UpperCAmelCase     :      Optional[int]														=       max_seq_length
           _UpperCAmelCase     :      Optional[int]														=       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
           _UpperCAmelCase     :      List[Any]														=       feature_size
           _UpperCAmelCase     :      Union[str, Any]														=       padding_value
           _UpperCAmelCase     :      Optional[Any]														=       sampling_rate
           _UpperCAmelCase     :      Union[str, Any]														=       return_attention_mask
           _UpperCAmelCase     :      int														=       do_normalize



      def 		snake_case_   (	self	:      Optional[int]		):
           return {
               "feature_size": self.feature_size,
               "padding_value": self.padding_value,
               "sampling_rate": self.sampling_rate,
               "return_attention_mask": self.return_attention_mask,
               "do_normalize": self.do_normalize,
           }



      def 		snake_case_   (	self	:      Optional[Any]	,      A	:      Union[str, Any]=False	,      A	:      Tuple=False		):
           def _flatten(A	:      Tuple		):
                return list(itertools.chain(*A		)		)

           if equal_length:
                _UpperCAmelCase     :      List[Any]														=       floats_list((self.batch_size, self.max_seq_length)		)
           else:
                # make sure that inputs increase in size
                _UpperCAmelCase     :      Any														=       [
                    _flatten(floats_list((x, self.feature_size)		)		)
                    for x in range(self.min_seq_length	,      self.max_seq_length	,      self.seq_length_diff		)
                ]

           if numpify:
                _UpperCAmelCase     :      Dict														=       [np.asarray(A		) for x in speech_inputs]

           return speech_inputs







@require_torch
@require_torchaudio
class       UpperCAmelCase_    (							_UpperCamelCase			,     unittest.TestCase		):
      __SCREAMING_SNAKE_CASE		:  List[str]						=					ASTFeatureExtractor



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      str														=       ASTFeatureExtractionTester(self		)



      def 		snake_case_   (	self	:      Optional[Any]		):
           # Tests that all call wrap to encode_plus and batch_encode_plus
           _UpperCAmelCase     :      List[Any]														=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
           # create three inputs of length 800, 1000, and 1200
           _UpperCAmelCase     :      List[Any]														=       [floats_list((1, x)		)[0] for x in range(8_0_0	,      1_4_0_0	,      2_0_0		)]
           _UpperCAmelCase     :      Tuple														=       [np.asarray(A		) for speech_input in speech_inputs]

           # Test not batched input
           _UpperCAmelCase     :      List[Any]														=       feat_extract(speech_inputs[0]	,      return_tensors="np"		).input_values
           _UpperCAmelCase     :      Tuple														=       feat_extract(np_speech_inputs[0]	,      return_tensors="np"		).input_values
           self.assertTrue(np.allclose(A	,      A	,      atol=1e-3		)		)

           # Test batched
           _UpperCAmelCase     :      List[str]														=       feat_extract(A	,      padding=A	,      return_tensors="np"		).input_values
           _UpperCAmelCase     :      str														=       feat_extract(A	,      padding=A	,      return_tensors="np"		).input_values
           for enc_seq_a, enc_seq_a in zip(A	,      A		):
                self.assertTrue(np.allclose(A	,      A	,      atol=1e-3		)		)

           # Test 2-D numpy arrays are batched.
           _UpperCAmelCase     :      Dict														=       [floats_list((1, x)		)[0] for x in (8_0_0, 8_0_0, 8_0_0)]
           _UpperCAmelCase     :      int														=       np.asarray(A		)
           _UpperCAmelCase     :      List[str]														=       feat_extract(A	,      return_tensors="np"		).input_values
           _UpperCAmelCase     :      List[Any]														=       feat_extract(A	,      return_tensors="np"		).input_values
           for enc_seq_a, enc_seq_a in zip(A	,      A		):
                self.assertTrue(np.allclose(A	,      A	,      atol=1e-3		)		)



      @require_torch
      def 		snake_case_   (	self	:      Tuple		):
           import torch

           _UpperCAmelCase     :      Dict														=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
           _UpperCAmelCase     :      Dict														=       np.random.rand(1_0_0		).astype(np.floataa		)
           _UpperCAmelCase     :      Dict														=       np_speech_inputs.tolist()

           for inputs in [py_speech_inputs, np_speech_inputs]:
                _UpperCAmelCase     :      Tuple														=       feature_extractor.pad([{"input_values": inputs}]	,      return_tensors="np"		)
                self.assertTrue(np_processed.input_values.dtype == np.floataa		)
                _UpperCAmelCase     :      int														=       feature_extractor.pad([{"input_values": inputs}]	,      return_tensors="pt"		)
                self.assertTrue(pt_processed.input_values.dtype == torch.floataa		)



      def 		snake_case_   (	self	:      Optional[Any]	,      A	:      Tuple		):
           from datasets import load_dataset

           _UpperCAmelCase     :      Union[str, Any]														=       load_dataset("hf-internal-testing/librispeech_asr_dummy"	,      "clean"	,      split="validation"		)
           # automatic decoding with librispeech
           _UpperCAmelCase     :      Optional[int]														=       ds.sort("id"		).select(range(A		)		)[:num_samples]["audio"]

           return [x["array"] for x in speech_samples]



      @require_torch
      def 		snake_case_   (	self	:      List[Any]		):
           # fmt: off
           _UpperCAmelCase     :      List[str]														=       torch.tensor(
               [-0.9_894, -1.2_776, -0.9_066, -1.2_776, -0.9_349, -1.2_609, -1.0_386, -1.2_776,
                -1.1_561, -1.2_776, -1.2_052, -1.2_723, -1.2_190, -1.2_132, -1.2_776, -1.1_133,
                -1.1_953, -1.1_343, -1.1_584, -1.2_203, -1.1_770, -1.2_474, -1.2_381, -1.1_936,
                -0.9_270, -0.8_317, -0.8_049, -0.7_706, -0.7_565, -0.7_869]		)
           # fmt: on

           _UpperCAmelCase     :      int														=       self._load_datasamples(1		)
           _UpperCAmelCase     :      Tuple														=       ASTFeatureExtractor()
           _UpperCAmelCase     :      Union[str, Any]														=       feature_extractor(A	,      return_tensors="pt"		).input_values
           self.assertEquals(input_values.shape	,      (1, 1_0_2_4, 1_2_8)		)
           self.assertTrue(torch.allclose(input_values[0, 0, :3_0]	,      A	,      atol=1e-4		)		)





"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_lowerCAmelCase							:   Any				=				logging.get_logger(__name__)

_lowerCAmelCase							:   Any				=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase							:   List[str]				=				{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase							:   List[str]				=				{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}






class       UpperCAmelCase_    (							_UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:  Optional[Any]						=					VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:  int						=					PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:  Dict						=					GPTaTokenizer



      def __init__(	self	:      Optional[Any]	,      A	:      Optional[int]=None	,      A	:      List[Any]=None	,      A	:      Optional[int]=None	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Union[str, Any]="<|endoftext|>"	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Optional[Any]=False	,      **A	:      str	,      ):
           super().__init__(
               A	,      A	,      tokenizer_file=A	,      unk_token=A	,      bos_token=A	,      eos_token=A	,      add_prefix_space=A	,      **A	,      )

           _UpperCAmelCase     :      List[Any]														=       kwargs.pop("add_bos_token"	,      A		)

           _UpperCAmelCase     :      List[str]														=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
           if pre_tok_state.get("add_prefix_space"	,      A		) != add_prefix_space:
                _UpperCAmelCase     :      Dict														=       getattr(A	,      pre_tok_state.pop("type"		)		)
                _UpperCAmelCase     :      Optional[Any]														=       add_prefix_space
                _UpperCAmelCase     :      Optional[Any]														=       pre_tok_class(**A		)

           _UpperCAmelCase     :      List[Any]														=       add_prefix_space



      def 		snake_case_   (	self	:      List[Any]	,      *A	:      List[Any]	,      **A	:      List[str]		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)
           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[Any]	,      *A	:      List[Any]	,      **A	:      int		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)

           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[int]	,      A	:      str	,      A	:      Optional[str] = None		):
           _UpperCAmelCase     :      List[Any]														=       self._tokenizer.model.save(A	,      name=A		)
           return tuple(A		)



      def 		snake_case_   (	self	:      int	,      A	:      "Conversation"		):
           _UpperCAmelCase     :      List[str]														=       []
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(A	,      add_special_tokens=A		) + [self.eos_token_id]		)

           if len(A		) > self.model_max_length:
                _UpperCAmelCase     :      Optional[int]														=       input_ids[-self.model_max_length :]
           return input_ids
