





'''simple docstring'''






import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


A__							:  Optional[int]    =random.Random()

if is_torch_available():
      import torch
def   UpperCamelCase__ (						lowerCAmelCase				,      lowerCAmelCase=1.0				,      lowerCAmelCase=None				,      lowerCAmelCase=None    ):





      """simple docstring"""


      if rng is None:
            _lowerCAmelCase					=				global_rng

      _lowerCAmelCase					=				[]
      for batch_idx in range(shape[0]    ):
            values.append([]    )
            for _ in range(shape[1]    ):
                  values[-1].append(rng.random() * scale    )

      return values




class 					UpperCAmelCase	(     unittest.TestCase  ):




      def __init__(     self     :					List[Any]     ,     __snake_case     :					List[str]     ,     __snake_case     :					Union[str, Any]=7     ,     __snake_case     :					Tuple=4_00     ,     __snake_case     :					Tuple=20_00     ,     __snake_case     :					Union[str, Any]=1     ,     __snake_case     :					Optional[int]=0.0     ,     __snake_case     :					int=1_60_00     ,     __snake_case     :					str=True     ,     __snake_case     :					Optional[Any]=True     ,     )							-> Dict:
            _lowerCAmelCase					=				parent
            _lowerCAmelCase					=				batch_size
            _lowerCAmelCase					=				min_seq_length
            _lowerCAmelCase					=				max_seq_length
            _lowerCAmelCase					=				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
            _lowerCAmelCase					=				feature_size
            _lowerCAmelCase					=				padding_value
            _lowerCAmelCase					=				sampling_rate
            _lowerCAmelCase					=				return_attention_mask
            _lowerCAmelCase					=				do_normalize




      def 							lowercase__       (     self     :					Tuple    )							-> Dict:
            return {
                "feature_size": self.feature_size,
                "padding_value": self.padding_value,
                "sampling_rate": self.sampling_rate,
                "return_attention_mask": self.return_attention_mask,
                "do_normalize": self.do_normalize,
            }




      def 							lowercase__       (     self     :					List[Any]     ,     __snake_case     :					str=False     ,     __snake_case     :					str=False    )							-> Optional[int]:
            def _flatten(__snake_case     :					List[Any]    ):
                  return list(itertools.chain(*__snake_case    )    )

            if equal_length:
                  _lowerCAmelCase					=				floats_list((self.batch_size, self.max_seq_length)    )
            else:
                  # make sure that inputs increase in size
                  _lowerCAmelCase					=				[
                      _flatten(floats_list((x, self.feature_size)    )    )
                      for x in range(self.min_seq_length     ,     self.max_seq_length     ,     self.seq_length_diff    )
                  ]

            if numpify:
                  _lowerCAmelCase					=				[np.asarray(__snake_case    ) for x in speech_inputs]

            return speech_inputs




@require_torch
@require_torchaudio
class 					UpperCAmelCase	(     snake_case_				,						unittest.TestCase  ):
      _lowercase:     str			=  ASTFeatureExtractor




      def 							lowercase__       (     self     :					Optional[int]    )							-> List[str]:
            _lowerCAmelCase					=				ASTFeatureExtractionTester(self    )




      def 							lowercase__       (     self     :					Any    )							-> Any:
            # Tests that all call wrap to encode_plus and batch_encode_plus
            _lowerCAmelCase					=				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()    )
            # create three inputs of length 800, 1000, and 1200
            _lowerCAmelCase					=				[floats_list((1, x)    )[0] for x in range(8_00     ,     14_00     ,     2_00    )]
            _lowerCAmelCase					=				[np.asarray(__snake_case    ) for speech_input in speech_inputs]

            # Test not batched input
            _lowerCAmelCase					=				feat_extract(speech_inputs[0]     ,     return_tensors="""np"""    ).input_values
            _lowerCAmelCase					=				feat_extract(np_speech_inputs[0]     ,     return_tensors="""np"""    ).input_values
            self.assertTrue(np.allclose(__snake_case     ,     __snake_case     ,     atol=1E-3    )    )

            # Test batched
            _lowerCAmelCase					=				feat_extract(__snake_case     ,     padding=__snake_case     ,     return_tensors="""np"""    ).input_values
            _lowerCAmelCase					=				feat_extract(__snake_case     ,     padding=__snake_case     ,     return_tensors="""np"""    ).input_values
            for enc_seq_a, enc_seq_a in zip(__snake_case     ,     __snake_case    ):
                  self.assertTrue(np.allclose(__snake_case     ,     __snake_case     ,     atol=1E-3    )    )

            # Test 2-D numpy arrays are batched.
            _lowerCAmelCase					=				[floats_list((1, x)    )[0] for x in (8_00, 8_00, 8_00)]
            _lowerCAmelCase					=				np.asarray(__snake_case    )
            _lowerCAmelCase					=				feat_extract(__snake_case     ,     return_tensors="""np"""    ).input_values
            _lowerCAmelCase					=				feat_extract(__snake_case     ,     return_tensors="""np"""    ).input_values
            for enc_seq_a, enc_seq_a in zip(__snake_case     ,     __snake_case    ):
                  self.assertTrue(np.allclose(__snake_case     ,     __snake_case     ,     atol=1E-3    )    )




      @require_torch
      def 							lowercase__       (     self     :					Optional[int]    )							-> Union[str, Any]:
            import torch

            _lowerCAmelCase					=				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()    )
            _lowerCAmelCase					=				np.random.rand(1_00    ).astype(np.floataa    )
            _lowerCAmelCase					=				np_speech_inputs.tolist()

            for inputs in [py_speech_inputs, np_speech_inputs]:
                  _lowerCAmelCase					=				feature_extractor.pad([{"""input_values""": inputs}]     ,     return_tensors="""np"""    )
                  self.assertTrue(np_processed.input_values.dtype == np.floataa    )
                  _lowerCAmelCase					=				feature_extractor.pad([{"""input_values""": inputs}]     ,     return_tensors="""pt"""    )
                  self.assertTrue(pt_processed.input_values.dtype == torch.floataa    )




      def 							lowercase__       (     self     :					int     ,     __snake_case     :					List[str]    )							-> Dict:
            from datasets import load_dataset

            _lowerCAmelCase					=				load_dataset("""hf-internal-testing/librispeech_asr_dummy"""     ,     """clean"""     ,     split="""validation"""    )
            # automatic decoding with librispeech
            _lowerCAmelCase					=				ds.sort("""id"""    ).select(range(__snake_case    )    )[:num_samples]["""audio"""]

            return [x["array"] for x in speech_samples]




      @require_torch
      def 							lowercase__       (     self     :					List[str]    )							-> str:
            # fmt: off
            _lowerCAmelCase					=				torch.tensor(
                [-0.98_94, -1.27_76, -0.90_66, -1.27_76, -0.93_49, -1.26_09, -1.03_86, -1.27_76,
                 -1.15_61, -1.27_76, -1.20_52, -1.27_23, -1.21_90, -1.21_32, -1.27_76, -1.11_33,
                 -1.19_53, -1.13_43, -1.15_84, -1.22_03, -1.17_70, -1.24_74, -1.23_81, -1.19_36,
                 -0.92_70, -0.83_17, -0.80_49, -0.77_06, -0.75_65, -0.78_69]    )
            # fmt: on

            _lowerCAmelCase					=				self._load_datasamples(1    )
            _lowerCAmelCase					=				ASTFeatureExtractor()
            _lowerCAmelCase					=				feature_extractor(__snake_case     ,     return_tensors="""pt"""    ).input_values
            self.assertEquals(input_values.shape     ,     (1, 10_24, 1_28)    )
            self.assertTrue(torch.allclose(input_values[0, 0, :30]     ,     __snake_case     ,     atol=1E-4    )    )













'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__							:  Any    =None


A__							:  Optional[int]    =logging.get_logger(__name__)

A__							:  Union[str, Any]    ={'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A__							:  List[str]    ={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

A__							:  List[str]    ={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

A__							:  List[str]    ='''▁'''

# Segments (not really needed)
A__							:  str    =0
A__							:  str    =1
A__							:  List[Any]    =2
A__							:  str    =3
A__							:  Optional[Any]    =4




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Optional[int]			=  VOCAB_FILES_NAMES
      _lowercase:     Optional[int]			=  PRETRAINED_VOCAB_FILES_MAP
      _lowercase:     Dict			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:     Optional[Any]			=  '''left'''
      _lowercase:     Dict			=  XLNetTokenizer




      def __init__(     self     :					List[str]     ,     __snake_case     :					Optional[Any]=None     ,     __snake_case     :					str=None     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					str=True     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					List[Any]="<s>"     ,     __snake_case     :					List[Any]="</s>"     ,     __snake_case     :					str="<unk>"     ,     __snake_case     :					int="<sep>"     ,     __snake_case     :					int="<pad>"     ,     __snake_case     :					Dict="<cls>"     ,     __snake_case     :					int="<mask>"     ,     __snake_case     :					Optional[int]=["<eop>", "<eod>"]     ,     **__snake_case     :					List[str]     ,     )							-> List[str]:
            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase					=				AddedToken(__snake_case     ,     lstrip=__snake_case     ,     rstrip=__snake_case    ) if isinstance(__snake_case     ,     __snake_case    ) else mask_token

            super().__init__(
                vocab_file=__snake_case     ,     tokenizer_file=__snake_case     ,     do_lower_case=__snake_case     ,     remove_space=__snake_case     ,     keep_accents=__snake_case     ,     bos_token=__snake_case     ,     eos_token=__snake_case     ,     unk_token=__snake_case     ,     sep_token=__snake_case     ,     pad_token=__snake_case     ,     cls_token=__snake_case     ,     mask_token=__snake_case     ,     additional_special_tokens=__snake_case     ,     **__snake_case     ,     )

            _lowerCAmelCase					=				3
            _lowerCAmelCase					=				do_lower_case
            _lowerCAmelCase					=				remove_space
            _lowerCAmelCase					=				keep_accents
            _lowerCAmelCase					=				vocab_file
            _lowerCAmelCase					=				False if not self.vocab_file else True




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[self.cls_token_id]
            if token_ids_a is None:
                  return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[2]

            if token_ids_a is None:
                  return len(token_ids_a + sep    ) * [0] + cls_segment_id
            return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str] = None    )							-> Tuple[str]:
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""    )

            if not os.path.isdir(__snake_case    ):
                  logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
                  return
            _lowerCAmelCase					=				os.path.join(
                __snake_case     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
                  copyfile(self.vocab_file     ,     __snake_case    )

            return (out_vocab_file,)






