
import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np
import torch

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


UpperCamelCase     :    Any       =			logging.get_logger(__name__)

class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):
     lowercase						       =						["""input_features""", """is_longer"""]





     def __init__(     self     ,					__UpperCAmelCase=64     ,					__UpperCAmelCase=4_8000     ,					__UpperCAmelCase=480     ,					__UpperCAmelCase=10     ,					__UpperCAmelCase=1024     ,					__UpperCAmelCase=0.0     ,					__UpperCAmelCase=False     ,					__UpperCAmelCase = 0     ,					__UpperCAmelCase = 1_4000     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = "fusion"     ,					__UpperCAmelCase = "repeatpad"     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      super().__init__(
          feature_size=_lowercase     ,					sampling_rate=_lowercase     ,					padding_value=_lowercase     ,					return_attention_mask=_lowercase     ,					**_lowercase     ,					)
      __UpperCamelCase      =	top_db
      __UpperCamelCase      =	truncation
      __UpperCamelCase      =	padding
      __UpperCamelCase      =	fft_window_size
      __UpperCamelCase      =	(fft_window_size >> 1) + 1
      __UpperCamelCase      =	hop_length
      __UpperCamelCase      =	max_length_s
      __UpperCamelCase      =	max_length_s * sampling_rate
      __UpperCamelCase      =	sampling_rate
      __UpperCamelCase      =	frequency_min
      __UpperCamelCase      =	frequency_max
      __UpperCamelCase      =	mel_filter_bank(
          num_frequency_bins=self.nb_frequency_bins     ,					num_mel_filters=_lowercase     ,					min_frequency=_lowercase     ,					max_frequency=_lowercase     ,					sampling_rate=_lowercase     ,					norm=_lowercase     ,					mel_scale='htk'     ,					)
      __UpperCamelCase      =	mel_filter_bank(
          num_frequency_bins=self.nb_frequency_bins     ,					num_mel_filters=_lowercase     ,					min_frequency=_lowercase     ,					max_frequency=_lowercase     ,					sampling_rate=_lowercase     ,					norm='slaney'     ,					mel_scale='slaney'     ,					)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	copy.deepcopy(self.__dict__				)
      __UpperCamelCase      =	self.__class__.__name__
      if "mel_filters" in output:
       del output["mel_filters"]
      if "mel_filters_slaney" in output:
       del output["mel_filters_slaney"]
      return output





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None				):

      '''simple docstring'''




      __UpperCamelCase      =	spectrogram(
          _lowercase     ,					window_function(self.fft_window_size     ,					'hann'				)     ,					frame_length=self.fft_window_size     ,					hop_length=self.hop_length     ,					power=2.0     ,					mel_filters=_lowercase     ,					log_mel='dB'     ,					)
      return log_mel_spectrogram.T





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				):

      '''simple docstring'''




      __UpperCamelCase      =	np.array_split(list(range(0     ,					total_frames - chunk_frames + 1				)				)     ,					3				)
      if len(ranges[1]				) == 0:
       # if the audio is too short, we just use the first chunk
       __UpperCamelCase      =	[0]
      if len(ranges[2]				) == 0:
       # if the audio is too short, we just use the first chunk
       __UpperCamelCase      =	[0]
      # randomly choose index for each part
      __UpperCamelCase      =	np.random.choice(ranges[0]				)
      __UpperCamelCase      =	np.random.choice(ranges[1]				)
      __UpperCamelCase      =	np.random.choice(ranges[2]				)

      __UpperCamelCase      =	mel[idx_front : idx_front + chunk_frames, :]
      __UpperCamelCase      =	mel[idx_middle : idx_middle + chunk_frames, :]
      __UpperCamelCase      =	mel[idx_back : idx_back + chunk_frames, :]

      __UpperCamelCase      =	torch.tensor(mel[None, None, :]				)
      __UpperCamelCase      =	torch.nn.functional.interpolate(
          _lowercase     ,					size=[chunk_frames, 64]     ,					mode='bilinear'     ,					align_corners=_lowercase				)
      __UpperCamelCase      =	mel_shrink[0][0].numpy()
      __UpperCamelCase      =	np.stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back]     ,					axis=0				)
      return mel_fusion





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				):

      '''simple docstring'''




      if waveform.shape[0] > max_length:
       if truncation == "rand_trunc":
        __UpperCamelCase      =	True
        # random crop to max_length (for compatibility) -> this should be handled by self.pad
        __UpperCamelCase      =	len(_lowercase				) - max_length
        __UpperCamelCase      =	np.random.randint(0     ,					overflow + 1				)
        __UpperCamelCase      =	waveform[idx : idx + max_length]
        __UpperCamelCase      =	self._np_extract_fbank_features(_lowercase     ,					self.mel_filters_slaney				)[None, :]
       elif truncation == "fusion":
        __UpperCamelCase      =	self._np_extract_fbank_features(_lowercase     ,					self.mel_filters				)
        __UpperCamelCase      =	max_length // self.hop_length + 1  # the +1 related to how the spectrogram is computed
        __UpperCamelCase      =	mel.shape[0]
        if chunk_frames == total_frames:
         # there is a corner case where the audio length is larger than max_length but smaller than max_length+hop_length.
         # In this case, we just use the whole audio.
         __UpperCamelCase      =	np.stack([mel, mel, mel, mel]     ,					axis=0				)
         __UpperCamelCase      =	False
        else:
         __UpperCamelCase      =	self._random_mel_fusion(_lowercase     ,					_lowercase     ,					_lowercase				)
         __UpperCamelCase      =	True
       else:
        raise NotImplementedError(F'data_truncating {truncation} not implemented'				)

      else:
       __UpperCamelCase      =	False
       # only use repeat as a new possible value for padding. you repeat the audio before applying the usual max_length padding
       if waveform.shape[0] < max_length:
        if padding == "repeat":
         __UpperCamelCase      =	int(max_length / len(_lowercase				)				)
         __UpperCamelCase      =	np.stack(np.tile(_lowercase     ,					n_repeat + 1				)				)[:max_length]
        if padding == "repeatpad":
         __UpperCamelCase      =	int(max_length / len(_lowercase				)				)
         __UpperCamelCase      =	np.stack(np.tile(_lowercase     ,					_lowercase				)				)
        __UpperCamelCase      =	np.pad(_lowercase     ,					(0, max_length - waveform.shape[0])     ,					mode='constant'     ,					constant_values=0				)

       if truncation == "fusion":
        __UpperCamelCase      =	self._np_extract_fbank_features(_lowercase     ,					self.mel_filters				)
        __UpperCamelCase      =	np.stack([input_mel, input_mel, input_mel, input_mel]     ,					axis=0				)
       else:
        __UpperCamelCase      =	self._np_extract_fbank_features(_lowercase     ,					self.mel_filters_slaney				)[None, :]

      return input_mel, longer







     def __call__(     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      __UpperCamelCase      =	truncation if truncation is not None else self.truncation
      __UpperCamelCase      =	padding if padding else self.padding

      if sampling_rate is not None:
       if sampling_rate != self.sampling_rate:
        raise ValueError(
            F'The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a'
            F' sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input'
            F' was sampled with {self.sampling_rate} and not {sampling_rate}.'				)
      else:
       logger.warning(
           'It is strongly recommended to pass the `sampling_rate` argument to this function. '
           'Failing to do so can result in silent errors that might be hard to debug.'				)

      __UpperCamelCase      =	isinstance(_lowercase     ,					np.ndarray				) and len(raw_speech.shape				) > 1
      if is_batched_numpy and len(raw_speech.shape				) > 2:
       raise ValueError(F'Only mono-channel audio is supported for input to {self}'				)
      __UpperCamelCase      =	is_batched_numpy or (
          isinstance(_lowercase     ,					(list, tuple)				) and (isinstance(raw_speech[0]     ,					(np.ndarray, tuple, list)				))
      )

      if is_batched:
       __UpperCamelCase      =	[np.asarray(_lowercase     ,					dtype=np.floataa				) for speech in raw_speech]
      elif not is_batched and not isinstance(_lowercase     ,					np.ndarray				):
       __UpperCamelCase      =	np.asarray(_lowercase     ,					dtype=np.floataa				)
      elif isinstance(_lowercase     ,					np.ndarray				) and raw_speech.dtype is np.dtype(np.floataa				):
       __UpperCamelCase      =	raw_speech.astype(np.floataa				)

      # always return batch
      if not is_batched:
       __UpperCamelCase      =	[np.asarray(_lowercase				)]

      # convert to mel spectrogram, truncate and pad if needed.
      __UpperCamelCase      =	[
          self._get_input_mel(_lowercase     ,					max_length if max_length else self.nb_max_samples     ,					_lowercase     ,					_lowercase				)
          for waveform in raw_speech
      ]

      __UpperCamelCase      =	[]
      __UpperCamelCase      =	[]
      for mel, longer in padded_inputs:
       input_mel.append(_lowercase				)
       is_longer.append(_lowercase				)

      if truncation == "fusion" and sum(_lowercase				) == 0:
       # if no audio is longer than 10s, then randomly select one audio to be longer
       __UpperCamelCase      =	np.random.randint(0     ,					len(_lowercase				)				)
       __UpperCamelCase      =	True

      if isinstance(input_mel[0]     ,					_lowercase				):
       __UpperCamelCase      =	[np.asarray(_lowercase     ,					dtype=np.floataa				) for feature in input_mel]

      # is_longer is a list of bool
      __UpperCamelCase      =	[[longer] for longer in is_longer]

      __UpperCamelCase      =	{'input_features': input_mel, 'is_longer': is_longer}
      __UpperCamelCase      =	BatchFeature(_lowercase				)

      if return_tensors is not None:
       __UpperCamelCase      =	input_features.convert_to_tensors(_lowercase				)

      return input_features





"""simple docstring"""

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCamelCase     :    List[Any]       =			logging.get_logger(__name__)

UpperCamelCase     :    List[str]       =			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase     :    List[Any]       =			{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase     :    List[Any]       =			{
    "Salesforce/codegen-350M-mono": 2_0_4_8,
}

class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):
     lowercase						       =						VOCAB_FILES_NAMES
     lowercase						       =						PRETRAINED_VOCAB_FILES_MAP
     lowercase						       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase						       =						["input_ids", "attention_mask"]
     lowercase						       =						CodeGenTokenizer





     def __init__(     self     ,					__UpperCAmelCase=None     ,					__UpperCAmelCase=None     ,					__UpperCAmelCase=None     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase="<|endoftext|>"     ,					__UpperCAmelCase=False     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      super().__init__(
          __UpperCAmelCase     ,					__UpperCAmelCase     ,					tokenizer_file=__UpperCAmelCase     ,					unk_token=__UpperCAmelCase     ,					bos_token=__UpperCAmelCase     ,					eos_token=__UpperCAmelCase     ,					add_prefix_space=__UpperCAmelCase     ,					**__UpperCAmelCase     ,					)

      if kwargs.pop('add_bos_token'     ,					__UpperCAmelCase				):
       __UpperCamelCase      =	kwargs.pop('name_or_path'     ,					''				)
       raise ValueError(
           'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
           'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
           F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
           F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
           'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
           ' so that the fast tokenizer works correctly.'				)

      __UpperCamelCase      =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
      if pre_tok_state.get('add_prefix_space'     ,					__UpperCAmelCase				) != add_prefix_space:
       __UpperCamelCase      =	getattr(__UpperCAmelCase     ,					pre_tok_state.pop('type'				)				)
       __UpperCamelCase      =	add_prefix_space
       __UpperCamelCase      =	pre_tok_class(**__UpperCAmelCase				)

      __UpperCamelCase      =	add_prefix_space





     def 						UpperCAmelCase (     self     ,					*__UpperCAmelCase     ,					**__UpperCAmelCase				):

      '''simple docstring'''




      __UpperCamelCase      =	kwargs.get('is_split_into_words'     ,					__UpperCAmelCase				)
      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__UpperCAmelCase     ,					**__UpperCAmelCase				)





     def 						UpperCAmelCase (     self     ,					*__UpperCAmelCase     ,					**__UpperCAmelCase				):

      '''simple docstring'''




      __UpperCamelCase      =	kwargs.get('is_split_into_words'     ,					__UpperCAmelCase				)

      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__UpperCAmelCase     ,					**__UpperCAmelCase				)





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None				):

      '''simple docstring'''




      __UpperCamelCase      =	self._tokenizer.model.save(__UpperCAmelCase     ,					name=__UpperCAmelCase				)
      return tuple(__UpperCAmelCase				)





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = False     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      __UpperCamelCase      =	super().decode(
          token_ids=__UpperCAmelCase     ,					skip_special_tokens=__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase     ,					**__UpperCAmelCase     ,					)

      if truncate_before_pattern is not None and len(__UpperCAmelCase				) > 0:
       __UpperCamelCase      =	self.truncate(__UpperCAmelCase     ,					__UpperCAmelCase				)

      return decoded_text







     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase				):

      '''simple docstring'''




      def find_re(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				):
       __UpperCamelCase      =	pattern.search(__UpperCAmelCase     ,					__UpperCAmelCase				)
       return m.start() if m else -1

      __UpperCamelCase      =	[re.compile(__UpperCAmelCase     ,					re.MULTILINE				) for pattern in truncate_before_pattern]

      __UpperCamelCase      =	list(re.finditer('^print'     ,					__UpperCAmelCase     ,					re.MULTILINE				)				)

      if len(__UpperCAmelCase				) > 1:
       __UpperCamelCase      =	completion[: prints[1].start()]

      __UpperCamelCase      =	list(re.finditer('^def'     ,					__UpperCAmelCase     ,					re.MULTILINE				)				)

      if len(__UpperCAmelCase				) > 1:
       __UpperCamelCase      =	completion[: defs[1].start()]

      __UpperCamelCase      =	0

      __UpperCamelCase      =	[
          pos for pos in [find_re(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				) for terminal in terminals] if pos != -1
      ]

      if len(__UpperCAmelCase				) > 0:
       return completion[: min(__UpperCAmelCase				)]
      else:
       return completion



