




'''simple docstring'''







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast


class     _a						(			unittest.TestCase				):
       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase     =	mock.Mock()
           UpperCAmelCase     =	500
           UpperCAmelCase     =	{}
           UpperCAmelCase     =	HTTPError
           UpperCAmelCase     =	{}

           # Download this model to make sure it's in the cache.
           UpperCAmelCase     =	BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''       )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch('''requests.Session.request'''     ,       return_value=lowercase       ) as mock_head:
               UpperCAmelCase     =	BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''       )
               # This check we did call the fake head request
               mock_head.assert_called()
       @require_tokenizers
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	mock.Mock()
           UpperCAmelCase     =	500
           UpperCAmelCase     =	{}
           UpperCAmelCase     =	HTTPError
           UpperCAmelCase     =	{}

           # Download this model to make sure it's in the cache.
           UpperCAmelCase     =	GPTaTokenizerFast.from_pretrained('''gpt2'''       )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch('''requests.Session.request'''     ,       return_value=lowercase       ) as mock_head:
               UpperCAmelCase     =	GPTaTokenizerFast.from_pretrained('''gpt2'''       )
               # This check we did call the fake head request
               mock_head.assert_called()
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           try:
               UpperCAmelCase     =	tempfile.mktemp()
               with open(lowercase     ,       '''wb'''       ) as f:
                   http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''     ,       lowercase       )

               UpperCAmelCase     =	AlbertTokenizer.from_pretrained(lowercase       )
           finally:
               os.remove(lowercase       )

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile('''tokenizer.json'''       ):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
           try:
               with open('''tokenizer.json'''     ,       '''wb'''       ) as f:
                   http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''     ,       lowercase       )
               UpperCAmelCase     =	AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''       )
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size     ,       1_000       )
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
               os.remove('''tokenizer.json'''       )







       def 					A   (			self		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''       )






@is_staging_test
class     _a						(			unittest.TestCase				):
       __a       :  Tuple            =      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]
       @classmethod
       def 					A   (			cls		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	TOKEN
           HfFolder.save_token(lowercase       )
       @classmethod
       def 					A   (			cls		:			List[Any]       ):



           '''simple docstring'''

           try:
               delete_repo(token=cls._token     ,       repo_id='''test-tokenizer'''       )
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token     ,       repo_id='''valid_org/test-tokenizer-org'''       )
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token     ,       repo_id='''test-dynamic-tokenizer'''       )
           except HTTPError:
               pass
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               UpperCAmelCase     =	os.path.join(lowercase     ,       '''vocab.txt'''       )
               with open(lowercase     ,       '''w'''     ,       encoding='''utf-8'''       ) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
               UpperCAmelCase     =	BertTokenizer(lowercase       )

           tokenizer.push_to_hub('''test-tokenizer'''     ,       use_auth_token=self._token       )
           UpperCAmelCase     =	BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"       )
           self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab       )

           # Reset repo
           delete_repo(token=self._token     ,       repo_id='''test-tokenizer'''       )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(lowercase     ,       repo_id='''test-tokenizer'''     ,       push_to_hub=lowercase     ,       use_auth_token=self._token       )

           UpperCAmelCase     =	BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"       )
           self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab       )
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               UpperCAmelCase     =	os.path.join(lowercase     ,       '''vocab.txt'''       )
               with open(lowercase     ,       '''w'''     ,       encoding='''utf-8'''       ) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
               UpperCAmelCase     =	BertTokenizer(lowercase       )

           tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''     ,       use_auth_token=self._token       )
           UpperCAmelCase     =	BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''       )
           self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab       )

           # Reset repo
           delete_repo(token=self._token     ,       repo_id='''valid_org/test-tokenizer-org'''       )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   lowercase     ,       repo_id='''valid_org/test-tokenizer-org'''     ,       push_to_hub=lowercase     ,       use_auth_token=self._token       )

           UpperCAmelCase     =	BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''       )
           self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab       )







       @require_tokenizers
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               UpperCAmelCase     =	os.path.join(lowercase     ,       '''vocab.txt'''       )
               with open(lowercase     ,       '''w'''     ,       encoding='''utf-8'''       ) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
               UpperCAmelCase     =	CustomTokenizer(lowercase       )

           # No fast custom tokenizer
           tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,       use_auth_token=self._token       )

           UpperCAmelCase     =	AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,       trust_remote_code=lowercase       )
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,       '''CustomTokenizer'''       )

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               UpperCAmelCase     =	os.path.join(lowercase     ,       '''vocab.txt'''       )
               with open(lowercase     ,       '''w'''     ,       encoding='''utf-8'''       ) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )

               UpperCAmelCase     =	BertTokenizerFast.from_pretrained(lowercase       )
               bert_tokenizer.save_pretrained(lowercase       )
               UpperCAmelCase     =	CustomTokenizerFast.from_pretrained(lowercase       )

           tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,       use_auth_token=self._token       )

           UpperCAmelCase     =	AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,       trust_remote_code=lowercase       )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,       '''CustomTokenizerFast'''       )
           UpperCAmelCase     =	AutoTokenizer.from_pretrained(
               f"{USER}/test-dynamic-tokenizer"     ,       use_fast=lowercase     ,       trust_remote_code=lowercase       )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,       '''CustomTokenizer'''       )






class     _a						(			unittest.TestCase				):
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''Hello 友達'''       )
           self.assertEqual(trie.data     ,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}       )
           trie.add('''Hello'''       )
           trie.data
           self.assertEqual(trie.data     ,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}       )
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''       )     ,       ['''[CLS] This is a extra_id_100''']       )
           trie.add('''[CLS]'''       )
           trie.add('''extra_id_1'''       )
           trie.add('''extra_id_100'''       )
           self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''       )     ,       ['''[CLS]''', ''' This is a ''', '''extra_id_100''']       )
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''A'''       )
           self.assertEqual(trie.split('''ABC'''       )     ,       ['''A''', '''BC''']       )
           self.assertEqual(trie.split('''BCA'''       )     ,       ['''BC''', '''A''']       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''TOKEN]'''       )
           trie.add('''[SPECIAL_TOKEN]'''       )
           self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''       )     ,       ['''This is something ''', '''[SPECIAL_TOKEN]''']       )
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''A'''       )
           trie.add('''P'''       )
           trie.add('''[SPECIAL_TOKEN]'''       )
           self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''       )     ,       ['''This is something ''', '''[SPECIAL_TOKEN]''']       )
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''AB'''       )
           trie.add('''B'''       )
           trie.add('''C'''       )
           self.assertEqual(trie.split('''ABC'''       )     ,       ['''AB''', '''C''']       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           trie.add('''ABC'''       )
           trie.add('''B'''       )
           trie.add('''CD'''       )
           self.assertEqual(trie.split('''ABCD'''       )     ,       ['''ABC''', '''D''']       )







       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	Trie()
           UpperCAmelCase     =	trie.cut_text('''ABC'''     ,       [0, 0, 2, 1, 2, 3]       )
           self.assertEqual(lowercase     ,       ['''AB''', '''C''']       )










'''simple docstring'''







import math
from typing import Optional

import numpy as np

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A									=logging.get_logger(__name__)

A									={
    'facebook/encodec_24khz': 'https://huggingface.co/facebook/encodec_24khz/resolve/main/config.json',
    'facebook/encodec_48khz': 'https://huggingface.co/facebook/encodec_48khz/resolve/main/config.json',
}


class     _a						(			__a				):
       __a       :  Union[str, Any]            =      """encodec"""
       def __init__(			self		:			Optional[int]     ,       lowercase		:			Union[str, Any]=[1.5, 3.0, 6.0, 12.0, 24.0]     ,       lowercase		:			str=24_000     ,       lowercase		:			int=1     ,       lowercase		:			int=False     ,       lowercase		:			List[str]=None     ,       lowercase		:			Tuple=None     ,       lowercase		:			int=128     ,       lowercase		:			List[Any]=32     ,       lowercase		:			Optional[int]=1     ,       lowercase		:			Optional[int]=[8, 5, 4, 2]     ,       lowercase		:			Union[str, Any]="weight_norm"     ,       lowercase		:			Optional[int]=7     ,       lowercase		:			Tuple=7     ,       lowercase		:			Tuple=3     ,       lowercase		:			Union[str, Any]=2     ,       lowercase		:			Optional[Any]=True     ,       lowercase		:			Tuple="reflect"     ,       lowercase		:			Any=2     ,       lowercase		:			List[str]=2     ,       lowercase		:			str=1.0     ,       lowercase		:			List[str]=1_024     ,       lowercase		:			Optional[Any]=None     ,       lowercase		:			Any=True     ,       **lowercase		:			Optional[Any]     ,       ):



           '''simple docstring'''

           UpperCAmelCase     =	target_bandwidths
           UpperCAmelCase     =	sampling_rate
           UpperCAmelCase     =	audio_channels
           UpperCAmelCase     =	normalize
           UpperCAmelCase     =	chunk_length_s
           UpperCAmelCase     =	overlap
           UpperCAmelCase     =	hidden_size
           UpperCAmelCase     =	num_filters
           UpperCAmelCase     =	num_residual_layers
           UpperCAmelCase     =	upsampling_ratios
           UpperCAmelCase     =	norm_type
           UpperCAmelCase     =	kernel_size
           UpperCAmelCase     =	last_kernel_size
           UpperCAmelCase     =	residual_kernel_size
           UpperCAmelCase     =	dilation_growth_rate
           UpperCAmelCase     =	use_causal_conv
           UpperCAmelCase     =	pad_mode
           UpperCAmelCase     =	compress
           UpperCAmelCase     =	num_lstm_layers
           UpperCAmelCase     =	trim_right_ratio
           UpperCAmelCase     =	codebook_size
           UpperCAmelCase     =	codebook_dim if codebook_dim is not None else hidden_size
           UpperCAmelCase     =	use_conv_shortcut

           if self.norm_type not in ["weight_norm", "time_group_norm"]:
               raise ValueError(
                   f"self.norm_type must be one of `\"weight_norm\"`, `\"time_group_norm\"`), got {self.norm_type}"       )

           super().__init__(**lowercase       )
       @property
       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           if self.chunk_length_s is None:
               return None
           else:
               return int(self.chunk_length_s * self.sampling_rate       )
       @property
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           if self.chunk_length_s is None or self.overlap is None:
               return None
           else:
               return max(1     ,       int((1.0 - self.overlap) * self.chunk_length       )       )
       @property
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	np.prod(self.upsampling_ratios       )
           return math.ceil(self.sampling_rate / hop_length       )







       @property
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           return int(1_000 * self.target_bandwidths[-1] // (self.frame_rate * 10)       )




