




'''simple docstring'''







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class     _a						(			_A							,							unittest.TestCase				):
       __a       :  Tuple            =      CodeGenTokenizer
       __a       :  Optional[int]            =      CodeGenTokenizerFast
       __a       :  int            =      True
       __a       :  List[Any]            =      {"""add_prefix_space""": True}
       __a       :  int            =      False
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           UpperCAmelCase     =	[
               "l",
               "o",
               "w",
               "e",
               "r",
               "s",
               "t",
               "i",
               "d",
               "n",
               "\u0120",
               "\u0120l",
               "\u0120n",
               "\u0120lo",
               "\u0120low",
               "er",
               "\u0120lowest",
               "\u0120newer",
               "\u0120wider",
               "<unk>",
               "<|endoftext|>",
           ]
           UpperCAmelCase     =	dict(zip(lowercase     ,       range(len(lowercase       )       )       )       )
           UpperCAmelCase     =	["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
           UpperCAmelCase     =	{"unk_token": "<unk>"}

           UpperCAmelCase     =	os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['''vocab_file''']       )
           UpperCAmelCase     =	os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['''merges_file''']       )
           with open(self.vocab_file     ,       '''w'''     ,       encoding='''utf-8'''       ) as fp:
               fp.write(json.dumps(lowercase       ) + '''\n'''       )
           with open(self.merges_file     ,       '''w'''     ,       encoding='''utf-8'''       ) as fp:
               fp.write('''\n'''.join(lowercase       )       )
       def 					A   (			self		:			List[str]     ,       **lowercase		:			Any       ):



           '''simple docstring'''

           kwargs.update(self.special_tokens_map       )
           return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       **lowercase       )
       def 					A   (			self		:			str     ,       **lowercase		:			Union[str, Any]       ):



           '''simple docstring'''

           kwargs.update(self.special_tokens_map       )
           return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,       **lowercase       )
       def 					A   (			self		:			Any     ,       lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	"lower newer"
           UpperCAmelCase     =	"lower newer"
           return input_text, output_text
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	CodeGenTokenizer(self.vocab_file     ,       self.merges_file     ,       **self.special_tokens_map       )
           UpperCAmelCase     =	"lower newer"
           UpperCAmelCase     =	["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
           UpperCAmelCase     =	tokenizer.tokenize(lowercase     ,       add_prefix_space=lowercase       )
           self.assertListEqual(lowercase     ,       lowercase       )

           UpperCAmelCase     =	tokens + [tokenizer.unk_token]
           UpperCAmelCase     =	[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase       )     ,       lowercase       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase     =	self.get_tokenizer()
           UpperCAmelCase     =	self.get_rust_tokenizer(add_prefix_space=lowercase       )

           UpperCAmelCase     =	"lower newer"

           # Testing tokenization
           UpperCAmelCase     =	tokenizer.tokenize(lowercase     ,       add_prefix_space=lowercase       )
           UpperCAmelCase     =	rust_tokenizer.tokenize(lowercase       )
           self.assertListEqual(lowercase     ,       lowercase       )

           # Testing conversion to ids without special tokens
           UpperCAmelCase     =	tokenizer.encode(lowercase     ,       add_special_tokens=lowercase     ,       add_prefix_space=lowercase       )
           UpperCAmelCase     =	rust_tokenizer.encode(lowercase     ,       add_special_tokens=lowercase       )
           self.assertListEqual(lowercase     ,       lowercase       )

           # Testing conversion to ids with special tokens
           UpperCAmelCase     =	self.get_rust_tokenizer(add_prefix_space=lowercase       )
           UpperCAmelCase     =	tokenizer.encode(lowercase     ,       add_prefix_space=lowercase       )
           UpperCAmelCase     =	rust_tokenizer.encode(lowercase       )
           self.assertListEqual(lowercase     ,       lowercase       )

           # Testing the unknown token
           UpperCAmelCase     =	tokens + [rust_tokenizer.unk_token]
           UpperCAmelCase     =	[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase       )     ,       lowercase       )
       def 					A   (			self		:			Any     ,       *lowercase		:			Dict     ,       **lowercase		:			List[str]       ):



           '''simple docstring'''

           pass
       def 					A   (			self		:			Tuple     ,       lowercase		:			List[str]=15       ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                   UpperCAmelCase     =	self.rust_tokenizer_class.from_pretrained(lowercase     ,       **lowercase       )

                   # Simple input
                   UpperCAmelCase     =	"This is a simple input"
                   UpperCAmelCase     =	["This is a simple input 1", "This is a simple input 2"]
                   UpperCAmelCase     =	("This is a simple input", "This is a pair")
                   UpperCAmelCase     =	[
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(lowercase     ,       tokenizer_r.encode     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''       )

                   # Simple input
                   self.assertRaises(lowercase     ,       tokenizer_r.encode_plus     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''       )

                   # Simple input
                   self.assertRaises(
                       lowercase     ,       tokenizer_r.batch_encode_plus     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''     ,       )

                   # Pair input
                   self.assertRaises(lowercase     ,       tokenizer_r.encode     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''       )

                   # Pair input
                   self.assertRaises(lowercase     ,       tokenizer_r.encode_plus     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''       )

                   # Pair input
                   self.assertRaises(
                       lowercase     ,       tokenizer_r.batch_encode_plus     ,       lowercase     ,       max_length=lowercase     ,       padding='''max_length'''     ,       )
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           UpperCAmelCase     =	CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       pad_token='''<pad>'''       )

           # Simple input
           UpperCAmelCase     =	"This is a simple input"
           UpperCAmelCase     =	["This is a simple input looooooooong", "This is a simple input"]
           UpperCAmelCase     =	("This is a simple input", "This is a pair")
           UpperCAmelCase     =	[
               ("This is a simple input loooooong", "This is a simple input"),
               ("This is a simple pair loooooong", "This is a simple pair"),
           ]

           UpperCAmelCase     =	tokenizer.pad_token_id

           UpperCAmelCase     =	tokenizer(lowercase     ,       padding='''max_length'''     ,       max_length=30     ,       return_tensors='''np'''       )
           UpperCAmelCase     =	tokenizer(lowercase     ,       padding=lowercase     ,       truncate=lowercase     ,       return_tensors='''np'''       )
           UpperCAmelCase     =	tokenizer(*lowercase     ,       padding='''max_length'''     ,       max_length=60     ,       return_tensors='''np'''       )
           UpperCAmelCase     =	tokenizer(lowercase     ,       padding=lowercase     ,       truncate=lowercase     ,       return_tensors='''np'''       )

           # s
           # test single string max_length padding
           self.assertEqual(out_s['''input_ids'''].shape[-1]     ,       30       )
           self.assertTrue(pad_token_id in out_s['''input_ids''']       )
           self.assertTrue(0 in out_s['''attention_mask''']       )

           # s2
           # test automatic padding
           self.assertEqual(out_sa['''input_ids'''].shape[-1]     ,       33       )
           # long slice doesn't have padding
           self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]       )
           self.assertFalse(0 in out_sa['''attention_mask'''][0]       )
           # short slice does have padding
           self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]       )
           self.assertTrue(0 in out_sa['''attention_mask'''][1]       )

           # p
           # test single pair max_length padding
           self.assertEqual(out_p['''input_ids'''].shape[-1]     ,       60       )
           self.assertTrue(pad_token_id in out_p['''input_ids''']       )
           self.assertTrue(0 in out_p['''attention_mask''']       )

           # p2
           # test automatic padding pair
           self.assertEqual(out_pa['''input_ids'''].shape[-1]     ,       52       )
           # long slice pair doesn't have padding
           self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]       )
           self.assertFalse(0 in out_pa['''attention_mask'''][0]       )
           # short slice pair does have padding
           self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]       )
           self.assertTrue(0 in out_pa['''attention_mask'''][1]       )
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           UpperCAmelCase     =	"$$$"
           UpperCAmelCase     =	CodeGenTokenizer.from_pretrained(self.tmpdirname     ,       bos_token=lowercase     ,       add_bos_token=lowercase       )

           UpperCAmelCase     =	"This is a simple input"
           UpperCAmelCase     =	["This is a simple input 1", "This is a simple input 2"]

           UpperCAmelCase     =	tokenizer.bos_token_id

           UpperCAmelCase     =	tokenizer(lowercase       )
           UpperCAmelCase     =	tokenizer(lowercase       )

           self.assertEqual(out_s.input_ids[0]     ,       lowercase       )
           self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

           UpperCAmelCase     =	tokenizer.decode(out_s.input_ids       )
           UpperCAmelCase     =	tokenizer.batch_decode(out_sa.input_ids       )

           self.assertEqual(decode_s.split()[0]     ,       lowercase       )
           self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )
       @slow
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''       )

           UpperCAmelCase     =	"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
           UpperCAmelCase     =	"\nif len_a > len_b:      result = a\nelse:      result = b"

           UpperCAmelCase     =	tokenizer.encode(lowercase       )
           UpperCAmelCase     =	["^#", re.escape('''<|endoftext|>'''       ), "^'''", "^\"\"\"", "\n\n\n"]
           UpperCAmelCase     =	tokenizer.decode(lowercase     ,       truncate_before_pattern=lowercase       )
           self.assertEqual(lowercase     ,       lowercase       )







       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           pass










'''simple docstring'''







import math
from typing import Optional

import numpy as np

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A									=logging.get_logger(__name__)

A									={
    'facebook/encodec_24khz': 'https://huggingface.co/facebook/encodec_24khz/resolve/main/config.json',
    'facebook/encodec_48khz': 'https://huggingface.co/facebook/encodec_48khz/resolve/main/config.json',
}


class     _a						(			__a				):
       __a       :  Union[str, Any]            =      """encodec"""
       def __init__(			self		:			Optional[int]     ,       lowercase		:			Union[str, Any]=[1.5, 3.0, 6.0, 12.0, 24.0]     ,       lowercase		:			str=24_000     ,       lowercase		:			int=1     ,       lowercase		:			int=False     ,       lowercase		:			List[str]=None     ,       lowercase		:			Tuple=None     ,       lowercase		:			int=128     ,       lowercase		:			List[Any]=32     ,       lowercase		:			Optional[int]=1     ,       lowercase		:			Optional[int]=[8, 5, 4, 2]     ,       lowercase		:			Union[str, Any]="weight_norm"     ,       lowercase		:			Optional[int]=7     ,       lowercase		:			Tuple=7     ,       lowercase		:			Tuple=3     ,       lowercase		:			Union[str, Any]=2     ,       lowercase		:			Optional[Any]=True     ,       lowercase		:			Tuple="reflect"     ,       lowercase		:			Any=2     ,       lowercase		:			List[str]=2     ,       lowercase		:			str=1.0     ,       lowercase		:			List[str]=1_024     ,       lowercase		:			Optional[Any]=None     ,       lowercase		:			Any=True     ,       **lowercase		:			Optional[Any]     ,       ):



           '''simple docstring'''

           UpperCAmelCase     =	target_bandwidths
           UpperCAmelCase     =	sampling_rate
           UpperCAmelCase     =	audio_channels
           UpperCAmelCase     =	normalize
           UpperCAmelCase     =	chunk_length_s
           UpperCAmelCase     =	overlap
           UpperCAmelCase     =	hidden_size
           UpperCAmelCase     =	num_filters
           UpperCAmelCase     =	num_residual_layers
           UpperCAmelCase     =	upsampling_ratios
           UpperCAmelCase     =	norm_type
           UpperCAmelCase     =	kernel_size
           UpperCAmelCase     =	last_kernel_size
           UpperCAmelCase     =	residual_kernel_size
           UpperCAmelCase     =	dilation_growth_rate
           UpperCAmelCase     =	use_causal_conv
           UpperCAmelCase     =	pad_mode
           UpperCAmelCase     =	compress
           UpperCAmelCase     =	num_lstm_layers
           UpperCAmelCase     =	trim_right_ratio
           UpperCAmelCase     =	codebook_size
           UpperCAmelCase     =	codebook_dim if codebook_dim is not None else hidden_size
           UpperCAmelCase     =	use_conv_shortcut

           if self.norm_type not in ["weight_norm", "time_group_norm"]:
               raise ValueError(
                   f"self.norm_type must be one of `\"weight_norm\"`, `\"time_group_norm\"`), got {self.norm_type}"       )

           super().__init__(**lowercase       )
       @property
       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           if self.chunk_length_s is None:
               return None
           else:
               return int(self.chunk_length_s * self.sampling_rate       )
       @property
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           if self.chunk_length_s is None or self.overlap is None:
               return None
           else:
               return max(1     ,       int((1.0 - self.overlap) * self.chunk_length       )       )
       @property
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	np.prod(self.upsampling_ratios       )
           return math.ceil(self.sampling_rate / hop_length       )







       @property
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           return int(1_000 * self.target_bandwidths[-1] // (self.frame_rate * 10)       )




