



"""simple docstring"""





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class  lowercase__  (       SCREAMING_SNAKE_CASE							,   unittest.TestCase ):







       '''simple docstring'''


       UpperCamelCase		=	CodeGenTokenizer
       UpperCamelCase		=	CodeGenTokenizerFast
       UpperCamelCase		=	True
       UpperCamelCase		=	{'''add_prefix_space''': True}
       UpperCamelCase		=	False



       def 	lowercase__							(				self     :    int       )					->		Any:


             '''simple docstring'''




             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             UpperCAmelCase_     					=				[
                 "l",
                 "o",
                 "w",
                 "e",
                 "r",
                 "s",
                 "t",
                 "i",
                 "d",
                 "n",
                 "\u0120",
                 "\u0120l",
                 "\u0120n",
                 "\u0120lo",
                 "\u0120low",
                 "er",
                 "\u0120lowest",
                 "\u0120newer",
                 "\u0120wider",
                 "<unk>",
                 "<|endoftext|>",
             ]
             UpperCAmelCase_     					=				dict(zip(_UpperCAmelCase						,							range(len(_UpperCAmelCase       )       )       )       )
             UpperCAmelCase_     					=				["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
             UpperCAmelCase_     					=				{"unk_token": "<unk>"}

             UpperCAmelCase_     					=				os.path.join(self.tmpdirname						,							VOCAB_FILES_NAMES["vocab_file"]       )
             UpperCAmelCase_     					=				os.path.join(self.tmpdirname						,							VOCAB_FILES_NAMES["merges_file"]       )
             with open(self.vocab_file						,							"w"						,							encoding="utf-8"       ) as fp:
                   fp.write(json.dumps(_UpperCAmelCase       ) + "\n"       )
             with open(self.merges_file						,							"w"						,							encoding="utf-8"       ) as fp:
                   fp.write("\n".join(_UpperCAmelCase       )       )



       def 	lowercase__							(				self     :    List[str]						,							**_UpperCAmelCase     :    Dict       )					->		List[Any]:


             '''simple docstring'''




             kwargs.update(self.special_tokens_map       )
             return CodeGenTokenizer.from_pretrained(self.tmpdirname						,							**_UpperCAmelCase       )



       def 	lowercase__							(				self     :    Any						,							**_UpperCAmelCase     :    int       )					->		int:


             '''simple docstring'''




             kwargs.update(self.special_tokens_map       )
             return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,							**_UpperCAmelCase       )



       def 	lowercase__							(				self     :    Dict						,							_UpperCAmelCase     :    Optional[int]       )					->		Dict:


             '''simple docstring'''




             UpperCAmelCase_     					=				"lower newer"
             UpperCAmelCase_     					=				"lower newer"
             return input_text, output_text



       def 	lowercase__							(				self     :    Union[str, Any]       )					->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_     					=				CodeGenTokenizer(self.vocab_file						,							self.merges_file						,							**self.special_tokens_map       )
             UpperCAmelCase_     					=				"lower newer"
             UpperCAmelCase_     					=				["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
             UpperCAmelCase_     					=				tokenizer.tokenize(_UpperCAmelCase						,							add_prefix_space=_UpperCAmelCase       )
             self.assertListEqual(_UpperCAmelCase						,							_UpperCAmelCase       )

             UpperCAmelCase_     					=				tokens + [tokenizer.unk_token]
             UpperCAmelCase_     					=				[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )						,							_UpperCAmelCase       )



       def 	lowercase__							(				self     :    List[Any]       )					->		List[str]:


             '''simple docstring'''




             if not self.test_rust_tokenizer:
                   return

             UpperCAmelCase_     					=				self.get_tokenizer()
             UpperCAmelCase_     					=				self.get_rust_tokenizer(add_prefix_space=_UpperCAmelCase       )

             UpperCAmelCase_     					=				"lower newer"

             # Testing tokenization
             UpperCAmelCase_     					=				tokenizer.tokenize(_UpperCAmelCase						,							add_prefix_space=_UpperCAmelCase       )
             UpperCAmelCase_     					=				rust_tokenizer.tokenize(_UpperCAmelCase       )
             self.assertListEqual(_UpperCAmelCase						,							_UpperCAmelCase       )

             # Testing conversion to ids without special tokens
             UpperCAmelCase_     					=				tokenizer.encode(_UpperCAmelCase						,							add_special_tokens=_UpperCAmelCase						,							add_prefix_space=_UpperCAmelCase       )
             UpperCAmelCase_     					=				rust_tokenizer.encode(_UpperCAmelCase						,							add_special_tokens=_UpperCAmelCase       )
             self.assertListEqual(_UpperCAmelCase						,							_UpperCAmelCase       )

             # Testing conversion to ids with special tokens
             UpperCAmelCase_     					=				self.get_rust_tokenizer(add_prefix_space=_UpperCAmelCase       )
             UpperCAmelCase_     					=				tokenizer.encode(_UpperCAmelCase						,							add_prefix_space=_UpperCAmelCase       )
             UpperCAmelCase_     					=				rust_tokenizer.encode(_UpperCAmelCase       )
             self.assertListEqual(_UpperCAmelCase						,							_UpperCAmelCase       )

             # Testing the unknown token
             UpperCAmelCase_     					=				tokens + [rust_tokenizer.unk_token]
             UpperCAmelCase_     					=				[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )						,							_UpperCAmelCase       )



       def 	lowercase__							(				self     :    str						,							*_UpperCAmelCase     :    Union[str, Any]						,							**_UpperCAmelCase     :    Union[str, Any]       )					->		Dict:


             '''simple docstring'''




             pass



       def 	lowercase__							(				self     :    Optional[Any]						,							_UpperCAmelCase     :    Any=15       )					->		Union[str, Any]:


             '''simple docstring'''




             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                         UpperCAmelCase_     					=				self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase						,							**_UpperCAmelCase       )

                         # Simple input
                         UpperCAmelCase_     					=				"This is a simple input"
                         UpperCAmelCase_     					=				["This is a simple input 1", "This is a simple input 2"]
                         UpperCAmelCase_     					=				("This is a simple input", "This is a pair")
                         UpperCAmelCase_     					=				[
                             ("This is a simple input 1", "This is a simple input 2"),
                             ("This is a simple pair 1", "This is a simple pair 2"),
                         ]

                         # Simple input tests
                         self.assertRaises(_UpperCAmelCase						,							tokenizer_r.encode						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"       )

                         # Simple input
                         self.assertRaises(_UpperCAmelCase						,							tokenizer_r.encode_plus						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"       )

                         # Simple input
                         self.assertRaises(
                             _UpperCAmelCase						,							tokenizer_r.batch_encode_plus						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"						,							)

                         # Pair input
                         self.assertRaises(_UpperCAmelCase						,							tokenizer_r.encode						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"       )

                         # Pair input
                         self.assertRaises(_UpperCAmelCase						,							tokenizer_r.encode_plus						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"       )

                         # Pair input
                         self.assertRaises(
                             _UpperCAmelCase						,							tokenizer_r.batch_encode_plus						,							_UpperCAmelCase						,							max_length=_UpperCAmelCase						,							padding="max_length"						,							)



       def 	lowercase__							(				self     :    Tuple       )					->		Any:


             '''simple docstring'''




             UpperCAmelCase_     					=				CodeGenTokenizer.from_pretrained(self.tmpdirname						,							pad_token="<pad>"       )

             # Simple input
             UpperCAmelCase_     					=				"This is a simple input"
             UpperCAmelCase_     					=				["This is a simple input looooooooong", "This is a simple input"]
             UpperCAmelCase_     					=				("This is a simple input", "This is a pair")
             UpperCAmelCase_     					=				[
                 ("This is a simple input loooooong", "This is a simple input"),
                 ("This is a simple pair loooooong", "This is a simple pair"),
             ]

             UpperCAmelCase_     					=				tokenizer.pad_token_id

             UpperCAmelCase_     					=				tokenizer(_UpperCAmelCase						,							padding="max_length"						,							max_length=30						,							return_tensors="np"       )
             UpperCAmelCase_     					=				tokenizer(_UpperCAmelCase						,							padding=_UpperCAmelCase						,							truncate=_UpperCAmelCase						,							return_tensors="np"       )
             UpperCAmelCase_     					=				tokenizer(*_UpperCAmelCase						,							padding="max_length"						,							max_length=60						,							return_tensors="np"       )
             UpperCAmelCase_     					=				tokenizer(_UpperCAmelCase						,							padding=_UpperCAmelCase						,							truncate=_UpperCAmelCase						,							return_tensors="np"       )

             # s
             # test single string max_length padding
             self.assertEqual(out_s["input_ids"].shape[-1]						,							30       )
             self.assertTrue(pad_token_id in out_s["input_ids"]       )
             self.assertTrue(0 in out_s["attention_mask"]       )

             # s2
             # test automatic padding
             self.assertEqual(out_sa["input_ids"].shape[-1]						,							33       )
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa["input_ids"][0]       )
             self.assertFalse(0 in out_sa["attention_mask"][0]       )
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa["input_ids"][1]       )
             self.assertTrue(0 in out_sa["attention_mask"][1]       )

             # p
             # test single pair max_length padding
             self.assertEqual(out_p["input_ids"].shape[-1]						,							60       )
             self.assertTrue(pad_token_id in out_p["input_ids"]       )
             self.assertTrue(0 in out_p["attention_mask"]       )

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa["input_ids"].shape[-1]						,							52       )
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa["input_ids"][0]       )
             self.assertFalse(0 in out_pa["attention_mask"][0]       )
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa["input_ids"][1]       )
             self.assertTrue(0 in out_pa["attention_mask"][1]       )



       def 	lowercase__							(				self     :    int       )					->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				"$$$"
             UpperCAmelCase_     					=				CodeGenTokenizer.from_pretrained(self.tmpdirname						,							bos_token=_UpperCAmelCase						,							add_bos_token=_UpperCAmelCase       )

             UpperCAmelCase_     					=				"This is a simple input"
             UpperCAmelCase_     					=				["This is a simple input 1", "This is a simple input 2"]

             UpperCAmelCase_     					=				tokenizer.bos_token_id

             UpperCAmelCase_     					=				tokenizer(_UpperCAmelCase       )
             UpperCAmelCase_     					=				tokenizer(_UpperCAmelCase       )

             self.assertEqual(out_s.input_ids[0]						,							_UpperCAmelCase       )
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

             UpperCAmelCase_     					=				tokenizer.decode(out_s.input_ids       )
             UpperCAmelCase_     					=				tokenizer.batch_decode(out_sa.input_ids       )

             self.assertEqual(decode_s.split()[0]						,							_UpperCAmelCase       )
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )



       @slow
       def 	lowercase__							(				self     :    List[str]       )					->		int:


             '''simple docstring'''




             UpperCAmelCase_     					=				CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"       )

             UpperCAmelCase_     					=				"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
             UpperCAmelCase_     					=				"\nif len_a > len_b:      result = a\nelse:      result = b"

             UpperCAmelCase_     					=				tokenizer.encode(_UpperCAmelCase       )
             UpperCAmelCase_     					=				["^#", re.escape("<|endoftext|>"       ), "^'''", "^\"\"\"", "\n\n\n"]
             UpperCAmelCase_     					=				tokenizer.decode(_UpperCAmelCase						,							truncate_before_pattern=_UpperCAmelCase       )
             self.assertEqual(_UpperCAmelCase						,							_UpperCAmelCase       )







       def 	lowercase__							(				self     :    Optional[Any]       )					->		List[str]:


             '''simple docstring'''




             pass












"""simple docstring"""



import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


_lowerCAmelCase	:	Union[str, Any]				=      threading.Lock()
_lowerCAmelCase	:	Optional[logging.Handler]				=      None

_lowerCAmelCase	:	Union[str, Any]				=      {
    """debug""": logging.DEBUG,
    """info""": logging.INFO,
    """warning""": logging.WARNING,
    """error""": logging.ERROR,
    """critical""": logging.CRITICAL,
}

_lowerCAmelCase	:	Dict				=      logging.WARNING

_lowerCAmelCase	:	Optional[Any]				=      True


def 	SCREAMING_SNAKE_CASE__       (  )->		List[str]:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							os.getenv("TRANSFORMERS_VERBOSITY"			,					snake_case				)
     if env_level_str:
          if env_level_str in log_levels:
               return log_levels[env_level_str]
          else:
               logging.getLogger().warning(
                   f'Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '
                   f'has to be one of: { ", ".join(log_levels.keys()				) }'				)
     return _default_log_level


def 	SCREAMING_SNAKE_CASE__       (  )->		str:






     '''simple docstring'''



     return __name__.split("."				)[0]


def 	SCREAMING_SNAKE_CASE__       (  )->		logging.Logger:






     '''simple docstring'''



     return logging.getLogger(_get_library_name()				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     global _default_handler

     with _lock:
          if _default_handler:
               # This library has already configured the library root logger.
               return
          UpperCAmelCase__						:    Any							=							logging.StreamHandler()  # Set sys.stderr as stream.
          UpperCAmelCase__						:    Union[str, Any]							=							sys.stderr.flush

          # Apply our default configuration to the library root logger.
          UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger()
          library_root_logger.addHandler(_default_handler				)
          library_root_logger.setLevel(_get_default_logging_level()				)
          UpperCAmelCase__						:    int							=							False


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     global _default_handler

     with _lock:
          if not _default_handler:
               return

          UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger()
          library_root_logger.removeHandler(_default_handler				)
          library_root_logger.setLevel(logging.NOTSET				)
          UpperCAmelCase__						:    Optional[int]							=							None


def 	SCREAMING_SNAKE_CASE__       (  )->		Optional[int]:






     '''simple docstring'''



     return log_levels


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : Optional[str] = None				)->		logging.Logger:






     '''simple docstring'''



     if name is None:
          UpperCAmelCase__						:    Tuple							=							_get_library_name()

     _configure_library_root_logger()
     return logging.getLogger(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		int:






     '''simple docstring'''



     _configure_library_root_logger()
     return _get_library_root_logger().getEffectiveLevel()


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : int				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     _get_library_root_logger().setLevel(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		Optional[Any]:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		List[str]:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		Tuple:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		str:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert _default_handler is not None
     _get_library_root_logger().removeHandler(_default_handler				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert _default_handler is not None
     _get_library_root_logger().addHandler(_default_handler				)


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : logging.Handler				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert handler is not None
     _get_library_root_logger().addHandler(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : logging.Handler				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert handler is not None and handler not in _get_library_root_logger().handlers
     _get_library_root_logger().removeHandler(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     UpperCAmelCase__						:    Dict							=							False


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     UpperCAmelCase__						:    List[Any]							=							True


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger().handlers

     for handler in handlers:
          UpperCAmelCase__						:    Union[str, Any]							=							logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"				)
          handler.setFormatter(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     UpperCAmelCase__						:    Any							=							_get_library_root_logger().handlers

     for handler in handlers:
          handler.setFormatter(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  self       : Optional[int]			,					*snake_case       : List[str]			,					**snake_case       : str				)->		Union[str, Any]:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS"			,					snake_case				)
     if no_advisory_warnings:
          return
     self.warning(*snake_case			,					**snake_case				)


_lowerCAmelCase	:	int				=      warning_advice


@functools.lru_cache(snake_case				)
def 	SCREAMING_SNAKE_CASE__       (  self       : Optional[Any]			,					*snake_case       : Dict			,					**snake_case       : Any				)->		Any:






     '''simple docstring'''



     self.warning(*snake_case			,					**snake_case				)


_lowerCAmelCase	:	Tuple				=      warning_once




class 			lowerCAmelCase__			:
   def __init__(		self      :	List[str]							,			*snake_case__      :	Any							,			**snake_case__      :	List[str]							):  # pylint: disable=unused-argument


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							args[0] if args else None
   def __iter__(		self      :	Any							):


        '''simple docstring'''
        return iter(self._iterator							)
   def __getattr__(		self      :	List[Any]							,			snake_case__      :	Tuple							):


        '''simple docstring'''

        def empty_fn(*snake_case__      :	Dict							,			**snake_case__      :	str							):  # pylint: disable=unused-argument
             return

        return empty_fn
   def __enter__(		self      :	Optional[Any]							):


        '''simple docstring'''
        return self




   def __exit__(		self      :	Tuple							,			snake_case__      :	Tuple							,			snake_case__      :	Any							,			snake_case__      :	Optional[int]							):


        '''simple docstring'''
        return




class 			lowerCAmelCase__			:
   def __call__(		self      :	Optional[Any]							,			*snake_case__      :	Optional[Any]							,			**snake_case__      :	Tuple							):


        '''simple docstring'''
        if _tqdm_active:
             return tqdm_lib.tqdm(*snake_case__							,			**snake_case__							)
        else:
             return EmptyTqdm(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	Dict							,			*snake_case__      :	List[str]							,			**snake_case__      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Dict							=							None
        if _tqdm_active:
             return tqdm_lib.tqdm.set_lock(*snake_case__							,			**snake_case__							)




   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        if _tqdm_active:
             return tqdm_lib.tqdm.get_lock()




_lowerCAmelCase	:	Optional[int]				=      _tqdm_cls()


def 	SCREAMING_SNAKE_CASE__       (  )->		bool:






     '''simple docstring'''



     global _tqdm_active
     return bool(_tqdm_active				)


def 	SCREAMING_SNAKE_CASE__       (  )->		List[Any]:






     '''simple docstring'''



     global _tqdm_active
     UpperCAmelCase__						:    int							=							True
     hf_hub_utils.enable_progress_bars()



def 	SCREAMING_SNAKE_CASE__       (  )->		List[Any]:






     '''simple docstring'''



     global _tqdm_active
     UpperCAmelCase__						:    Optional[Any]							=							False
     hf_hub_utils.disable_progress_bars()


