

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      CodeGenTokenizer
   lowerCamelCase__				=      CodeGenTokenizerFast
   lowerCamelCase__				=      True
   lowerCamelCase__				=      {"add_prefix_space": True}
   lowerCamelCase__				=      False





   def      _snake_case		(       self					:			Union[str, Any]			):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       SCREAMING_SNAKE_CASE										=   [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "<unk>",
           "<|endoftext|>",
       ]
       SCREAMING_SNAKE_CASE										=   dict(zip(__lowerCamelCase     , range(len(__lowerCamelCase			)			)			)			)
       SCREAMING_SNAKE_CASE										=   ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       SCREAMING_SNAKE_CASE										=   {"unk_token": "<unk>"}

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["vocab_file"]			)
       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["merges_file"]			)
       with open(self.vocab_file     , "w"     , encoding="utf-8"			) as fp:
           fp.write(json.dumps(__lowerCamelCase			) + "\n"			)
       with open(self.merges_file     , "w"     , encoding="utf-8"			) as fp:
           fp.write("\n".join(__lowerCamelCase			)			)





   def      _snake_case		(       self					:			Tuple     , **__lowerCamelCase					:			List[str]			):
       kwargs.update(self.special_tokens_map			)
       return CodeGenTokenizer.from_pretrained(self.tmpdirname     , **__lowerCamelCase			)





   def      _snake_case		(       self					:			List[str]     , **__lowerCamelCase					:			Optional[int]			):
       kwargs.update(self.special_tokens_map			)
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     , **__lowerCamelCase			)





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   "lower newer"
       SCREAMING_SNAKE_CASE										=   "lower newer"
       return input_text, output_text





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   CodeGenTokenizer(self.vocab_file     , self.merges_file     , **self.special_tokens_map			)
       SCREAMING_SNAKE_CASE										=   "lower newer"
       SCREAMING_SNAKE_CASE										=   ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase     , add_prefix_space=__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokens + [tokenizer.unk_token]
       SCREAMING_SNAKE_CASE										=   [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , __lowerCamelCase			)





   def      _snake_case		(       self					:			Tuple			):
       if not self.test_rust_tokenizer:
           return

       SCREAMING_SNAKE_CASE										=   self.get_tokenizer()
       SCREAMING_SNAKE_CASE										=   self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "lower newer"

       # Testing tokenization
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase     , add_prefix_space=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   rust_tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)

       # Testing conversion to ids without special tokens
       SCREAMING_SNAKE_CASE										=   tokenizer.encode(__lowerCamelCase     , add_special_tokens=__lowerCamelCase     , add_prefix_space=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   rust_tokenizer.encode(__lowerCamelCase     , add_special_tokens=__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)

       # Testing conversion to ids with special tokens
       SCREAMING_SNAKE_CASE										=   self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode(__lowerCamelCase     , add_prefix_space=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   rust_tokenizer.encode(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)

       # Testing the unknown token
       SCREAMING_SNAKE_CASE										=   tokens + [rust_tokenizer.unk_token]
       SCREAMING_SNAKE_CASE										=   [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , __lowerCamelCase			)





   def      _snake_case		(       self					:			Tuple     , *__lowerCamelCase					:			Dict     , **__lowerCamelCase					:			Optional[Any]			):
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			Any=15			):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
               SCREAMING_SNAKE_CASE										=   self.rust_tokenizer_class.from_pretrained(__lowerCamelCase     , **__lowerCamelCase			)

               # Simple input
               SCREAMING_SNAKE_CASE										=   "This is a simple input"
               SCREAMING_SNAKE_CASE										=   ["This is a simple input 1", "This is a simple input 2"]
               SCREAMING_SNAKE_CASE										=   ("This is a simple input", "This is a pair")
               SCREAMING_SNAKE_CASE										=   [
                   ("This is a simple input 1", "This is a simple input 2"),
                   ("This is a simple pair 1", "This is a simple pair 2"),
               ]

               # Simple input tests
               self.assertRaises(__lowerCamelCase     , tokenizer_r.encode     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"			)

               # Simple input
               self.assertRaises(__lowerCamelCase     , tokenizer_r.encode_plus     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"			)

               # Simple input
               self.assertRaises(
                   __lowerCamelCase     , tokenizer_r.batch_encode_plus     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"     , )

               # Pair input
               self.assertRaises(__lowerCamelCase     , tokenizer_r.encode     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"			)

               # Pair input
               self.assertRaises(__lowerCamelCase     , tokenizer_r.encode_plus     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"			)

               # Pair input
               self.assertRaises(
                   __lowerCamelCase     , tokenizer_r.batch_encode_plus     , __lowerCamelCase     , max_length=__lowerCamelCase     , padding="max_length"     , )





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   CodeGenTokenizer.from_pretrained(self.tmpdirname     , pad_token="<pad>"			)

       # Simple input
       SCREAMING_SNAKE_CASE										=   "This is a simple input"
       SCREAMING_SNAKE_CASE										=   ["This is a simple input looooooooong", "This is a simple input"]
       SCREAMING_SNAKE_CASE										=   ("This is a simple input", "This is a pair")
       SCREAMING_SNAKE_CASE										=   [
           ("This is a simple input loooooong", "This is a simple input"),
           ("This is a simple pair loooooong", "This is a simple pair"),
       ]

       SCREAMING_SNAKE_CASE										=   tokenizer.pad_token_id

       SCREAMING_SNAKE_CASE										=   tokenizer(__lowerCamelCase     , padding="max_length"     , max_length=30     , return_tensors="np"			)
       SCREAMING_SNAKE_CASE										=   tokenizer(__lowerCamelCase     , padding=__lowerCamelCase     , truncate=__lowerCamelCase     , return_tensors="np"			)
       SCREAMING_SNAKE_CASE										=   tokenizer(*__lowerCamelCase     , padding="max_length"     , max_length=60     , return_tensors="np"			)
       SCREAMING_SNAKE_CASE										=   tokenizer(__lowerCamelCase     , padding=__lowerCamelCase     , truncate=__lowerCamelCase     , return_tensors="np"			)

       # s
       # test single string max_length padding
       self.assertEqual(out_s["input_ids"].shape[-1]     , 30			)
       self.assertTrue(pad_token_id in out_s["input_ids"]			)
       self.assertTrue(0 in out_s["attention_mask"]			)

       # s2
       # test automatic padding
       self.assertEqual(out_sa["input_ids"].shape[-1]     , 33			)
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["input_ids"][0]			)
       self.assertFalse(0 in out_sa["attention_mask"][0]			)
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["input_ids"][1]			)
       self.assertTrue(0 in out_sa["attention_mask"][1]			)

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["input_ids"].shape[-1]     , 60			)
       self.assertTrue(pad_token_id in out_p["input_ids"]			)
       self.assertTrue(0 in out_p["attention_mask"]			)

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["input_ids"].shape[-1]     , 52			)
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["input_ids"][0]			)
       self.assertFalse(0 in out_pa["attention_mask"][0]			)
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["input_ids"][1]			)
       self.assertTrue(0 in out_pa["attention_mask"][1]			)





   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   "$$$"
       SCREAMING_SNAKE_CASE										=   CodeGenTokenizer.from_pretrained(self.tmpdirname     , bos_token=__lowerCamelCase     , add_bos_token=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "This is a simple input"
       SCREAMING_SNAKE_CASE										=   ["This is a simple input 1", "This is a simple input 2"]

       SCREAMING_SNAKE_CASE										=   tokenizer.bos_token_id

       SCREAMING_SNAKE_CASE										=   tokenizer(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer(__lowerCamelCase			)

       self.assertEqual(out_s.input_ids[0]     , __lowerCamelCase			)
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

       SCREAMING_SNAKE_CASE										=   tokenizer.decode(out_s.input_ids			)
       SCREAMING_SNAKE_CASE										=   tokenizer.batch_decode(out_sa.input_ids			)

       self.assertEqual(decode_s.split()[0]     , __lowerCamelCase			)
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)





   @slow
   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"			)

       SCREAMING_SNAKE_CASE										=   "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
       SCREAMING_SNAKE_CASE										=   "\nif len_a > len_b:      result = a\nelse:      result = b"

       SCREAMING_SNAKE_CASE										=   tokenizer.encode(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   ["^#", re.escape("<|endoftext|>"			), "^'''", "^\"\"\"", "\n\n\n"]
       SCREAMING_SNAKE_CASE										=   tokenizer.decode(__lowerCamelCase     , truncate_before_pattern=__lowerCamelCase			)
       self.assertEqual(__lowerCamelCase     , __lowerCamelCase			)





   def      _snake_case		(       self					:			int			):
       pass







"""simple docstring"""



import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


_lowerCAmelCase	:	Union[str, Any]				=      threading.Lock()
_lowerCAmelCase	:	Optional[logging.Handler]				=      None

_lowerCAmelCase	:	Union[str, Any]				=      {
    """debug""": logging.DEBUG,
    """info""": logging.INFO,
    """warning""": logging.WARNING,
    """error""": logging.ERROR,
    """critical""": logging.CRITICAL,
}

_lowerCAmelCase	:	Dict				=      logging.WARNING

_lowerCAmelCase	:	Optional[Any]				=      True


def 	SCREAMING_SNAKE_CASE__       (  )->		List[str]:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							os.getenv("TRANSFORMERS_VERBOSITY"			,					snake_case				)
     if env_level_str:
          if env_level_str in log_levels:
               return log_levels[env_level_str]
          else:
               logging.getLogger().warning(
                   f'Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '
                   f'has to be one of: { ", ".join(log_levels.keys()				) }'				)
     return _default_log_level


def 	SCREAMING_SNAKE_CASE__       (  )->		str:






     '''simple docstring'''



     return __name__.split("."				)[0]


def 	SCREAMING_SNAKE_CASE__       (  )->		logging.Logger:






     '''simple docstring'''



     return logging.getLogger(_get_library_name()				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     global _default_handler

     with _lock:
          if _default_handler:
               # This library has already configured the library root logger.
               return
          UpperCAmelCase__						:    Any							=							logging.StreamHandler()  # Set sys.stderr as stream.
          UpperCAmelCase__						:    Union[str, Any]							=							sys.stderr.flush

          # Apply our default configuration to the library root logger.
          UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger()
          library_root_logger.addHandler(_default_handler				)
          library_root_logger.setLevel(_get_default_logging_level()				)
          UpperCAmelCase__						:    int							=							False


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     global _default_handler

     with _lock:
          if not _default_handler:
               return

          UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger()
          library_root_logger.removeHandler(_default_handler				)
          library_root_logger.setLevel(logging.NOTSET				)
          UpperCAmelCase__						:    Optional[int]							=							None


def 	SCREAMING_SNAKE_CASE__       (  )->		Optional[int]:






     '''simple docstring'''



     return log_levels


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : Optional[str] = None				)->		logging.Logger:






     '''simple docstring'''



     if name is None:
          UpperCAmelCase__						:    Tuple							=							_get_library_name()

     _configure_library_root_logger()
     return logging.getLogger(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		int:






     '''simple docstring'''



     _configure_library_root_logger()
     return _get_library_root_logger().getEffectiveLevel()


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : int				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     _get_library_root_logger().setLevel(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		Optional[Any]:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		List[str]:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		Tuple:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		str:






     '''simple docstring'''



     return set_verbosity(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert _default_handler is not None
     _get_library_root_logger().removeHandler(_default_handler				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert _default_handler is not None
     _get_library_root_logger().addHandler(_default_handler				)


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : logging.Handler				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert handler is not None
     _get_library_root_logger().addHandler(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  snake_case       : logging.Handler				)->		None:






     '''simple docstring'''



     _configure_library_root_logger()

     assert handler is not None and handler not in _get_library_root_logger().handlers
     _get_library_root_logger().removeHandler(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     UpperCAmelCase__						:    Dict							=							False


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     _configure_library_root_logger()
     UpperCAmelCase__						:    List[Any]							=							True


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							_get_library_root_logger().handlers

     for handler in handlers:
          UpperCAmelCase__						:    Union[str, Any]							=							logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"				)
          handler.setFormatter(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  )->		None:






     '''simple docstring'''



     UpperCAmelCase__						:    Any							=							_get_library_root_logger().handlers

     for handler in handlers:
          handler.setFormatter(snake_case				)


def 	SCREAMING_SNAKE_CASE__       (  self       : Optional[int]			,					*snake_case       : List[str]			,					**snake_case       : str				)->		Union[str, Any]:






     '''simple docstring'''



     UpperCAmelCase__						:    Union[str, Any]							=							os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS"			,					snake_case				)
     if no_advisory_warnings:
          return
     self.warning(*snake_case			,					**snake_case				)


_lowerCAmelCase	:	int				=      warning_advice


@functools.lru_cache(snake_case				)
def 	SCREAMING_SNAKE_CASE__       (  self       : Optional[Any]			,					*snake_case       : Dict			,					**snake_case       : Any				)->		Any:






     '''simple docstring'''



     self.warning(*snake_case			,					**snake_case				)


_lowerCAmelCase	:	Tuple				=      warning_once




class 			lowerCAmelCase__			:
   def __init__(		self      :	List[str]							,			*snake_case__      :	Any							,			**snake_case__      :	List[str]							):  # pylint: disable=unused-argument


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							args[0] if args else None
   def __iter__(		self      :	Any							):


        '''simple docstring'''
        return iter(self._iterator							)
   def __getattr__(		self      :	List[Any]							,			snake_case__      :	Tuple							):


        '''simple docstring'''

        def empty_fn(*snake_case__      :	Dict							,			**snake_case__      :	str							):  # pylint: disable=unused-argument
             return

        return empty_fn
   def __enter__(		self      :	Optional[Any]							):


        '''simple docstring'''
        return self




   def __exit__(		self      :	Tuple							,			snake_case__      :	Tuple							,			snake_case__      :	Any							,			snake_case__      :	Optional[int]							):


        '''simple docstring'''
        return




class 			lowerCAmelCase__			:
   def __call__(		self      :	Optional[Any]							,			*snake_case__      :	Optional[Any]							,			**snake_case__      :	Tuple							):


        '''simple docstring'''
        if _tqdm_active:
             return tqdm_lib.tqdm(*snake_case__							,			**snake_case__							)
        else:
             return EmptyTqdm(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	Dict							,			*snake_case__      :	List[str]							,			**snake_case__      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Dict							=							None
        if _tqdm_active:
             return tqdm_lib.tqdm.set_lock(*snake_case__							,			**snake_case__							)




   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        if _tqdm_active:
             return tqdm_lib.tqdm.get_lock()




_lowerCAmelCase	:	Optional[int]				=      _tqdm_cls()


def 	SCREAMING_SNAKE_CASE__       (  )->		bool:






     '''simple docstring'''



     global _tqdm_active
     return bool(_tqdm_active				)


def 	SCREAMING_SNAKE_CASE__       (  )->		List[Any]:






     '''simple docstring'''



     global _tqdm_active
     UpperCAmelCase__						:    int							=							True
     hf_hub_utils.enable_progress_bars()



def 	SCREAMING_SNAKE_CASE__       (  )->		List[Any]:






     '''simple docstring'''



     global _tqdm_active
     UpperCAmelCase__						:    Optional[Any]							=							False
     hf_hub_utils.disable_progress_bars()


