



'''simple docstring'''





from ...utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_torch_available,
    is_transformers_available,
)


try:
 if not (is_transformers_available() and is_torch_available()):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
 from .multicontrolnet import MultiControlNetModel
 from .pipeline_controlnet import StableDiffusionControlNetPipeline
 from .pipeline_controlnet_imgaimg import StableDiffusionControlNetImgaImgPipeline
 from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline


if is_transformers_available() and is_flax_available():
 from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline









'''simple docstring'''





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 				snake_case (UpperCamelCase		,      unittest.TestCase   ):
       lowerCAmelCase__  :Optional[int]						   =					CodeGenTokenizer
       lowerCAmelCase__  :List[Any]						   =					CodeGenTokenizerFast
       lowerCAmelCase__  :str						   =					True
       lowerCAmelCase__  :Tuple						   =					{"add_prefix_space": True}
       lowerCAmelCase__  :Dict						   =					False



       def 							_a      (				self      )  ->							Union[str, Any]:
           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           lowercase__		       =						[
               "l",
               "o",
               "w",
               "e",
               "r",
               "s",
               "t",
               "i",
               "d",
               "n",
               "\u0120",
               "\u0120l",
               "\u0120n",
               "\u0120lo",
               "\u0120low",
               "er",
               "\u0120lowest",
               "\u0120newer",
               "\u0120wider",
               "<unk>",
               "<|endoftext|>",
           ]
           lowercase__		       =						dict(zip(UpperCAmelCase_     ,range(len(UpperCAmelCase_      )      )      )      )
           lowercase__		       =						["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
           lowercase__		       =						{"unk_token": "<unk>"}

           lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["vocab_file"]      )
           lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["merges_file"]      )
           with open(self.vocab_file     ,"w"     ,encoding="utf-8"      ) as fp:
               fp.write(json.dumps(UpperCAmelCase_      ) + "\n"      )
           with open(self.merges_file     ,"w"     ,encoding="utf-8"      ) as fp:
               fp.write("\n".join(UpperCAmelCase_      )      )



       def 							_a      (				self     ,**UpperCAmelCase_      )  ->							int:
           kwargs.update(self.special_tokens_map      )
           return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



       def 							_a      (				self     ,**UpperCAmelCase_      )  ->							Tuple:
           kwargs.update(self.special_tokens_map      )
           return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						"lower newer"
           lowercase__		       =						"lower newer"
           return input_text, output_text



       def 							_a      (				self      )  ->							Optional[int]:
           lowercase__		       =						CodeGenTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map      )
           lowercase__		       =						"lower newer"
           lowercase__		       =						["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						tokens + [tokenizer.unk_token]
           lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							int:
           if not self.test_rust_tokenizer:
               return

           lowercase__		       =						self.get_tokenizer()
           lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )

           lowercase__		       =						"lower newer"

           # Testing tokenization
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing conversion to ids without special tokens
           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing conversion to ids with special tokens
           lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing the unknown token
           lowercase__		       =						tokens + [rust_tokenizer.unk_token]
           lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



       def 							_a      (				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							Any:
           # It's very difficult to mix/test pretokenization with byte-level
           # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
           pass



       def 							_a      (				self     ,UpperCAmelCase_=15      )  ->							Union[str, Any]:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                   lowercase__		       =						self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_     ,**UpperCAmelCase_      )

                   # Simple input
                   lowercase__		       =						"This is a simple input"
                   lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]
                   lowercase__		       =						("This is a simple input", "This is a pair")
                   lowercase__		       =						[
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Simple input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Simple input
                   self.assertRaises(
                       UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)

                   # Pair input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Pair input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Pair input
                   self.assertRaises(
                       UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)



       def 							_a      (				self      )  ->							Optional[Any]:
           lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,pad_token="<pad>"      )

           # Simple input
           lowercase__		       =						"This is a simple input"
           lowercase__		       =						["This is a simple input looooooooong", "This is a simple input"]
           lowercase__		       =						("This is a simple input", "This is a pair")
           lowercase__		       =						[
               ("This is a simple input loooooong", "This is a simple input"),
               ("This is a simple pair loooooong", "This is a simple pair"),
           ]

           lowercase__		       =						tokenizer.pad_token_id

           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding="max_length"     ,max_length=30     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(*UpperCAmelCase_     ,padding="max_length"     ,max_length=60     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )

           # s
           # test single string max_length padding
           self.assertEqual(out_s["input_ids"].shape[-1]     ,30      )
           self.assertTrue(pad_token_id in out_s["input_ids"]      )
           self.assertTrue(0 in out_s["attention_mask"]      )

           # s2
           # test automatic padding
           self.assertEqual(out_sa["input_ids"].shape[-1]     ,33      )
           # long slice doesn't have padding
           self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
           self.assertFalse(0 in out_sa["attention_mask"][0]      )
           # short slice does have padding
           self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
           self.assertTrue(0 in out_sa["attention_mask"][1]      )

           # p
           # test single pair max_length padding
           self.assertEqual(out_p["input_ids"].shape[-1]     ,60      )
           self.assertTrue(pad_token_id in out_p["input_ids"]      )
           self.assertTrue(0 in out_p["attention_mask"]      )

           # p2
           # test automatic padding pair
           self.assertEqual(out_pa["input_ids"].shape[-1]     ,52      )
           # long slice pair doesn't have padding
           self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
           self.assertFalse(0 in out_pa["attention_mask"][0]      )
           # short slice pair does have padding
           self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
           self.assertTrue(0 in out_pa["attention_mask"][1]      )



       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						"$$$"
           lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,bos_token=UpperCAmelCase_     ,add_bos_token=UpperCAmelCase_      )

           lowercase__		       =						"This is a simple input"
           lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]

           lowercase__		       =						tokenizer.bos_token_id

           lowercase__		       =						tokenizer(UpperCAmelCase_      )
           lowercase__		       =						tokenizer(UpperCAmelCase_      )

           self.assertEqual(out_s.input_ids[0]     ,UpperCAmelCase_      )
           self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

           lowercase__		       =						tokenizer.decode(out_s.input_ids      )
           lowercase__		       =						tokenizer.batch_decode(out_sa.input_ids      )

           self.assertEqual(decode_s.split()[0]     ,UpperCAmelCase_      )
           self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )



       @slow
       def 							_a      (				self      )  ->							List[Any]:
           lowercase__		       =						CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

           lowercase__		       =						"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
           lowercase__		       =						"\nif len_a > len_b:      result = a\nelse:      result = b"

           lowercase__		       =						tokenizer.encode(UpperCAmelCase_      )
           lowercase__		       =						["^#", re.escape("<|endoftext|>"      ), "^'''", "^\"\"\"", "\n\n\n"]
           lowercase__		       =						tokenizer.decode(UpperCAmelCase_     ,truncate_before_pattern=UpperCAmelCase_      )
           self.assertEqual(UpperCAmelCase_     ,UpperCAmelCase_      )







       def 							_a      (				self      )  ->							Any:
           pass




