
import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class __snake_case		(				A__ ,     unittest.TestCase					):
    _a		:						Union[str, Any]=    CLIPTokenizer
    _a		:						Tuple=    CLIPTokenizerFast
    _a		:						List[str]=    True
    _a		:						Union[str, Any]=    {}
    _a		:						List[str]=    False
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           super().setUp()

           # fmt: off
           lowercase							:					Optional[int]	     =				["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
           # fmt: on
           lowercase							:					Dict	     =				dict(zip(__snake_case		,range(len(__snake_case				)				)				)				)
           lowercase							:					List[str]	     =				["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
           lowercase							:					Optional[Any]	     =				{"""unk_token""": """<unk>"""}

           lowercase							:					Tuple	     =				os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES["""vocab_file"""]				)
           lowercase							:					Tuple	     =				os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES["""merges_file"""]				)
           with open(self.vocab_file		,"""w"""		,encoding="""utf-8"""				) as fp:
                  fp.write(json.dumps(__snake_case				) + """\n"""				)
           with open(self.merges_file		,"""w"""		,encoding="""utf-8"""				) as fp:
                  fp.write("""\n""".join(__snake_case				)				)
    def 				_SCREAMING_SNAKE_CASE	(  self		,**snake_case				):



           '''simple docstring'''







           kwargs.update(self.special_tokens_map				)
           return CLIPTokenizer.from_pretrained(self.tmpdirname		,**__snake_case				)
    def 				_SCREAMING_SNAKE_CASE	(  self		,**snake_case				):



           '''simple docstring'''







           kwargs.update(self.special_tokens_map				)
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname		,**__snake_case				)
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case				):



           '''simple docstring'''







           lowercase							:					List[str]	     =				"""lower newer"""
           lowercase							:					str	     =				"""lower newer"""
           return input_text, output_text
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           lowercase							:					List[str]	     =				CLIPTokenizer(self.vocab_file		,self.merges_file		,**self.special_tokens_map				)
           lowercase							:					str	     =				"""lower newer"""
           lowercase							:					Optional[Any]	     =				["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
           lowercase							:					Any	     =				tokenizer.tokenize(__snake_case				)
           self.assertListEqual(__snake_case		,__snake_case				)

           lowercase							:					Optional[Any]	     =				tokens + [tokenizer.unk_token]
           lowercase							:					Tuple	     =				[10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case				)		,__snake_case				)
    @require_ftfy
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                         lowercase							:					List[str]	     =				self.tokenizer_class.from_pretrained(__snake_case		,**__snake_case				)
                         lowercase							:					str	     =				self.rust_tokenizer_class.from_pretrained(__snake_case		,**__snake_case				)

                         lowercase							:					Any	     =				"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
                         lowercase							:					Optional[Any]	     =				tokenizer_s.tokenize(__snake_case				)
                         lowercase							:					Optional[int]	     =				tokenizer_r.tokenize(__snake_case				)

                         self.assertListEqual(__snake_case		,__snake_case				)

                         # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                         # with Tilde) encoded in 2 different ways
                         lowercase							:					Union[str, Any]	     =				"""xa\u0303y""" + """ """ + """x\xe3y"""
                         lowercase							:					Optional[Any]	     =				tokenizer_s.tokenize(__snake_case				)
                         lowercase							:					Optional[int]	     =				tokenizer_r.tokenize(__snake_case				)

                         self.assertListEqual(__snake_case		,__snake_case				)

                         # Test that the tokenization is identical on unicode of space type
                         lowercase							:					str	     =				[
                             """\u0009""",  # (horizontal tab, '\t')
                             """\u000B""",  # (vertical tab)
                             """\u000C""",  # (form feed)
                             """\u0020""",  # (space, ' ')
                             """\u200E""",  # (left-to-right mark):w
                             """\u200F""",  # (right-to-left mark)
                         ]
                         for unicode_seq in spaces_unicodes:
                                lowercase							:					Tuple	     =				tokenizer_s.tokenize(__snake_case				)
                                lowercase							:					List[Any]	     =				tokenizer_r.tokenize(__snake_case				)

                                self.assertListEqual(__snake_case		,__snake_case				)

                         # Test that the tokenization is identical on unicode of line break type
                         lowercase							:					List[Any]	     =				[
                             """\u000A""",  # (line feed, '\n')
                             """\r\n""",  # (carriage return and line feed, '\r\n')
                             """\u000D""",  # (carriage return, '\r')
                             """\r""",  # (carriage return, '\r')
                             """\u000D""",  # (carriage return, '\r')
                             """\u2028""",  # (line separator)
                             """\u2029""",  # (paragraph separator)
                             # "\u0085", # (next line)
                         ]

                         # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                         # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                         # space (and thus into an empty list).

                         for unicode_seq in line_break_unicodes:
                                lowercase							:					Optional[Any]	     =				tokenizer_s.tokenize(__snake_case				)
                                lowercase							:					Optional[Any]	     =				tokenizer_r.tokenize(__snake_case				)

                                self.assertListEqual(__snake_case		,__snake_case				)
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                         lowercase							:					Dict	     =				"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                         lowercase							:					Optional[int]	     =				f"{text_of_1_token} {text_of_1_token}"

                         lowercase							:					List[Any]	     =				self.rust_tokenizer_class.from_pretrained(
                             __snake_case		,use_fast=__snake_case		,)
                         lowercase							:					Any	     =				tokenizer_r(__snake_case		,return_offsets_mapping=__snake_case		,add_special_tokens=__snake_case				)
                         self.assertEqual(encoding.offset_mapping[0]		,(0, len(__snake_case				))				)
                         self.assertEqual(
                             encoding.offset_mapping[1]		,(len(__snake_case				) + 1, len(__snake_case				) + 1 + len(__snake_case				))		,)

                         lowercase							:					Optional[int]	     =				f" {text}"

                         lowercase							:					int	     =				self.rust_tokenizer_class.from_pretrained(
                             __snake_case		,use_fast=__snake_case		,)
                         lowercase							:					Optional[int]	     =				tokenizer_r(__snake_case		,return_offsets_mapping=__snake_case		,add_special_tokens=__snake_case				)
                         self.assertEqual(encoding.offset_mapping[0]		,(1, 1 + len(__snake_case				))				)
                         self.assertEqual(
                             encoding.offset_mapping[1]		,(1 + len(__snake_case				) + 1, 1 + len(__snake_case				) + 1 + len(__snake_case				))		,)
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           with self.assertRaises(__snake_case				) as context:
                  self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""				)

           self.assertTrue(
               context.exception.args[0].startswith(
                   """The `backend_tokenizer` provided does not match the expected format."""				)				)
    @require_ftfy
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           super().test_tokenization_python_rust_equals()
    def 				_SCREAMING_SNAKE_CASE	(  self				):



           '''simple docstring'''







           pass


from ...processing_utils import ProcessorMixin

class __snake_case		(				lowerCAmelCase					):
    _a		:						Union[str, Any]=    "WhisperFeatureExtractor"
    _a		:						int=    "WhisperTokenizer"
    def __init__(  self		,snake_case		,snake_case				):



           '''simple docstring'''







           super().__init__(snake_case		,snake_case				)
           lowercase							:					Optional[int]	     =				self.feature_extractor
           lowercase							:					Tuple	     =				False
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case=None		,snake_case=None		,snake_case=True				):



           '''simple docstring'''







           return self.tokenizer.get_decoder_prompt_ids(task=snake_case		,language=snake_case		,no_timestamps=snake_case				)
    def __call__(  self		,*snake_case		,**snake_case				):



           '''simple docstring'''







           if self._in_target_context_manager:
                  return self.current_processor(*snake_case		,**snake_case				)

           lowercase							:					Optional[Any]	     =				kwargs.pop("""audio"""		,snake_case				)
           lowercase							:					str	     =				kwargs.pop("""sampling_rate"""		,snake_case				)
           lowercase							:					Dict	     =				kwargs.pop("""text"""		,snake_case				)
           if len(snake_case				) > 0:
                  lowercase							:					List[Any]	     =				args[0]
                  lowercase							:					Tuple	     =				args[1:]

           if audio is None and text is None:
                  raise ValueError("""You need to specify either an `audio` or `text` input to process."""				)

           if audio is not None:
                  lowercase							:					Any	     =				self.feature_extractor(snake_case		,*snake_case		,sampling_rate=snake_case		,**snake_case				)
           if text is not None:
                  lowercase							:					str	     =				self.tokenizer(snake_case		,**snake_case				)

           if text is None:
                  return inputs

           elif audio is None:
                  return encodings
           else:
                  lowercase							:					List[Any]	     =				encodings["""input_ids"""]
                  return inputs
    def 				_SCREAMING_SNAKE_CASE	(  self		,*snake_case		,**snake_case				):



           '''simple docstring'''







           return self.tokenizer.batch_decode(*snake_case		,**snake_case				)
    def 				_SCREAMING_SNAKE_CASE	(  self		,*snake_case		,**snake_case				):



           '''simple docstring'''







           return self.tokenizer.decode(*snake_case		,**snake_case				)
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case		,snake_case="np"				):



           '''simple docstring'''







           return self.tokenizer.get_prompt_ids(snake_case		,return_tensors=snake_case				)
