


import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							a						(_lowerCAmelCase	,							unittest.TestCase ):




							"""simple docstring"""


							__UpperCAmelCase       :				int	    =    GPTaTokenizer
							__UpperCAmelCase       :				Optional[int]	    =    GPTaTokenizerFast
							__UpperCAmelCase       :				Optional[int]	    =    True
							__UpperCAmelCase       :				List[str]	    =    {"add_prefix_space": True}
							__UpperCAmelCase       :				int	    =    False

							def        __snake_case   (				self			:    Union[str, Any]							)      ->       Dict:
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														__snake_case						:     int									=  [
														    "l",
														    "o",
														    "w",
														    "e",
														    "r",
														    "s",
														    "t",
														    "i",
														    "d",
														    "n",
														    "\u0120",
														    "\u0120l",
														    "\u0120n",
														    "\u0120lo",
														    "\u0120low",
														    "er",
														    "\u0120lowest",
														    "\u0120newer",
														    "\u0120wider",
														    "<unk>",
														    "<|endoftext|>",
														]
														__snake_case						:     Optional[Any]									=  dict(zip(lowerCamelCase       ,      range(len(lowerCamelCase							)							)							)							)
														__snake_case						:     Any									=  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
														__snake_case						:     str									=  {"unk_token": "<unk>"}

														__snake_case						:     Dict									=  os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["vocab_file"]							)
														__snake_case						:     Union[str, Any]									=  os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["merges_file"]							)
														with open(self.vocab_file       ,      "w"       ,      encoding="utf-8"							) as fp:
																					fp.write(json.dumps(lowerCamelCase							) + "\n"							)
														with open(self.merges_file       ,      "w"       ,      encoding="utf-8"							) as fp:
																					fp.write("\n".join(lowerCamelCase							)							)

							def        __snake_case   (				self			:    str       ,      **lowerCamelCase			:    int							)      ->       str:
														kwargs.update(self.special_tokens_map							)
														return GPTaTokenizer.from_pretrained(self.tmpdirname       ,      **lowerCamelCase							)

							def        __snake_case   (				self			:    Dict       ,      **lowerCamelCase			:    int							)      ->       Tuple:
														kwargs.update(self.special_tokens_map							)
														return GPTaTokenizerFast.from_pretrained(self.tmpdirname       ,      **lowerCamelCase							)

							def        __snake_case   (				self			:    Optional[int]       ,      lowerCamelCase			:    int							)      ->       Optional[int]:
														__snake_case						:     Dict									=  "lower newer"
														__snake_case						:     Optional[int]									=  "lower newer"
														return input_text, output_text

							def        __snake_case   (				self			:    List[Any]							)      ->       Dict:
														__snake_case						:     Dict									=  GPTaTokenizer(self.vocab_file       ,      self.merges_file       ,      **self.special_tokens_map							)
														__snake_case						:     List[str]									=  "lower newer"
														__snake_case						:     int									=  ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
														__snake_case						:     str									=  tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase							)
														self.assertListEqual(lowerCamelCase       ,      lowerCamelCase							)

														__snake_case						:     Union[str, Any]									=  tokens + [tokenizer.unk_token]
														__snake_case						:     Any									=  [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase							)       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    Union[str, Any]							)      ->       List[str]:
														if not self.test_rust_tokenizer:
																					return

														__snake_case						:     Tuple									=  self.get_tokenizer()
														__snake_case						:     List[str]									=  self.get_rust_tokenizer(add_prefix_space=lowerCamelCase							)

														__snake_case						:     str									=  "lower newer"

														# Testing tokenization
														__snake_case						:     Dict									=  tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase							)
														__snake_case						:     List[Any]									=  rust_tokenizer.tokenize(lowerCamelCase							)
														self.assertListEqual(lowerCamelCase       ,      lowerCamelCase							)

														# Testing conversion to ids without special tokens
														__snake_case						:     int									=  tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase       ,      add_prefix_space=lowerCamelCase							)
														__snake_case						:     int									=  rust_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase							)
														self.assertListEqual(lowerCamelCase       ,      lowerCamelCase							)

														# Testing conversion to ids with special tokens
														__snake_case						:     int									=  self.get_rust_tokenizer(add_prefix_space=lowerCamelCase							)
														__snake_case						:     int									=  tokenizer.encode(lowerCamelCase       ,      add_prefix_space=lowerCamelCase							)
														__snake_case						:     str									=  rust_tokenizer.encode(lowerCamelCase							)
														self.assertListEqual(lowerCamelCase       ,      lowerCamelCase							)

														# Testing the unknown token
														__snake_case						:     Optional[Any]									=  tokens + [rust_tokenizer.unk_token]
														__snake_case						:     str									=  [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase							)       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    int       ,      *lowerCamelCase			:    Dict       ,      **lowerCamelCase			:    Dict							)      ->       Tuple:
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass

							def        __snake_case   (				self			:    List[Any]       ,      lowerCamelCase			:    str=15							)      ->       str:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
																												__snake_case						:     Any									=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,      **lowerCamelCase							)

																												# Simple input
																												__snake_case						:     Tuple									=  "This is a simple input"
																												__snake_case						:     Optional[int]									=  ["This is a simple input 1", "This is a simple input 2"]
																												__snake_case						:     int									=  ("This is a simple input", "This is a pair")
																												__snake_case						:     Optional[int]									=  [
																												    ("This is a simple input 1", "This is a simple input 2"),
																												    ("This is a simple pair 1", "This is a simple pair 2"),
																												]

																												# Simple input tests
																												self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"							)

																												# Simple input
																												self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"							)

																												# Simple input
																												self.assertRaises(
																												    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"       ,      )

																												# Pair input
																												self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"							)

																												# Pair input
																												self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"							)

																												# Pair input
																												self.assertRaises(
																												    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding="max_length"       ,      )

							def        __snake_case   (				self			:    Union[str, Any]							)      ->       int:
														__snake_case						:     int									=  GPTaTokenizer.from_pretrained(self.tmpdirname       ,      pad_token="<pad>"							)

														# Simple input
														__snake_case						:     Any									=  "This is a simple input"
														__snake_case						:     int									=  ["This is a simple input looooooooong", "This is a simple input"]
														__snake_case						:     List[Any]									=  ("This is a simple input", "This is a pair")
														__snake_case						:     Any									=  [
														    ("This is a simple input loooooong", "This is a simple input"),
														    ("This is a simple pair loooooong", "This is a simple pair"),
														]

														__snake_case						:     Optional[Any]									=  tokenizer.pad_token_id

														__snake_case						:     Dict									=  tokenizer(lowerCamelCase       ,      padding="max_length"       ,      max_length=30       ,      return_tensors="np"							)
														__snake_case						:     Optional[Any]									=  tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors="np"							)
														__snake_case						:     List[Any]									=  tokenizer(*lowerCamelCase       ,      padding="max_length"       ,      max_length=60       ,      return_tensors="np"							)
														__snake_case						:     Union[str, Any]									=  tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors="np"							)

														# s
														# test single string max_length padding
														self.assertEqual(out_s["input_ids"].shape[-1]       ,      30							)
														self.assertTrue(pad_token_id in out_s["input_ids"]							)
														self.assertTrue(0 in out_s["attention_mask"]							)

														# s2
														# test automatic padding
														self.assertEqual(out_sa["input_ids"].shape[-1]       ,      33							)
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["input_ids"][0]							)
														self.assertFalse(0 in out_sa["attention_mask"][0]							)
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["input_ids"][1]							)
														self.assertTrue(0 in out_sa["attention_mask"][1]							)

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["input_ids"].shape[-1]       ,      60							)
														self.assertTrue(pad_token_id in out_p["input_ids"]							)
														self.assertTrue(0 in out_p["attention_mask"]							)

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["input_ids"].shape[-1]       ,      52							)
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["input_ids"][0]							)
														self.assertFalse(0 in out_pa["attention_mask"][0]							)
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["input_ids"][1]							)
														self.assertTrue(0 in out_pa["attention_mask"][1]							)

							def        __snake_case   (				self			:    Optional[Any]							)      ->       Optional[Any]:
														__snake_case						:     List[str]									=  "$$$"
														__snake_case						:     int									=  GPTaTokenizer.from_pretrained(self.tmpdirname       ,      bos_token=lowerCamelCase       ,      add_bos_token=lowerCamelCase							)

														__snake_case						:     Union[str, Any]									=  "This is a simple input"
														__snake_case						:     List[Any]									=  ["This is a simple input 1", "This is a simple input 2"]

														__snake_case						:     Dict									=  tokenizer.bos_token_id

														__snake_case						:     List[Any]									=  tokenizer(lowerCamelCase							)
														__snake_case						:     Tuple									=  tokenizer(lowerCamelCase							)

														self.assertEqual(out_s.input_ids[0]       ,      lowerCamelCase							)
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

														__snake_case						:     Tuple									=  tokenizer.decode(out_s.input_ids							)
														__snake_case						:     List[str]									=  tokenizer.batch_decode(out_sa.input_ids							)

														self.assertEqual(decode_s.split()[0]       ,      lowerCamelCase							)
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)

							def        __snake_case   (				self			:    Dict							)      ->       Dict:
														pass






							def        __snake_case   (				self			:    Dict							)      ->       str:
														# TODO: change to self.get_tokenizers() when the fast version is implemented
														__snake_case						:     Optional[Any]									=  [self.get_tokenizer(do_lower_case=lowerCamelCase       ,      add_bos_token=lowerCamelCase							)]
														for tokenizer in tokenizers:
																					with self.subTest(F'{tokenizer.__class__.__name__}'							):
																												__snake_case						:     Tuple									=  "Encode this."
																												__snake_case						:     int									=  "This one too please."
																												__snake_case						:     Tuple									=  tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase							)
																												encoded_sequence += tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase							)
																												__snake_case						:     List[Any]									=  tokenizer.encode_plus(
																												    lowerCamelCase       ,      lowerCamelCase       ,      add_special_tokens=lowerCamelCase       ,      return_special_tokens_mask=lowerCamelCase       ,      )
																												__snake_case						:     Dict									=  encoded_sequence_dict["input_ids"]
																												__snake_case						:     str									=  encoded_sequence_dict["special_tokens_mask"]
																												self.assertEqual(len(lowerCamelCase							)       ,      len(lowerCamelCase							)							)

																												__snake_case						:     List[str]									=  [
																												    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCamelCase							)
																												]
																												__snake_case						:     Optional[Any]									=  [x for x in filtered_sequence if x is not None]
																												self.assertEqual(lowerCamelCase       ,      lowerCamelCase							)

@require_tokenizers
class 							a						(unittest.TestCase ):




							"""simple docstring"""


							def        __snake_case   (				self			:    int							)      ->       List[Any]:
														# More context:
														# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
														# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
														# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

														__snake_case						:     Optional[int]									=  AutoTokenizer.from_pretrained("facebook/opt-350m"       ,      from_slow=lowerCamelCase							)
														__snake_case						:     Optional[int]									=  "A photo of a cat"

														__snake_case						:     Union[str, Any]									=  tokenizer.encode(
														    lowerCamelCase       ,      )
														self.assertEqual(lowerCamelCase       ,      [2, 250, 1345, 9, 10, 4758]							)
														tokenizer.save_pretrained("test_opt"							)

														__snake_case						:     int									=  AutoTokenizer.from_pretrained("./test_opt"							)
														__snake_case						:     Tuple									=  tokenizer.encode(
														    lowerCamelCase       ,      )
														self.assertEqual(lowerCamelCase       ,      [2, 250, 1345, 9, 10, 4758]							)

							def        __snake_case   (				self			:    Union[str, Any]							)      ->       Tuple:
														__snake_case						:     Tuple									=  AutoTokenizer.from_pretrained("facebook/opt-350m"       ,      use_slow=lowerCamelCase							)
														__snake_case						:     List[str]									=  "A photo of a cat"

														__snake_case						:     Dict									=  tokenizer.encode(
														    lowerCamelCase       ,      )
														# Same as above
														self.assertEqual(lowerCamelCase       ,      [2, 250, 1345, 9, 10, 4758]							)






							@unittest.skip("This test is failing because of a bug in the fast tokenizer"							)
							def        __snake_case   (				self			:    Optional[int]							)      ->       str:
														__snake_case						:     int									=  AutoTokenizer.from_pretrained("facebook/opt-350m"       ,      from_slow=lowerCamelCase							)

														__snake_case						:     Any									=  "bos"
														__snake_case						:     str									=  tokenizer.get_vocab()["bos"]

														__snake_case						:     Any									=  "A photo of a cat"
														__snake_case						:     List[str]									=  tokenizer.encode(
														    lowerCamelCase       ,      )
														# We changed the bos token
														self.assertEqual(lowerCamelCase       ,      [31957, 250, 1345, 9, 10, 4758]							)
														tokenizer.save_pretrained("./tok"							)
														__snake_case						:     Union[str, Any]									=  AutoTokenizer.from_pretrained("./tok"							)
														self.assertTrue(tokenizer.is_fast							)
														__snake_case						:     Optional[Any]									=  tokenizer.encode(
														    lowerCamelCase       ,      )
														self.assertEqual(lowerCamelCase       ,      [31957, 250, 1345, 9, 10, 4758]							)






'''simple docstring'''






# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..models.auto import AutoModelForVisionaSeq
from ..utils import requires_backends
from .base import PipelineTool


if TYPE_CHECKING:
     from PIL import Image

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       """Salesforce/blip-image-captioning-base"""
    lowercase__					=       (
        """This is a tool that generates a description of an image. It takes an input named `image` which should be the """
        """image to caption, and returns a text that contains the description in English."""
    )
    lowercase__					=       """image_captioner"""
    lowercase__					=       AutoModelForVisionaSeq

    lowercase__					=       ["""image"""]
    lowercase__					=       ["""text"""]







    def __init__( self					:    Tuple		,     *__snake_case					:    int		,     **__snake_case					:    List[Any]      ):
      requires_backends(self		,     ['vision']      )
      super().__init__(*__snake_case		,     **__snake_case      )







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    "Image"      ):
      return self.pre_processor(images=__snake_case		,     return_tensors='pt'      )







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    str      ):
      return self.model.generate(**__snake_case      )







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    Tuple      ):
      return self.pre_processor.batch_decode(__snake_case		,     skip_special_tokens=__snake_case      )[0].strip()