





'''simple docstring'''




import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
 from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							     =			get_tests_dir("fixtures/test_sentencepiece.model")

_SCREAMING_SNAKE_CASE							     =			{"target_lang": "fi", "source_lang": "en"}
_SCREAMING_SNAKE_CASE							     =			">>zh<<"
_SCREAMING_SNAKE_CASE							     =			"Helsinki-NLP/"

if is_torch_available():
 _SCREAMING_SNAKE_CASE							     =			"pt"
elif is_tf_available():
 _SCREAMING_SNAKE_CASE							     =			"tf"
else:
 _SCREAMING_SNAKE_CASE							     =			"jax"
@require_sentencepiece
class 			_lowerCAmelCase		(			A__							,     unittest.TestCase	):






     """simple docstring"""






     snake_case_						      =   MarianTokenizer
     snake_case_						      =   False
     snake_case_						      =   True



     def      lowerCAmelCase			(   self							:			Any						)->	List[Any]:
        super().setUp()
        snake_case 						=   ["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
        snake_case 						=   dict(zip(__snake_case   ,       range(len(__snake_case						)						)						)						)
        snake_case 						=   Path(self.tmpdirname						)
        save_json(__snake_case   ,       save_dir / VOCAB_FILES_NAMES["""vocab"""]						)
        save_json(__snake_case   ,       save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""]						)
        if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
           copyfile(__snake_case   ,       save_dir / VOCAB_FILES_NAMES["""source_spm"""]						)
           copyfile(__snake_case   ,       save_dir / VOCAB_FILES_NAMES["""target_spm"""]						)

        snake_case 						=   MarianTokenizer.from_pretrained(self.tmpdirname						)
        tokenizer.save_pretrained(self.tmpdirname						)



     def      lowerCAmelCase			(   self							:			List[Any]   ,       **__snake_case							:			str						)->	MarianTokenizer:
        return MarianTokenizer.from_pretrained(self.tmpdirname   ,       **__snake_case						)



     def      lowerCAmelCase			(   self							:			int   ,       __snake_case							:			List[Any]						)->	List[Any]:
        return (
            "This is a test",
            "This is a test",
        )



     def      lowerCAmelCase			(   self							:			List[str]						)->	Any:
        snake_case 						=   """</s>"""
        snake_case 						=   0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case						)   ,       __snake_case						)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case						)   ,       __snake_case						)



     def      lowerCAmelCase			(   self							:			str						)->	int:
        snake_case 						=   list(self.get_tokenizer().get_vocab().keys()						)

        self.assertEqual(vocab_keys[0]   ,       """</s>"""						)
        self.assertEqual(vocab_keys[1]   ,       """<unk>"""						)
        self.assertEqual(vocab_keys[-1]   ,       """<pad>"""						)
        self.assertEqual(len(__snake_case						)   ,       9						)



     def      lowerCAmelCase			(   self							:			Optional[int]						)->	Dict:
        self.assertEqual(self.get_tokenizer().vocab_size   ,       9						)



     def      lowerCAmelCase			(   self							:			Tuple						)->	Tuple:
        snake_case 						=   MarianTokenizer.from_pretrained(f'''{ORG_NAME}opus-mt-en-de'''						)
        snake_case 						=   en_de_tokenizer(["""I am a small frog"""]   ,       return_tensors=__snake_case						)
        self.assertIsInstance(__snake_case   ,       __snake_case						)
        snake_case 						=   [38, 1_21, 14, 6_97, 3_88_48, 0]
        self.assertListEqual(__snake_case   ,       batch.input_ids[0]						)

        snake_case 						=   tempfile.mkdtemp()
        en_de_tokenizer.save_pretrained(__snake_case						)
        snake_case 						=   [x.name for x in Path(__snake_case						).glob("""*"""						)]
        self.assertIn("""source.spm"""   ,       __snake_case						)
        MarianTokenizer.from_pretrained(__snake_case						)



     def      lowerCAmelCase			(   self							:			List[Any]						)->	Optional[Any]:
        snake_case 						=   self.get_tokenizer()

        snake_case 						=   tok(
            ["""I am a small frog""" * 10_00, """I am a small frog"""]   ,       padding=__snake_case   ,       truncation=__snake_case   ,       return_tensors=__snake_case						)
        self.assertIsInstance(__snake_case   ,       __snake_case						)
        self.assertEqual(batch.input_ids.shape   ,       (2, 5_12)						)



     def      lowerCAmelCase			(   self							:			int						)->	int:
        snake_case 						=   self.get_tokenizer()
        snake_case 						=   tok(["""I am a tiny frog""", """I am a small frog"""]   ,       padding=__snake_case   ,       return_tensors=__snake_case						)
        self.assertIsInstance(__snake_case   ,       __snake_case						)
        self.assertEqual(batch_smaller.input_ids.shape   ,       (2, 10)						)



     @slow
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	int:
        # fmt: off
        snake_case 						=   {"""input_ids""": [[4_34_95, 4_62, 20, 4_21_64, 13_69, 52, 4_64, 1_32, 17_03, 4_92, 13, 74_91, 3_89_99, 6, 8, 4_64, 1_32, 17_03, 4_92, 13, 46_69, 3_78_67, 13, 75_25, 27, 15_93, 9_88, 13, 3_39_72, 70_29, 6, 20, 82_51, 3_83, 2, 2_70, 58_66, 37_88, 2, 23_53, 82_51, 1_23_38, 2, 1_39_58, 3_87, 2, 36_29, 69_53, 1_88, 29_00, 2, 1_39_58, 80_11, 1_15_01, 23, 84_60, 40_73, 3_40_09, 20, 4_35, 1_14_39, 27, 8, 84_60, 40_73, 60_04, 20, 99_88, 3_75, 27, 33, 2_66, 19_45, 10_76, 13_50, 3_78_67, 32_88, 5, 5_77, 10_76, 43_74, 8, 50_82, 5, 2_64_53, 2_57, 5_56, 4_03, 2, 2_42, 1_32, 3_83, 3_16, 4_92, 8, 1_07_67, 6, 3_16, 3_04, 42_39, 3, 0], [1_48, 1_57_22, 19, 18_39, 12, 13_50, 13, 2_23_27, 50_82, 54_18, 4_75_67, 3_59_38, 59, 3_18, 1_95_52, 1_08, 21_83, 54, 1_49_76, 48_35, 32, 5_47, 11_14, 8, 3_15, 24_17, 5, 92, 1_90_88, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00], [36, 63_95, 1_25_70, 3_91_47, 1_15_97, 6, 2_66, 4, 4_54_05, 72_96, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__snake_case   ,       model_name="""Helsinki-NLP/opus-mt-en-de"""   ,       revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""   ,       decode_kwargs={"""use_source_tokenizer""": True}   ,       )



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Union[str, Any]:
        snake_case 						=   MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs"""						)

        snake_case 						=   """Tämä on testi"""
        snake_case 						=   """This is a test"""

        snake_case 						=   [76, 7, 20_47, 2]
        snake_case 						=   [69, 12, 11, 9_40, 2]

        snake_case 						=   tokenizer(__snake_case						).input_ids
        self.assertListEqual(__snake_case   ,       __snake_case						)

        snake_case 						=   tokenizer(text_target=__snake_case						).input_ids
        self.assertListEqual(__snake_case   ,       __snake_case						)

        snake_case 						=   tokenizer.decode(__snake_case   ,       skip_special_tokens=__snake_case						)
        self.assertEqual(__snake_case   ,       __snake_case						)








'''simple docstring'''




from __future__ import annotations





def 	__lowerCamelCase			(      __lowerCAmelCase       :							Any      ,    __lowerCAmelCase       :							str      ,    __lowerCAmelCase       :							Any      ,    __lowerCAmelCase       :							int		)		->  Tuple:  # noqa: E741
   while r - l > 1:
      snake_case 						=   (l + r) // 2
      if v[m] >= key:
         snake_case 						=   m
      else:
         snake_case 						=   m  # noqa: E741
   return r





def 	__lowerCamelCase			(      __lowerCAmelCase       :							list[int]		)		->  int:
   if len(__lowerCAmelCase		) == 0:
      return 0

   snake_case 						=   [0] * len(__lowerCAmelCase		)
   snake_case 						=   1

   snake_case 						=   v[0]

   for i in range(1      ,    len(__lowerCAmelCase		)		):
      if v[i] < tail[0]:
         snake_case 						=   v[i]
      elif v[i] > tail[length - 1]:
         snake_case 						=   v[i]
         length += 1
      else:
         snake_case 						=   v[i]

   return length


if __name__ == "__main__":
 import doctest

 doctest.testmod()

