




'''simple docstring'''







import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_      =			get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')


@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase_						(       snake_case__		,    unittest.TestCase      ):





  """simple docstring"""



  a_       :Optional[Any]												=GPTSwaTokenizer
  a_       :Union[str, Any]												=False
  a_       :Tuple												=True
  a_       :List[Any]												=False

  def __a						(					self     :							Optional[Any]						):





       '''simple docstring'''

       super().setUp()

       # We have a SentencePiece fixture for testing
       __a       =   GPTSwaTokenizer(__a				,							eos_token="""<unk>"""				,							bos_token="""<unk>"""				,							pad_token="""<unk>"""						)

       tokenizer.save_pretrained(self.tmpdirname						)

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   """This is a test"""
       __a       =   """This is a test"""
       return input_text, output_text

  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a       =   """<s>"""
       __a       =   1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a						)				,							__a						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a						)				,							__a						)

  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       __a       =   list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]				,							"""<unk>"""						)
       self.assertEqual(vocab_keys[1]				,							"""<s>"""						)
       self.assertEqual(vocab_keys[-1]				,							"""j"""						)
       self.assertEqual(len(__a						)				,							2_0_0_0						)

  def __a						(					self     :							Dict						):





       '''simple docstring'''

       self.assertEqual(self.get_tokenizer().vocab_size				,							2_0_0_0						)

  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   GPTSwaTokenizer(__a						)

       __a       =   tokenizer.tokenize("""This is a test"""						)
       self.assertListEqual(__a				,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)				,							[4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]						)

       __a       =   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
       # fmt: off
       self.assertListEqual(
           __a				,							["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]				,							)
       # fmt: on

       __a       =   tokenizer.convert_tokens_to_ids(__a						)
       self.assertListEqual(
           __a				,							[2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]				,							)

       __a       =   tokenizer.convert_ids_to_tokens(__a						)
       # fmt: off
       self.assertListEqual(
           __a				,							["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]						)
       # fmt: on

  def __a						(					self     :							Dict						):





       '''simple docstring'''

       __a       =   GPTSwaTokenizer(__a						)
       __a       =   ["""This is a test""", """I was born in 92000, and this is falsé."""]
       __a       =   [
           [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
           [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
       ]

       # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
       for text, expected_ids in zip(__a				,							__a						):
            self.assertListEqual(tokenizer.encode_fast(__a						)				,							__a						)

       # Test that decode_fast returns the input text
       for text, token_ids in zip(__a				,							__a						):
            self.assertEqual(tokenizer.decode_fast(__a						)				,							__a						)


  @slow
  def __a						(					self     :							Any						):





       '''simple docstring'''

       __a       =   [
           """<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')""",
           """Hey there, how are you doing this fine day?""",
           """This is a text with a trailing spaces followed by a dot     .""",
           """Häj sväjs lillebrör! =)""",
           """Det är inget fel på Mr. Cool""",
       ]

       # fmt: off
       __a       =   {"""input_ids""": [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
       # fmt: on
       self.tokenizer_integration_test_util(
           expected_encoding=__a				,							model_name="""AI-Sweden/gpt-sw3-126m"""				,							sequences=__a				,							)










'''simple docstring'''







import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
SCREAMING_SNAKE_CASE_      =			{'LayoutLMv2Config', 'LayoutLMv3Config'}


@is_pipeline_test
class  lowerCAmelCase_						(       unittest.TestCase      ):





  """simple docstring"""



  a_       :List[Any]												=MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
  a_       :List[Any]												=TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

  if model_mapping is not None:
    a_       :List[Any]												={config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
  if tf_model_mapping is not None:
    a_       :Tuple												={
        config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
    }

  @require_torch
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""pt"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       __a       =   text_classifier("""This is great !"""				,							top_k=2						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}]						)

       __a       =   text_classifier(["""This is great !""", """This is bad"""]				,							top_k=2						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
           ]				,							)

       __a       =   text_classifier("""This is great !"""				,							top_k=1						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       # Legacy behavior
       __a       =   text_classifier("""This is great !"""				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

       __a       =   text_classifier("""This is great !"""				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[[{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}]]						)

       __a       =   text_classifier(["""This is great !""", """Something else"""]				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
               [{"""label""": """LABEL_0""", """score""": 0.5_0_4}, {"""label""": """LABEL_1""", """score""": 0.4_9_6}],
           ]				,							)

       __a       =   text_classifier(["""This is great !""", """Something else"""]				,							return_all_scores=SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[
               {"""label""": """LABEL_0""", """score""": 0.5_0_4},
               {"""label""": """LABEL_0""", """score""": 0.5_0_4},
           ]				,							)

  @require_torch
  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       import torch

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""pt"""				,							device=torch.device("""cpu"""						)				,							)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

  @require_tf
  def __a						(					self     :							List[str]						):





       '''simple docstring'''

       __a       =   pipeline(
           task="""text-classification"""				,							model="""hf-internal-testing/tiny-random-distilbert"""				,							framework="""tf"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """LABEL_0""", """score""": 0.5_0_4}]						)

  @slow
  @require_torch
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   pipeline("""text-classification"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""This is bad !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """NEGATIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""Birds are a type of animal"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 0.9_8_8}]						)

  @slow
  @require_tf
  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a       =   pipeline("""text-classification"""				,							framework="""tf"""						)

       __a       =   text_classifier("""This is great !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""This is bad !"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """NEGATIVE""", """score""": 1.0}]						)
       __a       =   text_classifier("""Birds are a type of animal"""						)
       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": """POSITIVE""", """score""": 0.9_8_8}]						)

  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							int				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   TextClassificationPipeline(model=SCREAMING_SNAKE_CASE__				,							tokenizer=SCREAMING_SNAKE_CASE__						)
       return text_classifier, ["HuggingFace is in", "This is another test"]


  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   text_classifier.model
       # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
       __a       =   """HuggingFace is in"""
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)

       self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]						)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)

       __a       =   ["""HuggingFace is in """, """Paris is in France"""]
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}, {"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]				,							)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)
       self.assertTrue(outputs[1]["""label"""] in model.config.idalabel.values()						)

       # Forcing to get all results with `top_k=None`
       # This is NOT the legacy format
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__				,							top_k=SCREAMING_SNAKE_CASE__						)
       __a       =   len(model.config.idalabel.values()						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}] * N, [{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}] * N]				,							)

       __a       =   {"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
       __a       =   text_classifier(SCREAMING_SNAKE_CASE__						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}				,							)
       self.assertTrue(outputs["""label"""] in model.config.idalabel.values()						)

       # This might be used a text pair, but tokenizer + pipe interaction
       # makes it hard to understand that it's not using the pair properly
       # https://github.com/huggingface/transformers/issues/17305
       # We disabled this usage instead as it was outputting wrong outputs.
       __a       =   [["""HuggingFace is in """, """Paris is in France"""]]
       with self.assertRaises(SCREAMING_SNAKE_CASE__						):
            text_classifier(SCREAMING_SNAKE_CASE__						)

       # This used to be valid for doing text pairs
       # We're keeping it working because of backward compatibility
       __a       =   text_classifier([[["""HuggingFace is in """, """Paris is in France"""]]]						)
       self.assertEqual(
           nested_simplify(SCREAMING_SNAKE_CASE__						)				,							[{"""label""": ANY(SCREAMING_SNAKE_CASE__						), """score""": ANY(SCREAMING_SNAKE_CASE__						)}]				,							)
       self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()						)






