
"""simple docstring"""


import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
A_    							=					{"""LayoutLMv2Config""", """LayoutLMv3Config"""}




@is_pipeline_test
class  __lowerCamelCase  ( unittest.TestCase			):
				a__:      Optional[Any]          =    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
				a__:      Dict          =    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

				if model_mapping is not None:
								a__:      Tuple          =    {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
				if tf_model_mapping is not None:
								a__:      List[Any]          =    {
								    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
								}




				@require_torch
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  pipeline(
							    task='''text-classification'''							,						model='''hf-internal-testing/tiny-random-distilbert'''							,						framework='''pt'''   )

							lowerCamelCase_					=  text_classifier('''This is great !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}]   )

							lowerCamelCase_					=  text_classifier('''This is great !'''							,						top_k=2   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}]   )

							lowerCamelCase_					=  text_classifier(['''This is great !''', '''This is bad''']							,						top_k=2   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[
							        [{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}],
							        [{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}],
							    ]							,						)

							lowerCamelCase_					=  text_classifier('''This is great !'''							,						top_k=1   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}]   )

							# Legacy behavior
							lowerCamelCase_					=  text_classifier('''This is great !'''							,						return_all_scores=UpperCAmelCase   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}]   )

							lowerCamelCase_					=  text_classifier('''This is great !'''							,						return_all_scores=UpperCAmelCase   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}]]   )

							lowerCamelCase_					=  text_classifier(['''This is great !''', '''Something else''']							,						return_all_scores=UpperCAmelCase   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[
							        [{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}],
							        [{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}, {'''label''': '''LABEL_1''', '''score''': 0.4_9_6}],
							    ]							,						)

							lowerCamelCase_					=  text_classifier(['''This is great !''', '''Something else''']							,						return_all_scores=UpperCAmelCase   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[
							        {'''label''': '''LABEL_0''', '''score''': 0.5_0_4},
							        {'''label''': '''LABEL_0''', '''score''': 0.5_0_4},
							    ]							,						)




				@require_torch
				def        UpperCAmelCase__						(				self   ):
							import torch

							lowerCamelCase_					=  pipeline(
							    task='''text-classification'''							,						model='''hf-internal-testing/tiny-random-distilbert'''							,						framework='''pt'''							,						device=torch.device('''cpu'''   )							,						)

							lowerCamelCase_					=  text_classifier('''This is great !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}]   )




				@require_tf
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  pipeline(
							    task='''text-classification'''							,						model='''hf-internal-testing/tiny-random-distilbert'''							,						framework='''tf'''   )

							lowerCamelCase_					=  text_classifier('''This is great !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''LABEL_0''', '''score''': 0.5_0_4}]   )




				@slow
				@require_torch
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  pipeline('''text-classification'''   )

							lowerCamelCase_					=  text_classifier('''This is great !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''POSITIVE''', '''score''': 1.0}]   )
							lowerCamelCase_					=  text_classifier('''This is bad !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''NEGATIVE''', '''score''': 1.0}]   )
							lowerCamelCase_					=  text_classifier('''Birds are a type of animal'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''POSITIVE''', '''score''': 0.9_8_8}]   )




				@slow
				@require_tf
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  pipeline('''text-classification'''							,						framework='''tf'''   )

							lowerCamelCase_					=  text_classifier('''This is great !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''POSITIVE''', '''score''': 1.0}]   )
							lowerCamelCase_					=  text_classifier('''This is bad !'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''NEGATIVE''', '''score''': 1.0}]   )
							lowerCamelCase_					=  text_classifier('''Birds are a type of animal'''   )
							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': '''POSITIVE''', '''score''': 0.9_8_8}]   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase   ):
							lowerCamelCase_					=  TextClassificationPipeline(model=UpperCAmelCase							,						tokenizer=UpperCAmelCase   )
							return text_classifier, ["HuggingFace is in", "This is another test"]





				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase   ):
							lowerCamelCase_					=  text_classifier.model
							# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
							lowerCamelCase_					=  '''HuggingFace is in'''
							lowerCamelCase_					=  text_classifier(UpperCAmelCase   )

							self.assertEqual(nested_simplify(UpperCAmelCase   )							,						[{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}]   )
							self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()   )

							lowerCamelCase_					=  ['''HuggingFace is in ''', '''Paris is in France''']
							lowerCamelCase_					=  text_classifier(UpperCAmelCase   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}, {'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}]							,						)
							self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()   )
							self.assertTrue(outputs[1]['''label'''] in model.config.idalabel.values()   )

							# Forcing to get all results with `top_k=None`
							# This is NOT the legacy format
							lowerCamelCase_					=  text_classifier(UpperCAmelCase							,						top_k=UpperCAmelCase   )
							lowerCamelCase_					=  len(model.config.idalabel.values()   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[[{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}] * N, [{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}] * N]							,						)

							lowerCamelCase_					=  {'''text''': '''HuggingFace is in ''', '''text_pair''': '''Paris is in France'''}
							lowerCamelCase_					=  text_classifier(UpperCAmelCase   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}							,						)
							self.assertTrue(outputs['''label'''] in model.config.idalabel.values()   )

							# This might be used a text pair, but tokenizer + pipe interaction
							# makes it hard to understand that it's not using the pair properly
							# https://github.com/huggingface/transformers/issues/17305
							# We disabled this usage instead as it was outputting wrong outputs.
							lowerCamelCase_					=  [['''HuggingFace is in ''', '''Paris is in France''']]
							with self.assertRaises(UpperCAmelCase   ):
										text_classifier(UpperCAmelCase   )

							# This used to be valid for doing text pairs
							# We're keeping it working because of backward compatibility
							lowerCamelCase_					=  text_classifier([[['''HuggingFace is in ''', '''Paris is in France''']]]   )
							self.assertEqual(
							    nested_simplify(UpperCAmelCase   )							,						[{'''label''': ANY(UpperCAmelCase   ), '''score''': ANY(UpperCAmelCase   )}]							,						)
							self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()   )







import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							    =							'▁'

SCREAMING_SNAKE_CASE							    =							{'vocab_file': 'prophetnet.tokenizer'}

SCREAMING_SNAKE_CASE							    =							{
    'vocab_file': {
        'microsoft/xprophetnet-large-wiki100-cased': (
            'https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'
        ),
    }
}

SCREAMING_SNAKE_CASE							    =							{
    'microsoft/xprophetnet-large-wiki100-cased': {'do_lower_case': False},
}

SCREAMING_SNAKE_CASE							    =							{
    'microsoft/xprophetnet-large-wiki100-cased': 5_1_2,
}




def  a   (lowerCAmelCase__   ):
   __a					=  collections.OrderedDict()
   with open(lowerCAmelCase__		,				"""r"""		,				encoding="""utf-8"""   ) as reader:
      __a					=  reader.readlines()
   for index, token in enumerate(lowerCAmelCase__   ):
      __a					=  token.rstrip("""\n"""   )
      __a					=  index
   return vocab


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      VOCAB_FILES_NAMES
     _lowerCamelCase   =      PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase   =      ["""input_ids""", """attention_mask"""]


     def __init__(	self  ,       __A  ,       __A="[SEP]"  ,       __A="[SEP]"  ,       __A="[SEP]"  ,       __A="[UNK]"  ,       __A="[PAD]"  ,       __A="[CLS]"  ,       __A="[MASK]"  ,       __A = None  ,       **__A  ,       ):
        __a					=  {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=__A  ,       eos_token=__A  ,       sep_token=__A  ,       unk_token=__A  ,       pad_token=__A  ,       cls_token=__A  ,       mask_token=__A  ,       sp_model_kwargs=self.sp_model_kwargs  ,       **__A  ,       )

        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""    )
           raise

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(str(__A    )    )
        __a					=  vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # put special tokens and [unused] tokens into the vocab
        __a					=  {"""[PAD]""": 0, """[CLS]""": 1, """[SEP]""": 2, """[UNK]""": 3, """[MASK]""": 4}

        for i in range(10    ):
           __a					=  f'''[unused{i}]'''
           __a					=  5 + i

        # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
        __a					=  12
        __a					=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}
        for k in self.fairseq_tokens_to_ids.keys():
           self.unique_no_split_tokens.append(__A    )


     def __getstate__(	self    ):
        __a					=  self.__dict__.copy()
        __a					=  None
        return state


     def __setstate__(	self  ,       __A    ):
        __a					=  d
        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""    )
           raise

        # for backward compatibility
        if not hasattr(self  ,       """sp_model_kwargs"""    ):
           __a					=  {}

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(self.vocab_file    )


     def 	snake_case_  (	self  ,       __A  ,       __A = None  ,       __A = False    ):

        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__A  ,       token_ids_a=__A  ,       already_has_special_tokens=__A    )

        if token_ids_a is None:
           return ([0] * len(__A    )) + [1]
        return ([0] * len(__A    )) + [1] + ([0] * len(__A    )) + [1]


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  [self.sep_token_id]

        if token_ids_a is None:
           return len(token_ids_a + sep    ) * [0]
        return len(token_ids_a + sep + sep + token_ids_a + sep    ) * [0]


     @property
     def 	snake_case_  (	self    ):
        return len(self.sp_model    ) + self.fairseq_offset


     def 	snake_case_  (	self    ):
        __a					=  {self.convert_ids_to_tokens(__A    ): i for i in range(self.vocab_size    )}
        vocab.update(self.added_tokens_encoder    )
        return vocab


     def 	snake_case_  (	self  ,       __A    ):
        return self.sp_model.encode(__A  ,       out_type=__A    )


     def 	snake_case_  (	self  ,       __A    ):
        if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
        __a					=  self.sp_model.PieceToId(__A    )

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


     def 	snake_case_  (	self  ,       __A    ):
        if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset    )


     def 	snake_case_  (	self  ,       __A    ):
        __a					=  """""".join(__A    ).replace(__A  ,       """ """    ).strip()
        return out_string


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        if not os.path.isdir(__A    ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
           return
        __a					=  os.path.join(
            __A  ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

        if os.path.abspath(self.vocab_file    ) != os.path.abspath(__A    ) and os.path.isfile(self.vocab_file    ):
           copyfile(self.vocab_file  ,       __A    )
        elif not os.path.isfile(self.vocab_file    ):
           with open(__A  ,       """wb"""    ) as fi:
              __a					=  self.sp_model.serialized_model_proto()
              fi.write(__A    )

        return (out_vocab_file,)


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):

        if token_ids_a is None:
           return token_ids_a + [self.sep_token_id]
        __a					=  [self.sep_token_id]
        return token_ids_a + sep + token_ids_a + sep



