


"""simple docstring"""







import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_						,		lowerCAmelCase_     )
					assert dataset.num_rows == 4
					assert dataset.num_columns == 1
					assert dataset.column_names == ["text"]
					for feature, expected_dtype in expected_features.items():
										assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory"						,		[False, True]     )
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										__SCREAMING_SNAKE_CASE		     =       TextDatasetReader(lowerCAmelCase_						,		cache_dir=lowerCAmelCase_						,		keep_in_memory=lowerCAmelCase_     ).read()
					_check_text_dataset(lowerCAmelCase_						,		lowerCAmelCase_     )




@pytest.mark.parametrize(
    "features"						,		[
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]						,		)
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					__SCREAMING_SNAKE_CASE		     =       features.copy() if features else default_expected_features
					__SCREAMING_SNAKE_CASE		     =       (
					    Features({feature: Value(lowerCAmelCase_     ) for feature, dtype in features.items()}     ) if features is not None else None
					)
					__SCREAMING_SNAKE_CASE		     =       TextDatasetReader(lowerCAmelCase_						,		features=lowerCAmelCase_						,		cache_dir=lowerCAmelCase_     ).read()
					_check_text_dataset(lowerCAmelCase_						,		lowerCAmelCase_     )




@pytest.mark.parametrize("split"						,		[None, NamedSplit("train"     ), "train", "test"]     )
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					__SCREAMING_SNAKE_CASE		     =       TextDatasetReader(lowerCAmelCase_						,		cache_dir=lowerCAmelCase_						,		split=lowerCAmelCase_     ).read()
					_check_text_dataset(lowerCAmelCase_						,		lowerCAmelCase_     )
					assert dataset.split == split if split else "train"




@pytest.mark.parametrize("path_type"						,		[str, list]     )
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					if issubclass(lowerCAmelCase_						,		lowerCAmelCase_     ):
										__SCREAMING_SNAKE_CASE		     =       text_path
					elif issubclass(lowerCAmelCase_						,		lowerCAmelCase_     ):
										__SCREAMING_SNAKE_CASE		     =       [text_path]
					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					__SCREAMING_SNAKE_CASE		     =       TextDatasetReader(lowerCAmelCase_						,		cache_dir=lowerCAmelCase_     ).read()
					_check_text_dataset(lowerCAmelCase_						,		lowerCAmelCase_     )




def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_=("train",)     ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_						,		lowerCAmelCase_     )
					for split in splits:
										__SCREAMING_SNAKE_CASE		     =       dataset_dict[split]
										assert dataset.num_rows == 4
										assert dataset.num_columns == 1
										assert dataset.column_names == ["text"]
										for feature, expected_dtype in expected_features.items():
															assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory"						,		[False, True]     )
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										__SCREAMING_SNAKE_CASE		     =       TextDatasetReader({"train": text_path}						,		cache_dir=lowerCAmelCase_						,		keep_in_memory=lowerCAmelCase_     ).read()
					_check_text_datasetdict(lowerCAmelCase_						,		lowerCAmelCase_     )




@pytest.mark.parametrize(
    "features"						,		[
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]						,		)
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					__SCREAMING_SNAKE_CASE		     =       features.copy() if features else default_expected_features
					__SCREAMING_SNAKE_CASE		     =       (
					    Features({feature: Value(lowerCAmelCase_     ) for feature, dtype in features.items()}     ) if features is not None else None
					)
					__SCREAMING_SNAKE_CASE		     =       TextDatasetReader({"train": text_path}						,		features=lowerCAmelCase_						,		cache_dir=lowerCAmelCase_     ).read()
					_check_text_datasetdict(lowerCAmelCase_						,		lowerCAmelCase_     )




@pytest.mark.parametrize("split"						,		[None, NamedSplit("train"     ), "train", "test"]     )
def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					if split:
										__SCREAMING_SNAKE_CASE		     =       {split: text_path}
					else:
										__SCREAMING_SNAKE_CASE		     =       "train"
										__SCREAMING_SNAKE_CASE		     =       {"train": text_path, "test": text_path}
					__SCREAMING_SNAKE_CASE		     =       tmp_path / "cache"
					__SCREAMING_SNAKE_CASE		     =       {"text": "string"}
					__SCREAMING_SNAKE_CASE		     =       TextDatasetReader(lowerCAmelCase_						,		cache_dir=lowerCAmelCase_     ).read()
					_check_text_datasetdict(lowerCAmelCase_						,		lowerCAmelCase_						,		splits=list(path.keys()     )     )
					assert all(dataset[split].split == split for split in path.keys()     )




"""simple docstring"""







import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
a__       :							Dict        =  {'''LayoutLMv2Config''', '''LayoutLMv3Config'''}




@is_pipeline_test
class 				UpperCamelCase_			(				unittest.TestCase):

	"""simple docstring"""





	snake_case__       :						str					      =    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
	snake_case__       :						Tuple					      =    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

	if model_mapping is not None:
		snake_case__       :						List[Any]					      =    {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
	if tf_model_mapping is not None:
		snake_case__       :						Dict					      =    {
		    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
		}




	@require_torch
	def 						UpperCAmelCase_   (				self    :       str     ) ->			Tuple:
						__SCREAMING_SNAKE_CASE		     =       pipeline(
						    task="text-classification"    ,    model="hf-internal-testing/tiny-random-distilbert"    ,    framework="pt"     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}]     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"    ,    top_k=2     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}]     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier(["This is great !", "This is bad"]    ,    top_k=2     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [
						        [{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}],
						        [{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}],
						    ]    ,    )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"    ,    top_k=1     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}]     )

						# Legacy behavior
						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"    ,    return_all_scores=UpperCAmelCase__     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}]     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"    ,    return_all_scores=UpperCAmelCase__     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [[{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}]]     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier(["This is great !", "Something else"]    ,    return_all_scores=UpperCAmelCase__     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [
						        [{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}],
						        [{"label": "LABEL_0", "score": 0.504}, {"label": "LABEL_1", "score": 0.496}],
						    ]    ,    )

						__SCREAMING_SNAKE_CASE		     =       text_classifier(["This is great !", "Something else"]    ,    return_all_scores=UpperCAmelCase__     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [
						        {"label": "LABEL_0", "score": 0.504},
						        {"label": "LABEL_0", "score": 0.504},
						    ]    ,    )




	@require_torch
	def 						UpperCAmelCase_   (				self    :       Union[str, Any]     ) ->			Optional[int]:
						import torch

						__SCREAMING_SNAKE_CASE		     =       pipeline(
						    task="text-classification"    ,    model="hf-internal-testing/tiny-random-distilbert"    ,    framework="pt"    ,    device=torch.device("cpu"     )    ,    )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}]     )




	@require_tf
	def 						UpperCAmelCase_   (				self    :       Dict     ) ->			Optional[int]:
						__SCREAMING_SNAKE_CASE		     =       pipeline(
						    task="text-classification"    ,    model="hf-internal-testing/tiny-random-distilbert"    ,    framework="tf"     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "LABEL_0", "score": 0.504}]     )




	@slow
	@require_torch
	def 						UpperCAmelCase_   (				self    :       Optional[Any]     ) ->			int:
						__SCREAMING_SNAKE_CASE		     =       pipeline("text-classification"     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "POSITIVE", "score": 1.0}]     )
						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is bad !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "NEGATIVE", "score": 1.0}]     )
						__SCREAMING_SNAKE_CASE		     =       text_classifier("Birds are a type of animal"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "POSITIVE", "score": 0.988}]     )




	@slow
	@require_tf
	def 						UpperCAmelCase_   (				self    :       List[str]     ) ->			List[str]:
						__SCREAMING_SNAKE_CASE		     =       pipeline("text-classification"    ,    framework="tf"     )

						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is great !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "POSITIVE", "score": 1.0}]     )
						__SCREAMING_SNAKE_CASE		     =       text_classifier("This is bad !"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "NEGATIVE", "score": 1.0}]     )
						__SCREAMING_SNAKE_CASE		     =       text_classifier("Birds are a type of animal"     )
						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": "POSITIVE", "score": 0.988}]     )




	def 						UpperCAmelCase_   (				self    :       Any    ,    UpperCAmelCase__    :       str    ,    UpperCAmelCase__    :       Any    ,    UpperCAmelCase__    :       Tuple     ) ->			Dict:
						__SCREAMING_SNAKE_CASE		     =       TextClassificationPipeline(model=UpperCAmelCase__    ,    tokenizer=UpperCAmelCase__     )
						return text_classifier, ["HuggingFace is in", "This is another test"]




	def 						UpperCAmelCase_   (				self    :       str    ,    UpperCAmelCase__    :       Optional[int]    ,    UpperCAmelCase__    :       Optional[int]     ) ->			Dict:
						__SCREAMING_SNAKE_CASE		     =       text_classifier.model
						# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
						__SCREAMING_SNAKE_CASE		     =       "HuggingFace is in"
						__SCREAMING_SNAKE_CASE		     =       text_classifier(UpperCAmelCase__     )

						self.assertEqual(nested_simplify(UpperCAmelCase__     )    ,    [{"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}]     )
						self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()     )

						__SCREAMING_SNAKE_CASE		     =       ["HuggingFace is in ", "Paris is in France"]
						__SCREAMING_SNAKE_CASE		     =       text_classifier(UpperCAmelCase__     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [{"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}, {"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}]    ,    )
						self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()     )
						self.assertTrue(outputs[1]["label"] in model.config.idalabel.values()     )

						# Forcing to get all results with `top_k=None`
						# This is NOT the legacy format
						__SCREAMING_SNAKE_CASE		     =       text_classifier(UpperCAmelCase__    ,    top_k=UpperCAmelCase__     )
						__SCREAMING_SNAKE_CASE		     =       len(model.config.idalabel.values()     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [[{"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}] * N, [{"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}] * N]    ,    )

						__SCREAMING_SNAKE_CASE		     =       {"text": "HuggingFace is in ", "text_pair": "Paris is in France"}
						__SCREAMING_SNAKE_CASE		     =       text_classifier(UpperCAmelCase__     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    {"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}    ,    )
						self.assertTrue(outputs["label"] in model.config.idalabel.values()     )

						# This might be used a text pair, but tokenizer + pipe interaction
						# makes it hard to understand that it's not using the pair properly
						# https://github.com/huggingface/transformers/issues/17305
						# We disabled this usage instead as it was outputting wrong outputs.
						__SCREAMING_SNAKE_CASE		     =       [["HuggingFace is in ", "Paris is in France"]]
						with self.assertRaises(UpperCAmelCase__     ):
											text_classifier(UpperCAmelCase__     )

						# This used to be valid for doing text pairs
						# We're keeping it working because of backward compatibility
						__SCREAMING_SNAKE_CASE		     =       text_classifier([[["HuggingFace is in ", "Paris is in France"]]]     )
						self.assertEqual(
						    nested_simplify(UpperCAmelCase__     )    ,    [{"label": ANY(UpperCAmelCase__     ), "score": ANY(UpperCAmelCase__     )}]    ,    )
						self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()     )
