






import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
			from sagemaker import Session, TrainingJobAnalytics
			from sagemaker.huggingface import HuggingFace




@pytest.mark.skipif(
    literal_eval(os.getenv("""TEST_SAGEMAKER"""		,							"""False"""      )      ) is not True		,							reason="""Skipping test because should only be run when releasing minor transformers version"""		,							)
@pytest.mark.usefixtures("""sm_env"""      )
@parameterized_class(
    [
        {
            """framework""": """pytorch""",
            """script""": """run_glue_model_parallelism.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_600, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
        {
            """framework""": """pytorch""",
            """script""": """run_glue.py""",
            """model_name_or_path""": """roberta-large""",
            """instance_type""": """ml.p3dn.24xlarge""",
            """results""": {"""train_runtime""": 1_600, """eval_accuracy""": 0.3, """eval_loss""": 1.2},
        },
    ]      )
class 							snake_case_       (    unittest.TestCase      ):

						"""simple docstring"""



						def        UpperCAmelCase__					(self:					Any   )    ->     Optional[int]:




											'''simple docstring'''



											if self.framework == "pytorch":
																subprocess.run(
																    f'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split() ,					encoding="utf-8" ,					check=__UpperCAmelCase ,					)
											assert hasattr(self ,					"env"   )



						def        UpperCAmelCase__					(self:					Optional[Any] ,					__UpperCAmelCase:					Any   )    ->     Tuple:




											'''simple docstring'''



											__a					:						int      =					{
											    "enabled": True,
											    "processes_per_host": 8,
											}
											__a					:						Union[str, Any]      =					{
											    "enabled": True,
											    "parameters": {
											        "microbatches": 4,
											        "placement_strategy": "spread",
											        "pipeline": "interleaved",
											        "optimize": "speed",
											        "partitions": 4,
											        "ddp": True,
											    },
											}

											__a					:						Union[str, Any]      =					{"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

											__a					:						int      =					"trainer" if self.script == "run_glue.py" else "smtrainer"
											# creates estimator
											return HuggingFace(
											    entry_point=self.script ,					source_dir=self.env.test_path ,					role=self.env.role ,					image_uri=self.env.image_uri ,					base_job_name=f'{self.env.base_job_name}-{instance_count}-smp-{name_extension}' ,					instance_count=__UpperCAmelCase ,					instance_type=self.instance_type ,					debugger_hook_config=__UpperCAmelCase ,					hyperparameters={
											        **self.env.hyperparameters,
											        "model_name_or_path": self.model_name_or_path,
											        "max_steps": 500,
											    } ,					metric_definitions=self.env.metric_definitions ,					distribution=__UpperCAmelCase ,					py_version="py36" ,					)



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					Union[str, Any]   )    ->     str:




											'''simple docstring'''



											TrainingJobAnalytics(__UpperCAmelCase   ).export_csv(f'{self.env.test_path}/{job_name}_metrics.csv'   )



						@parameterized.expand([(1,)]   )
						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					Optional[int]   )    ->     Optional[int]:




											'''simple docstring'''



											__a					:						Optional[int]      =					self.create_estimator(__UpperCAmelCase   )

											# run training
											estimator.fit()

											# result dataframe
											__a					:						Optional[Any]      =					TrainingJobAnalytics(estimator.latest_training_job.name   ).dataframe()

											# extract kpis
											__a					:						List[str]      =					list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]   )
											__a					:						Optional[int]      =					list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]   )
											# get train time from SageMaker job, this includes starting, preprocessing, stopping
											__a					:						str      =					(
											    Session().describe_training_job(estimator.latest_training_job.name   ).get("TrainingTimeInSeconds" ,					999999   )
											)

											# assert kpis
											assert train_runtime <= self.results["train_runtime"]
											assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy   )
											assert all(t <= self.results["eval_loss"] for t in eval_loss   )

											# dump tests result into json file to share in PR
											with open(f'{estimator.latest_training_job.name}.json' ,					"w"   ) as outfile:
																json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss} ,					__UpperCAmelCase   )













from collections import defaultdict
from typing import Optional

from ..image_utils import load_image
from ..utils import (
    add_end_docstrings,
    is_torch_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ChunkPipeline


if is_torch_available():
			import torch

			from ..models.auto.modeling_auto import MODEL_FOR_MASK_GENERATION_MAPPING

UpperCAmelCase__      			=						logging.get_logger(__name__)




@add_end_docstrings(__UpperCamelCase      )
class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					int ,					**__UpperCAmelCase:					Optional[Any]   )    ->     str:




											'''simple docstring'''



											super().__init__(**__UpperCAmelCase   )
											requires_backends(self ,					"vision"   )
											requires_backends(self ,					"torch"   )

											if self.framework != "pt":
																raise ValueError(f'The {self.__class__} is only available in PyTorch.'   )

											self.check_model_type(__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Optional[int] ,					**__UpperCAmelCase:					Optional[int]   )    ->     List[Any]:




											'''simple docstring'''



											__a					:						Any      =					{}
											__a					:						Union[str, Any]      =					{}
											__a					:						List[str]      =					{}
											# preprocess args
											if "points_per_batch" in kwargs:
																__a					:						Dict      =					kwargs["points_per_batch"]
											if "points_per_crop" in kwargs:
																__a					:						Tuple      =					kwargs["points_per_crop"]
											if "crops_n_layers" in kwargs:
																__a					:						Dict      =					kwargs["crops_n_layers"]
											if "crop_overlap_ratio" in kwargs:
																__a					:						int      =					kwargs["crop_overlap_ratio"]
											if "crop_n_points_downscale_factor" in kwargs:
																__a					:						Optional[int]      =					kwargs["crop_n_points_downscale_factor"]
											# postprocess args
											if "pred_iou_thresh" in kwargs:
																__a					:						int      =					kwargs["pred_iou_thresh"]
											if "stability_score_offset" in kwargs:
																__a					:						Any      =					kwargs["stability_score_offset"]
											if "mask_threshold" in kwargs:
																__a					:						int      =					kwargs["mask_threshold"]
											if "stability_score_thresh" in kwargs:
																__a					:						Any      =					kwargs["stability_score_thresh"]
											if "crops_nms_thresh" in kwargs:
																__a					:						int      =					kwargs["crops_nms_thresh"]
											if "output_rle_mask" in kwargs:
																__a					:						Optional[Any]      =					kwargs["output_rle_mask"]
											if "output_bboxes_mask" in kwargs:
																__a					:						Optional[Any]      =					kwargs["output_bboxes_mask"]
											return preprocess_kwargs, forward_params, postprocess_kwargs



						def __call__(self:					Union[str, Any] ,					__UpperCAmelCase:					str ,					*__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					str=None ,					__UpperCAmelCase:					Union[str, Any]=None ,					**__UpperCAmelCase:					Optional[int]   )    ->     Tuple:




											'''simple docstring'''



											return super().__call__(__UpperCAmelCase ,					*__UpperCAmelCase ,					num_workers=__UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					**__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Optional[Any] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					int=64 ,					__UpperCAmelCase:					int = 0 ,					__UpperCAmelCase:					float = 512 / 1500 ,					__UpperCAmelCase:					Optional[int] = 32 ,					__UpperCAmelCase:					Optional[int] = 1 ,					)    ->     str:




											'''simple docstring'''



											__a					:						Union[str, Any]      =					load_image(__UpperCAmelCase   )
											__a					:						List[str]      =					self.image_processor.size["longest_edge"]
											__a      ,			__a      ,			__a      ,			__a					:						int      =					self.image_processor.generate_crop_boxes(
											    __UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )
											__a					:						Union[str, Any]      =					self.image_processor(images=__UpperCAmelCase ,					return_tensors="pt"   )

											with self.device_placement():
																if self.framework == "pt":
																					__a					:						str      =					self.get_inference_context()
																					with inference_context():
																										__a					:						int      =					self._ensure_tensor_on_device(__UpperCAmelCase ,					device=self.device   )
																										__a					:						Union[str, Any]      =					self.model.get_image_embeddings(model_inputs.pop("pixel_values"   )   )
																										__a					:						Optional[Any]      =					image_embeddings

											__a					:						Dict      =					grid_points.shape[1]
											__a					:						Any      =					points_per_batch if points_per_batch is not None else n_points

											if points_per_batch <= 0:
																raise ValueError(
																    "Cannot have points_per_batch<=0. Must be >=1 to returned batched outputs. "
																    "To return all points at once, set points_per_batch to None"   )

											for i in range(0 ,					__UpperCAmelCase ,					__UpperCAmelCase   ):
																__a					:						List[str]      =					grid_points[:, i : i + points_per_batch, :, :]
																__a					:						str      =					input_labels[:, i : i + points_per_batch]
																__a					:						str      =					i == n_points - points_per_batch
																yield {
																    "input_points": batched_points,
																    "input_labels": labels,
																    "input_boxes": crop_boxes,
																    "is_last": is_last,
																    **model_inputs,
																}



						def        UpperCAmelCase__					(self:					Any ,					__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					Any=0.88 ,					__UpperCAmelCase:					Optional[int]=0.95 ,					__UpperCAmelCase:					Optional[int]=0 ,					__UpperCAmelCase:					int=1 ,					)    ->     List[str]:




											'''simple docstring'''



											__a					:						List[Any]      =					model_inputs.pop("input_boxes"   )
											__a					:						Optional[Any]      =					model_inputs.pop("is_last"   )
											__a					:						Dict      =					model_inputs.pop("original_sizes"   ).tolist()
											__a					:						int      =					model_inputs.pop("reshaped_input_sizes"   ).tolist()

											__a					:						Optional[Any]      =					self.model(**__UpperCAmelCase   )

											# post processing happens here in order to avoid CPU GPU copies of ALL the masks
											__a					:						Union[str, Any]      =					model_outputs["pred_masks"]
											__a					:						Optional[Any]      =					self.image_processor.post_process_masks(
											    __UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					binarize=__UpperCAmelCase   )
											__a					:						Optional[Any]      =					model_outputs["iou_scores"]
											__a      ,			__a      ,			__a					:						Dict      =					self.image_processor.filter_masks(
											    masks[0] ,					iou_scores[0] ,					original_sizes[0] ,					input_boxes[0] ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					)
											return {
											    "masks": masks,
											    "is_last": is_last,
											    "boxes": boxes,
											    "iou_scores": iou_scores,
											}



						def        UpperCAmelCase__					(self:					Any ,					__UpperCAmelCase:					str ,					__UpperCAmelCase:					List[str]=False ,					__UpperCAmelCase:					int=False ,					__UpperCAmelCase:					List[str]=0.7 ,					)    ->     Dict:




											'''simple docstring'''



											__a					:						Dict      =					[]
											__a					:						Optional[Any]      =					[]
											__a					:						Optional[Any]      =					[]
											for model_output in model_outputs:
																all_scores.append(model_output.pop("iou_scores"   )   )
																all_masks.extend(model_output.pop("masks"   )   )
																all_boxes.append(model_output.pop("boxes"   )   )

											__a					:						Any      =					torch.cat(__UpperCAmelCase   )
											__a					:						Union[str, Any]      =					torch.cat(__UpperCAmelCase   )
											__a      ,			__a      ,			__a      ,			__a					:						Tuple      =					self.image_processor.post_process_for_mask_generation(
											    __UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )

											__a					:						Optional[int]      =					defaultdict(__UpperCAmelCase   )
											for output in model_outputs:
																for k, v in output.items():
																					extra[k].append(__UpperCAmelCase   )

											__a					:						int      =					{}
											if output_rle_mask:
																__a					:						Optional[int]      =					rle_mask

											if output_bboxes_mask:
																__a					:						List[str]      =					bounding_boxes

											return {"masks": output_masks, "scores": iou_scores, **optional, **extra}





