





from sklearn.metrics import precision_score

import datasets


_snake_case				= """
Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation:
Precision = TP / (TP + FP)
where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive).
"""


_snake_case				= """
Args:
    predictions (`list` of `int`): Predicted class labels.
    references (`list` of `int`): Actual class labels.
    labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None.
    pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1.
    average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`.

        - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary.
        - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives.
        - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.
        - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall.
        - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification).
    sample_weight (`list` of `float`): Sample weights Defaults to None.
    zero_division (`int` or `string`): Sets the value to return when there is a zero division. Defaults to 'warn'.

        - 0: Returns 0 when there is a zero division.
        - 1: Returns 1 when there is a zero division.
        - 'warn': Raises warnings and then returns 0 when there is a zero division.

Returns:
    precision (`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better.

Examples:

    Example 1-A simple binary example
        >>> precision_metric = datasets.load_metric(\"precision\")
        >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0])
        >>> print(results)
        {'precision': 0.5}

    Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`.
        >>> precision_metric = datasets.load_metric(\"precision\")
        >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0)
        >>> print(round(results['precision'], 2))
        0.67

    Example 3-The same simple binary example as in Example 1, but with `sample_weight` included.
        >>> precision_metric = datasets.load_metric(\"precision\")
        >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3])
        >>> print(results)
        {'precision': 0.23529411764705882}

    Example 4-A multiclass example, with different values for the `average` input.
        >>> predictions = [0, 2, 1, 0, 0, 1]
        >>> references = [0, 1, 2, 0, 1, 2]
        >>> results = precision_metric.compute(predictions=predictions, references=references, average='macro')
        >>> print(results)
        {'precision': 0.2222222222222222}
        >>> results = precision_metric.compute(predictions=predictions, references=references, average='micro')
        >>> print(results)
        {'precision': 0.3333333333333333}
        >>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted')
        >>> print(results)
        {'precision': 0.2222222222222222}
        >>> results = precision_metric.compute(predictions=predictions, references=references, average=None)
        >>> print([round(res, 2) for res in results['precision']])
        [0.67, 0.0, 0.0]
"""


_snake_case				= """
@article{scikit-learn,
    title={Scikit-learn: Machine Learning in {P}ython},
    author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
    and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
    and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
    Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
    journal={Journal of Machine Learning Research},
    volume={12},
    pages={2825--2830},
    year={2011}
}
"""





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,							_KWARGS_DESCRIPTION					)
class 					lowerCAmelCase							(   datasets.Metric					):







  def   UpperCAmelCase     (			self  :str					):






     '''simple docstring'''





     return datasets.MetricInfo(
         description=_DESCRIPTION    ,							citation=_CITATION    ,							inputs_description=_KWARGS_DESCRIPTION    ,							features=datasets.Features(
             {
                 "predictions": datasets.Sequence(datasets.Value("int32"					)					),
                 "references": datasets.Sequence(datasets.Value("int32"					)					),
             }
             if self.config_name == "multilabel"
             else {
                 "predictions": datasets.Value("int32"					),
                 "references": datasets.Value("int32"					),
             }					)    ,							reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html"]    ,							)







  def   UpperCAmelCase     (			self  :List[str]    ,							_lowercase  :Optional[Any]    ,							_lowercase  :Dict    ,							_lowercase  :Union[str, Any]=None    ,							_lowercase  :int=1    ,							_lowercase  :List[str]="binary"    ,							_lowercase  :List[Any]=None    ,							_lowercase  :Optional[Any]="warn"    ,							):






     '''simple docstring'''





     lowercase__    =  precision_score(
         _lowercase    ,							_lowercase    ,							labels=_lowercase    ,							pos_label=_lowercase    ,							average=_lowercase    ,							sample_weight=_lowercase    ,							zero_division=_lowercase    ,							)
     return {"precision": float(_lowercase					) if score.size == 1 else score}













import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
  from tests.trainer.test_trainer import (  # noqa
      RegressionModelConfig,
      RegressionPreTrainedModel,
      get_regression_trainer,
  )


set_seed(42)

_snake_case				= get_tests_dir("""fixtures""")
_snake_case				= os.path.join(dirname(get_tests_dir()))
_snake_case				= dirname(os.path.abspath(__file__))

# default torch.distributed port
_snake_case				= """10999"""

_snake_case				= """t5-small"""

# *** Working Models ***
_snake_case				= """hf-internal-testing/tiny-albert"""
_snake_case				= """sshleifer/bart-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-bert"""
_snake_case				= """hf-internal-testing/tiny-random-bigbird_pegasus"""
_snake_case				= """hf-internal-testing/tiny-random-big_bird"""
_snake_case				= """hf-internal-testing/tiny-random-blenderbot"""
_snake_case				= """bigscience/bigscience-small-testing"""
_snake_case				= """hf-internal-testing/tiny-random-deberta"""
_snake_case				= """hf-internal-testing/tiny-random-deberta-v2"""
_snake_case				= """sshleifer/tiny-distilbert-base-cased"""
_snake_case				= """hf-internal-testing/tiny-electra"""
_snake_case				= """hf-internal-testing/tiny-random-flaubert"""
_snake_case				= """stas/tiny-wmt19-en-de"""
_snake_case				= """hf-internal-testing/tiny-random-funnel"""
_snake_case				= """sshleifer/tiny-gpt2"""
_snake_case				= """hf-internal-testing/tiny-random-gptj"""
_snake_case				= """hf-internal-testing/tiny-random-gpt_neo"""
_snake_case				= """hf-internal-testing/tiny-layoutlm"""
_snake_case				= """hf-internal-testing/tiny-random-led"""
_snake_case				= """hf-internal-testing/tiny-random-longformer"""
_snake_case				= """stas/tiny-m2m_100"""  # hf tiny model is unsuitable
_snake_case				= """sshleifer/tiny-marian-en-de"""
_snake_case				= """sshleifer/tiny-mbart"""
_snake_case				= """hf-internal-testing/tiny-random-mobilebert"""
_snake_case				= """hf-internal-testing/tiny-random-mpnet"""
_snake_case				= """stas/pegasus-cnn_dailymail-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-random-prophetnet"""
_snake_case				= """sshleifer/tiny-distilroberta-base"""
_snake_case				= """hf-internal-testing/tiny-random-squeezebert"""
_snake_case				= """patrickvonplaten/t5-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-random-t5-v1.1"""
_snake_case				= """hf-internal-testing/tiny-random-vit"""
_snake_case				= """hf-internal-testing/tiny-xlm-roberta"""
_snake_case				= """sshleifer/tiny-xlnet-base-cased"""


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
_snake_case				= """hf-internal-testing/tiny-random-mt5"""
_snake_case				= """hf-internal-testing/tiny-random-camembert"""
_snake_case				= """hf-internal-testing/tiny-random-openai-gpt"""

# missing tokenizer files
_snake_case				= """hf-internal-testing/tiny-random-convbert"""
_snake_case				= """hf-internal-testing/tiny-random-layoutlmv2"""
_snake_case				= """hf-internal-testing/tiny-random-hubert"""

# issues with tokenizer
_snake_case				= """hf-internal-testing/tiny-random-ctrl"""
_snake_case				= """hf-internal-testing/tiny-random-transfo-xl"""  # same as ctrl

# other issues with tiny models
_snake_case				= """hf-internal-testing/tiny-random-ibert"""  # multiple issues with either mlm/qa/clas
_snake_case				= """hf-internal-testing/tiny-random-reformer"""  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
_snake_case				= """hf-internal-testing/tiny-random-dpr"""
# - "dpr"  examples/research_projects/rag-end2end-retriever/
_snake_case				= """hf-internal-testing/tiny-random-rag"""
# - "rag" research_projects
_snake_case				= """"""
# - "luke" Entities classes - no plan to make such example
_snake_case				= """hf-internal-testing/tiny-random-lxmert"""
# - "lxmert" doesn't work with run_qa.py
_snake_case				= """hf-internal-testing/tiny-random-clip"""
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
_snake_case				= """hf-internal-testing/tiny-random-speech_to_text"""
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
_snake_case				= """hf-internal-testing/tiny-random-tapas"""


def _A   (    __magic_name__=False       ):
   # 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
   # - it won't be able to handle that
   # 2. for now testing with just 2 gpus max (since some quality tests may give different
   # results with mode gpus because we use very little data)
   lowercase__    =  min(2  ,					get_gpu_count()       ) if distributed else 1
   lowercase__    =  os.environ.get("DS_TEST_PORT"  ,					__magic_name__       )
   return f'''deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}'''.split()


def _A   (    ):
   lowercase__    =  f'''{FIXTURE_DIRECTORY}/tests_samples'''
   lowercase__    =  f'''{data_dir_samples}/wmt_en_ro'''
   lowercase__    =  f'''{data_dir_samples}/xsum'''
   lowercase__    =  "\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        ".split()

   # try to cover as many models as possible once (it's enough to run on one task per model)
   # but need a tiny model for each
   #
   # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
   lowercase__    =  {
       "trans": [
           "bart",
           "fsmt",
           "m2m_100",
           "marian",
           "mbart",
           "t5",
           "t5_v1",
           # "mt5", missing model files
       ],
       "sum": [
           "pegasus",
       ],
       "clm": [
           "big_bird",
           "bigbird_pegasus",
           "blenderbot",
           "bloom",
           "gpt2",
           "gpt_neo",
           "gptj",
           "xlm-roberta",
           "prophetnet",
           # "camembert", missing model files
       ],
       "mlm": [
           "albert",
           "deberta",
           "deberta-v2",
           "distilbert",
           "electra",
           "flaubert",
           "funnel",
           "layoutlm",
           # "reformer", # multiple issues with either mlm/qa/clas
       ],
       "qa": [
           "led",
           "longformer",
           "mobilebert",
           "mpnet",
           "roberta",
           "squeezebert",
           # "convbert", # missing tokenizer files
           # "layoutlmv2", missing model files
       ],
       "clas": [
           "bert",
           "xlnet",
           # "hubert", # missing tokenizer files
           # "ibert", # multiple issues with either mlm/qa/clas
           # "transfo-xl", # tokenizer issues as ctrl
           # "ctrl", # tokenizer issues
           # "openai-gpt", missing model files
           # "tapas", multiple issues
       ],
       "img_clas": [
           "vit",
       ],
   }

   lowercase__    =  f'''{ROOT_DIRECTORY}/examples/pytorch'''

   lowercase__    =  {
       "trans": f'''
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        ''',
       "sum": f'''
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        ''',
       "clm": f'''
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        ''',
       "mlm": f'''
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        ''',
       "qa": f'''
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        ''',
       "clas": f'''
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        ''',
       "img_clas": f'''
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        ''',
   }

   lowercase__    =  get_launcher(distributed=__magic_name__       )

   lowercase__    =  {}
   for task, args in tasks.items():
      lowercase__    =  args.split()
      for model in tasksamodels[task]:
         lowercase__    =  globals()[f'''{model.upper().replace('-'  ,					'_'       )}_TINY''']
         lowercase__    =  f'''--model_name_or_path {model_name}'''.split()
         lowercase__    =  launcher + args + args_model + args_main

         # # generation special case
         # if task == "gen":
         #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
         #     args_model += f"--model_type {model}".split()
         #     cmds[f"{task}_{model}"] = launcher + args + args_model
         # else:

   return cmds


_snake_case				= make_task_cmds()

_snake_case				= """zero2"""
_snake_case				= """zero3"""

_snake_case				= [ZEROa, ZEROa]


def _A   (    __magic_name__  ,					__magic_name__  ,					__magic_name__       ):
   # customize the test name generator function as we want both params to appear in the sub-test
   # name, as by default it shows only the first param
   lowercase__    =  parameterized.to_safe_name("_".join(str(__magic_name__       ) for x in param.args       )       )
   return f'''{func.__name__}_{param_based_name}'''


# Cartesian-product of zero stages with models to test
_snake_case				= list(itertools.product(stages, task_cmds.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class 					lowerCAmelCase							(   lowercase_					):







  def   UpperCAmelCase     (			self  :Tuple    ,							_lowercase  :int    ,							_lowercase  :Any					):






     '''simple docstring'''





     if task not in task_cmds:
        raise ValueError(f'''don\'t know of task {task}, have {task_cmds.keys()}'''					)

     lowercase__    =  task_cmds[task]
     lowercase__    =  f'''--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json'''.split()

     lowercase__    =  self.get_auto_remove_tmp_dir()
     lowercase__    =  f'''--output_dir {output_dir}'''.split()

     cmd += args_ds + args_out

     return cmd, output_dir







  @parameterized.expand(_lowercase    ,							name_func=_lowercase					)
  def   UpperCAmelCase     (			self  :str    ,							_lowercase  :Dict    ,							_lowercase  :Tuple					):






     '''simple docstring'''





     lowercase__   ,							lowercase__    =  self.get_task_cmd(_lowercase    ,							_lowercase					)

     # 1. generate the checkpoint
     cmd += "--save_steps 1".split()
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
     execute_subprocess_async(_lowercase    ,							env=self.get_env()					)

     # 2. test that the fp32 weights get reconsolidated
     lowercase__    =  f'''{output_dir}/checkpoint-1'''
     lowercase__    =  f'''{chkpt_dir}/out.bin'''
     lowercase__    =  f'''{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}'''
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
     subprocess.check_call(_lowercase    ,							shell=_lowercase					)
     assert os.path.exists(_lowercase					), f'''{recovered_model_path} was not found'''

     # possibly could also test that the resulting saved model is usable but given that we use
     # random models we won't know if it's any good






