


'''simple docstring'''




import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
       raise RuntimeError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training")

# TF training parameters
_a   :   Optional[int]												=   False
_a   :   Dict												=   False





def   _a		(lowercase__			:   Namespace						)			->			int:






     """simple docstring"""
     return TrainCommand(lowercase__						)




class 			_lowercase (	__lowercase							):
   @staticmethod
   def a  (						SCREAMING_SNAKE_CASE_       :      ArgumentParser      ) ->							Any:
        __snake_case           =						parser.add_parser('train'       ,   help='CLI tool to train a model on a task.'      )

        train_parser.add_argument(
            '--train_data'       ,   type=SCREAMING_SNAKE_CASE_       ,   required=SCREAMING_SNAKE_CASE_       ,   help='path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences.'       ,   )
        train_parser.add_argument(
            '--column_label'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=0       ,   help='Column of the dataset csv file with example labels.'      )
        train_parser.add_argument(
            '--column_text'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=1       ,   help='Column of the dataset csv file with example texts.'      )
        train_parser.add_argument(
            '--column_id'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=2       ,   help='Column of the dataset csv file with example ids.'      )
        train_parser.add_argument(
            '--skip_first_row'       ,   action='store_true'       ,   help='Skip the first row of the csv file (headers).'      )

        train_parser.add_argument('--validation_data'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=''       ,   help='path to validation dataset.'      )
        train_parser.add_argument(
            '--validation_split'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=0.1       ,   help='if validation dataset is not provided, fraction of train dataset to use as validation dataset.'       ,   )

        train_parser.add_argument('--output'       ,   type=SCREAMING_SNAKE_CASE_       ,   default='./'       ,   help='path to saved the trained model.'      )

        train_parser.add_argument(
            '--task'       ,   type=SCREAMING_SNAKE_CASE_       ,   default='text_classification'       ,   help='Task to train the model on.'      )
        train_parser.add_argument(
            '--model'       ,   type=SCREAMING_SNAKE_CASE_       ,   default='bert-base-uncased'       ,   help='Model\'s name or path to stored model.'      )
        train_parser.add_argument('--train_batch_size'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=32       ,   help='Batch size for training.'      )
        train_parser.add_argument('--valid_batch_size'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=64       ,   help='Batch size for validation.'      )
        train_parser.add_argument('--learning_rate'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=3e-5       ,   help='Learning rate.'      )
        train_parser.add_argument('--adam_epsilon'       ,   type=SCREAMING_SNAKE_CASE_       ,   default=1e-08       ,   help='Epsilon for Adam optimizer.'      )
        train_parser.set_defaults(func=SCREAMING_SNAKE_CASE_      )
   def __init__(						self       :      str       ,   SCREAMING_SNAKE_CASE_       :      Namespace      ) ->							Optional[int]:
        __snake_case           =						logging.get_logger('transformers-cli/training'      )

        __snake_case           =						'tf' if is_tf_available() else 'torch'

        os.makedirs(args.output       ,   exist_ok=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						args.output

        __snake_case           =						args.column_label
        __snake_case           =						args.column_text
        __snake_case           =						args.column_id

        self.logger.info(f'Loading {args.task} pipeline for {args.model}'      )
        if args.task == "text_classification":
             __snake_case           =						TextClassificationPipeline.from_pretrained(args.model      )
        elif args.task == "token_classification":
             raise NotImplementedError
        elif args.task == "question_answering":
             raise NotImplementedError

        self.logger.info(f'Loading dataset from {args.train_data}'      )
        __snake_case           =						Processor.create_from_csv(
            args.train_data       ,   column_label=args.column_label       ,   column_text=args.column_text       ,   column_id=args.column_id       ,   skip_first_row=args.skip_first_row       ,   )
        __snake_case           =						None
        if args.validation_data:
             self.logger.info(f'Loading validation dataset from {args.validation_data}'      )
             __snake_case           =						Processor.create_from_csv(
                 args.validation_data       ,   column_label=args.column_label       ,   column_text=args.column_text       ,   column_id=args.column_id       ,   skip_first_row=args.skip_first_row       ,   )

        __snake_case           =						args.validation_split
        __snake_case           =						args.train_batch_size
        __snake_case           =						args.valid_batch_size
        __snake_case           =						args.learning_rate
        __snake_case           =						args.adam_epsilon
   def a  (						self       :      Optional[int]      ) ->							int:
        if self.framework == "tf":
             return self.run_tf()
        return self.run_torch()
   def a  (						self       :      Tuple      ) ->							List[str]:
        raise NotImplementedError





   def a  (						self       :      Dict      ) ->							Optional[int]:
        self.pipeline.fit(
            self.train_dataset       ,   validation_data=self.valid_dataset       ,   validation_split=self.validation_split       ,   learning_rate=self.learning_rate       ,   adam_epsilon=self.adam_epsilon       ,   train_batch_size=self.train_batch_size       ,   valid_batch_size=self.valid_batch_size       ,   )

        # Save trained pipeline
        self.pipeline.save_pretrained(self.output      )








'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import CLIPSegProcessor, ViTImageProcessor





@require_vision
class        lowercase__    (		unittest.TestCase   ):





							'''simple docstring'''


							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= tempfile.mkdtemp()

											# fmt: off
											_SCREAMING_SNAKE_CASE					:			Any							= ["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
											# fmt: on
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= dict(zip(__snake_case       ,   range(len(__snake_case    )    )    )    )
											_SCREAMING_SNAKE_CASE					:			str							= ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
											_SCREAMING_SNAKE_CASE					:			Any							= {"""unk_token""": """<unk>"""}

											_SCREAMING_SNAKE_CASE					:			Optional[int]							= os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES["""vocab_file"""]    )
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES["""merges_file"""]    )
											with open(self.vocab_file       ,   """w"""       ,   encoding="""utf-8"""    ) as fp:
															fp.write(json.dumps(__snake_case    ) + """\n"""    )
											with open(self.merges_file       ,   """w"""       ,   encoding="""utf-8"""    ) as fp:
															fp.write("""\n""".join(__snake_case    )    )

											_SCREAMING_SNAKE_CASE					:			Any							= {
											    """do_resize""": True,
											    """size""": 20,
											    """do_center_crop""": True,
											    """crop_size""": 18,
											    """do_normalize""": True,
											    """image_mean""": [0.48145466, 0.4578275, 0.40821073],
											    """image_std""": [0.26862954, 0.26130258, 0.27577711],
											}
											_SCREAMING_SNAKE_CASE					:			List[str]							= os.path.join(self.tmpdirname       ,   __snake_case    )
											with open(self.image_processor_file       ,   """w"""       ,   encoding="""utf-8"""    ) as fp:
															json.dump(__snake_case       ,   __snake_case    )






							def UpperCAmelCase_      (       self       ,   **__snake_case    ):
											return CLIPTokenizer.from_pretrained(self.tmpdirname       ,   **__snake_case    )






							def UpperCAmelCase_      (       self       ,   **__snake_case    ):
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,   **__snake_case    )






							def UpperCAmelCase_      (       self       ,   **__snake_case    ):
											return ViTImageProcessor.from_pretrained(self.tmpdirname       ,   **__snake_case    )






							def UpperCAmelCase_      (       self    ):
											shutil.rmtree(self.tmpdirname    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [np.random.randint(255       ,   size=(3, 30, 400)       ,   dtype=np.uinta    )]

											_SCREAMING_SNAKE_CASE					:			List[str]							= [Image.fromarray(np.moveaxis(__snake_case       ,   0       ,   -1    )    ) for x in image_inputs]

											return image_inputs






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.get_tokenizer()
											_SCREAMING_SNAKE_CASE					:			int							= self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE					:			Dict							= self.get_image_processor()

											_SCREAMING_SNAKE_CASE					:			str							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )
											processor_slow.save_pretrained(self.tmpdirname    )
											_SCREAMING_SNAKE_CASE					:			List[str]							= CLIPSegProcessor.from_pretrained(self.tmpdirname       ,   use_fast=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Any							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )
											processor_fast.save_pretrained(self.tmpdirname    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= CLIPSegProcessor.from_pretrained(self.tmpdirname    )

											self.assertEqual(processor_slow.tokenizer.get_vocab()       ,   tokenizer_slow.get_vocab()    )
											self.assertEqual(processor_fast.tokenizer.get_vocab()       ,   tokenizer_fast.get_vocab()    )
											self.assertEqual(tokenizer_slow.get_vocab()       ,   tokenizer_fast.get_vocab()    )
											self.assertIsInstance(processor_slow.tokenizer       ,   __snake_case    )
											self.assertIsInstance(processor_fast.tokenizer       ,   __snake_case    )

											self.assertEqual(processor_slow.image_processor.to_json_string()       ,   image_processor.to_json_string()    )
											self.assertEqual(processor_fast.image_processor.to_json_string()       ,   image_processor.to_json_string()    )
											self.assertIsInstance(processor_slow.image_processor       ,   __snake_case    )
											self.assertIsInstance(processor_fast.image_processor       ,   __snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			int							= CLIPSegProcessor(tokenizer=self.get_tokenizer()       ,   image_processor=self.get_image_processor()    )
											processor.save_pretrained(self.tmpdirname    )

											_SCREAMING_SNAKE_CASE					:			str							= self.get_tokenizer(bos_token="""(BOS)"""       ,   eos_token="""(EOS)"""    )
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.get_image_processor(do_normalize=__snake_case       ,   padding_value=1.0    )

											_SCREAMING_SNAKE_CASE					:			Tuple							= CLIPSegProcessor.from_pretrained(
											    self.tmpdirname       ,   bos_token="""(BOS)"""       ,   eos_token="""(EOS)"""       ,   do_normalize=__snake_case       ,   padding_value=1.0    )

											self.assertEqual(processor.tokenizer.get_vocab()       ,   tokenizer_add_kwargs.get_vocab()    )
											self.assertIsInstance(processor.tokenizer       ,   __snake_case    )

											self.assertEqual(processor.image_processor.to_json_string()       ,   image_processor_add_kwargs.to_json_string()    )
											self.assertIsInstance(processor.image_processor       ,   __snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= self.get_image_processor()
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= self.get_tokenizer()

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			List[Any]							= self.prepare_image_inputs()

											_SCREAMING_SNAKE_CASE					:			List[Any]							= image_processor(__snake_case       ,   return_tensors="""np"""    )
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= processor(images=__snake_case       ,   return_tensors="""np"""    )

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()       ,   input_processor[key].sum()       ,   delta=1e-2    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= self.get_image_processor()
											_SCREAMING_SNAKE_CASE					:			Dict							= self.get_tokenizer()

											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			str							= """lower newer"""

											_SCREAMING_SNAKE_CASE					:			Any							= processor(text=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Tuple							= tokenizer(__snake_case    )

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]       ,   encoded_processor[key]    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.get_image_processor()
											_SCREAMING_SNAKE_CASE					:			List[Any]							= self.get_tokenizer()

											_SCREAMING_SNAKE_CASE					:			Dict							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Any							= """lower newer"""
											_SCREAMING_SNAKE_CASE					:			List[Any]							= self.prepare_image_inputs()

											_SCREAMING_SNAKE_CASE					:			List[str]							= processor(text=__snake_case       ,   images=__snake_case    )

											self.assertListEqual(list(inputs.keys()    )       ,   ["""input_ids""", """attention_mask""", """pixel_values"""]    )

											# test if it raises when no input is passed
											with pytest.raises(__snake_case    ):
															processor()






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Dict							= self.get_image_processor()
											_SCREAMING_SNAKE_CASE					:			Tuple							= self.get_tokenizer()

											_SCREAMING_SNAKE_CASE					:			List[Any]							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			str							= self.prepare_image_inputs()
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= self.prepare_image_inputs()

											_SCREAMING_SNAKE_CASE					:			List[Any]							= processor(images=__snake_case       ,   visual_prompt=__snake_case    )

											self.assertListEqual(list(inputs.keys()    )       ,   ["""pixel_values""", """conditional_pixel_values"""]    )

											# test if it raises when no input is passed
											with pytest.raises(__snake_case    ):
															processor()






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.get_image_processor()
											_SCREAMING_SNAKE_CASE					:			Dict							= self.get_tokenizer()

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= CLIPSegProcessor(tokenizer=__snake_case       ,   image_processor=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											_SCREAMING_SNAKE_CASE					:			int							= processor.batch_decode(__snake_case    )
											_SCREAMING_SNAKE_CASE					:			int							= tokenizer.batch_decode(__snake_case    )

											self.assertListEqual(__snake_case       ,   __snake_case    )




