





"""simple docstring"""





import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_lowerCamelCase    	=      16
_lowerCamelCase    	=      32




def 						lowerCAmelCase_      (    lowercase_  :				Any					, lowercase_  :				Optional[int]					, lowercase_  :				Tuple					, lowercase_  :				Optional[int]					, lowercase_  :				List[str] = 16						):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			AutoTokenizer.from_pretrained('''bert-base-cased'''						)
 __SCREAMING_SNAKE_CASE					:     int   				=			DatasetDict(
     {
         '''train''': dataset['''train'''].select(_A						),
         '''validation''': dataset['''train'''].select(_A						),
         '''test''': dataset['''validation'''],
     }						)

 def tokenize_function(lowercase_  :				List[Any]						):
  # max_length=None => use the model max length (it's actually the default)
  __SCREAMING_SNAKE_CASE					:     List[Any]   				=			tokenizer(examples['''sentence1''']					, examples['''sentence2''']					, truncation=_A					, max_length=_A						)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			datasets.map(
      _A					, batched=_A					, remove_columns=['''idx''', '''sentence1''', '''sentence2''']					, )

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			tokenized_datasets.rename_column('''label'''					, '''labels'''						)

 def collate_fn(lowercase_  :				Optional[int]						):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			16
  elif accelerator.mixed_precision != "no":
   __SCREAMING_SNAKE_CASE					:     List[str]   				=			8
  else:
   __SCREAMING_SNAKE_CASE					:     Any   				=			None

  return tokenizer.pad(
      _A					, padding='''longest'''					, max_length=_A					, pad_to_multiple_of=_A					, return_tensors='''pt'''					, )

 # Instantiate dataloaders.
 __SCREAMING_SNAKE_CASE					:     Tuple   				=			DataLoader(
     tokenized_datasets['''train''']					, shuffle=_A					, collate_fn=_A					, batch_size=_A						)
 __SCREAMING_SNAKE_CASE					:     int   				=			DataLoader(
     tokenized_datasets['''validation''']					, shuffle=_A					, collate_fn=_A					, batch_size=_A						)

 __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			DataLoader(
     tokenized_datasets['''test''']					, shuffle=_A					, collate_fn=_A					, batch_size=_A						)

 return train_dataloader, eval_dataloader, test_dataloader




def 						lowerCAmelCase_      (    lowercase_  :				Tuple					, lowercase_  :				Tuple						):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE					:     List[str]   				=			[]
 # Download the dataset
 __SCREAMING_SNAKE_CASE					:     List[Any]   				=			load_dataset('''glue'''					, '''mrpc'''						)
 # Create our splits
 __SCREAMING_SNAKE_CASE					:     Any   				=			StratifiedKFold(n_splits=int(args.num_folds						)						)
 # Initialize accelerator
 __SCREAMING_SNAKE_CASE					:     List[Any]   				=			Accelerator(cpu=args.cpu					, mixed_precision=args.mixed_precision						)
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			config['''lr''']
 __SCREAMING_SNAKE_CASE					:     Dict   				=			int(config['''num_epochs''']						)
 __SCREAMING_SNAKE_CASE					:     str   				=			int(config['''seed''']						)
 __SCREAMING_SNAKE_CASE					:     List[str]   				=			int(config['''batch_size''']						)

 __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			evaluate.load('''glue'''					, '''mrpc'''						)

 # If the batch size is too big we use gradient accumulation
 __SCREAMING_SNAKE_CASE					:     int   				=			1
 if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  __SCREAMING_SNAKE_CASE					:     Tuple   				=			batch_size // MAX_GPU_BATCH_SIZE
  __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			MAX_GPU_BATCH_SIZE

 set_seed(_A						)

 # New Code #
 # Create our folds:
 __SCREAMING_SNAKE_CASE					:     List[Any]   				=			kfold.split(np.zeros(datasets['''train'''].num_rows						)					, datasets['''train''']['''label''']						)
 __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			[]
 # Iterate over them
 for i, (train_idxs, valid_idxs) in enumerate(_A						):
  __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE					:     List[Any]   				=			get_fold_dataloaders(
      _A					, _A					, _A					, _A					, )
  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __SCREAMING_SNAKE_CASE					:     List[str]   				=			AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''					, return_dict=_A						)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __SCREAMING_SNAKE_CASE					:     List[str]   				=			model.to(accelerator.device						)

  # Instantiate optimizer
  __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			AdamW(params=model.parameters()					, lr=_A						)

  # Instantiate scheduler
  __SCREAMING_SNAKE_CASE					:     Any   				=			get_linear_schedule_with_warmup(
      optimizer=_A					, num_warmup_steps=100					, num_training_steps=(len(_A						) * num_epochs) // gradient_accumulation_steps					, )

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE					:     Any   				=			accelerator.prepare(
      _A					, _A					, _A					, _A					, _A						)

  # Now we train the model
  for epoch in range(_A						):
   model.train()
   for step, batch in enumerate(_A						):
    # We could avoid this line since we set the accelerator with `device_placement=True`.
    batch.to(accelerator.device						)
    __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			model(**_A						)
    __SCREAMING_SNAKE_CASE					:     str   				=			outputs.loss
    __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			loss / gradient_accumulation_steps
    accelerator.backward(_A						)
    if step % gradient_accumulation_steps == 0:
     optimizer.step()
     lr_scheduler.step()
     optimizer.zero_grad()

   model.eval()
   for step, batch in enumerate(_A						):
    # We could avoid this line since we set the accelerator with `device_placement=True`.
    batch.to(accelerator.device						)
    with torch.no_grad():
     __SCREAMING_SNAKE_CASE					:     Any   				=			model(**_A						)
    __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			outputs.logits.argmax(dim=-1						)
    __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			accelerator.gather_for_metrics((predictions, batch['''labels'''])						)
    metric.add_batch(
        predictions=_A					, references=_A					, )

   __SCREAMING_SNAKE_CASE					:     List[Any]   				=			metric.compute()
   # Use accelerator.print to print only on the main process.
   accelerator.print(F'''epoch {epoch}:'''					, _A						)

  # New Code #
  # We also run predictions on the test set at the very end
  __SCREAMING_SNAKE_CASE					:     List[Any]   				=			[]
  for step, batch in enumerate(_A						):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device						)
   with torch.no_grad():
    __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			model(**_A						)
   __SCREAMING_SNAKE_CASE					:     Dict   				=			outputs.logits
   __SCREAMING_SNAKE_CASE				, __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			accelerator.gather_for_metrics((predictions, batch['''labels'''])						)
   fold_predictions.append(predictions.cpu()						)
   if i == 0:
    # We need all of the test predictions
    test_references.append(references.cpu()						)
        # Use accelerator.print to print only on the main process.
  test_predictions.append(torch.cat(_A					, dim=0						)						)
  # We now need to release all our memory and get rid of the current model, optimizer, etc
  accelerator.free_memory()
 # New Code #
 # Finally we check the accuracy of our folded results:
 __SCREAMING_SNAKE_CASE					:     int   				=			torch.cat(_A					, dim=0						)
 __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			torch.stack(_A					, dim=0						).sum(dim=0						).div(int(args.num_folds						)						).argmax(dim=-1						)
 __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			metric.compute(predictions=_A					, references=_A						)
 accelerator.print('''Average test metrics from all folds:'''					, _A						)




def 						lowerCAmelCase_      (    ):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			argparse.ArgumentParser(description='''Simple example of training script.'''						)
 parser.add_argument(
     '''--mixed_precision'''					, type=_A					, default=_A					, choices=['''no''', '''fp16''', '''bf16''', '''fp8''']					, help='''Whether to use mixed precision. Choose'''
     '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
     '''and an Nvidia Ampere GPU.'''					, )
 parser.add_argument('''--cpu'''					, action='''store_true'''					, help='''If passed, will train on the CPU.'''						)
 # New Code #
 parser.add_argument('''--num_folds'''					, type=_A					, default=3					, help='''The number of splits to perform across the dataset'''						)
 __SCREAMING_SNAKE_CASE					:     str   				=			parser.parse_args()
 __SCREAMING_SNAKE_CASE					:     Tuple   				=			{'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
 training_function(_A					, _A						)


if __name__ == "__main__":
    main()



"""simple docstring"""



import math




class 		a_    :







    def   _snake_case       (      self      :							List[Any]  ,       __UpperCamelCase      :							list[list[float]]  ,       __UpperCamelCase      :							list[int]  )		->int:






        '''simple docstring'''






        _UpperCAmelCase        = 0.0
        _UpperCAmelCase        = 0.0
        for i in range(len(__UpperCamelCase  )  ):
            da += math.pow((sample[i] - weights[0][i])  ,       2  )
            da += math.pow((sample[i] - weights[1][i])  ,       2  )
            return 0 if da > da else 1
        return 0







    def   _snake_case       (      self      :							Dict  ,       __UpperCamelCase      :							list[list[int | float]]  ,       __UpperCamelCase      :							list[int]  ,       __UpperCamelCase      :							int  ,       __UpperCamelCase      :							float  )		->list[list[int | float]]:






        '''simple docstring'''






        for i in range(len(__UpperCamelCase  )  ):
            weights[j][i] += alpha * (sample[i] - weights[j][i])
        return weights







def 			_UpperCamelCase						(    )     ->	None:





    """simple docstring"""

    _UpperCAmelCase        = [[1, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 1]]

    # weight initialization ( n, C )
    _UpperCAmelCase        = [[0.2, 0.6, 0.5, 0.9], [0.8, 0.4, 0.7, 0.3]]

    # training
    _UpperCAmelCase        = SelfOrganizingMap()
    _UpperCAmelCase        = 3
    _UpperCAmelCase        = 0.5

    for _ in range(_A   ):
        for j in range(len(_A   )   ):
            # training sample
            _UpperCAmelCase        = training_samples[j]

            # Compute the winning vector
            _UpperCAmelCase        = self_organizing_map.get_winner(_A       ,   _A   )

            # Update the winning vector
            _UpperCAmelCase        = self_organizing_map.update(_A       ,   _A       ,   _A       ,   _A   )

    # classify test sample
    _UpperCAmelCase        = [0, 0, 0, 1]
    _UpperCAmelCase        = self_organizing_map.get_winner(_A       ,   _A   )

    # results
    print(F"""Clusters that the test sample belongs to : {winner}"""   )
    print(F"""Weights that have been trained : {weights}"""   )


# running the main() function
if __name__ == "__main__":
 main()