


'''simple docstring'''
import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__SCREAMING_SNAKE_CASE					:				Union[str, Any]									=		16
__SCREAMING_SNAKE_CASE					:				List[Any]									=		32







def       a_				(	UpperCamelCase_						,  UpperCamelCase_ = 1_6 ):
							A_			    =		AutoTokenizer.from_pretrained("bert-base-cased" )
							A_			    =		load_dataset("glue"						,  "mrpc" )

							def tokenize_function(UpperCamelCase_ ):
														# max_length=None => use the model max length (it's actually the default)
														A_			    =		tokenizer(examples["sentence1"]						,  examples["sentence2"]						,  truncation=UpperCamelCase_						,  max_length=UpperCamelCase_ )
														return outputs

							# Apply the method we just defined to all the examples in all the splits of the dataset
							# starting with the main process first:
							with accelerator.main_process_first():
														A_			    =		datasets.map(
														    UpperCamelCase_						,  batched=UpperCamelCase_						,  remove_columns=["idx", "sentence1", "sentence2"]						,  )

							# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
							# transformers library
							A_			    =		tokenized_datasets.rename_column("label"						,  "labels" )

							def collate_fn(UpperCamelCase_ ):
														# On TPU it's best to pad everything to the same length or training will be very slow.
														A_			    =		1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
														# When using mixed precision we want round multiples of 8/16
														if accelerator.mixed_precision == "fp8":
																					A_			    =		1_6
														elif accelerator.mixed_precision != "no":
																					A_			    =		8
														else:
																					A_			    =		None

														return tokenizer.pad(
														    UpperCamelCase_						,  padding="longest"						,  max_length=UpperCamelCase_						,  pad_to_multiple_of=UpperCamelCase_						,  return_tensors="pt"						,  )

							# Instantiate dataloaders.
							A_			    =		DataLoader(
							    tokenized_datasets["train"]						,  shuffle=UpperCamelCase_						,  collate_fn=UpperCamelCase_						,  batch_size=UpperCamelCase_ )
							A_			    =		DataLoader(
							    tokenized_datasets["validation"]						,  shuffle=UpperCamelCase_						,  collate_fn=UpperCamelCase_						,  batch_size=UpperCamelCase_ )

							return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							__SCREAMING_SNAKE_CASE					:				str									=		mocked_dataloaders  # noqa: F811







def       a_				(	UpperCamelCase_						,  UpperCamelCase_ ):
							# For testing only
							if os.environ.get("TESTING_MOCKED_DATALOADERS"						,  UpperCamelCase_ ) == "1":
														A_			    =		2
							# Initialize accelerator
							A_			    =		Accelerator(cpu=args.cpu						,  mixed_precision=args.mixed_precision )
							# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
							A_			    =		config["lr"]
							A_			    =		int(config["num_epochs"] )
							A_			    =		int(config["seed"] )
							A_			    =		int(config["batch_size"] )

							A_			    =		evaluate.load("glue"						,  "mrpc" )

							# If the batch size is too big we use gradient accumulation
							A_			    =		1
							if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
														A_			    =		batch_size // MAX_GPU_BATCH_SIZE
														A_			    =		MAX_GPU_BATCH_SIZE

							set_seed(UpperCamelCase_ )
							A_    ,					A_			    =		get_dataloaders(UpperCamelCase_						,  UpperCamelCase_ )
							# Instantiate the model (we build the model here so that the seed also control new weights initialization)
							A_			    =		AutoModelForSequenceClassification.from_pretrained("bert-base-cased"						,  return_dict=UpperCamelCase_ )

							# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
							# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
							# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
							A_			    =		model.to(accelerator.device )

							# Instantiate optimizer
							A_			    =		AdamW(params=model.parameters()						,  lr=UpperCamelCase_ )

							# Instantiate scheduler
							A_			    =		get_linear_schedule_with_warmup(
							    optimizer=UpperCamelCase_						,  num_warmup_steps=1_0_0						,  num_training_steps=(len(UpperCamelCase_ ) * num_epochs) // gradient_accumulation_steps						,  )

							# Prepare everything
							# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
							# prepare method.
							A_    ,					A_    ,					A_    ,					A_    ,					A_			    =		accelerator.prepare(
							    UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_ )

							# Now we train the model
							for epoch in range(UpperCamelCase_ ):
														model.train()
														for step, batch in enumerate(UpperCamelCase_ ):
																					# We could avoid this line since we set the accelerator with `device_placement=True`.
																					batch.to(accelerator.device )
																					A_			    =		model(**UpperCamelCase_ )
																					A_			    =		outputs.loss
																					A_			    =		loss / gradient_accumulation_steps
																					accelerator.backward(UpperCamelCase_ )
																					if step % gradient_accumulation_steps == 0:
																												optimizer.step()
																												lr_scheduler.step()
																												optimizer.zero_grad()

														model.eval()
														A_			    =		0
														for step, batch in enumerate(UpperCamelCase_ ):
																					# We could avoid this line since we set the accelerator with `device_placement=True`.
																					batch.to(accelerator.device )
																					with torch.no_grad():
																												A_			    =		model(**UpperCamelCase_ )
																					A_			    =		outputs.logits.argmax(dim=-1 )
																					A_    ,					A_			    =		accelerator.gather((predictions, batch["labels"]) )
																					# New Code #
																					# First we check if it's a distributed system
																					if accelerator.use_distributed:
																												# Then see if we're on the last batch of our eval dataloader
																												if step == len(UpperCamelCase_ ) - 1:
																																			# Last batch needs to be truncated on distributed systems as it contains additional samples
																																			A_			    =		predictions[: len(eval_dataloader.dataset ) - samples_seen]
																																			A_			    =		references[: len(eval_dataloader.dataset ) - samples_seen]
																												else:
																																			# Otherwise we add the number of samples seen
																																			samples_seen += references.shape[0]
            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
            # accelerator.gather_for_metrics((predictions, batch["labels"]))
																					metric.add_batch(
																					    predictions=UpperCamelCase_						,  references=UpperCamelCase_						,  )

														A_			    =		metric.compute()
														# Use accelerator.print to print only on the main process.
														accelerator.print(f"epoch {epoch}:"						,  UpperCamelCase_ )







def       a_				(	):
							A_			    =		argparse.ArgumentParser(description="Simple example of training script." )
							parser.add_argument(
							    "--mixed_precision"						,  type=UpperCamelCase_						,  default=UpperCamelCase_						,  choices=["no", "fp16", "bf16", "fp8"]						,  help="Whether to use mixed precision. Choose"
							    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
							    "and an Nvidia Ampere GPU."						,  )
							parser.add_argument("--cpu"						,  action="store_true"						,  help="If passed, will train on the CPU." )
							A_			    =		parser.parse_args()
							A_			    =		{"lr": 2e-5, "num_epochs": 3, "seed": 4_2, "batch_size": 1_6}
							training_function(UpperCamelCase_						,  UpperCamelCase_ )


if __name__ == "__main__":
							main()









'''simple docstring'''
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
							import botoa  # noqa: F401







def       a_				(	UpperCamelCase_ ):
							A_			    =		botoa.client("iam" )

							A_			    =		{
							    "Version": "2012-10-17",
							    "Statement": [
							        {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
							    ],
							}
							try:
														# create the role, associated with the chosen trust policy
														iam_client.create_role(
														    RoleName=UpperCamelCase_						,  AssumeRolePolicyDocument=json.dumps(UpperCamelCase_						,  indent=2 ) )
														A_			    =		{
														    "Version": "2012-10-17",
														    "Statement": [
														        {
														            "Effect": "Allow",
														            "Action": [
														                "sagemaker:*",
														                "ecr:GetDownloadUrlForLayer",
														                "ecr:BatchGetImage",
														                "ecr:BatchCheckLayerAvailability",
														                "ecr:GetAuthorizationToken",
														                "cloudwatch:PutMetricData",
														                "cloudwatch:GetMetricData",
														                "cloudwatch:GetMetricStatistics",
														                "cloudwatch:ListMetrics",
														                "logs:CreateLogGroup",
														                "logs:CreateLogStream",
														                "logs:DescribeLogStreams",
														                "logs:PutLogEvents",
														                "logs:GetLogEvents",
														                "s3:CreateBucket",
														                "s3:ListBucket",
														                "s3:GetBucketLocation",
														                "s3:GetObject",
														                "s3:PutObject",
														            ],
														            "Resource": "*",
														        }
														    ],
														}
														# attach policy to role
														iam_client.put_role_policy(
														    RoleName=UpperCamelCase_						,  PolicyName=f"{role_name}_policy_permission"						,  PolicyDocument=json.dumps(UpperCamelCase_						,  indent=2 )						,  )
							except iam_client.exceptions.EntityAlreadyExistsException:
														print(f"role {role_name} already exists. Using existing one" )







def       a_				(	UpperCamelCase_ ):
							A_			    =		botoa.client("iam" )
							return iam_client.get_role(RoleName=UpperCamelCase_ )["Role"]["Arn"]







def       a_				(	):
							A_			    =		_ask_options(
							    "How do you want to authorize?"						,  ["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "]						,  UpperCamelCase_						,  )
							A_			    =		None
							if credentials_configuration == 0:
														A_			    =		_ask_field("Enter your AWS Profile name: [default] "						,  default="default" )
														A_			    =		aws_profile
							else:
														print(
														    "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"
														    "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`" )
														A_			    =		_ask_field("AWS Access Key ID: " )
														A_			    =		aws_access_key_id

														A_			    =		_ask_field("AWS Secret Access Key: " )
														A_			    =		aws_secret_access_key

							A_			    =		_ask_field("Enter your AWS Region: [us-east-1]"						,  default="us-east-1" )
							A_			    =		aws_region

							A_			    =		_ask_options(
							    "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"						,  ["Provide IAM Role name", "Create new IAM role using credentials"]						,  UpperCamelCase_						,  )
							if role_management == 0:
														A_			    =		_ask_field("Enter your IAM role name: " )
							else:
														A_			    =		"accelerate_sagemaker_execution_role"
														print(f"Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials" )
														_create_iam_role_for_sagemaker(UpperCamelCase_ )

							A_			    =		_ask_field(
							    "Do you want to use custom Docker image? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_custom_docker_image:
														A_			    =		_ask_field("Enter your Docker image: "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower() )

							A_			    =		_ask_field(
							    "Do you want to provide SageMaker input channels with data locations? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_sagemaker_inputs_enabled:
														A_			    =		_ask_field(
														    "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  )

							A_			    =		_ask_field(
							    "Do you want to enable SageMaker metrics? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_sagemaker_metrics_enabled:
														A_			    =		_ask_field(
														    "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  )

							A_			    =		_ask_options(
							    "What is the distributed mode?"						,  ["No distributed training", "Data parallelism"]						,  _convert_sagemaker_distributed_mode						,  )
							A_			    =		{}
							A_			    =		_ask_field(
							    "Do you wish to optimize your script with torch dynamo?[yes/NO]:"						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							if use_dynamo:
														A_			    =		"dynamo_"
														A_			    =		_ask_options(
														    "Which dynamo backend would you like to use?"						,  [x.lower() for x in DYNAMO_BACKENDS]						,  _convert_dynamo_backend						,  default=2						,  )
														A_			    =		_ask_field(
														    "Do you want to customize the defaults sent to torch.compile? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )

														if use_custom_options:
																					A_			    =		_ask_options(
																					    "Which mode do you want to use?"						,  UpperCamelCase_						,  lambda UpperCamelCase_ : TORCH_DYNAMO_MODES[int(UpperCamelCase_ )]						,  default="default"						,  )
																					A_			    =		_ask_field(
																					    "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
																					A_			    =		_ask_field(
																					    "Do you want to enable dynamic shape tracing? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		"Which EC2 instance type you want to use for your training?"
							if distributed_type != SageMakerDistributedType.NO:
														A_			    =		_ask_options(
														    UpperCamelCase_						,  UpperCamelCase_						,  lambda UpperCamelCase_ : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(UpperCamelCase_ )] )
							else:
														eca_instance_query += "? [ml.p3.2xlarge]:"
														A_			    =		_ask_field(UpperCamelCase_						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  default="ml.p3.2xlarge" )

							A_			    =		1
							if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
														A_			    =		_ask_field(
														    "How many machines do you want use? [1]: "						,  UpperCamelCase_						,  default=1						,  )

							A_			    =		_ask_options(
							    "Do you wish to use FP16 or BF16 (mixed precision)?"						,  ["no", "fp16", "bf16", "fp8"]						,  _convert_mixed_precision						,  )

							if use_dynamo and mixed_precision == "no":
														print(
														    "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts." )

							return SageMakerConfig(
							    image_uri=UpperCamelCase_						,  compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER						,  distributed_type=UpperCamelCase_						,  use_cpu=UpperCamelCase_						,  dynamo_config=UpperCamelCase_						,  eca_instance_type=UpperCamelCase_						,  profile=UpperCamelCase_						,  region=UpperCamelCase_						,  iam_role_name=UpperCamelCase_						,  mixed_precision=UpperCamelCase_						,  num_machines=UpperCamelCase_						,  sagemaker_inputs_file=UpperCamelCase_						,  sagemaker_metrics_file=UpperCamelCase_						,  )





