



# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


a_      :      Tuple    =     'Run commands across TPU VMs for initial setup before running `accelerate launch`.'






def     __a					(		__UpperCAmelCase=None       ):
 if subparsers is not None:
  a__												= subparsers.add_parser('''tpu-config'''							,  description=_description       )
 else:
  a__												= argparse.ArgumentParser('''Accelerate tpu-config command'''							,  description=_description       )
 # Core arguments
 a__												= parser.add_argument_group(
     '''Config Arguments'''							,  '''Arguments that can be configured through `accelerate config`.'''       )
 config_args.add_argument(
     '''--config_file'''							,  type=__UpperCAmelCase							,  default=__UpperCAmelCase							,  help='''Path to the config file to use for accelerate.'''							,  )
 config_args.add_argument(
     '''--tpu_name'''							,  default=__UpperCAmelCase							,  help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.'''							,  )
 config_args.add_argument(
     '''--tpu_zone'''							,  default=__UpperCAmelCase							,  help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.'''							,  )
 a__												= parser.add_argument_group('''TPU Arguments'''							,  '''Arguments for options ran inside the TPU.'''       )
 pod_args.add_argument(
     '''--use_alpha'''							,  action='''store_true'''							,  help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'''							,  )
 pod_args.add_argument(
     '''--command_file'''							,  default=__UpperCAmelCase							,  help='''The path to the file containing the commands to run on the pod on startup.'''							,  )
 pod_args.add_argument(
     '''--command'''							,  action='''append'''							,  nargs='''+'''							,  help='''A command to run on the pod. Can be passed multiple times.'''							,  )
 pod_args.add_argument(
     '''--install_accelerate'''							,  action='''store_true'''							,  help='''Whether to install accelerate on the pod. Defaults to False.'''							,  )
 pod_args.add_argument(
     '''--accelerate_version'''							,  default='''latest'''							,  help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'''							,  )
 pod_args.add_argument(
     '''--debug'''							,  action='''store_true'''							,  help='''If set, will print the command that would be run instead of running it.'''       )

 if subparsers is not None:
  parser.set_defaults(func=__UpperCAmelCase       )
 return parser






def     __a					(		__UpperCAmelCase       ):
 a__												= None

 # Get the default from the config file if it exists.
 if args.config_file is not None or os.path.isfile(__UpperCAmelCase       ):
  a__												= load_config_from_file(args.config_file       )
  if not args.command_file and defaults.command_file is not None and not args.command:
   a__												= defaults.command_file
  if not args.command and defaults.commands is not None:
   a__												= defaults.commands
  if not args.tpu_name:
   a__												= defaults.tpu_name
  if not args.tpu_zone:
   a__												= defaults.tpu_zone
 if args.accelerate_version == "dev":
  a__												= '''git+https://github.com/huggingface/accelerate.git'''
 elif args.accelerate_version == "latest":
  a__												= '''accelerate -U'''
 elif isinstance(parse(args.accelerate_version       )							,  __UpperCAmelCase       ):
  a__												= f"accelerate=={args.accelerate_version}"

 if not args.command_file and not args.command:
  raise ValueError('''You must specify either a command file or a command to run on the pod.'''       )

 if args.command_file:
  with open(args.command_file							,  '''r'''       ) as f:
   a__												= [f.read().splitlines()]

    # To turn list of lists into list of strings
 if isinstance(args.command[0]							,  __UpperCAmelCase       ):
  a__												= [line for cmd in args.command for line in cmd]
 # Default to the shared folder and install accelerate
 a__												= ['''cd /usr/share''']
 if args.install_accelerate:
  new_cmd += [f"pip install {args.accelerate_version}"]
 new_cmd += args.command
 a__												= '''; '''.join(__UpperCAmelCase       )

 # Then send it to gcloud
 # Eventually try to use google-api-core to do this instead of subprocess
 a__												= ['''gcloud''']
 if args.use_alpha:
  cmd += ["alpha"]
 cmd += [
     "compute",
     "tpus",
     "tpu-vm",
     "ssh",
     args.tpu_name,
     "--zone",
     args.tpu_zone,
     "--command",
     args.command,
     "--worker",
     "all",
 ]
 if args.debug:
  print(f"Running {' '.join(__UpperCAmelCase       )}"       )
  return
 subprocess.run(__UpperCAmelCase       )
 print('''Successfully setup pod.'''       )






def     __a					(		):
 a__												= tpu_command_parser()
 a__												= parser.parse_args()

 tpu_command_launcher(__UpperCAmelCase       )










import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForPreTraining,
         AlbertForQuestionAnswering,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertModel,
     )
     from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class    __UpperCamelCase			:

 """simple docstring"""



 def __init__(	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=1_3		,				SCREAMING_SNAKE_CASE=7		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=9_9		,				SCREAMING_SNAKE_CASE=1_6		,				SCREAMING_SNAKE_CASE=3_6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=3_7		,				SCREAMING_SNAKE_CASE="gelu"		,				SCREAMING_SNAKE_CASE=0.1		,				SCREAMING_SNAKE_CASE=0.1		,				SCREAMING_SNAKE_CASE=5_1_2		,				SCREAMING_SNAKE_CASE=1_6		,				SCREAMING_SNAKE_CASE=2		,				SCREAMING_SNAKE_CASE=0.02		,				SCREAMING_SNAKE_CASE=3		,				SCREAMING_SNAKE_CASE=4		,				SCREAMING_SNAKE_CASE=None		,				) ->			Union[str, Any]:
  a__												= parent
  a__												= batch_size
  a__												= seq_length
  a__												= is_training
  a__												= use_input_mask
  a__												= use_token_type_ids
  a__												= use_labels
  a__												= vocab_size
  a__												= embedding_size
  a__												= hidden_size
  a__												= num_hidden_layers
  a__												= num_hidden_groups
  a__												= num_attention_heads
  a__												= intermediate_size
  a__												= hidden_act
  a__												= hidden_dropout_prob
  a__												= attention_probs_dropout_prob
  a__												= max_position_embeddings
  a__												= type_vocab_size
  a__												= type_sequence_label_size
  a__												= initializer_range
  a__												= num_labels
  a__												= num_choices
  a__												= scope

 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size      )

  a__												= None
  if self.use_input_mask:
   a__												= random_attention_mask([self.batch_size, self.seq_length]      )

  a__												= None
  if self.use_token_type_ids:
   a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size      )

  a__												= None
  a__												= None
  a__												= None
  if self.use_labels:
   a__												= ids_tensor([self.batch_size]		,				self.type_sequence_label_size      )
   a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.num_labels      )
   a__												= ids_tensor([self.batch_size]		,				self.num_choices      )

  a__												= self.get_config()

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

 def 						_UpperCAmelCase       (	self      ) ->			Any:
  return AlbertConfig(
      vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				initializer_range=self.initializer_range		,				num_hidden_groups=self.num_hidden_groups		,				)

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Optional[int]:
  a__												= AlbertModel(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE      )
  a__												= model(SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE      )
  a__												= model(SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.last_hidden_state.shape		,				(self.batch_size, self.seq_length, self.hidden_size)      )
  self.parent.assertEqual(result.pooler_output.shape		,				(self.batch_size, self.hidden_size)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			List[Any]:
  a__												= AlbertForPreTraining(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE		,				sentence_order_label=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.prediction_logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)      )
  self.parent.assertEqual(result.sop_logits.shape		,				(self.batch_size, config.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Dict:
  a__												= AlbertForMaskedLM(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Any:
  a__												= AlbertForQuestionAnswering(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				start_positions=SCREAMING_SNAKE_CASE		,				end_positions=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.start_logits.shape		,				(self.batch_size, self.seq_length)      )
  self.parent.assertEqual(result.end_logits.shape		,				(self.batch_size, self.seq_length)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Union[str, Any]:
  a__												= self.num_labels
  a__												= AlbertForSequenceClassification(SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Tuple:
  a__												= self.num_labels
  a__												= AlbertForTokenClassification(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Optional[int]:
  a__												= self.num_choices
  a__												= AlbertForMultipleChoice(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= input_ids.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= token_type_ids.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= input_mask.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_choices)      )



 def 						_UpperCAmelCase       (	self      ) ->			Tuple:
  a__												= self.prepare_config_and_inputs()
  (
      (
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			
  )												= config_and_inputs
  a__												= {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
  return config, inputs_dict





@require_torch
class    __UpperCamelCase			(       _lowercase	,	_lowercase	,	unittest.TestCase				):

 """simple docstring"""



 _lowercase   :    List[str]      						=					(
     (
         AlbertModel,
         AlbertForPreTraining,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 _lowercase   :    List[str]      						=					(
     {
         '''feature-extraction''': AlbertModel,
         '''fill-mask''': AlbertForMaskedLM,
         '''question-answering''': AlbertForQuestionAnswering,
         '''text-classification''': AlbertForSequenceClassification,
         '''token-classification''': AlbertForTokenClassification,
         '''zero-shot''': AlbertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 _lowercase   :    List[str]      						=					True

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=False      ) ->			List[str]:
  a__												= super()._prepare_for_class(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				return_labels=SCREAMING_SNAKE_CASE      )

  if return_labels:
   if model_class in get_values(SCREAMING_SNAKE_CASE      ):
    a__												= torch.zeros(
        (self.model_tester.batch_size, self.model_tester.seq_length)		,				dtype=torch.long		,				device=SCREAMING_SNAKE_CASE      )
    a__												= torch.zeros(
        self.model_tester.batch_size		,				dtype=torch.long		,				device=SCREAMING_SNAKE_CASE      )
  return inputs_dict

 def 						_UpperCAmelCase       (	self      ) ->			Dict:
  a__												= AlbertModelTester(self      )
  a__												= ConfigTester(self		,				config_class=SCREAMING_SNAKE_CASE		,				hidden_size=3_7      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  self.config_tester.run_common_tests()

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Any:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Optional[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  for type in ["absolute", "relative_key", "relative_key_query"]:
   a__												= type
   self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE      )



 @slow
 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   a__												= AlbertModel.from_pretrained(SCREAMING_SNAKE_CASE      )
   self.assertIsNotNone(SCREAMING_SNAKE_CASE      )





@require_torch
class    __UpperCamelCase			(       unittest.TestCase				):

 """simple docstring"""



 @slow
 def 						_UpperCAmelCase       (	self      ) ->			Optional[Any]:
  a__												= AlbertModel.from_pretrained('''albert-base-v2'''      )
  a__												= torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]      )
  a__												= torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
  with torch.no_grad():
   a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE      )[0]
  a__												= torch.Size((1, 1_1, 7_6_8)      )
  self.assertEqual(output.shape		,				SCREAMING_SNAKE_CASE      )
  a__												= torch.tensor(
      [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]      )

  self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,				SCREAMING_SNAKE_CASE		,				atol=1e-4      )      )





