


"""simple docstring"""
# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


UpperCAmelCase				=						"""Run commands across TPU VMs for initial setup before running `accelerate launch`."""

def 				_snake_case      (   __snake_case							:      Tuple=None   ):




   """simple docstring"""







   if subparsers is not None:
      _lowerCamelCase	:					str   =		subparsers.add_parser("""tpu-config"""						, description=_description   )
   else:
      _lowerCamelCase	:					Union[str, Any]   =		argparse.ArgumentParser("""Accelerate tpu-config command"""						, description=_description   )
   # Core arguments
   _lowerCamelCase	:					Optional[int]   =		parser.add_argument_group(
       """Config Arguments"""						, """Arguments that can be configured through `accelerate config`."""   )
   config_args.add_argument(
       """--config_file"""						, type=__snake_case						, default=__snake_case						, help="""Path to the config file to use for accelerate."""						, )
   config_args.add_argument(
       """--tpu_name"""						, default=__snake_case						, help="""The name of the TPU to use. If not specified, will use the TPU specified in the config file."""						, )
   config_args.add_argument(
       """--tpu_zone"""						, default=__snake_case						, help="""The zone of the TPU to use. If not specified, will use the zone specified in the config file."""						, )
   _lowerCamelCase	:					List[Any]   =		parser.add_argument_group("""TPU Arguments"""						, """Arguments for options ran inside the TPU."""   )
   pod_args.add_argument(
       """--use_alpha"""						, action="""store_true"""						, help="""Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`."""						, )
   pod_args.add_argument(
       """--command_file"""						, default=__snake_case						, help="""The path to the file containing the commands to run on the pod on startup."""						, )
   pod_args.add_argument(
       """--command"""						, action="""append"""						, nargs="""+"""						, help="""A command to run on the pod. Can be passed multiple times."""						, )
   pod_args.add_argument(
       """--install_accelerate"""						, action="""store_true"""						, help="""Whether to install accelerate on the pod. Defaults to False."""						, )
   pod_args.add_argument(
       """--accelerate_version"""						, default="""latest"""						, help="""The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify 'dev' to install from GitHub."""						, )
   pod_args.add_argument(
       """--debug"""						, action="""store_true"""						, help="""If set, will print the command that would be run instead of running it."""   )

   if subparsers is not None:
      parser.set_defaults(func=__snake_case   )
   return parser

def 				_snake_case      (   __snake_case							:      int   ):




   """simple docstring"""







   _lowerCamelCase	:					str   =		None

   # Get the default from the config file if it exists.
   if args.config_file is not None or os.path.isfile(__snake_case   ):
      _lowerCamelCase	:					Optional[int]   =		load_config_from_file(args.config_file   )
      if not args.command_file and defaults.command_file is not None and not args.command:
         _lowerCamelCase	:					Dict   =		defaults.command_file
      if not args.command and defaults.commands is not None:
         _lowerCamelCase	:					List[Any]   =		defaults.commands
      if not args.tpu_name:
         _lowerCamelCase	:					Optional[Any]   =		defaults.tpu_name
      if not args.tpu_zone:
         _lowerCamelCase	:					Optional[Any]   =		defaults.tpu_zone
   if args.accelerate_version == "dev":
      _lowerCamelCase	:					Optional[int]   =		"""git+https://github.com/huggingface/accelerate.git"""
   elif args.accelerate_version == "latest":
      _lowerCamelCase	:					Dict   =		"""accelerate -U"""
   elif isinstance(parse(args.accelerate_version   )						, __snake_case   ):
      _lowerCamelCase	:					Any   =		F'accelerate=={args.accelerate_version}'

   if not args.command_file and not args.command:
      raise ValueError("""You must specify either a command file or a command to run on the pod."""   )

   if args.command_file:
      with open(args.command_file						, """r"""   ) as f:
         _lowerCamelCase	:					Dict   =		[f.read().splitlines()]

    # To turn list of lists into list of strings
   if isinstance(args.command[0]						, __snake_case   ):
      _lowerCamelCase	:					Tuple   =		[line for cmd in args.command for line in cmd]
   # Default to the shared folder and install accelerate
   _lowerCamelCase	:					Optional[int]   =		["""cd /usr/share"""]
   if args.install_accelerate:
      new_cmd += [F'pip install {args.accelerate_version}']
   new_cmd += args.command
   _lowerCamelCase	:					List[str]   =		"""; """.join(__snake_case   )

   # Then send it to gcloud
   # Eventually try to use google-api-core to do this instead of subprocess
   _lowerCamelCase	:					Dict   =		["""gcloud"""]
   if args.use_alpha:
      cmd += ["alpha"]
   cmd += [
       "compute",
       "tpus",
       "tpu-vm",
       "ssh",
       args.tpu_name,
       "--zone",
       args.tpu_zone,
       "--command",
       args.command,
       "--worker",
       "all",
   ]
   if args.debug:
      print(F'Running {" ".join(__snake_case   )}'   )
      return
   subprocess.run(__snake_case   )
   print("""Successfully setup pod."""   )





def 				_snake_case      (   ):




   """simple docstring"""







   _lowerCamelCase	:					Optional[int]   =		tpu_command_parser()
   _lowerCamelCase	:					Optional[Any]   =		parser.parse_args()

   tpu_command_launcher(__snake_case   )


import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()



def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 __a							=	torch.load(_A				,					map_location="cpu"       )

 __a							=	chkpt["model"]

 # We have the base model one level deeper than the original XLM repository
 __a							=	{}
 for k, v in state_dict.items():
  if "pred_layer" in k:
   __a							=	v
  else:
   __a							=	v

 __a							=	chkpt["params"]
 __a							=	{n: v for n, v in config.items() if not isinstance(_A				,					(torch.FloatTensor, numpy.ndarray)       )}

 __a							=	chkpt["dico_word2id"]
 __a							=	{s + "</w>" if s.find("@@"       ) == -1 and i > 13 else s.replace("@@"				,					""       ): i for s, i in vocab.items()}

 # Save pytorch-model
 __a							=	pytorch_dump_folder_path + "/" + WEIGHTS_NAME
 __a							=	pytorch_dump_folder_path + "/" + CONFIG_NAME
 __a							=	pytorch_dump_folder_path + "/" + VOCAB_FILES_NAMES["vocab_file"]

 print(f"""Save PyTorch model to {pytorch_weights_dump_path}"""       )
 torch.save(_A				,					_A       )

 print(f"""Save configuration file to {pytorch_config_dump_path}"""       )
 with open(_A				,					"w"				,					encoding="utf-8"       ) as f:
  f.write(json.dumps(_A				,					indent=2       ) + "\n"       )

 print(f"""Save vocab file to {pytorch_config_dump_path}"""       )
 with open(_A				,					"w"				,					encoding="utf-8"       ) as f:
  f.write(json.dumps(_A				,					indent=2       ) + "\n"       )


if __name__ == "__main__":
  SCREAMING_SNAKE_CASE			:						Union[str, Any]				  =		argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--xlm_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
  )
  SCREAMING_SNAKE_CASE			:						Any				  =		parser.parse_args()
  convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)


