
'''simple docstring'''
import argparse
import json
import os
import re
from collections import OrderedDict
from os.path import basename, dirname

import fairseq
import torch
from fairseq import hub_utils
from fairseq.data.dictionary import Dictionary

from transformers import FSMTConfig, FSMTForConditionalGeneration
from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES
from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE
from transformers.utils import WEIGHTS_NAME, logging


logging.set_verbosity_warning()

__lowerCAmelCase												=	2

# based on the results of a search on a range of `num_beams`, `length_penalty` and `early_stopping`
# values against wmt19 test data to obtain the best BLEU scores, we will use the following defaults:
#
# * `num_beams`: 5 (higher scores better, but requires more memory/is slower, can be adjusted by users)
# * `early_stopping`: `False` consistently scored better
# * `length_penalty` varied, so will assign the best one depending on the model
__lowerCAmelCase												=	{
    # fairseq:
    'wmt19-ru-en': {'length_penalty': 1.1},
    'wmt19-en-ru': {'length_penalty': 1.15},
    'wmt19-en-de': {'length_penalty': 1.0},
    'wmt19-de-en': {'length_penalty': 1.1},
    # allenai:
    'wmt16-en-de-dist-12-1': {'length_penalty': 0.6},
    'wmt16-en-de-dist-6-1': {'length_penalty': 0.6},
    'wmt16-en-de-12-1': {'length_penalty': 0.8},
    'wmt19-de-en-6-6-base': {'length_penalty': 0.6},
    'wmt19-de-en-6-6-big': {'length_penalty': 0.6},
}

# this remaps the different models to their organization names
__lowerCAmelCase												=	{}
for m in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
						__lowerCAmelCase												=	'facebook'
for m in [
    "wmt16-en-de-dist-12-1",
    "wmt16-en-de-dist-6-1",
    "wmt16-en-de-12-1",
    "wmt19-de-en-6-6-base",
    "wmt19-de-en-6-6-big",
]:
						__lowerCAmelCase												=	'allenai'






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):
							_snake_case       =  dict((re.sub(R"""@@$"""							, """"""							, snake_case_				), v) if k.endswith("""@@"""				) else (re.sub(R"""$"""							, """</w>"""							, snake_case_				), v) for k, v in d.items()				)
							_snake_case       =  """<s> <pad> </s> <unk>""".split()
							# restore the special tokens
							for k in keep_keys:
														del da[f"""{k}</w>"""]
														_snake_case       =  d[k]  # restore
							return da






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							assert os.path.exists(snake_case_				)
							os.makedirs(snake_case_							, exist_ok=snake_case_				)
							print(f"""Writing results to {pytorch_dump_folder_path}"""				)

							# handle various types of models

							_snake_case       =  basename(snake_case_				)
							_snake_case       =  dirname(snake_case_				)

							_snake_case       =  fairseq.model_parallel.models.transformer.ModelParallelTransformerModel
							_snake_case       =  cls.hub_models()
							_snake_case       =  {"""bpe""": """fastbpe""", """tokenizer""": """moses"""}
							_snake_case       =  """."""
							# note: since the model dump is old, fairseq has upgraded its model some
							# time later, and it does a whole lot of rewrites and splits on the saved
							# weights, therefore we can't use torch.load() directly on the model file.
							# see: upgrade_state_dict(state_dict) in fairseq_model.py
							print(f"""using checkpoint {checkpoint_file}"""				)
							_snake_case       =  hub_utils.from_pretrained(
							    snake_case_							, snake_case_							, snake_case_							, archive_map=snake_case_							, **snake_case_				)

							_snake_case       =  vars(chkpt["""args"""]["""model"""]				)

							_snake_case       =  args["""source_lang"""]
							_snake_case       =  args["""target_lang"""]

							_snake_case       =  dirname(snake_case_				)
							_snake_case       =  basename(snake_case_				)

							# dicts
							_snake_case       =  os.path.join(snake_case_							, f"""dict.{src_lang}.txt"""				)
							_snake_case       =  os.path.join(snake_case_							, f"""dict.{tgt_lang}.txt"""				)

							_snake_case       =  Dictionary.load(snake_case_				)
							_snake_case       =  rewrite_dict_keys(src_dict.indices				)
							_snake_case       =  len(snake_case_				)
							_snake_case       =  os.path.join(snake_case_							, """vocab-src.json"""				)
							print(f"""Generating {src_vocab_file} of {src_vocab_size} of {src_lang} records"""				)
							with open(snake_case_							, """w"""							, encoding="""utf-8"""				) as f:
														f.write(json.dumps(snake_case_							, ensure_ascii=snake_case_							, indent=snake_case_				)				)

							# detect whether this is a do_lower_case situation, which can be derived by checking whether we
							# have at least one uppercase letter in the source vocab
							_snake_case       =  True
							for k in src_vocab.keys():
														if not k.islower():
																					_snake_case       =  False
																					break

							_snake_case       =  Dictionary.load(snake_case_				)
							_snake_case       =  rewrite_dict_keys(tgt_dict.indices				)
							_snake_case       =  len(snake_case_				)
							_snake_case       =  os.path.join(snake_case_							, """vocab-tgt.json"""				)
							print(f"""Generating {tgt_vocab_file} of {tgt_vocab_size} of {tgt_lang} records"""				)
							with open(snake_case_							, """w"""							, encoding="""utf-8"""				) as f:
														f.write(json.dumps(snake_case_							, ensure_ascii=snake_case_							, indent=snake_case_				)				)

							# merges_file (bpecodes)
							_snake_case       =  os.path.join(snake_case_							, VOCAB_FILES_NAMES["""merges_file"""]				)
							for fn in ["bpecodes", "code"]:  # older fairseq called the merges file "code"
														_snake_case       =  os.path.join(snake_case_							, snake_case_				)
														if os.path.exists(snake_case_				):
																					break
							with open(snake_case_							, encoding="""utf-8"""				) as fin:
														_snake_case       =  fin.read()
							_snake_case       =  re.sub(R""" \d+$"""							, """"""							, snake_case_							, 0							, re.M				)  # remove frequency number
							print(f"""Generating {merges_file}"""				)
							with open(snake_case_							, """w"""							, encoding="""utf-8"""				) as fout:
														fout.write(snake_case_				)

							# model config
							_snake_case       =  os.path.join(snake_case_							, """config.json"""				)

							# validate bpe/tokenizer config, as currently it's hardcoded to moses+fastbpe -
							# may have to modify the tokenizer if a different type is used by a future model
							assert args["bpe"] == "fastbpe", f"""need to extend tokenizer to support bpe={args["bpe"]}"""
							assert args["tokenizer"] == "moses", f"""need to extend tokenizer to support bpe={args["tokenizer"]}"""

							_snake_case       =  {
							    """architectures""": ["""FSMTForConditionalGeneration"""],
							    """model_type""": """fsmt""",
							    """activation_dropout""": args["""activation_dropout"""],
							    """activation_function""": """relu""",
							    """attention_dropout""": args["""attention_dropout"""],
							    """d_model""": args["""decoder_embed_dim"""],
							    """dropout""": args["""dropout"""],
							    """init_std""": 0.02,
							    """max_position_embeddings""": args["""max_source_positions"""],
							    """num_hidden_layers""": args["""encoder_layers"""],
							    """src_vocab_size""": src_vocab_size,
							    """tgt_vocab_size""": tgt_vocab_size,
							    """langs""": [src_lang, tgt_lang],
							    """encoder_attention_heads""": args["""encoder_attention_heads"""],
							    """encoder_ffn_dim""": args["""encoder_ffn_embed_dim"""],
							    """encoder_layerdrop""": args["""encoder_layerdrop"""],
							    """encoder_layers""": args["""encoder_layers"""],
							    """decoder_attention_heads""": args["""decoder_attention_heads"""],
							    """decoder_ffn_dim""": args["""decoder_ffn_embed_dim"""],
							    """decoder_layerdrop""": args["""decoder_layerdrop"""],
							    """decoder_layers""": args["""decoder_layers"""],
							    """bos_token_id""": 0,
							    """pad_token_id""": 1,
							    """eos_token_id""": 2,
							    """is_encoder_decoder""": True,
							    """scale_embedding""": not args["""no_scale_embedding"""],
							    """tie_word_embeddings""": args["""share_all_embeddings"""],
							}

							# good hparam defaults to start with
							_snake_case       =  5
							_snake_case       =  False
							if model_dir in best_score_hparams and "length_penalty" in best_score_hparams[model_dir]:
														_snake_case       =  best_score_hparams[model_dir]["""length_penalty"""]
							else:
														_snake_case       =  1.0

							print(f"""Generating {fsmt_model_config_file}"""				)
							with open(snake_case_							, """w"""							, encoding="""utf-8"""				) as f:
														f.write(json.dumps(snake_case_							, ensure_ascii=snake_case_							, indent=snake_case_				)				)

							# tokenizer config
							_snake_case       =  os.path.join(snake_case_							, snake_case_				)

							_snake_case       =  {
							    """langs""": [src_lang, tgt_lang],
							    """model_max_length""": 1024,
							    """do_lower_case""": do_lower_case,
							}

							print(f"""Generating {fsmt_tokenizer_config_file}"""				)
							with open(snake_case_							, """w"""							, encoding="""utf-8"""				) as f:
														f.write(json.dumps(snake_case_							, ensure_ascii=snake_case_							, indent=snake_case_				)				)

							# model
							_snake_case       =  chkpt["""models"""][0]
							_snake_case       =  model.state_dict()

							# rename keys to start with 'model.'
							_snake_case       =  OrderedDict(("""model.""" + k, v) for k, v in model_state_dict.items()				)

							# remove unneeded keys
							_snake_case       =  [
							    """model.model""",
							    """model.encoder.version""",
							    """model.decoder.version""",
							    """model.encoder_embed_tokens.weight""",
							    """model.decoder_embed_tokens.weight""",
							    """model.encoder.embed_positions._float_tensor""",
							    """model.decoder.embed_positions._float_tensor""",
							]
							for k in ignore_keys:
														model_state_dict.pop(snake_case_							, snake_case_				)

							_snake_case       =  FSMTConfig.from_pretrained(snake_case_				)
							_snake_case       =  FSMTForConditionalGeneration(snake_case_				)

							# check that it loads ok
							model_new.load_state_dict(snake_case_							, strict=snake_case_				)

							# save
							_snake_case       =  os.path.join(snake_case_							, snake_case_				)
							print(f"""Generating {pytorch_weights_dump_path}"""				)
							torch.save(snake_case_							, snake_case_				)

							print("""Conversion is done!"""				)
							print("""\nLast step is to upload the files to s3"""				)
							print(f"""cd {data_root}"""				)
							print(f"""transformers-cli upload {model_dir}"""				)


if __name__ == "__main__":
						__lowerCAmelCase												=	argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--fsmt_checkpoint_path',
						    default=None,
						    type=str,
						    required=True,
						    help=(
						        'Path to the official PyTorch checkpoint file which is expected to reside in the dump dir with dicts,'
						        ' bpecodes, etc.'
						    ),
						)
						parser.add_argument(
						    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
						)
						__lowerCAmelCase												=	parser.parse_args()
						convert_fsmt_checkpoint_to_pytorch(args.fsmt_checkpoint_path, args.pytorch_dump_folder_path)

'''simple docstring'''
import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

__lowerCAmelCase												=	'bert-base-cased'
__lowerCAmelCase												=	'fp16'
__lowerCAmelCase												=	'bf16'
__lowerCAmelCase												=	[FPaa, BFaa]


@require_fsdp
@require_cuda
class       _lowerCAmelCase   (   __snake_case				):







							'''simple docstring'''

							def 	lowercase							(self  )					->       Optional[Any]:
														super().setUp()

														_snake_case       =  dict(
														    ACCELERATE_USE_FSDP="""true"""  ,  MASTER_ADDR="""localhost"""  ,  MASTER_PORT="""10999"""  ,  RANK="""0"""  ,  LOCAL_RANK="""0"""  ,  WORLD_SIZE="""1"""  ,  )

							def 	lowercase							(self  )					->       Optional[int]:
														from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

														for i, strategy in enumerate(UpperCAmelCase  ):
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  f"""{i + 1}"""
																					_snake_case       =  strategy
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  FullyShardedDataParallelPlugin()
																												self.assertEqual(fsdp_plugin.sharding_strategy  ,  ShardingStrategy(i + 1  )  )

							def 	lowercase							(self  )					->       List[str]:
														from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

														for i, prefetch_policy in enumerate(UpperCAmelCase  ):
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  prefetch_policy
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  FullyShardedDataParallelPlugin()
																												if prefetch_policy == "NO_PREFETCH":
																																			self.assertIsNone(fsdp_plugin.backward_prefetch  )
																												else:
																																			self.assertEqual(fsdp_plugin.backward_prefetch  ,  BackwardPrefetch(i + 1  )  )

							def 	lowercase							(self  )					->       int:
														from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

														for i, state_dict_type in enumerate(UpperCAmelCase  ):
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  state_dict_type
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  FullyShardedDataParallelPlugin()
																												self.assertEqual(fsdp_plugin.state_dict_type  ,  StateDictType(i + 1  )  )
																												if state_dict_type == "FULL_STATE_DICT":
																																			self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu  )
																																			self.assertTrue(fsdp_plugin.state_dict_config.ranka_only  )

							def 	lowercase							(self  )					->       Optional[Any]:
														_snake_case       =  AutoModel.from_pretrained(UpperCAmelCase  )
														for policy in FSDP_AUTO_WRAP_POLICY:
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  policy
																					if policy == "TRANSFORMER_BASED_WRAP":
																												_snake_case       =  """BertLayer"""
																					elif policy == "SIZE_BASED_WRAP":
																												_snake_case       =  """2000"""
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  FullyShardedDataParallelPlugin()
																												fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase  )
																												if policy == "NO_WRAP":
																																			self.assertIsNone(fsdp_plugin.auto_wrap_policy  )
																												else:
																																			self.assertIsNotNone(fsdp_plugin.auto_wrap_policy  )

														_snake_case       =  self.dist_env.copy()
														_snake_case       =  """TRANSFORMER_BASED_WRAP"""
														_snake_case       =  """T5Layer"""
														with mockenv_context(**UpperCAmelCase  ):
																					_snake_case       =  FullyShardedDataParallelPlugin()
																					with self.assertRaises(UpperCAmelCase  ) as cm:
																												fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase  )
																					self.assertTrue("""Could not find the transformer layer class to wrap in the model.""" in str(cm.exception  )  )

														_snake_case       =  self.dist_env.copy()
														_snake_case       =  """SIZE_BASED_WRAP"""
														_snake_case       =  """0"""
														with mockenv_context(**UpperCAmelCase  ):
																					_snake_case       =  FullyShardedDataParallelPlugin()
																					fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase  )
																					self.assertIsNone(fsdp_plugin.auto_wrap_policy  )

							def 	lowercase							(self  )					->       int:
														from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
														from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

														for mp_dtype in dtypes:
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  mp_dtype
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  Accelerator()
																												if mp_dtype == "fp16":
																																			_snake_case       =  torch.floataa
																												elif mp_dtype == "bf16":
																																			_snake_case       =  torch.bfloataa
																												_snake_case       =  MixedPrecision(param_dtype=UpperCAmelCase  ,  reduce_dtype=UpperCAmelCase  ,  buffer_dtype=UpperCAmelCase  )
																												self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy  ,  UpperCAmelCase  )
																												if mp_dtype == FPaa:
																																			self.assertTrue(isinstance(accelerator.scaler  ,  UpperCAmelCase  )  )
																												elif mp_dtype == BFaa:
																																			self.assertIsNone(accelerator.scaler  )
																												AcceleratorState._reset_state(UpperCAmelCase  )





							def 	lowercase							(self  )					->       Optional[int]:
														from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

														for flag in [True, False]:
																					_snake_case       =  self.dist_env.copy()
																					_snake_case       =  str(UpperCAmelCase  ).lower()
																					with mockenv_context(**UpperCAmelCase  ):
																												_snake_case       =  FullyShardedDataParallelPlugin()
																												self.assertEqual(fsdp_plugin.cpu_offload  ,  CPUOffload(offload_params=UpperCAmelCase  )  )


@require_fsdp
@require_multi_gpu
@slow
class       _lowerCAmelCase   (   __snake_case				):







							'''simple docstring'''

							def 	lowercase							(self  )					->       List[Any]:
														super().setUp()
														_snake_case       =  0.82
														_snake_case       =  [
														    """fsdp_shard_grad_op_transformer_based_wrap""",
														    """fsdp_full_shard_transformer_based_wrap""",
														]
														_snake_case       =  {
														    """multi_gpu_fp16""": 3200,
														    """fsdp_shard_grad_op_transformer_based_wrap_fp16""": 2000,
														    """fsdp_full_shard_transformer_based_wrap_fp16""": 1900,
														    # Disabling below test as it overwhelms the RAM memory usage
														    # on CI self-hosted runner leading to tests getting killed.
														    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
														}
														_snake_case       =  160
														_snake_case       =  160

														_snake_case       =  inspect.getfile(accelerate.test_utils  )
														_snake_case       =  os.path.sep.join(mod_file.split(os.path.sep  )[:-1] + ["""scripts""", """external_deps"""]  )

							def 	lowercase							(self  )					->       Any:
														_snake_case       =  os.path.join(self.test_scripts_folder  ,  """test_performance.py"""  )
														_snake_case       =  ["""accelerate""", """launch""", """--num_processes=2""", """--num_machines=1""", """--machine_rank=0""", """--use_fsdp"""]
														for config in self.performance_configs:
																					_snake_case       =  cmd.copy()
																					for i, strategy in enumerate(UpperCAmelCase  ):
																												if strategy.lower() in config:
																																			cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""  )
																																			break

																					if "fp32" in config:
																												cmd_config.append("""--mixed_precision=no"""  )
																					else:
																												cmd_config.append("""--mixed_precision=fp16"""  )

																					if "cpu_offload" in config:
																												cmd_config.append("""--fsdp_offload_params=True"""  )

																					for policy in FSDP_AUTO_WRAP_POLICY:
																												if policy.lower() in config:
																																			cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}"""  )
																																			break

																					if policy == "TRANSFORMER_BASED_WRAP":
																												cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""  )
																					elif policy == "SIZE_BASED_WRAP":
																												cmd_config.append("""--fsdp_min_num_params=2000"""  )

																					cmd_config.extend(
																					    [
																					        self.test_file_path,
																					        f"""--output_dir={self.tmpdir}""",
																					        f"""--performance_lower_bound={self.performance_lower_bound}""",
																					    ]  )
																					with patch_environment(omp_num_threads=1  ):
																												execute_subprocess_async(UpperCAmelCase  ,  env=os.environ.copy()  )

							def 	lowercase							(self  )					->       Union[str, Any]:
														_snake_case       =  os.path.join(self.test_scripts_folder  ,  """test_checkpointing.py"""  )
														_snake_case       =  [
														    """accelerate""",
														    """launch""",
														    """--num_processes=2""",
														    """--num_machines=1""",
														    """--machine_rank=0""",
														    """--use_fsdp""",
														    """--mixed_precision=fp16""",
														    """--fsdp_transformer_layer_cls_to_wrap=BertLayer""",
														]

														for i, strategy in enumerate(UpperCAmelCase  ):
																					_snake_case       =  cmd.copy()
																					cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""  )
																					if strategy != "FULL_SHARD":
																												continue
																					_snake_case       =  len(UpperCAmelCase  )
																					for state_dict_type in FSDP_STATE_DICT_TYPE:
																												_snake_case       =  cmd_config[:state_dict_config_index]
																												cmd_config.append(f"""--fsdp_state_dict_type={state_dict_type}"""  )
																												cmd_config.extend(
																												    [
																												        self.test_file_path,
																												        f"""--output_dir={self.tmpdir}""",
																												        """--partial_train_epoch=1""",
																												    ]  )
																												with patch_environment(omp_num_threads=1  ):
																																			execute_subprocess_async(UpperCAmelCase  ,  env=os.environ.copy()  )

																												_snake_case       =  cmd_config[:-1]
																												_snake_case       =  os.path.join(self.tmpdir  ,  """epoch_0"""  )
																												cmd_config.extend(
																												    [
																												        f"""--resume_from_checkpoint={resume_from_checkpoint}""",
																												    ]  )
																												with patch_environment(omp_num_threads=1  ):
																																			execute_subprocess_async(UpperCAmelCase  ,  env=os.environ.copy()  )





							def 	lowercase							(self  )					->       Optional[Any]:
														_snake_case       =  os.path.join(self.test_scripts_folder  ,  """test_peak_memory_usage.py"""  )
														_snake_case       =  [
														    """accelerate""",
														    """launch""",
														    """--num_processes=2""",
														    """--num_machines=1""",
														    """--machine_rank=0""",
														]
														for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
																					_snake_case       =  cmd.copy()
																					if "fp16" in spec:
																												cmd_config.extend(["""--mixed_precision=fp16"""]  )
																					else:
																												cmd_config.extend(["""--mixed_precision=no"""]  )

																					if "multi_gpu" in spec:
																												continue
																					else:
																												cmd_config.extend(["""--use_fsdp"""]  )
																												for i, strategy in enumerate(UpperCAmelCase  ):
																																			if strategy.lower() in spec:
																																										cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""  )
																																										break

																												if "cpu_offload" in spec:
																																			cmd_config.append("""--fsdp_offload_params=True"""  )

																												for policy in FSDP_AUTO_WRAP_POLICY:
																																			if policy.lower() in spec:
																																										cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}"""  )
																																										break

																												if policy == "TRANSFORMER_BASED_WRAP":
																																			cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""  )
																												elif policy == "SIZE_BASED_WRAP":
																																			cmd_config.append("""--fsdp_min_num_params=2000"""  )

																					cmd_config.extend(
																					    [
																					        self.test_file_path,
																					        f"""--output_dir={self.tmpdir}""",
																					        f"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
																					        f"""--n_train={self.n_train}""",
																					        f"""--n_val={self.n_val}""",
																					    ]  )
																					with patch_environment(omp_num_threads=1  ):
																												execute_subprocess_async(UpperCAmelCase  ,  env=os.environ.copy()  )