


import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowerCAmelCase__			:      Tuple								=[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

lowerCAmelCase__			:      Optional[Any]								=[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]






def 						a__			(				A__,      A__				):
			SCREAMING_SNAKE_CASE_		:     str					=					{
			    'word_embeddings.weight': 'word_embeddings.weight',
			    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
			    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
			    'weight': 'ln_f.weight',
			    'bias': 'ln_f.bias',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					int(re.match(r'.*layer_(\d*).*',      A__				)[1]				)
			layer_number -= 3
			return F'''h.{layer_number}.''' + key






def 						a__			(				A__				):
			if dtype == torch.bool:
						return 1 / 8
			SCREAMING_SNAKE_CASE_		:     str					=					re.search(r'[^\d](\d+)$',      str(A__				)				)
			if bit_search is None:
						raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''				)
			SCREAMING_SNAKE_CASE_		:     Dict					=					int(bit_search.groups()[0]				)
			return bit_size // 8






def 						a__			(				A__,      A__,      A__,      A__,      A__				):
			# Construct model
			if bloom_config_file == "":
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					BloomConfig()
			else:
						SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					BloomConfig.from_json_file(A__				)

			if shard_model:
						SCREAMING_SNAKE_CASE_		:     str					=					os.listdir(A__				)
						SCREAMING_SNAKE_CASE_		:     str					=					sorted(filter(lambda A__				: s.startswith('layer'				) and "model_00" in s,      A__				)				)

						SCREAMING_SNAKE_CASE_		:     Tuple					=					{'weight_map': {}, 'metadata': {}}
						SCREAMING_SNAKE_CASE_		:     int					=					0

						SCREAMING_SNAKE_CASE_		:     str					=					None

						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					BloomConfig()

						for j, file in enumerate(A__				):
									print('Processing file: {}'.format(A__				)				)
									SCREAMING_SNAKE_CASE_		:     Dict					=					None

									for i in range(A__				):
												# load all TP files
												SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					file.replace('model_00',      F'''model_0{i}'''				)
												SCREAMING_SNAKE_CASE_		:     List[Any]					=					torch.load(os.path.join(A__,      A__				),      map_location='cpu'				)

												# Rename keys in the transformers names
												SCREAMING_SNAKE_CASE_		:     Dict					=					list(temp.keys()				)
												for key in keys:
															SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					temp.pop(A__				)

												if tensors is None:
															SCREAMING_SNAKE_CASE_		:     Tuple					=					temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(A__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					SCREAMING_SNAKE_CASE_		:     int					=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
																					# We concatenate these weights accross TP ranks
																					SCREAMING_SNAKE_CASE_		:     Optional[int]					=					torch.cat([tensors[key], temp[key]],      dim=A__				)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(A__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
															SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tensors[key] / pretraining_tp
									torch.save(
									    A__,      os.path.join(
									        A__,      'pytorch_model_{}-of-{}.bin'.format(str(j + 1				).zfill(5				),      str(len(A__				)				).zfill(5				)				),      ),      )

									for key in tensors.keys():
												SCREAMING_SNAKE_CASE_		:     str					=					tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype				)
												if key not in index_dict["weight_map"]:
															SCREAMING_SNAKE_CASE_		:     List[Any]					=					'pytorch_model_{}-of-{}.bin'.format(
															    str(j + 1				).zfill(5				),      str(len(A__				)				).zfill(5				)				)

						SCREAMING_SNAKE_CASE_		:     Any					=					BloomConfig()
						SCREAMING_SNAKE_CASE_		:     str					=					pytorch_dump_folder_path + '/' + CONFIG_NAME
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					total_size
						with open(A__,      'w',      encoding='utf-8'				) as f:
									f.write(config.to_json_string()				)
						with open(os.path.join(A__,      WEIGHTS_NAME + '.index.json'				),      'w',      encoding='utf-8'				) as f:
									SCREAMING_SNAKE_CASE_		:     Dict					=					json.dumps(A__,      indent=2,      sort_keys=A__				) + '\n'
									f.write(A__				)
			else:
						SCREAMING_SNAKE_CASE_		:     Tuple					=					BloomModel(A__				)

						SCREAMING_SNAKE_CASE_		:     int					=					os.listdir(A__				)
						SCREAMING_SNAKE_CASE_		:     Tuple					=					sorted(filter(lambda A__				: s.startswith('layer'				) and "model_00" in s,      A__				)				)

						SCREAMING_SNAKE_CASE_		:     Tuple					=					None
						for i, file in enumerate(A__				):
									SCREAMING_SNAKE_CASE_		:     Dict					=					None
									for i in range(A__				):
												# load all TP files
												SCREAMING_SNAKE_CASE_		:     Dict					=					file.replace('model_00',      F'''model_0{i}'''				)
												SCREAMING_SNAKE_CASE_		:     Tuple					=					torch.load(os.path.join(A__,      A__				),      map_location='cpu'				)

												# Rename keys in the transformers names
												SCREAMING_SNAKE_CASE_		:     str					=					list(temp.keys()				)
												for key in keys:
															SCREAMING_SNAKE_CASE_		:     int					=					temp.pop(A__				)

												if tensors is None:
															SCREAMING_SNAKE_CASE_		:     List[Any]					=					temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(A__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					SCREAMING_SNAKE_CASE_		:     Dict					=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
																					# We concatenate these weights accross TP ranks
																					SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					torch.cat([tensors[key], temp[key]],      dim=A__				)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(A__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
															SCREAMING_SNAKE_CASE_		:     Any					=					tensors[key] / pretraining_tp

									SCREAMING_SNAKE_CASE_		:     Any					=					model.load_state_dict(A__,      strict=A__				)
									assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												SCREAMING_SNAKE_CASE_		:     Tuple					=					set(other_keys.missing_keys				)
									else:
												SCREAMING_SNAKE_CASE_		:     str					=					missing_keys.intersection(set(other_keys.missing_keys				)				)

						assert not missing_keys, F'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(A__,      exist_ok=A__				)
						SCREAMING_SNAKE_CASE_		:     Any					=					pytorch_dump_folder_path + '/' + WEIGHTS_NAME
						SCREAMING_SNAKE_CASE_		:     List[str]					=					pytorch_dump_folder_path + '/' + CONFIG_NAME
						print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''				)
						if config.torch_dtype is not None:
									SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					model.to(config.torch_dtype				)
						torch.save(model.state_dict(),      A__				)
						print(F'''Save configuration file to {pytorch_config_dump_path}'''				)
						with open(A__,      'w',      encoding='utf-8'				) as f:
									f.write(config.to_json_string()				)


if __name__ == "__main__":
				lowerCAmelCase__			:      Optional[Any]								=argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--bloom_checkpoint_path',
				    default=None,
				    type=str,
				    required=True,
				    help='Path to the Megatron-LM checkpoint path.',
				)
				parser.add_argument(
				    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
				)
				parser.add_argument(
				    '--bloom_config_file',
				    default='',
				    type=str,
				    help=(
				        'An optional config json file corresponding to the pre-trained model. \n'
				        'This specifies the model architecture.'
				    ),
				)
				parser.add_argument(
				    '--shard_model',
				    action='store_true',
				    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
				)
				parser.add_argument(
				    '--pretraining_tp',
				    default=4,
				    type=int,
				    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
				)
				lowerCAmelCase__			:      List[str]								=parser.parse_args()
				convert_bloom_checkpoint_to_pytorch(
				    args.bloom_checkpoint_path,
				    args.bloom_config_file,
				    args.pytorch_dump_folder_path,
				    args.shard_model,
				    args.pretraining_tp,
				)









import unittest

from transformers import MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, is_vision_available, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
				from PIL import Image
else:







				class 	__lowercase    :





											"""simple docstring"""







											@staticmethod
											def 					UpperCamelCase__     (      *lowerCAmelCase__			,						**lowerCAmelCase__       ):





														"""simple docstring"""




														pass







@is_pipeline_test
@require_vision
@require_torch
class 	__lowercase    (unittest.TestCase  ):





							"""simple docstring"""







							_UpperCAmelCase      =						MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					pipeline(
										    'zero-shot-object-detection'			,						model='hf-internal-testing/tiny-random-owlvit-object-detection'       )

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					[
										    {
										        'image': './tests/fixtures/tests_samples/COCO/000000039769.png',
										        'candidate_labels': ['cat', 'remote', 'couch'],
										    }
										]
										return object_detector, examples






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					object_detector(examples[0]			,						threshold=0.0       )

										SCREAMING_SNAKE_CASE_		:     int					=					len(lowerCAmelCase__       )
										self.assertGreater(lowerCAmelCase__			,						0       )
										self.assertEqual(
										    lowerCAmelCase__			,						[
										        {
										            'score': ANY(lowerCAmelCase__       ),
										            'label': ANY(lowerCAmelCase__       ),
										            'box': {'xmin': ANY(lowerCAmelCase__       ), 'ymin': ANY(lowerCAmelCase__       ), 'xmax': ANY(lowerCAmelCase__       ), 'ymax': ANY(lowerCAmelCase__       )},
										        }
										        for i in range(lowerCAmelCase__       )
										    ]			,						)






							@require_tf
							@unittest.skip('Zero Shot Object Detection not implemented in TF'       )
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							@require_torch
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					pipeline(
										    'zero-shot-object-detection'			,						model='hf-internal-testing/tiny-random-owlvit-object-detection'       )

										SCREAMING_SNAKE_CASE_		:     str					=					object_detector(
										    './tests/fixtures/tests_samples/COCO/000000039769.png'			,						candidate_labels=['cat', 'remote', 'couch']			,						threshold=0.64			,						)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        {'score': 0.7_235, 'label': 'cat', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										        {'score': 0.7_218, 'label': 'remote', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										        {'score': 0.7_184, 'label': 'couch', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										        {'score': 0.6_748, 'label': 'remote', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										        {'score': 0.6_656, 'label': 'cat', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										        {'score': 0.6_614, 'label': 'couch', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										        {'score': 0.6_456, 'label': 'remote', 'box': {'xmin': 4_9_4, 'ymin': 1_0_5, 'xmax': 5_2_1, 'ymax': 1_2_7}},
										        {'score': 0.642, 'label': 'remote', 'box': {'xmin': 6_7, 'ymin': 2_7_4, 'xmax': 9_3, 'ymax': 2_9_7}},
										        {'score': 0.6_419, 'label': 'cat', 'box': {'xmin': 4_9_4, 'ymin': 1_0_5, 'xmax': 5_2_1, 'ymax': 1_2_7}},
										    ]			,						)

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					object_detector(
										    [
										        {
										            'image': './tests/fixtures/tests_samples/COCO/000000039769.png',
										            'candidate_labels': ['cat', 'remote', 'couch'],
										        }
										    ]			,						threshold=0.64			,						)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        [
										            {'score': 0.7_235, 'label': 'cat', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										            {'score': 0.7_218, 'label': 'remote', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										            {'score': 0.7_184, 'label': 'couch', 'box': {'xmin': 2_0_4, 'ymin': 1_6_7, 'xmax': 2_3_2, 'ymax': 1_9_0}},
										            {'score': 0.6_748, 'label': 'remote', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										            {'score': 0.6_656, 'label': 'cat', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										            {'score': 0.6_614, 'label': 'couch', 'box': {'xmin': 5_7_1, 'ymin': 8_3, 'xmax': 5_9_8, 'ymax': 1_0_3}},
										            {'score': 0.6_456, 'label': 'remote', 'box': {'xmin': 4_9_4, 'ymin': 1_0_5, 'xmax': 5_2_1, 'ymax': 1_2_7}},
										            {'score': 0.642, 'label': 'remote', 'box': {'xmin': 6_7, 'ymin': 2_7_4, 'xmax': 9_3, 'ymax': 2_9_7}},
										            {'score': 0.6_419, 'label': 'cat', 'box': {'xmin': 4_9_4, 'ymin': 1_0_5, 'xmax': 5_2_1, 'ymax': 1_2_7}},
										        ]
										    ]			,						)






							@require_torch
							@slow
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					pipeline('zero-shot-object-detection'       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					object_detector(
										    'http://images.cocodataset.org/val2017/000000039769.jpg'			,						candidate_labels=['cat', 'remote', 'couch']			,						)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        {'score': 0.2_868, 'label': 'cat', 'box': {'xmin': 3_2_4, 'ymin': 2_0, 'xmax': 6_4_0, 'ymax': 3_7_3}},
										        {'score': 0.277, 'label': 'remote', 'box': {'xmin': 4_0, 'ymin': 7_2, 'xmax': 1_7_7, 'ymax': 1_1_5}},
										        {'score': 0.2_537, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 5_5, 'xmax': 3_1_5, 'ymax': 4_7_2}},
										        {'score': 0.1_474, 'label': 'remote', 'box': {'xmin': 3_3_5, 'ymin': 7_4, 'xmax': 3_7_1, 'ymax': 1_8_7}},
										        {'score': 0.1_208, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_4_2, 'ymax': 4_7_6}},
										    ]			,						)

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					object_detector(
										    [
										        {
										            'image': 'http://images.cocodataset.org/val2017/000000039769.jpg',
										            'candidate_labels': ['cat', 'remote', 'couch'],
										        },
										        {
										            'image': 'http://images.cocodataset.org/val2017/000000039769.jpg',
										            'candidate_labels': ['cat', 'remote', 'couch'],
										        },
										    ]			,						)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        [
										            {'score': 0.2_868, 'label': 'cat', 'box': {'xmin': 3_2_4, 'ymin': 2_0, 'xmax': 6_4_0, 'ymax': 3_7_3}},
										            {'score': 0.277, 'label': 'remote', 'box': {'xmin': 4_0, 'ymin': 7_2, 'xmax': 1_7_7, 'ymax': 1_1_5}},
										            {'score': 0.2_537, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 5_5, 'xmax': 3_1_5, 'ymax': 4_7_2}},
										            {'score': 0.1_474, 'label': 'remote', 'box': {'xmin': 3_3_5, 'ymin': 7_4, 'xmax': 3_7_1, 'ymax': 1_8_7}},
										            {'score': 0.1_208, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_4_2, 'ymax': 4_7_6}},
										        ],
										        [
										            {'score': 0.2_868, 'label': 'cat', 'box': {'xmin': 3_2_4, 'ymin': 2_0, 'xmax': 6_4_0, 'ymax': 3_7_3}},
										            {'score': 0.277, 'label': 'remote', 'box': {'xmin': 4_0, 'ymin': 7_2, 'xmax': 1_7_7, 'ymax': 1_1_5}},
										            {'score': 0.2_537, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 5_5, 'xmax': 3_1_5, 'ymax': 4_7_2}},
										            {'score': 0.1_474, 'label': 'remote', 'box': {'xmin': 3_3_5, 'ymin': 7_4, 'xmax': 3_7_1, 'ymax': 1_8_7}},
										            {'score': 0.1_208, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_4_2, 'ymax': 4_7_6}},
										        ],
										    ]			,						)






							@require_tf
							@unittest.skip('Zero Shot Object Detection not implemented in TF'       )
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										pass






							@require_torch
							@slow
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     str					=					0.2
										SCREAMING_SNAKE_CASE_		:     str					=					pipeline('zero-shot-object-detection'       )

										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					object_detector(
										    'http://images.cocodataset.org/val2017/000000039769.jpg'			,						candidate_labels=['cat', 'remote', 'couch']			,						threshold=lowerCAmelCase__			,						)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        {'score': 0.2_868, 'label': 'cat', 'box': {'xmin': 3_2_4, 'ymin': 2_0, 'xmax': 6_4_0, 'ymax': 3_7_3}},
										        {'score': 0.277, 'label': 'remote', 'box': {'xmin': 4_0, 'ymin': 7_2, 'xmax': 1_7_7, 'ymax': 1_1_5}},
										        {'score': 0.2_537, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 5_5, 'xmax': 3_1_5, 'ymax': 4_7_2}},
										    ]			,						)






							@require_torch
							@slow
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					2
										SCREAMING_SNAKE_CASE_		:     int					=					pipeline('zero-shot-object-detection'       )

										SCREAMING_SNAKE_CASE_		:     Tuple					=					object_detector(
										    'http://images.cocodataset.org/val2017/000000039769.jpg'			,						candidate_labels=['cat', 'remote', 'couch']			,						top_k=lowerCAmelCase__			,						)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__			,						decimals=4       )			,						[
										        {'score': 0.2_868, 'label': 'cat', 'box': {'xmin': 3_2_4, 'ymin': 2_0, 'xmax': 6_4_0, 'ymax': 3_7_3}},
										        {'score': 0.277, 'label': 'remote', 'box': {'xmin': 4_0, 'ymin': 7_2, 'xmax': 1_7_7, 'ymax': 1_1_5}},
										    ]			,						)





