





import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('0.8.3'):
							raise Exception('requires gluonnlp == 0.8.3')

if version.parse(mx.__version__) != version.parse('1.5.0'):
							raise Exception('requires mxnet == 1.5.0')

logging.set_verbosity_info()
_UpperCamelCase							=      logging.get_logger(__name__)

_UpperCamelCase							=      'The Nymphenburg Palace is a beautiful palace in Munich!'



def 				lowerCAmelCase_			(      SCREAMING_SNAKE_CASE       :							str    ,						SCREAMING_SNAKE_CASE       :							str   ):


							'''simple docstring'''







							__lowerCamelCase      :      Union[str, Any]         ={
							    '''attention_cell''': '''multi_head''',
							    '''num_layers''': 4,
							    '''units''': 1024,
							    '''hidden_size''': 768,
							    '''max_length''': 512,
							    '''num_heads''': 8,
							    '''scaled''': True,
							    '''dropout''': 0.1,
							    '''use_residual''': True,
							    '''embed_size''': 1024,
							    '''embed_dropout''': 0.1,
							    '''word_embed''': None,
							    '''layer_norm_eps''': 1E-5,
							    '''token_type_vocab_size''': 2,
							}

							__lowerCamelCase      :      Optional[Any]         =bort_4_8_768_1024_hparams

							# Let's construct the original Bort model here
							# Taken from official BERT implementation, see:
							# https://github.com/alexa/bort/blob/master/bort/bort.py
							__lowerCamelCase      :      Any         =BERTEncoder(
							    attention_cell=predefined_args['''attention_cell''']    ,						num_layers=predefined_args['''num_layers''']    ,						units=predefined_args['''units''']    ,						hidden_size=predefined_args['''hidden_size''']    ,						max_length=predefined_args['''max_length''']    ,						num_heads=predefined_args['''num_heads''']    ,						scaled=predefined_args['''scaled''']    ,						dropout=predefined_args['''dropout''']    ,						output_attention=SCREAMING_SNAKE_CASE    ,						output_all_encodings=SCREAMING_SNAKE_CASE    ,						use_residual=predefined_args['''use_residual''']    ,						activation=predefined_args.get('''activation'''    ,						'''gelu'''   )    ,						layer_norm_eps=predefined_args.get('''layer_norm_eps'''    ,						SCREAMING_SNAKE_CASE   )    ,						)

							# Vocab information needs to be fetched first
							# It's the same as RoBERTa, so RobertaTokenizer can be used later
							__lowerCamelCase      :      Union[str, Any]         ='''openwebtext_ccnews_stories_books_cased'''

							# Specify download folder to Gluonnlp's vocab
							__lowerCamelCase      :      Dict         =os.path.join(get_home_dir()    ,						'''models'''   )
							__lowerCamelCase      :      Any         =_load_vocab(SCREAMING_SNAKE_CASE    ,						SCREAMING_SNAKE_CASE    ,						SCREAMING_SNAKE_CASE    ,						cls=SCREAMING_SNAKE_CASE   )

							__lowerCamelCase      :      str         =nlp.model.BERTModel(
							    SCREAMING_SNAKE_CASE    ,						len(SCREAMING_SNAKE_CASE   )    ,						units=predefined_args['''units''']    ,						embed_size=predefined_args['''embed_size''']    ,						embed_dropout=predefined_args['''embed_dropout''']    ,						word_embed=predefined_args['''word_embed''']    ,						use_pooler=SCREAMING_SNAKE_CASE    ,						use_token_type_embed=SCREAMING_SNAKE_CASE    ,						token_type_vocab_size=predefined_args['''token_type_vocab_size''']    ,						use_classifier=SCREAMING_SNAKE_CASE    ,						use_decoder=SCREAMING_SNAKE_CASE    ,						)

							original_bort.load_parameters(SCREAMING_SNAKE_CASE    ,						cast_dtype=SCREAMING_SNAKE_CASE    ,						ignore_extra=SCREAMING_SNAKE_CASE   )
							__lowerCamelCase      :      List[Any]         =original_bort._collect_params_with_prefix()

							# Build our config 🤗
							__lowerCamelCase      :      Any         ={
							    '''architectures''': ['''BertForMaskedLM'''],
							    '''attention_probs_dropout_prob''': predefined_args['''dropout'''],
							    '''hidden_act''': '''gelu''',
							    '''hidden_dropout_prob''': predefined_args['''dropout'''],
							    '''hidden_size''': predefined_args['''embed_size'''],
							    '''initializer_range''': 0.02,
							    '''intermediate_size''': predefined_args['''hidden_size'''],
							    '''layer_norm_eps''': predefined_args['''layer_norm_eps'''],
							    '''max_position_embeddings''': predefined_args['''max_length'''],
							    '''model_type''': '''bort''',
							    '''num_attention_heads''': predefined_args['''num_heads'''],
							    '''num_hidden_layers''': predefined_args['''num_layers'''],
							    '''pad_token_id''': 1,  # 2 = BERT, 1 = RoBERTa
							    '''type_vocab_size''': 1,  # 2 = BERT, 1 = RoBERTa
							    '''vocab_size''': len(SCREAMING_SNAKE_CASE   ),
							}

							__lowerCamelCase      :      Union[str, Any]         =BertConfig.from_dict(SCREAMING_SNAKE_CASE   )
							__lowerCamelCase      :      Any         =BertForMaskedLM(SCREAMING_SNAKE_CASE   )
							hf_bort_model.eval()

							# Parameter mapping table (Gluonnlp to Transformers)
							# * denotes layer index
							#
							# | Gluon Parameter                                                | Transformers Parameter
							# | -------------------------------------------------------------- | ----------------------
							# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
							# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
							# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
							# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
							# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
							# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
							# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
							# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
							# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
							# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
							# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
							# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
							# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
							# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

							# Helper function to convert MXNET Arrays to PyTorch
							def to_torch(SCREAMING_SNAKE_CASE       :							List[Any]   ) -> nn.Parameter:
														return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()   )   )

							# Check param shapes and map new HF param back
							def check_and_map_params(SCREAMING_SNAKE_CASE       :							Optional[int]    ,						SCREAMING_SNAKE_CASE       :							Optional[Any]   ):
														__lowerCamelCase      :      Dict         =hf_param.shape

														__lowerCamelCase      :      Dict         =to_torch(params[gluon_param]   )
														__lowerCamelCase      :      int         =gluon_param.shape

														assert (
														    shape_hf == shape_gluon
														), F'The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers'

														return gluon_param

							__lowerCamelCase      :      Tuple         =check_and_map_params(
							    hf_bort_model.bert.embeddings.word_embeddings.weight    ,						'''word_embed.0.weight'''   )
							__lowerCamelCase      :      str         =check_and_map_params(
							    hf_bort_model.bert.embeddings.position_embeddings.weight    ,						'''encoder.position_weight'''   )
							__lowerCamelCase      :      str         =check_and_map_params(
							    hf_bort_model.bert.embeddings.LayerNorm.bias    ,						'''encoder.layer_norm.beta'''   )
							__lowerCamelCase      :      Union[str, Any]         =check_and_map_params(
							    hf_bort_model.bert.embeddings.LayerNorm.weight    ,						'''encoder.layer_norm.gamma'''   )

							# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
							__lowerCamelCase      :      List[Any]         =torch.zeros_like(
							    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data   )

							for i in range(hf_bort_config.num_hidden_layers   ):
														__lowerCamelCase      :      BertLayer         =hf_bort_model.bert.encoder.layer[i]

														# self attention
														__lowerCamelCase      :      BertSelfAttention         =layer.attention.self

														__lowerCamelCase      :      str         =check_and_map_params(
														    self_attn.key.bias.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_key.bias'   )

														__lowerCamelCase      :      Any         =check_and_map_params(
														    self_attn.key.weight.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_key.weight'   )
														__lowerCamelCase      :      Any         =check_and_map_params(
														    self_attn.query.bias.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_query.bias'   )
														__lowerCamelCase      :      Dict         =check_and_map_params(
														    self_attn.query.weight.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_query.weight'   )
														__lowerCamelCase      :      Dict         =check_and_map_params(
														    self_attn.value.bias.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_value.bias'   )
														__lowerCamelCase      :      Optional[Any]         =check_and_map_params(
														    self_attn.value.weight.data    ,						F'encoder.transformer_cells.{i}.attention_cell.proj_value.weight'   )

														# self attention output
														__lowerCamelCase      :      BertSelfOutput         =layer.attention.output

														__lowerCamelCase      :      Any         =check_and_map_params(
														    self_output.dense.bias    ,						F'encoder.transformer_cells.{i}.proj.bias'   )
														__lowerCamelCase      :      Dict         =check_and_map_params(
														    self_output.dense.weight    ,						F'encoder.transformer_cells.{i}.proj.weight'   )
														__lowerCamelCase      :      int         =check_and_map_params(
														    self_output.LayerNorm.bias    ,						F'encoder.transformer_cells.{i}.layer_norm.beta'   )
														__lowerCamelCase      :      Any         =check_and_map_params(
														    self_output.LayerNorm.weight    ,						F'encoder.transformer_cells.{i}.layer_norm.gamma'   )

														# intermediate
														__lowerCamelCase      :      BertIntermediate         =layer.intermediate

														__lowerCamelCase      :      Optional[int]         =check_and_map_params(
														    intermediate.dense.bias    ,						F'encoder.transformer_cells.{i}.ffn.ffn_1.bias'   )
														__lowerCamelCase      :      str         =check_and_map_params(
														    intermediate.dense.weight    ,						F'encoder.transformer_cells.{i}.ffn.ffn_1.weight'   )

														# output
														__lowerCamelCase      :      BertOutput         =layer.output

														__lowerCamelCase      :      int         =check_and_map_params(
														    bert_output.dense.bias    ,						F'encoder.transformer_cells.{i}.ffn.ffn_2.bias'   )
														__lowerCamelCase      :      Optional[Any]         =check_and_map_params(
														    bert_output.dense.weight    ,						F'encoder.transformer_cells.{i}.ffn.ffn_2.weight'   )
														__lowerCamelCase      :      Tuple         =check_and_map_params(
														    bert_output.LayerNorm.bias    ,						F'encoder.transformer_cells.{i}.ffn.layer_norm.beta'   )
														__lowerCamelCase      :      Optional[Any]         =check_and_map_params(
														    bert_output.LayerNorm.weight    ,						F'encoder.transformer_cells.{i}.ffn.layer_norm.gamma'   )

							# Save space and energy 🎄
							hf_bort_model.half()

							# Compare output of both models
							__lowerCamelCase      :      Union[str, Any]         =RobertaTokenizer.from_pretrained('''roberta-base'''   )

							__lowerCamelCase      :      str         =tokenizer.encode_plus(SCREAMING_SNAKE_CASE   )['''input_ids''']

							# Get gluon output
							__lowerCamelCase      :      Tuple         =mx.nd.array([input_ids]   )
							__lowerCamelCase      :      Union[str, Any]         =original_bort(inputs=SCREAMING_SNAKE_CASE    ,						token_types=[]   )

							# Get Transformer output (save and reload model again)
							hf_bort_model.save_pretrained(SCREAMING_SNAKE_CASE   )
							__lowerCamelCase      :      Dict         =BertModel.from_pretrained(SCREAMING_SNAKE_CASE   )
							hf_bort_model.eval()

							__lowerCamelCase      :      List[Any]         =tokenizer.encode_plus(SCREAMING_SNAKE_CASE    ,						return_tensors='''pt'''   )
							__lowerCamelCase      :      List[Any]         =hf_bort_model(**SCREAMING_SNAKE_CASE   )[0]

							__lowerCamelCase      :      List[str]         =output_gluon[0].asnumpy()
							__lowerCamelCase      :      Optional[Any]         =output_hf[0].detach().numpy()

							__lowerCamelCase      :      List[Any]         =np.max(np.abs(hf_layer - gluon_layer   )   ).item()
							__lowerCamelCase      :      Tuple         =np.allclose(SCREAMING_SNAKE_CASE    ,						SCREAMING_SNAKE_CASE    ,						atol=1E-3   )

							if success:
														print('''✔️ Both model do output the same tensors'''   )
							else:
														print('''❌ Both model do **NOT** output the same tensors'''   )
														print('''Absolute difference is:'''    ,						SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
							_UpperCamelCase							=      argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bort_checkpoint_path', default=None, type=str, required=True, help='Path the official Bort params file.'
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							_UpperCamelCase							=      parser.parse_args()
							convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)











"""simple docstring"""





import unittest

from .lib import (
    Matrix,
    Vector,
    axpy,
    square_zero_matrix,
    unit_basis_vector,
    zero_vector,
)
class 		SCREAMING_SNAKE_CASE_							(    unittest.TestCase   ):
				"""simple docstring"""




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Any         =Vector([1, 2, 3]			)
											self.assertEqual(x.component(0			)						,    1			)
											self.assertEqual(x.component(2			)						,    3			)
											__lowerCamelCase      :      Any         =Vector()




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      Tuple         =Vector([0, 0, 0, 0, 0, 1]			)
											self.assertEqual(str(__lowercase			)						,    '''(0,0,0,0,0,1)'''			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      int         =Vector([1, 2, 3, 4]			)
											self.assertEqual(len(__lowercase			)						,    4			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      Optional[Any]         =Vector([1, 2]			)
											__lowerCamelCase      :      Dict         =Vector([1, 2, 3, 4, 5]			)
											__lowerCamelCase      :      List[Any]         =Vector([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]			)
											__lowerCamelCase      :      int         =Vector([1, -1, 1, -1, 2, -3, 4, -5]			)
											self.assertAlmostEqual(x.euclidean_length()						,    2.236						,    3			)
											self.assertAlmostEqual(y.euclidean_length()						,    7.416						,    3			)
											self.assertEqual(z.euclidean_length()						,    0			)
											self.assertAlmostEqual(w.euclidean_length()						,    7.616						,    3			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      Tuple         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      Any         =Vector([1, 1, 1]			)
											self.assertEqual((x + y).component(0			)						,    2			)
											self.assertEqual((x + y).component(1			)						,    3			)
											self.assertEqual((x + y).component(2			)						,    4			)




				def 					__lowercase  (     self  :str			):
											__lowerCamelCase      :      Union[str, Any]         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      int         =Vector([1, 1, 1]			)
											self.assertEqual((x - y).component(0			)						,    0			)
											self.assertEqual((x - y).component(1			)						,    1			)
											self.assertEqual((x - y).component(2			)						,    2			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      List[Any]         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      List[Any]         =Vector([2, -1, 4]			)  # for test of dot product
											__lowerCamelCase      :      Any         =Vector([1, -2, -1]			)
											self.assertEqual(str(x * 3.0			)						,    '''(3.0,6.0,9.0)'''			)
											self.assertEqual((a * b)						,    0			)




				def 					__lowercase  (     self  :List[Any]			):
											self.assertEqual(str(zero_vector(10			)			).count('''0'''			)						,    10			)




				def 					__lowercase  (     self  :Union[str, Any]			):
											self.assertEqual(str(unit_basis_vector(3						,    1			)			)						,    '''(0,1,0)'''			)




				def 					__lowercase  (     self  :List[Any]			):
											__lowerCamelCase      :      Any         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      Optional[int]         =Vector([1, 0, 1]			)
											self.assertEqual(str(axpy(2						,    __lowercase						,    __lowercase			)			)						,    '''(3,4,7)'''			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      List[Any]         =Vector([1, 0, 0, 0, 0, 0]			)
											__lowerCamelCase      :      Optional[int]         =x.copy()
											self.assertEqual(str(__lowercase			)						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      str         =Vector([1, 0, 0]			)
											x.change_component(0						,    0			)
											x.change_component(1						,    1			)
											self.assertEqual(str(__lowercase			)						,    '''(0,1,0)'''			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Any         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual('''|1,2,3|\n|2,4,5|\n|6,7,8|\n'''						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Tuple         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      List[Any]         =[[-3, -14, -10], [-5, -10, -5], [-2, -1, 0]]
											for x in range(a.height()			):
																		for y in range(a.width()			):
																									self.assertEqual(minors[x][y]						,    a.minor(__lowercase						,    __lowercase			)			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      Optional[Any]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      Tuple         =[[-3, 14, -10], [5, -10, 5], [-2, 1, 0]]
											for x in range(a.height()			):
																		for y in range(a.width()			):
																									self.assertEqual(cofactors[x][y]						,    a.cofactor(__lowercase						,    __lowercase			)			)




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Tuple         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual(-5						,    a.determinant()			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Union[str, Any]         =Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]						,    3						,    3			)
											__lowerCamelCase      :      Tuple         =Vector([1, 2, 3]			)
											self.assertEqual('''(14,32,50)'''						,    str(a * x			)			)
											self.assertEqual('''|2,4,6|\n|8,10,12|\n|14,16,18|\n'''						,    str(a * 2			)			)




				def 					__lowercase  (     self  :Optional[Any]			):
											__lowerCamelCase      :      Optional[int]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											a.change_component(0						,    2						,    5			)
											self.assertEqual('''|1,2,5|\n|2,4,5|\n|6,7,8|\n'''						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :str			):
											__lowerCamelCase      :      str         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual(7						,    a.component(2						,    1			)						,    0.01			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      List[str]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      List[str]         =Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]						,    3						,    3			)
											self.assertEqual('''|2,4,10|\n|4,8,10|\n|12,14,18|\n'''						,    str(a + b			)			)




				def 					__lowercase  (     self  :Union[str, Any]			):
											__lowerCamelCase      :      int         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      Optional[int]         =Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]						,    3						,    3			)
											self.assertEqual('''|0,0,-4|\n|0,0,0|\n|0,0,-2|\n'''						,    str(a - b			)			)




				def 					__lowercase  (     self  :Any			):
											self.assertEqual(
											    '''|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n'''						,    str(square_zero_matrix(5			)			)						,    )


if __name__ == "__main__":
							unittest.main()




