


import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)






def 		_A		(		_lowercase )					-> List[str]:


     """simple docstring"""


     if model.config.model_type == "gpt2":
          return model.transformer.h[0].mlp.c_fc
     return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
    import torch
    import torch.nn as nn



    class 	__lowerCamelCase			(nn.Module						):







           def __init__(			self: Union[str, Any],A_: nn.Module,A_: int					):






                '''simple docstring'''




                super().__init__()
                __UpperCamelCase =			module
                __UpperCamelCase =			nn.Sequential(
                    nn.Linear(module.in_features,A_,bias=A_					),nn.Linear(A_,module.out_features,bias=A_					),)
                __UpperCamelCase =			(2.0 / (5 * min(module.in_features,module.out_features					))) ** 0.5
                nn.init.normal_(self.adapter[0].weight,std=A_					)
                nn.init.zeros_(self.adapter[1].weight					)
                self.adapter.to(module.weight.device					)







           def 		snake_case_	(			self: Optional[Any],A_: int,*A_: Union[str, Any],**A_: str					):






                '''simple docstring'''




                return self.module(A_,*A_,**A_					) + self.adapter(A_					)



@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	__lowerCamelCase			(unittest.TestCase						):
       # We keep the constants inside the init function and model loading inside setUp function

       # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
       # Therefore here we use only bloom-1b3 to test our module
       _lowercase											=     """bigscience/bloom-1b7"""

       # Constant values
       _lowercase											=     2.109659552692574

       _lowercase											=     """Hello my name is"""
       _lowercase											=     set()
       EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I"""						)
       EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n"""						)
       EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University"""						)
       _lowercase											=     10







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            __UpperCamelCase =			AutoTokenizer.from_pretrained(self.model_name					)



class 	__lowerCamelCase			(_a						):







       def 		snake_case_	(			self: int					):






            '''simple docstring'''




            super().setUp()

            # Models and tokenizer
            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(
                self.model_name,torch_dtype=torch.floataa,device_map='auto'					)
            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(self.model_name,load_in_abit=A_,device_map='auto'					)







       def 		snake_case_	(			self: Optional[Any]					):






            '''simple docstring'''




            del self.model_fpaa
            del self.model_abit

            gc.collect()
            torch.cuda.empty_cache()







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            __UpperCamelCase =			self.model_abit.config

            self.assertTrue(hasattr(A_,'quantization_config'					)					)

            __UpperCamelCase =			config.to_dict()
            __UpperCamelCase =			config.to_diff_dict()

            __UpperCamelCase =			config.to_json_string()







       def 		snake_case_	(			self: Optional[int]					):






            '''simple docstring'''




            from bitsandbytes.nn import Paramsabit

            __UpperCamelCase =			self.model_fpaa.get_memory_footprint()
            __UpperCamelCase =			self.model_abit.get_memory_footprint()

            self.assertAlmostEqual(mem_fpaa / mem_abit,self.EXPECTED_RELATIVE_DIFFERENCE					)
            __UpperCamelCase =			get_some_linear_layer(self.model_abit					)
            self.assertTrue(linear.weight.__class__ == Paramsabit					)







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            from transformers import TaPreTrainedModel

            self.model_fpaa.get_memory_footprint()
            self.model_abit.get_memory_footprint()

            for name, module in self.model_abit.named_modules():
                 if isinstance(A_,torch.nn.Linear					):
                      if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                           # 4-bit parameters are packed in uint8 variables
                           self.assertTrue(module.weight.dtype == torch.uinta					)







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					)
            __UpperCamelCase =			self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0					),max_new_tokens=10					)

            self.assertIn(self.tokenizer.decode(output_sequences[0],skip_special_tokens=A_					),self.EXPECTED_OUTPUTS					)







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            __UpperCamelCase =			BitsAndBytesConfig()
            __UpperCamelCase =			True

            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(
                self.model_name,quantization_config=A_,device_map='auto'					)

            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					)
            __UpperCamelCase =			model_abit_from_config.generate(
                input_ids=encoded_input['input_ids'].to(0					),max_new_tokens=10					)

            self.assertIn(self.tokenizer.decode(output_sequences[0],skip_special_tokens=A_					),self.EXPECTED_OUTPUTS					)







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            with self.assertRaises(A_					), tempfile.TemporaryDirectory() as tmpdirname:
                 self.model_abit.save_pretrained(A_					)







       def 		snake_case_	(			self: Dict					):






            '''simple docstring'''




            __UpperCamelCase =			BitsAndBytesConfig()

            with self.assertRaises(A_					):
                 __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(
                     self.model_name,quantization_config=A_,load_in_abit=A_,device_map='auto',bnb_abit_quant_type='nf4',)







       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            with self.assertRaises(A_					):
                 # Tries with `str`
                 self.model_abit.to('cpu'					)

            with self.assertRaises(A_					):
                 # Tries with a `dtype``
                 self.model_abit.to(torch.floataa					)

            with self.assertRaises(A_					):
                 # Tries with a `device`
                 self.model_abit.to(torch.device('cuda:0'					)					)

            with self.assertRaises(A_					):
                 # Tries with a `device`
                 self.model_abit.float()

            with self.assertRaises(A_					):
                 # Tries with a `device`
                 self.model_abit.half()

            # Test if we did not break anything
            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					)

            __UpperCamelCase =			self.model_fpaa.to(torch.floataa					)
            __UpperCamelCase =			self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0					),max_new_tokens=10					)

            # Check this does not throw an error
            __UpperCamelCase =			self.model_fpaa.to('cpu'					)

            # Check this does not throw an error
            __UpperCamelCase =			self.model_fpaa.half()

            # Check this does not throw an error
            __UpperCamelCase =			self.model_fpaa.float()







       def 		snake_case_	(			self: Optional[int]					):






            '''simple docstring'''




            __UpperCamelCase =			AutoModelForSeqaSeqLM.from_pretrained('t5-small',load_in_abit=A_,device_map='auto'					)
            self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa					)



@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	__lowerCamelCase			(unittest.TestCase						):







       @classmethod
       def 		snake_case_	(			cls: Optional[Any]					):






            '''simple docstring'''




            __UpperCamelCase =			't5-small'
            __UpperCamelCase =			'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
            __UpperCamelCase =			AutoTokenizer.from_pretrained(cls.model_name					)
            __UpperCamelCase =			'Translate in German: Hello, my dog is cute'







       def 		snake_case_	(			self: Dict					):






            '''simple docstring'''




            gc.collect()
            torch.cuda.empty_cache()







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            from transformers import TaForConditionalGeneration

            __UpperCamelCase =			TaForConditionalGeneration._keep_in_fpaa_modules
            __UpperCamelCase =			None

            # test with `t5-small`
            __UpperCamelCase =			TaForConditionalGeneration.from_pretrained(self.model_name,load_in_abit=A_,device_map='auto'					)
            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					).to(0					)
            __UpperCamelCase =			model.generate(**A_					)

            # test with `flan-t5-small`
            __UpperCamelCase =			TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name,load_in_abit=A_,device_map='auto'					)
            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					).to(0					)
            __UpperCamelCase =			model.generate(**A_					)
            __UpperCamelCase =			modules







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            import bitsandbytes as bnb

            from transformers import TaForConditionalGeneration

            # test with `t5-small`
            __UpperCamelCase =			TaForConditionalGeneration.from_pretrained(self.model_name,load_in_abit=A_,device_map='auto'					)

            # there was a bug with decoders - this test checks that it is fixed
            self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q,bnb.nn.Linearabit					)					)

            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					).to(0					)
            __UpperCamelCase =			model.generate(**A_					)

            # test with `flan-t5-small`
            __UpperCamelCase =			TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name,load_in_abit=A_,device_map='auto'					)
            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					).to(0					)
            __UpperCamelCase =			model.generate(**A_					)



class 	__lowerCamelCase			(_a						):







       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            super().setUp()
            # model_name
            __UpperCamelCase =			'bigscience/bloom-560m'
            __UpperCamelCase =			't5-small'

            # Different types of model

            __UpperCamelCase =			AutoModel.from_pretrained(self.model_name,load_in_abit=A_,device_map='auto'					)
            # Sequence classification model
            __UpperCamelCase =			AutoModelForSequenceClassification.from_pretrained(
                self.model_name,load_in_abit=A_,device_map='auto'					)
            # CausalLM model
            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(self.model_name,load_in_abit=A_,device_map='auto'					)
            # Seq2seq model
            __UpperCamelCase =			AutoModelForSeqaSeqLM.from_pretrained(
                self.seq_to_seq_name,load_in_abit=A_,device_map='auto'					)







       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            del self.base_model
            del self.sequence_model
            del self.model_abit
            del self.seq_to_seq_model

            gc.collect()
            torch.cuda.empty_cache()







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            from bitsandbytes.nn import Paramsabit

            self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit					)

            # Other heads should be nn.Parameter
            self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter					)
            self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter					)
            self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter					)



class 	__lowerCamelCase			(_a						):







       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            super().setUp()







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            del self.pipe

            gc.collect()
            torch.cuda.empty_cache()







       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            __UpperCamelCase =			pipeline(
                'text-generation',model=self.model_name,model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa},max_new_tokens=self.MAX_NEW_TOKENS,)

            # Real second forward pass
            __UpperCamelCase =			self.pipe(self.input_text					)
            self.assertIn(pipeline_output[0]['generated_text'],self.EXPECTED_OUTPUTS					)



@require_torch_multi_gpu
class 	__lowerCamelCase			(_a						):







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            super().setUp()







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(
                self.model_name,load_in_abit=A_,device_map='balanced'					)

            # Check correct device map
            self.assertEqual(set(model_parallel.hf_device_map.values()					),{0, 1}					)

            # Check that inference pass works on the model
            __UpperCamelCase =			self.tokenizer(self.input_text,return_tensors='pt'					)

            # Second real batch
            __UpperCamelCase =			model_parallel.generate(input_ids=encoded_input['input_ids'].to(0					),max_new_tokens=10					)
            self.assertIn(self.tokenizer.decode(output_parallel[0],skip_special_tokens=A_					),self.EXPECTED_OUTPUTS					)



class 	__lowerCamelCase			(_a						):







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            __UpperCamelCase =			'facebook/opt-350m'
            super().setUp()







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            if version.parse(importlib.metadata.version('bitsandbytes'					)					) < version.parse('0.37.0'					):
                 return

            # Step 1: freeze all parameters
            __UpperCamelCase =			AutoModelForCausalLM.from_pretrained(self.model_name,load_in_abit=A_					)

            self.assertEqual(set(model.hf_device_map.values()					),{torch.cuda.current_device()}					)

            for param in model.parameters():
                 __UpperCamelCase =			False  # freeze the model - train adapters later
                 if param.ndim == 1:
                      # cast the small parameters (e.g. layernorm) to fp32 for stability
                      __UpperCamelCase =			param.data.to(torch.floataa					)

        # Step 2: add adapters
            for _, module in model.named_modules():
                 if "OPTAttention" in repr(type(A_					)					):
                      __UpperCamelCase =			LoRALayer(module.q_proj,rank=16					)
                      __UpperCamelCase =			LoRALayer(module.k_proj,rank=16					)
                      __UpperCamelCase =			LoRALayer(module.v_proj,rank=16					)

        # Step 3: dummy batch
            __UpperCamelCase =			self.tokenizer('Test batch ',return_tensors='pt'					).to(0					)

            # Step 4: Check if the gradient is not None
            with torch.cuda.amp.autocast():
                 __UpperCamelCase =			model.forward(**A_					)
                 out.logits.norm().backward()

            for module in model.modules():
                 if isinstance(A_,A_					):
                      self.assertTrue(module.adapter[1].weight.grad is not None					)
                      self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0					)
                 elif isinstance(A_,nn.Embedding					):
                      self.assertTrue(module.weight.grad is None					)



class 	__lowerCamelCase			(_a						):
       _lowercase											=     """gpt2-xl"""
       _lowercase											=     3.3191854854152187






import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


__snake_case   							=						1_6
__snake_case   							=						3_2






def 		_A		(		_lowercase					,  _lowercase = 16					,  _lowercase = "bert-base-cased" )					-> Union[str, Any]:


     """simple docstring"""


     __UpperCamelCase =			AutoTokenizer.from_pretrained(_lowercase )
     __UpperCamelCase =			load_dataset('glue'					,  'mrpc' )

     def tokenize_function(_lowercase ):
          # max_length=None => use the model max length (it's actually the default)
          __UpperCamelCase =			tokenizer(examples['sentence1']					,  examples['sentence2']					,  truncation=_lowercase					,  max_length=_lowercase )
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     __UpperCamelCase =			datasets.map(
         _lowercase					,  batched=_lowercase					,  remove_columns=['idx', 'sentence1', 'sentence2']					,  load_from_cache_file=_lowercase )

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     __UpperCamelCase =			tokenized_datasets.rename_column('label'					,  'labels' )

     def collate_fn(_lowercase ):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          if accelerator.distributed_type == DistributedType.TPU:
               return tokenizer.pad(_lowercase					,  padding='max_length'					,  max_length=1_28					,  return_tensors='pt' )
          return tokenizer.pad(_lowercase					,  padding='longest'					,  return_tensors='pt' )

     # Instantiate dataloaders.
     __UpperCamelCase =			DataLoader(
         tokenized_datasets['train']					,  shuffle=_lowercase					,  collate_fn=_lowercase					,  batch_size=_lowercase )
     __UpperCamelCase =			DataLoader(
         tokenized_datasets['validation']					,  shuffle=_lowercase					,  collate_fn=_lowercase					,  batch_size=_lowercase )

     return train_dataloader, eval_dataloader






def 		_A		(		_lowercase					,  _lowercase )					-> int:


     """simple docstring"""


     __UpperCamelCase =			Accelerator()

     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     __UpperCamelCase =			config['lr']
     __UpperCamelCase =			int(config['num_epochs'] )
     __UpperCamelCase =			int(config['seed'] )
     __UpperCamelCase =			int(config['batch_size'] )
     __UpperCamelCase =			args.model_name_or_path

     set_seed(_lowercase )
     __UpperCamelCase,    __UpperCamelCase =			get_dataloaders(_lowercase					,  _lowercase					,  _lowercase )

     # Instantiate the model (we build the model here so that the seed also control new weights initialization)
     __UpperCamelCase =			AutoModelForSequenceClassification.from_pretrained(_lowercase					,  return_dict=_lowercase )

     # Instantiate optimizer
     __UpperCamelCase =			(
         AdamW
         if accelerator.state.deepspeed_plugin is None
         or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
         else DummyOptim
     )
     __UpperCamelCase =			optimizer_cls(params=model.parameters()					,  lr=_lowercase )

     if accelerator.state.deepspeed_plugin is not None:
          __UpperCamelCase =			accelerator.state.deepspeed_plugin.deepspeed_config[
              'gradient_accumulation_steps'
          ]
     else:
          __UpperCamelCase =			1
     __UpperCamelCase =			(len(_lowercase ) * num_epochs) // gradient_accumulation_steps

     # Instantiate scheduler
     if (
         accelerator.state.deepspeed_plugin is None
         or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
     ):
          __UpperCamelCase =			get_linear_schedule_with_warmup(
              optimizer=_lowercase					,  num_warmup_steps=0					,  num_training_steps=_lowercase					,  )
     else:
          __UpperCamelCase =			DummyScheduler(_lowercase					,  total_num_steps=_lowercase					,  warmup_num_steps=0 )

     # Prepare everything
     # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
     # prepare method.
     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase =			accelerator.prepare(
         _lowercase					,  _lowercase					,  _lowercase					,  _lowercase					,  _lowercase )

     # We need to keep track of how many total steps we have iterated over
     __UpperCamelCase =			0
     # We also need to keep track of the stating epoch so files are named properly
     __UpperCamelCase =			0

     # Now we train the model
     __UpperCamelCase =			evaluate.load('glue'					,  'mrpc' )
     __UpperCamelCase =			0
     __UpperCamelCase =			{}
     for epoch in range(_lowercase					,  _lowercase ):
          model.train()
          for step, batch in enumerate(_lowercase ):
               __UpperCamelCase =			model(**_lowercase )
               __UpperCamelCase =			outputs.loss
               __UpperCamelCase =			loss / gradient_accumulation_steps
               accelerator.backward(_lowercase )
               if step % gradient_accumulation_steps == 0:
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

               overall_step += 1

          model.eval()
          __UpperCamelCase =			0
          for step, batch in enumerate(_lowercase ):
               # We could avoid this line since we set the accelerator with `device_placement=True`.
               batch.to(accelerator.device )
               with torch.no_grad():
                    __UpperCamelCase =			model(**_lowercase )
               __UpperCamelCase =			outputs.logits.argmax(dim=-1 )
               # It is slightly faster to call this once, than multiple times
               __UpperCamelCase,    __UpperCamelCase =			accelerator.gather(
                   (predictions, batch['labels']) )  # If we are in a multiprocess environment, the last batch has duplicates
               if accelerator.use_distributed:
                    if step == len(_lowercase ) - 1:
                         __UpperCamelCase =			predictions[: len(eval_dataloader.dataset ) - samples_seen]
                         __UpperCamelCase =			references[: len(eval_dataloader.dataset ) - samples_seen]
                    else:
                         samples_seen += references.shape[0]
               metric.add_batch(
                   predictions=_lowercase					,  references=_lowercase					,  )

          __UpperCamelCase =			metric.compute()
          # Use accelerator.print to print only on the main process.
          accelerator.print(f'''epoch {epoch}:'''					,  _lowercase )
          __UpperCamelCase =			eval_metric['accuracy']

          if best_performance < eval_metric["accuracy"]:
               __UpperCamelCase =			eval_metric['accuracy']

     if args.performance_lower_bound is not None:
          assert (
              args.performance_lower_bound <= best_performance
          ), f'''Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}'''

     accelerator.wait_for_everyone()
     if accelerator.is_main_process:
          with open(os.path.join(args.output_dir					,  'all_results.json' )					,  'w' ) as f:
               json.dump(_lowercase					,  _lowercase )






def 		_A		(		)					-> List[str]:


     """simple docstring"""


     __UpperCamelCase =			argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.' )
     parser.add_argument(
         '--model_name_or_path'					,  type=_lowercase					,  default='bert-base-cased'					,  help='Path to pretrained model or model identifier from huggingface.co/models.'					,  required=_lowercase					,  )
     parser.add_argument(
         '--output_dir'					,  type=_lowercase					,  default='.'					,  help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'					,  )
     parser.add_argument(
         '--performance_lower_bound'					,  type=_lowercase					,  default=_lowercase					,  help='Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value.'					,  )
     parser.add_argument(
         '--num_epochs'					,  type=_lowercase					,  default=3					,  help='Number of train epochs.'					,  )
     __UpperCamelCase =			parser.parse_args()
     __UpperCamelCase =			{'lr': 2e-5, 'num_epochs': args.num_epochs, 'seed': 42, 'batch_size': 16}
     training_function(_lowercase					,  _lowercase )


if __name__ == "__main__":
    main()


