'''simple docstring'''





import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)


def     __lowerCAmelCase						(       a_		)      ->	Tuple:





     '''simple docstring'''





     if model.config.model_type == "gpt2":
          return model.transformer.h[0].mlp.c_fc
     return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn






 class 				UpperCAmelCase (       nn.Module ):




     '''simple docstring'''







     def __init__(       self    ,							lowercase__    ,							lowercase__ )							-> List[Any]:
          super().__init__()
          SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       module
          SCREAMING_SNAKE_CASE					:			Dict    	=       nn.Sequential(
              nn.Linear(module.in_features    ,							lowercase__    ,							bias=lowercase__ )    ,							nn.Linear(lowercase__    ,							module.out_features    ,							bias=lowercase__ )    ,							)
          SCREAMING_SNAKE_CASE					:			List[Any]    	=       (2.0 / (5 * min(module.in_features    ,							module.out_features ))) ** 0.5
          nn.init.normal_(self.adapter[0].weight    ,							std=lowercase__ )
          nn.init.zeros_(self.adapter[1].weight )
          self.adapter.to(module.weight.device )






     def       _UpperCamelCase    (       self    ,							lowercase__    ,							*lowercase__    ,							**lowercase__ )							-> Tuple:
          return self.module(lowercase__    ,							*lowercase__    ,							**lowercase__ ) + self.adapter(lowercase__ )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				UpperCAmelCase (       unittest.TestCase ):




    '''simple docstring'''







    snake_case__							:   Optional[int]							=						"bigscience/bloom-1b7"

    # Constant values
    snake_case__							:   Union[str, Any]							=						2.109659552692574

    snake_case__							:   Tuple							=						"Hello my name is"
    snake_case__							:   Optional[Any]							=						set()
    EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I" )
    EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n" )
    EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University" )
    snake_case__							:   Any							=						1_0






    def       _UpperCamelCase    (       self )							-> Union[str, Any]:
         SCREAMING_SNAKE_CASE					:			Tuple    	=       AutoTokenizer.from_pretrained(self.model_name )






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Dict:
         super().setUp()

         # Models and tokenizer
         SCREAMING_SNAKE_CASE					:			int    	=       AutoModelForCausalLM.from_pretrained(
             self.model_name    ,							torch_dtype=torch.floataa    ,							device_map='auto' )
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       AutoModelForCausalLM.from_pretrained(self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )






    def       _UpperCamelCase    (       self )							-> Tuple:
         del self.model_fpaa
         del self.model_abit

         gc.collect()
         torch.cuda.empty_cache()






    def       _UpperCamelCase    (       self )							-> Tuple:
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       self.model_abit.config

         self.assertTrue(hasattr(lowercase__    ,							'quantization_config' ) )

         SCREAMING_SNAKE_CASE					:			List[Any]    	=       config.to_dict()
         SCREAMING_SNAKE_CASE					:			List[str]    	=       config.to_diff_dict()

         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       config.to_json_string()






    def       _UpperCamelCase    (       self )							-> List[str]:
         from bitsandbytes.nn import Paramsabit

         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       self.model_fpaa.get_memory_footprint()
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.model_abit.get_memory_footprint()

         self.assertAlmostEqual(mem_fpaa / mem_abit    ,							self.EXPECTED_RELATIVE_DIFFERENCE )
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       get_some_linear_layer(self.model_abit )
         self.assertTrue(linear.weight.__class__ == Paramsabit )






    def       _UpperCamelCase    (       self )							-> Dict:
         from transformers import TaPreTrainedModel

         self.model_fpaa.get_memory_footprint()
         self.model_abit.get_memory_footprint()

         for name, module in self.model_abit.named_modules():
              if isinstance(lowercase__    ,							torch.nn.Linear ):
                   if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                        # 4-bit parameters are packed in uint8 variables
                        self.assertTrue(module.weight.dtype == torch.uinta )






    def       _UpperCamelCase    (       self )							-> Any:
         SCREAMING_SNAKE_CASE					:			str    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' )
         SCREAMING_SNAKE_CASE					:			List[str]    	=       self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0 )    ,							max_new_tokens=10 )

         self.assertIn(self.tokenizer.decode(output_sequences[0]    ,							skip_special_tokens=lowercase__ )    ,							self.EXPECTED_OUTPUTS )






    def       _UpperCamelCase    (       self )							-> Any:
         SCREAMING_SNAKE_CASE					:			List[str]    	=       BitsAndBytesConfig()
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       True

         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       AutoModelForCausalLM.from_pretrained(
             self.model_name    ,							quantization_config=lowercase__    ,							device_map='auto' )

         SCREAMING_SNAKE_CASE					:			int    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' )
         SCREAMING_SNAKE_CASE					:			int    	=       model_abit_from_config.generate(
             input_ids=encoded_input['input_ids'].to(0 )    ,							max_new_tokens=10 )

         self.assertIn(self.tokenizer.decode(output_sequences[0]    ,							skip_special_tokens=lowercase__ )    ,							self.EXPECTED_OUTPUTS )






    def       _UpperCamelCase    (       self )							-> Optional[Any]:







         with self.assertRaises(lowercase__ ), tempfile.TemporaryDirectory() as tmpdirname:
              self.model_abit.save_pretrained(lowercase__ )






    def       _UpperCamelCase    (       self )							-> List[Any]:
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       BitsAndBytesConfig()

         with self.assertRaises(lowercase__ ):
              SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       AutoModelForCausalLM.from_pretrained(
                  self.model_name    ,							quantization_config=lowercase__    ,							load_in_abit=lowercase__    ,							device_map='auto'    ,							bnb_abit_quant_type='nf4'    ,							)






    def       _UpperCamelCase    (       self )							-> List[Any]:







         with self.assertRaises(lowercase__ ):
              # Tries with `str`
              self.model_abit.to('cpu' )

         with self.assertRaises(lowercase__ ):
              # Tries with a `dtype``
              self.model_abit.to(torch.floataa )

         with self.assertRaises(lowercase__ ):
              # Tries with a `device`
              self.model_abit.to(torch.device('cuda:0' ) )

         with self.assertRaises(lowercase__ ):
              # Tries with a `device`
              self.model_abit.float()

         with self.assertRaises(lowercase__ ):
              # Tries with a `device`
              self.model_abit.half()

         # Test if we did not break anything
         SCREAMING_SNAKE_CASE					:			str    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' )

         SCREAMING_SNAKE_CASE					:			List[str]    	=       self.model_fpaa.to(torch.floataa )
         SCREAMING_SNAKE_CASE					:			Tuple    	=       self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0 )    ,							max_new_tokens=10 )

         # Check this does not throw an error
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       self.model_fpaa.to('cpu' )

         # Check this does not throw an error
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       self.model_fpaa.half()

         # Check this does not throw an error
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       self.model_fpaa.float()






    def       _UpperCamelCase    (       self )							-> List[Any]:
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       AutoModelForSeqaSeqLM.from_pretrained('t5-small'    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				UpperCAmelCase (       unittest.TestCase ):




    '''simple docstring'''







    @classmethod
    def       _UpperCamelCase    (       cls )							-> str:
         SCREAMING_SNAKE_CASE					:			str    	=       't5-small'
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       AutoTokenizer.from_pretrained(cls.model_name )
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       'Translate in German: Hello, my dog is cute'






    def       _UpperCamelCase    (       self )							-> List[str]:
         gc.collect()
         torch.cuda.empty_cache()






    def       _UpperCamelCase    (       self )							-> Union[str, Any]:
         from transformers import TaForConditionalGeneration

         SCREAMING_SNAKE_CASE					:			Any    	=       TaForConditionalGeneration._keep_in_fpaa_modules
         SCREAMING_SNAKE_CASE					:			Tuple    	=       None

         # test with `t5-small`
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       TaForConditionalGeneration.from_pretrained(self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' ).to(0 )
         SCREAMING_SNAKE_CASE					:			Any    	=       model.generate(**lowercase__ )

         # test with `flan-t5-small`
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       TaForConditionalGeneration.from_pretrained(
             self.dense_act_model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' ).to(0 )
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       model.generate(**lowercase__ )
         SCREAMING_SNAKE_CASE					:			Any    	=       modules






    def       _UpperCamelCase    (       self )							-> Union[str, Any]:
         import bitsandbytes as bnb

         from transformers import TaForConditionalGeneration

         # test with `t5-small`
         SCREAMING_SNAKE_CASE					:			Tuple    	=       TaForConditionalGeneration.from_pretrained(self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )

         # there was a bug with decoders - this test checks that it is fixed
         self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q    ,							bnb.nn.Linearabit ) )

         SCREAMING_SNAKE_CASE					:			List[Any]    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' ).to(0 )
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       model.generate(**lowercase__ )

         # test with `flan-t5-small`
         SCREAMING_SNAKE_CASE					:			Any    	=       TaForConditionalGeneration.from_pretrained(
             self.dense_act_model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         SCREAMING_SNAKE_CASE					:			Tuple    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' ).to(0 )
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       model.generate(**lowercase__ )






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Optional[int]:
         super().setUp()
         # model_name
         SCREAMING_SNAKE_CASE					:			Dict    	=       'bigscience/bloom-560m'
         SCREAMING_SNAKE_CASE					:			int    	=       't5-small'

         # Different types of model

         SCREAMING_SNAKE_CASE					:			str    	=       AutoModel.from_pretrained(self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         # Sequence classification model
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       AutoModelForSequenceClassification.from_pretrained(
             self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         # CausalLM model
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       AutoModelForCausalLM.from_pretrained(self.model_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )
         # Seq2seq model
         SCREAMING_SNAKE_CASE					:			str    	=       AutoModelForSeqaSeqLM.from_pretrained(
             self.seq_to_seq_name    ,							load_in_abit=lowercase__    ,							device_map='auto' )






    def       _UpperCamelCase    (       self )							-> Optional[int]:
         del self.base_model
         del self.sequence_model
         del self.model_abit
         del self.seq_to_seq_model

         gc.collect()
         torch.cuda.empty_cache()






    def       _UpperCamelCase    (       self )							-> Dict:
         from bitsandbytes.nn import Paramsabit

         self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit )

         # Other heads should be nn.Parameter
         self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter )
         self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter )
         self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter )






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Tuple:
         super().setUp()






    def       _UpperCamelCase    (       self )							-> List[str]:
         del self.pipe

         gc.collect()
         torch.cuda.empty_cache()






    def       _UpperCamelCase    (       self )							-> Dict:
         SCREAMING_SNAKE_CASE					:			int    	=       pipeline(
             'text-generation'    ,							model=self.model_name    ,							model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}    ,							max_new_tokens=self.MAX_NEW_TOKENS    ,							)

         # Real second forward pass
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.pipe(self.input_text )
         self.assertIn(pipeline_output[0]['generated_text']    ,							self.EXPECTED_OUTPUTS )






@require_torch_multi_gpu
class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Dict:
         super().setUp()






    def       _UpperCamelCase    (       self )							-> Dict:
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       AutoModelForCausalLM.from_pretrained(
             self.model_name    ,							load_in_abit=lowercase__    ,							device_map='balanced' )

         # Check correct device map
         self.assertEqual(set(model_parallel.hf_device_map.values() )    ,							{0, 1} )

         # Check that inference pass works on the model
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.tokenizer(self.input_text    ,							return_tensors='pt' )

         # Second real batch
         SCREAMING_SNAKE_CASE					:			Any    	=       model_parallel.generate(input_ids=encoded_input['input_ids'].to(0 )    ,							max_new_tokens=10 )
         self.assertIn(self.tokenizer.decode(output_parallel[0]    ,							skip_special_tokens=lowercase__ )    ,							self.EXPECTED_OUTPUTS )






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> str:
         SCREAMING_SNAKE_CASE					:			Tuple    	=       'facebook/opt-350m'
         super().setUp()






    def       _UpperCamelCase    (       self )							-> Tuple:







         if version.parse(importlib.metadata.version('bitsandbytes' ) ) < version.parse('0.37.0' ):
              return

         # Step 1: freeze all parameters
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       AutoModelForCausalLM.from_pretrained(self.model_name    ,							load_in_abit=lowercase__ )

         self.assertEqual(set(model.hf_device_map.values() )    ,							{torch.cuda.current_device()} )

         for param in model.parameters():
              SCREAMING_SNAKE_CASE					:			Any    	=       False  # freeze the model - train adapters later
              if param.ndim == 1:
                   # cast the small parameters (e.g. layernorm) to fp32 for stability
                   SCREAMING_SNAKE_CASE					:			List[str]    	=       param.data.to(torch.floataa )

        # Step 2: add adapters
         for _, module in model.named_modules():
              if "OPTAttention" in repr(type(lowercase__ ) ):
                   SCREAMING_SNAKE_CASE					:			Optional[int]    	=       LoRALayer(module.q_proj    ,							rank=16 )
                   SCREAMING_SNAKE_CASE					:			Optional[int]    	=       LoRALayer(module.k_proj    ,							rank=16 )
                   SCREAMING_SNAKE_CASE					:			Any    	=       LoRALayer(module.v_proj    ,							rank=16 )

        # Step 3: dummy batch
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       self.tokenizer('Test batch '    ,							return_tensors='pt' ).to(0 )

         # Step 4: Check if the gradient is not None
         with torch.cuda.amp.autocast():
              SCREAMING_SNAKE_CASE					:			Tuple    	=       model.forward(**lowercase__ )
              out.logits.norm().backward()

         for module in model.modules():
              if isinstance(lowercase__    ,							lowercase__ ):
                   self.assertTrue(module.adapter[1].weight.grad is not None )
                   self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0 )
              elif isinstance(lowercase__    ,							nn.Embedding ):
                   self.assertTrue(module.weight.grad is None )






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    snake_case__							:   Any							=						"gpt2-xl"
    snake_case__							:   Optional[Any]							=						3.3191854854152187




'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase  :int  				=    {
    """configuration_albert""": ["""ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """AlbertConfig""", """AlbertOnnxConfig"""],
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase  :str  				=    ["""AlbertTokenizer"""]

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase  :Optional[int]  				=    ["""AlbertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase  :List[str]  				=    [
     """ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """AlbertForMaskedLM""",
     """AlbertForMultipleChoice""",
     """AlbertForPreTraining""",
     """AlbertForQuestionAnswering""",
     """AlbertForSequenceClassification""",
     """AlbertForTokenClassification""",
     """AlbertModel""",
     """AlbertPreTrainedModel""",
     """load_tf_weights_in_albert""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase  :Any  				=    [
     """TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFAlbertForMaskedLM""",
     """TFAlbertForMultipleChoice""",
     """TFAlbertForPreTraining""",
     """TFAlbertForQuestionAnswering""",
     """TFAlbertForSequenceClassification""",
     """TFAlbertForTokenClassification""",
     """TFAlbertMainLayer""",
     """TFAlbertModel""",
     """TFAlbertPreTrainedModel""",
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase  :Optional[int]  				=    [
     """FlaxAlbertForMaskedLM""",
     """FlaxAlbertForMultipleChoice""",
     """FlaxAlbertForPreTraining""",
     """FlaxAlbertForQuestionAnswering""",
     """FlaxAlbertForSequenceClassification""",
     """FlaxAlbertForTokenClassification""",
     """FlaxAlbertModel""",
     """FlaxAlbertPreTrainedModel""",
 ]

if TYPE_CHECKING:
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_albert import AlbertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_albert_fast import AlbertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_albert import (
      ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      AlbertForMaskedLM,
      AlbertForMultipleChoice,
      AlbertForPreTraining,
      AlbertForQuestionAnswering,
      AlbertForSequenceClassification,
      AlbertForTokenClassification,
      AlbertModel,
      AlbertPreTrainedModel,
      load_tf_weights_in_albert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_albert import (
      TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFAlbertForMaskedLM,
      TFAlbertForMultipleChoice,
      TFAlbertForPreTraining,
      TFAlbertForQuestionAnswering,
      TFAlbertForSequenceClassification,
      TFAlbertForTokenClassification,
      TFAlbertMainLayer,
      TFAlbertModel,
      TFAlbertPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_albert import (
      FlaxAlbertForMaskedLM,
      FlaxAlbertForMultipleChoice,
      FlaxAlbertForPreTraining,
      FlaxAlbertForQuestionAnswering,
      FlaxAlbertForSequenceClassification,
      FlaxAlbertForTokenClassification,
      FlaxAlbertModel,
      FlaxAlbertPreTrainedModel,
  )
else:
 import sys

 _lowerCAmelCase  :Union[str, Any]  				=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



