
"""simple docstring"""
import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


lowercase_       :						Dict     					=	logging.get_logger(__name__)

lowercase_       :						Optional[Any]     					=	OrderedDict(
    [
        ('''audio-spectrogram-transformer''', '''ASTFeatureExtractor'''),
        ('''beit''', '''BeitFeatureExtractor'''),
        ('''chinese_clip''', '''ChineseCLIPFeatureExtractor'''),
        ('''clap''', '''ClapFeatureExtractor'''),
        ('''clip''', '''CLIPFeatureExtractor'''),
        ('''clipseg''', '''ViTFeatureExtractor'''),
        ('''conditional_detr''', '''ConditionalDetrFeatureExtractor'''),
        ('''convnext''', '''ConvNextFeatureExtractor'''),
        ('''cvt''', '''ConvNextFeatureExtractor'''),
        ('''data2vec-audio''', '''Wav2Vec2FeatureExtractor'''),
        ('''data2vec-vision''', '''BeitFeatureExtractor'''),
        ('''deformable_detr''', '''DeformableDetrFeatureExtractor'''),
        ('''deit''', '''DeiTFeatureExtractor'''),
        ('''detr''', '''DetrFeatureExtractor'''),
        ('''dinat''', '''ViTFeatureExtractor'''),
        ('''donut-swin''', '''DonutFeatureExtractor'''),
        ('''dpt''', '''DPTFeatureExtractor'''),
        ('''encodec''', '''EncodecFeatureExtractor'''),
        ('''flava''', '''FlavaFeatureExtractor'''),
        ('''glpn''', '''GLPNFeatureExtractor'''),
        ('''groupvit''', '''CLIPFeatureExtractor'''),
        ('''hubert''', '''Wav2Vec2FeatureExtractor'''),
        ('''imagegpt''', '''ImageGPTFeatureExtractor'''),
        ('''layoutlmv2''', '''LayoutLMv2FeatureExtractor'''),
        ('''layoutlmv3''', '''LayoutLMv3FeatureExtractor'''),
        ('''levit''', '''LevitFeatureExtractor'''),
        ('''maskformer''', '''MaskFormerFeatureExtractor'''),
        ('''mctct''', '''MCTCTFeatureExtractor'''),
        ('''mobilenet_v1''', '''MobileNetV1FeatureExtractor'''),
        ('''mobilenet_v2''', '''MobileNetV2FeatureExtractor'''),
        ('''mobilevit''', '''MobileViTFeatureExtractor'''),
        ('''nat''', '''ViTFeatureExtractor'''),
        ('''owlvit''', '''OwlViTFeatureExtractor'''),
        ('''perceiver''', '''PerceiverFeatureExtractor'''),
        ('''poolformer''', '''PoolFormerFeatureExtractor'''),
        ('''regnet''', '''ConvNextFeatureExtractor'''),
        ('''resnet''', '''ConvNextFeatureExtractor'''),
        ('''segformer''', '''SegformerFeatureExtractor'''),
        ('''sew''', '''Wav2Vec2FeatureExtractor'''),
        ('''sew-d''', '''Wav2Vec2FeatureExtractor'''),
        ('''speech_to_text''', '''Speech2TextFeatureExtractor'''),
        ('''speecht5''', '''SpeechT5FeatureExtractor'''),
        ('''swiftformer''', '''ViTFeatureExtractor'''),
        ('''swin''', '''ViTFeatureExtractor'''),
        ('''swinv2''', '''ViTFeatureExtractor'''),
        ('''table-transformer''', '''DetrFeatureExtractor'''),
        ('''timesformer''', '''VideoMAEFeatureExtractor'''),
        ('''tvlt''', '''TvltFeatureExtractor'''),
        ('''unispeech''', '''Wav2Vec2FeatureExtractor'''),
        ('''unispeech-sat''', '''Wav2Vec2FeatureExtractor'''),
        ('''van''', '''ConvNextFeatureExtractor'''),
        ('''videomae''', '''VideoMAEFeatureExtractor'''),
        ('''vilt''', '''ViltFeatureExtractor'''),
        ('''vit''', '''ViTFeatureExtractor'''),
        ('''vit_mae''', '''ViTFeatureExtractor'''),
        ('''vit_msn''', '''ViTFeatureExtractor'''),
        ('''wav2vec2''', '''Wav2Vec2FeatureExtractor'''),
        ('''wav2vec2-conformer''', '''Wav2Vec2FeatureExtractor'''),
        ('''wavlm''', '''Wav2Vec2FeatureExtractor'''),
        ('''whisper''', '''WhisperFeatureExtractor'''),
        ('''xclip''', '''CLIPFeatureExtractor'''),
        ('''yolos''', '''YolosFeatureExtractor'''),
    ]
)

lowercase_       :						Union[str, Any]     					=	_LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)





def    _lowerCAmelCase    (     lowerCamelCase__							:							str				)  ->    Dict:
  for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
    if class_name in extractors:
      _SCREAMING_SNAKE_CASE   :				Any							    = model_type_to_module_name(lowerCamelCase__				)

      _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = importlib.import_module(f'''.{module_name}''',						"transformers.models"				)
      try:
        return getattr(lowerCamelCase__,						lowerCamelCase__				)
      except AttributeError:
        continue

  for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
    if getattr(lowerCamelCase__,						"__name__",						lowerCamelCase__				) == class_name:
      return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  _SCREAMING_SNAKE_CASE   :				List[str]							    = importlib.import_module("transformers"				)
  if hasattr(lowerCamelCase__,						lowerCamelCase__				):
    return getattr(lowerCamelCase__,						lowerCamelCase__				)

  return None





def    _lowerCAmelCase    (     lowerCamelCase__							:							Union[str, os.PathLike],						lowerCamelCase__							:							Optional[Union[str, os.PathLike]] = None,						lowerCamelCase__							:							bool = False,						lowerCamelCase__							:							bool = False,						lowerCamelCase__							:							Optional[Dict[str, str]] = None,						lowerCamelCase__							:							Optional[Union[bool, str]] = None,						lowerCamelCase__							:							Optional[str] = None,						lowerCamelCase__							:							bool = False,						**lowerCamelCase__							:							Union[str, Any],						)  ->    List[Any]:
  _SCREAMING_SNAKE_CASE   :				List[str]							    = get_file_from_repo(
      lowerCamelCase__,						lowerCamelCase__,						cache_dir=lowerCamelCase__,						force_download=lowerCamelCase__,						resume_download=lowerCamelCase__,						proxies=lowerCamelCase__,						use_auth_token=lowerCamelCase__,						revision=lowerCamelCase__,						local_files_only=lowerCamelCase__,						)
  if resolved_config_file is None:
    logger.info(
        "Could not locate the feature extractor configuration file, will try to use the model config instead."				)
    return {}

  with open(lowerCamelCase__,						encoding="utf-8"				) as reader:
    return json.load(lowerCamelCase__				)





class 			UpperCamelCase      :
     def __init__(     self				):




       """simple docstring"""


       raise EnvironmentError(
           "AutoFeatureExtractor is designed to be instantiated "
           "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."				)
     @classmethod
     @replace_list_option_in_docstrings(snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     cls  ,     snake_case__  ,     **snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = kwargs.pop("config"  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = kwargs.pop("trust_remote_code"  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				str							    = True

       _SCREAMING_SNAKE_CASE   :				Any							    = FeatureExtractionMixin.get_feature_extractor_dict(snake_case__  ,     **snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = config_dict.get("feature_extractor_type"  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = None
       if "AutoFeatureExtractor" in config_dict.get("auto_map"  ,     {}				):
         _SCREAMING_SNAKE_CASE   :				int							    = config_dict["auto_map"]["AutoFeatureExtractor"]

       # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
       if feature_extractor_class is None and feature_extractor_auto_map is None:
         if not isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				List[Any]							    = AutoConfig.from_pretrained(snake_case__  ,     **snake_case__				)
         # It could be in `config.feature_extractor_type``
         _SCREAMING_SNAKE_CASE   :				List[str]							    = getattr(snake_case__  ,     "feature_extractor_type"  ,     snake_case__				)
         if hasattr(snake_case__  ,     "auto_map"				) and "AutoFeatureExtractor" in config.auto_map:
           _SCREAMING_SNAKE_CASE   :				str							    = config.auto_map["AutoFeatureExtractor"]

       if feature_extractor_class is not None:
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = feature_extractor_class_from_name(snake_case__				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = feature_extractor_auto_map is not None
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = feature_extractor_class is not None or type(snake_case__				) in FEATURE_EXTRACTOR_MAPPING
       _SCREAMING_SNAKE_CASE   :				int							    = resolve_trust_remote_code(
           snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__				)

       if has_remote_code and trust_remote_code:
         _SCREAMING_SNAKE_CASE   :				Optional[int]							    = get_class_from_dynamic_module(
             snake_case__  ,     snake_case__  ,     **snake_case__				)
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = kwargs.pop("code_revision"  ,     snake_case__				)
         if os.path.isdir(snake_case__				):
           feature_extractor_class.register_for_auto_class()
         return feature_extractor_class.from_dict(snake_case__  ,     **snake_case__				)
       elif feature_extractor_class is not None:
         return feature_extractor_class.from_dict(snake_case__  ,     **snake_case__				)
       # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
       elif type(snake_case__				) in FEATURE_EXTRACTOR_MAPPING:
         _SCREAMING_SNAKE_CASE   :				Optional[int]							    = FEATURE_EXTRACTOR_MAPPING[type(snake_case__				)]
         return feature_extractor_class.from_dict(snake_case__  ,     **snake_case__				)

       raise ValueError(
           F'''Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a '''
           F'''`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following '''
           F'''`model_type` keys in its {CONFIG_NAME}: {", ".join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()				)}'''				)




     @staticmethod
     def 				__SCREAMING_SNAKE_CASE							(     snake_case__  ,     snake_case__				):




       """simple docstring"""


       FEATURE_EXTRACTOR_MAPPING.register(snake_case__  ,     snake_case__				)





"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be trained."""}  )
     A__      =			field(
         default="""./"""      ,       metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path of training dataset."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for training."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for evaluation."""}  )
     A__      =			field(default=0.1      ,       metadata={"""help""": """Value of weight decay."""}  )
     A__      =			field(
         default=10000      ,       metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""}  )
     A__      =			field(default=2e-4      ,       metadata={"""help""": """Learning rate fo training."""}  )
     A__      =			field(default="""cosine"""      ,       metadata={"""help""": """Learning rate."""}  )
     A__      =			field(
         default=750      ,       metadata={"""help""": """Number of warmup steps in the learning rate schedule."""}  )
     A__      =			field(
         default=16      ,       metadata={"""help""": """Number of gradient accumulation steps."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""}  )
     A__      =			field(default=50000      ,       metadata={"""help""": """Maximum number of training steps."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Sequence lengths used for training."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Training seed."""}  )
     A__      =			field(
         default=1024      ,       metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """States path if the training should continue from a checkpoint folder."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True the data is pretokenized."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size used for evaluation."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Length of sequences to be evaluated."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Sample from the language model's output distribution."""}  )
     A__      =			field(default=0.2      ,       metadata={"""help""": """Sampling temperature used for generation."""}  )
     A__      =			field(default=256      ,       metadata={"""help""": """Maximum number of newly generated tokens."""}  )
     A__      =			field(default=0      ,       metadata={"""help""": """Top-k parameter used for generation."""}  )
     A__      =			field(default=0.95      ,       metadata={"""help""": """Top-p parameter used for nucleus sampling."""}  )
     A__      =			field(default=10      ,       metadata={"""help""": """Number of generations to run in parallel."""}  )
     A__      =			field(
         default=200      ,       metadata={"""help""": """Number of completions to generate for each sample."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""eval_results.json"""      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""0"""      ,       metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""}  )
     A__      =			field(
         default=-1      ,       metadata={
             """help""": (
                 """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
                 """ number corresponds to which GPU device id to run on."""
             )
         }      ,       )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={
             """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
         }      ,       )
     A__      =			field(
         default="""transformersbook/codeparrot"""      ,       metadata={"""help""": """Folder or name of dataset to process."""}  )
     A__      =			field(
         default="""codeparrot-clean"""      ,       metadata={"""help""": """Folder to save processed processed dataset."""}  )
     A__      =			field(
         default=100000      ,       metadata={"""help""": """Number of files to save per JSON output file."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(
         default=1000      ,       metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=100      ,       metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.25      ,       metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""}  )
     A__      =			field(
         default=1.5      ,       metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.7      ,       metadata={"""help""": """Probability for filtering config, test and uncommon files."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True, near-duplicate samples are removed."""}  )
     A__      =			field(
         default=0.85      ,       metadata={"""help""": """Jaccard threshold for near-duplicate samples."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2"""      ,       metadata={"""help""": """Base tokenizer to build new tokenizer from."""}  )
     A__      =			field(
         default="""transformersbook/codeparrot-train"""      ,       metadata={"""help""": """Dataset to train tokenizer on."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(default=200000      ,       metadata={"""help""": """Number of examples to train tokenizer on."""}  )
     A__      =			field(
         default=32768      ,       metadata={"""help""": """Number of examples to train the tokenizer on."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of new tokenizer."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path to the dataset to pretokenize."""}  )
     A__      =			field(
         default="""tokenized-codeparrot-train"""      ,       metadata={"""help""": """Repo name of the pretokenized data."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2-large"""      ,       metadata={"""help""": """Configuration to use for model initialization."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Tokenizer attached to model."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of the created model."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )



