
import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
  from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
  lowercase  : List[str]						   =							True
  from torch.cuda.amp import autocast


lowercase  : Optional[Any]						   =							logging.getLogger(__name__)

@dataclass
class __snake_case		:
    _a		:						str=    field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}					)
    _a		:						Optional[str]=    field(
        default=lowerCAmelCase ,     metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"} ,     )
    _a		:						Optional[bool]=    field(
        default=lowerCAmelCase ,     metadata={"help": "Whether to freeze the feature extractor layers of the model."}					)
    _a		:						Optional[bool]=    field(
        default=lowerCAmelCase ,     metadata={"help": "Whether to log verbose messages or not."} ,     )
    _a		:						Optional[float]=    field(
        default=2.0 ,     metadata={"help": "Maximum temperature for gumbel softmax."}					)
    _a		:						Optional[float]=    field(
        default=0.5 ,     metadata={"help": "Minimum temperature for gumbel softmax."}					)
    _a		:						Optional[float]=    field(
        default=0.99_99_95 ,     metadata={"help": "Decay of gumbel temperature during training."}					)







def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )			->   Dict:
       logging.basicConfig(
           format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""" ,     datefmt="""%m/%d/%Y %H:%M:%S""" ,     handlers=[logging.StreamHandler(sys.stdout    )] ,     )
       lowercase							:					Union[str, Any]	     =				logging.WARNING
       if model_args.verbose_logging:
              lowercase							:					Union[str, Any]	     =				logging.DEBUG
       elif trainer_utils.is_main_process(training_args.local_rank    ):
              lowercase							:					int	     =				logging.INFO
       logger.setLevel(SCREAMING_SNAKE_CASE__    )

@dataclass
class __snake_case		:
    _a		:						str=    field(
        default=lowerCAmelCase ,     metadata={"help": "The name of the dataset to use (via the datasets library)."}					)
    _a		:						Optional[str]=    field(
        default=lowerCAmelCase ,     metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}					)
    _a		:						Optional[str]=    field(
        default="train" ,     metadata={
            "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
        } ,     )
    _a		:						Optional[str]=    field(
        default="validation" ,     metadata={
            "help": (
                "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
            )
        } ,     )
    _a		:						Optional[str]=    field(
        default="file" ,     metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"} ,     )
    _a		:						bool=    field(
        default=lowerCAmelCase ,     metadata={"help": "Overwrite the cached preprocessed datasets or not."}					)
    _a		:						Optional[int]=    field(
        default=1 ,     metadata={
            "help": "The percentage of the train set used as validation set in case there's no validation split"
        } ,     )
    _a		:						Optional[int]=    field(
        default=lowerCAmelCase ,     metadata={"help": "The number of processes to use for the preprocessing."} ,     )
    _a		:						Optional[float]=    field(
        default=20.0 ,     metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}					)






@dataclass
class __snake_case		:
    _a		:						WavaVecaForPreTraining
    _a		:						WavaVecaFeatureExtractor
    _a		:						Union[bool, str]=    "longest"
    _a		:						Optional[int]=    None
    _a		:						Optional[int]=    None
    def __call__(  self		,snake_case				):



           '''simple docstring'''







           lowercase							:					List[Any]	     =				self.feature_extractor.pad(
               snake_case		,max_length=self.max_length		,padding=self.padding		,pad_to_multiple_of=self.pad_to_multiple_of		,return_tensors="""pt"""		,)
           lowercase							:					Optional[int]	     =				self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]				)

           lowercase							:					Optional[int]	     =				batch["""input_values"""].shape[0]

           # make sure that no loss is computed on padded inputs
           if batch["attention_mask"] is not None:
                  # compute real output lengths according to convolution formula
                  lowercase							:					Union[str, Any]	     =				self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1				)				).to(
                      torch.long				)

                  lowercase							:					str	     =				torch.zeros(
                      (batch_size, mask_indices_seq_length)		,dtype=torch.long		,device=batch["""input_values"""].device				)

                  # these two operations makes sure that all values
                  # before the output lengths indices are attended to
                  lowercase							:					Union[str, Any]	     =				1
                  lowercase							:					Optional[int]	     =				attention_mask.flip([-1]				).cumsum(-1				).flip([-1]				).bool()

           # sample randomly masked indices
           lowercase							:					int	     =				_compute_mask_indices(
               (batch_size, mask_indices_seq_length)		,self.model.config.mask_time_prob		,self.model.config.mask_time_length		,attention_mask=snake_case		,min_masks=2		,)

           return batch






class __snake_case		(				lowerCAmelCase					):
    def __init__(  self		,*snake_case		,snake_case=1		,snake_case=0		,snake_case=1.0		,**snake_case				):



           '''simple docstring'''







           super().__init__(*snake_case		,**snake_case				)
           lowercase							:					Optional[int]	     =				0
           lowercase							:					Optional[Any]	     =				max_gumbel_temp
           lowercase							:					List[str]	     =				min_gumbel_temp
           lowercase							:					Dict	     =				gumbel_temp_decay
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case		,snake_case				):



           '''simple docstring'''







           model.train()
           lowercase							:					int	     =				self._prepare_inputs(snake_case				)

           if self.use_amp:
                  with autocast():
                         lowercase							:					Optional[Any]	     =				self.compute_loss(snake_case		,snake_case				)
           else:
                  lowercase							:					Optional[Any]	     =				self.compute_loss(snake_case		,snake_case				)

           if self.args.n_gpu > 1 or self.deepspeed:
                  if model.module.config.ctc_loss_reduction == "mean":
                         lowercase							:					Tuple	     =				loss.mean()
                  elif model.module.config.ctc_loss_reduction == "sum":
                         lowercase							:					Optional[int]	     =				loss.sum() / (inputs["""mask_time_indices"""]).sum()
                  else:
                         raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"				)

           if self.args.gradient_accumulation_steps > 1:
                  lowercase							:					str	     =				loss / self.args.gradient_accumulation_steps

           if self.use_amp:
                  self.scaler.scale(snake_case				).backward()
           elif self.use_apex:
                  with amp.scale_loss(snake_case		,self.optimizer				) as scaled_loss:
                         scaled_loss.backward()
           elif self.deepspeed:
                  self.deepspeed.backward(snake_case				)
           else:
                  loss.backward()

           self.num_update_step += 1
           # make sure gumbel softmax temperature is decayed
           if self.args.n_gpu > 1 or self.deepspeed:
                  model.module.set_gumbel_temperature(
                      max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,self.min_gumbel_temp				)				)
           else:
                  model.set_gumbel_temperature(
                      max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,self.min_gumbel_temp				)				)

           return loss.detach()







def 						_snake_case(  )			->   str:
       # See all possible arguments in src/transformers/training_args.py
       # or by passing the --help flag to this script.
       # We now keep distinct sets of args, for a cleaner separation of concerns.

       lowercase							:					Optional[Any]	     =				HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)    )

       lowercase     ,      lowercase     ,      lowercase							:					int	     =				parser.parse_args_into_dataclasses()
       configure_logger(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )

       # Downloading and loading a dataset from the hub.
       lowercase							:					Dict	     =				load_dataset(data_args.dataset_name ,     data_args.dataset_config_name ,     cache_dir=model_args.cache_dir    )

       if "validation" not in datasets.keys():
              # make sure only "validation" and "train" keys remain"
              lowercase							:					Optional[int]	     =				DatasetDict()
              lowercase							:					Dict	     =				load_dataset(
                  data_args.dataset_name ,     data_args.dataset_config_name ,     split=f"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]" ,     cache_dir=model_args.cache_dir ,     )
              lowercase							:					Optional[Any]	     =				load_dataset(
                  data_args.dataset_name ,     data_args.dataset_config_name ,     split=f"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]" ,     cache_dir=model_args.cache_dir ,     )
       else:
              # make sure only "validation" and "train" keys remain"
              lowercase							:					List[Any]	     =				DatasetDict()
              lowercase							:					int	     =				load_dataset(
                  data_args.dataset_name ,     data_args.dataset_config_name ,     split="""validation""" ,     cache_dir=model_args.cache_dir ,     )
              lowercase							:					List[str]	     =				load_dataset(
                  data_args.dataset_name ,     data_args.dataset_config_name ,     split=f"{data_args.train_split_name}" ,     cache_dir=model_args.cache_dir ,     )

       # only normalized-inputs-training is supported
       lowercase							:					int	     =				WavaVecaFeatureExtractor.from_pretrained(
           model_args.model_name_or_path ,     cache_dir=model_args.cache_dir ,     do_normalize=SCREAMING_SNAKE_CASE__    )

       def prepare_dataset(SCREAMING_SNAKE_CASE__    ):
              # check that all files have the correct sampling rate
              lowercase     ,      lowercase							:					int	     =				librosa.load(batch[data_args.speech_file_column] ,     sr=feature_extractor.sampling_rate    )
              return batch

       # load audio files into numpy arrays
       lowercase							:					Tuple	     =				datasets.map(
           SCREAMING_SNAKE_CASE__ ,     num_proc=data_args.preprocessing_num_workers ,     remove_columns=datasets["""train"""].column_names    )

       # filter audio files that are too long
       lowercase							:					Optional[int]	     =				vectorized_datasets.filter(
           lambda SCREAMING_SNAKE_CASE__    : len(data["""speech"""]    ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate    )    )

       def normalize(SCREAMING_SNAKE_CASE__    ):
              return feature_extractor(batch["""speech"""] ,     sampling_rate=feature_extractor.sampling_rate    )

       # normalize and transform to `BatchFeatures`
       lowercase							:					List[Any]	     =				vectorized_datasets.map(
           SCREAMING_SNAKE_CASE__ ,     batched=SCREAMING_SNAKE_CASE__ ,     num_proc=data_args.preprocessing_num_workers ,     load_from_cache_file=not data_args.overwrite_cache ,     remove_columns=vectorized_datasets["""train"""].column_names ,     )

       # pretraining is only supported for "newer" stable layer norm architecture
       # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
       lowercase							:					Optional[int]	     =				WavaVecaConfig.from_pretrained(
           model_args.model_name_or_path ,     cache_dir=model_args.cache_dir ,     gradient_checkpointing=training_args.gradient_checkpointing ,     )

       if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
              raise ValueError(
                  """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
                  """ ``config.feat_extract_norm='layer'"""    )

       lowercase							:					List[str]	     =				WavaVecaForPreTraining(SCREAMING_SNAKE_CASE__    )

       lowercase							:					str	     =				DataCollatorForWavaVecaPretraining(model=SCREAMING_SNAKE_CASE__ ,     feature_extractor=SCREAMING_SNAKE_CASE__    )

       lowercase							:					Optional[int]	     =				WavaVecaPreTrainer(
           model=SCREAMING_SNAKE_CASE__ ,     data_collator=SCREAMING_SNAKE_CASE__ ,     args=SCREAMING_SNAKE_CASE__ ,     train_dataset=vectorized_datasets["""train"""] ,     eval_dataset=vectorized_datasets["""validation"""] ,     tokenizer=SCREAMING_SNAKE_CASE__ ,     max_gumbel_temp=model_args.max_gumbel_temperature ,     min_gumbel_temp=model_args.min_gumbel_temperature ,     gumbel_temp_decay=model_args.gumbel_temperature_decay ,     )
       trainer.train()


if __name__ == "__main__":
  main()


import argparse

import requests
import torch
from PIL import Image

from transformers import CLIPProcessor, GroupViTConfig, GroupViTModel







def 						_snake_case(  SCREAMING_SNAKE_CASE__    )			->   List[str]:
       # vision encoder
       if "img_encoder.pos_embed" in name:
              lowercase							:					Dict	     =				name.replace("""img_encoder.pos_embed""" ,     """vision_model.embeddings.position_embeddings"""    )
       if "img_encoder.patch_embed.proj" in name:
              lowercase							:					Any	     =				name.replace("""img_encoder.patch_embed.proj""" ,     """vision_model.embeddings.patch_embeddings.projection"""    )
       if "img_encoder.patch_embed.norm" in name:
              lowercase							:					Tuple	     =				name.replace("""img_encoder.patch_embed.norm""" ,     """vision_model.embeddings.layernorm"""    )
       if "img_encoder.layers" in name:
              lowercase							:					Tuple	     =				name.replace("""img_encoder.layers""" ,     """vision_model.encoder.stages"""    )
       if "blocks" in name and "res" not in name:
              lowercase							:					Optional[Any]	     =				name.replace("""blocks""" ,     """layers"""    )
       if "attn" in name and "pre_assign" not in name:
              lowercase							:					Union[str, Any]	     =				name.replace("""attn""" ,     """self_attn"""    )
       if "proj" in name and "self_attn" in name and "text" not in name:
              lowercase							:					Dict	     =				name.replace("""proj""" ,     """out_proj"""    )
       if "pre_assign_attn.attn.proj" in name:
              lowercase							:					Dict	     =				name.replace("""pre_assign_attn.attn.proj""" ,     """pre_assign_attn.attn.out_proj"""    )
       if "norm1" in name:
              lowercase							:					Tuple	     =				name.replace("""norm1""" ,     """layer_norm1"""    )
       if "norm2" in name and "pre_assign" not in name:
              lowercase							:					Tuple	     =				name.replace("""norm2""" ,     """layer_norm2"""    )
       if "img_encoder.norm" in name:
              lowercase							:					List[Any]	     =				name.replace("""img_encoder.norm""" ,     """vision_model.layernorm"""    )
       # text encoder
       if "text_encoder.token_embedding" in name:
              lowercase							:					Tuple	     =				name.replace("""text_encoder.token_embedding""" ,     """text_model.embeddings.token_embedding"""    )
       if "text_encoder.positional_embedding" in name:
              lowercase							:					Any	     =				name.replace("""text_encoder.positional_embedding""" ,     """text_model.embeddings.position_embedding.weight"""    )
       if "text_encoder.transformer.resblocks." in name:
              lowercase							:					Dict	     =				name.replace("""text_encoder.transformer.resblocks.""" ,     """text_model.encoder.layers."""    )
       if "ln_1" in name:
              lowercase							:					List[str]	     =				name.replace("""ln_1""" ,     """layer_norm1"""    )
       if "ln_2" in name:
              lowercase							:					Tuple	     =				name.replace("""ln_2""" ,     """layer_norm2"""    )
       if "c_fc" in name:
              lowercase							:					str	     =				name.replace("""c_fc""" ,     """fc1"""    )
       if "c_proj" in name:
              lowercase							:					Union[str, Any]	     =				name.replace("""c_proj""" ,     """fc2"""    )
       if "text_encoder" in name:
              lowercase							:					Tuple	     =				name.replace("""text_encoder""" ,     """text_model"""    )
       if "ln_final" in name:
              lowercase							:					Optional[Any]	     =				name.replace("""ln_final""" ,     """final_layer_norm"""    )
       # projection layers
       if "img_projector.linear_hidden." in name:
              lowercase							:					List[str]	     =				name.replace("""img_projector.linear_hidden.""" ,     """visual_projection."""    )
       if "img_projector.linear_out." in name:
              lowercase							:					Tuple	     =				name.replace("""img_projector.linear_out.""" ,     """visual_projection.3."""    )
       if "text_projector.linear_hidden" in name:
              lowercase							:					List[Any]	     =				name.replace("""text_projector.linear_hidden""" ,     """text_projection"""    )
       if "text_projector.linear_out" in name:
              lowercase							:					List[Any]	     =				name.replace("""text_projector.linear_out""" ,     """text_projection.3"""    )

       return name







def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )			->   List[str]:
       for key in orig_state_dict.copy().keys():
              lowercase							:					List[Any]	     =				orig_state_dict.pop(SCREAMING_SNAKE_CASE__    )

              if "qkv" in key:
                     # weights and biases of the key, value and query projections of vision encoder's attention layers require special treatment:
                     # we need to split them up into separate matrices/vectors
                     lowercase							:					Dict	     =				key.split("""."""    )
                     lowercase     ,      lowercase							:					Optional[Any]	     =				int(key_split[2]    ), int(key_split[4]    )
                     lowercase							:					List[Any]	     =				config.vision_config.hidden_size
                     if "weight" in key:
                            lowercase							:					str	     =				val[:dim, :]
                            lowercase							:					List[str]	     =				val[dim : dim * 2, :]
                            lowercase							:					Optional[int]	     =				val[-dim:, :]
                     else:
                            lowercase							:					Dict	     =				val[:dim]
                            lowercase							:					Dict	     =				val[dim : dim * 2]
                            lowercase							:					Dict	     =				val[-dim:]
              elif "in_proj" in key:
                     # weights and biases of the key, value and query projections of text encoder's attention layers require special treatment:
                     # we need to split them up into separate matrices/vectors
                     lowercase							:					int	     =				key.split("""."""    )
                     lowercase							:					Tuple	     =				int(key_split[3]    )
                     lowercase							:					str	     =				config.text_config.hidden_size
                     if "weight" in key:
                            lowercase							:					Optional[int]	     =				val[:dim, :]
                            lowercase							:					Optional[Any]	     =				val[
                                dim : dim * 2, :
                            ]
                            lowercase							:					Optional[int]	     =				val[-dim:, :]
                     else:
                            lowercase							:					Optional[int]	     =				val[:dim]
                            lowercase							:					Dict	     =				val[dim : dim * 2]
                            lowercase							:					List[str]	     =				val[-dim:]
              else:
                     lowercase							:					Tuple	     =				rename_key(SCREAMING_SNAKE_CASE__    )
                     # squeeze if necessary
                     if (
                         "text_projection.0" in new_name
                         or "text_projection.3" in new_name
                         or "visual_projection.0" in new_name
                         or "visual_projection.3" in new_name
                     ):
                            lowercase							:					str	     =				val.squeeze_()
                     else:
                            lowercase							:					Any	     =				val

       return orig_state_dict







def 						_snake_case(  )			->   List[Any]:
       lowercase							:					Tuple	     =				"""http://images.cocodataset.org/val2017/000000039769.jpg"""
       lowercase							:					Any	     =				Image.open(requests.get(SCREAMING_SNAKE_CASE__ ,     stream=SCREAMING_SNAKE_CASE__    ).raw    )
       return im







@torch.no_grad()
def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__="groupvit-gcc-yfcc" ,     SCREAMING_SNAKE_CASE__=False    )			->   str:
       lowercase							:					Dict	     =				GroupViTConfig()
       lowercase							:					Tuple	     =				GroupViTModel(SCREAMING_SNAKE_CASE__    ).eval()

       lowercase							:					List[str]	     =				torch.load(SCREAMING_SNAKE_CASE__ ,     map_location="""cpu"""    )["""model"""]
       lowercase							:					Optional[Any]	     =				convert_state_dict(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )
       lowercase     ,      lowercase							:					int	     =				model.load_state_dict(SCREAMING_SNAKE_CASE__ ,     strict=SCREAMING_SNAKE_CASE__    )
       assert missing_keys == ["text_model.embeddings.position_ids"]
       assert (unexpected_keys == ["multi_label_logit_scale"]) or (len(SCREAMING_SNAKE_CASE__    ) == 0)

       # verify result
       lowercase							:					Tuple	     =				CLIPProcessor.from_pretrained("""openai/clip-vit-base-patch32"""    )
       lowercase							:					Optional[Any]	     =				prepare_img()
       lowercase							:					List[Any]	     =				processor(text=["""a photo of a cat""", """a photo of a dog"""] ,     images=SCREAMING_SNAKE_CASE__ ,     padding=SCREAMING_SNAKE_CASE__ ,     return_tensors="""pt"""    )

       with torch.no_grad():
              lowercase							:					Optional[Any]	     =				model(**SCREAMING_SNAKE_CASE__    )

       if model_name == "groupvit-gcc-yfcc":
              lowercase							:					Optional[int]	     =				torch.tensor([[13.3523, 6.3629]]    )
       elif model_name == "groupvit-gcc-redcaps":
              lowercase							:					int	     =				torch.tensor([[16.1873, 8.6230]]    )
       else:
              raise ValueError(f"Model name {model_name} not supported."    )
       assert torch.allclose(outputs.logits_per_image ,     SCREAMING_SNAKE_CASE__ ,     atol=1e-3    )

       processor.save_pretrained(SCREAMING_SNAKE_CASE__    )
       model.save_pretrained(SCREAMING_SNAKE_CASE__    )
       print("""Successfully saved processor and model to""" ,     SCREAMING_SNAKE_CASE__    )

       if push_to_hub:
              print("""Pushing to the hub..."""    )
              processor.push_to_hub(SCREAMING_SNAKE_CASE__ ,     organization="""nielsr"""    )
              model.push_to_hub(SCREAMING_SNAKE_CASE__ ,     organization="""nielsr"""    )


if __name__ == "__main__":
  lowercase  : List[Any]						   =							argparse.ArgumentParser()
  parser.add_argument(
      """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to dump the processor and PyTorch model."""
  )
  parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to GroupViT checkpoint""")
  parser.add_argument(
      """--model_name""",
      default="""groupvit-gccy-fcc""",
      type=str,
      help="""Name of the model. Expecting either 'groupvit-gcc-yfcc' or 'groupvit-gcc-redcaps'""",
  )
  parser.add_argument(
      """--push_to_hub""",
      action="""store_true""",
      help="""Whether or not to push the converted model and processor to the 🤗 hub using the provided `model_name`.""",
  )
  lowercase  : Union[str, Any]						   =							parser.parse_args()

  convert_groupvit_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub)
