





'''simple docstring'''




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
  from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
  lowercase__       : str         =       True
  from torch.cuda.amp import autocast


lowercase__       : Optional[int]         =       logging.getLogger(__name__)


@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Whether to log verbose messages or not.'''}   ,						)
     lowerCAmelCase						     =			field(
         default=2.0   ,						metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}				)
     lowerCAmelCase						     =			field(
         default=0.5   ,						metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}				)
     lowerCAmelCase						     =			field(
         default=0.999_995   ,						metadata={'''help''': '''Decay of gumbel temperature during training.'''}				)






def 			_lowerCAmelCase     (    __snake_case    :					ModelArguments					,				__snake_case    :					TrainingArguments     )	->	Optional[int]:
     logging.basicConfig(
         format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'					,				datefmt='%m/%d/%Y %H:%M:%S'					,				handlers=[logging.StreamHandler(sys.stdout     )]					,				)
     __A		:					Union[str, Any]						=			logging.WARNING
     if model_args.verbose_logging:
          __A		:					Optional[int]						=			logging.DEBUG
     elif trainer_utils.is_main_process(training_args.local_rank     ):
          __A		:					str						=			logging.INFO
     logger.setLevel(__snake_case     )


@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}				)
     lowerCAmelCase						     =			field(
         default='''train'''   ,						metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         }   ,						)
     lowerCAmelCase						     =			field(
         default='''validation'''   ,						metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         }   ,						)
     lowerCAmelCase						     =			field(
         default='''file'''   ,						metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}				)
     lowerCAmelCase						     =			field(
         default=1   ,						metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         }   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The number of processes to use for the preprocessing.'''}   ,						)
     lowerCAmelCase						     =			field(
         default=20.0   ,						metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}				)




@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			42
     lowerCAmelCase						     =			42
     lowerCAmelCase						     =			"longest"
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			None


     def __call__(		self		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Optional[int]						=			self.feature_extractor.pad(
              _UpperCAmelCase		,      max_length=self.max_length		,      padding=self.padding		,      pad_to_multiple_of=self.pad_to_multiple_of		,      return_tensors='pt'		,      )
          __A		:					Tuple						=			self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1])

          __A		:					Any						=			batch['input_values'].shape[0]

          # make sure that no loss is computed on padded inputs
          if batch["attention_mask"] is not None:
               # compute real output lengths according to convolution formula
               __A		:					Union[str, Any]						=			self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1)).to(
                   torch.long)

               __A		:					Dict						=			torch.zeros(
                   (batch_size, mask_indices_seq_length)		,      dtype=torch.long		,      device=batch['input_values'].device)

               # these two operations makes sure that all values
               # before the output lengths indices are attended to
               __A		:					List[str]						=			1
               __A		:					Optional[Any]						=			attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()

          # sample randomly masked indices
          __A		:					Optional[Any]						=			_compute_mask_indices(
              (batch_size, mask_indices_seq_length)		,      self.model.config.mask_time_prob		,      self.model.config.mask_time_length		,      attention_mask=_UpperCAmelCase		,      min_masks=2		,      )

          return batch




class 					SCREAMING_SNAKE_CASE       (a__				):


     def __init__(		self		,      *_UpperCAmelCase		,      _UpperCAmelCase=1		,      _UpperCAmelCase=0		,      _UpperCAmelCase=1.0		,      **_UpperCAmelCase):

          '''simple docstring'''






          super().__init__(*_UpperCAmelCase		,      **_UpperCAmelCase)
          __A		:					Tuple						=			0
          __A		:					List[Any]						=			max_gumbel_temp
          __A		:					int						=			min_gumbel_temp
          __A		:					Union[str, Any]						=			gumbel_temp_decay


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          model.train()
          __A		:					Any						=			self._prepare_inputs(_UpperCAmelCase)

          if self.use_amp:
               with autocast():
                    __A		:					Any						=			self.compute_loss(_UpperCAmelCase		,      _UpperCAmelCase)
          else:
               __A		:					Tuple						=			self.compute_loss(_UpperCAmelCase		,      _UpperCAmelCase)

          if self.args.n_gpu > 1 or self.deepspeed:
               if model.module.config.ctc_loss_reduction == "mean":
                    __A		:					Optional[Any]						=			loss.mean()
               elif model.module.config.ctc_loss_reduction == "sum":
                    __A		:					List[Any]						=			loss.sum() / (inputs['mask_time_indices']).sum()
               else:
                    raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']')

          if self.args.gradient_accumulation_steps > 1:
               __A		:					Dict						=			loss / self.args.gradient_accumulation_steps

          if self.use_amp:
               self.scaler.scale(_UpperCAmelCase).backward()
          elif self.use_apex:
               with amp.scale_loss(_UpperCAmelCase		,      self.optimizer) as scaled_loss:
                    scaled_loss.backward()
          elif self.deepspeed:
               self.deepspeed.backward(_UpperCAmelCase)
          else:
               loss.backward()

          self.num_update_step += 1
          # make sure gumbel softmax temperature is decayed
          if self.args.n_gpu > 1 or self.deepspeed:
               model.module.set_gumbel_temperature(
                   max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,      self.min_gumbel_temp))
          else:
               model.set_gumbel_temperature(
                   max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,      self.min_gumbel_temp))

          return loss.detach()






def 			_lowerCAmelCase     (    )	->	List[Any]:
     # See all possible arguments in src/transformers/training_args.py
     # or by passing the --help flag to this script.
     # We now keep distinct sets of args, for a cleaner separation of concerns.

     __A		:					Optional[int]						=			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

     __A					,__A					,__A		:					Any						=			parser.parse_args_into_dataclasses()
     configure_logger(__snake_case					,				__snake_case     )

     # Downloading and loading a dataset from the hub.
     __A		:					Optional[int]						=			load_dataset(data_args.dataset_name					,				data_args.dataset_config_name					,				cache_dir=model_args.cache_dir     )

     if "validation" not in datasets.keys():
          # make sure only "validation" and "train" keys remain"
          __A		:					Optional[int]						=			DatasetDict()
          __A		:					Optional[int]						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'					,				cache_dir=model_args.cache_dir					,				)
          __A		:					str						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'					,				cache_dir=model_args.cache_dir					,				)
     else:
          # make sure only "validation" and "train" keys remain"
          __A		:					Dict						=			DatasetDict()
          __A		:					List[Any]						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split='validation'					,				cache_dir=model_args.cache_dir					,				)
          __A		:					Any						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}'					,				cache_dir=model_args.cache_dir					,				)

     # only normalized-inputs-training is supported
     __A		:					List[Any]						=			WavaVecaFeatureExtractor.from_pretrained(
         model_args.model_name_or_path					,				cache_dir=model_args.cache_dir					,				do_normalize=__snake_case     )

     def prepare_dataset(__snake_case    :					Optional[Any]     ):
          # check that all files have the correct sampling rate
          __A					,__A		:					Tuple						=			librosa.load(batch[data_args.speech_file_column]					,				sr=feature_extractor.sampling_rate     )
          return batch

     # load audio files into numpy arrays
     __A		:					List[str]						=			datasets.map(
         __snake_case					,				num_proc=data_args.preprocessing_num_workers					,				remove_columns=datasets['train'].column_names     )

     # filter audio files that are too long
     __A		:					List[str]						=			vectorized_datasets.filter(
         lambda __snake_case     : len(data['speech']     ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate     )     )

     def normalize(__snake_case    :					int     ):
          return feature_extractor(batch['speech']					,				sampling_rate=feature_extractor.sampling_rate     )

     # normalize and transform to `BatchFeatures`
     __A		:					str						=			vectorized_datasets.map(
         __snake_case					,				batched=__snake_case					,				num_proc=data_args.preprocessing_num_workers					,				load_from_cache_file=not data_args.overwrite_cache					,				remove_columns=vectorized_datasets['train'].column_names					,				)

     # pretraining is only supported for "newer" stable layer norm architecture
     # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
     __A		:					int						=			WavaVecaConfig.from_pretrained(
         model_args.model_name_or_path					,				cache_dir=model_args.cache_dir					,				gradient_checkpointing=training_args.gradient_checkpointing					,				)

     if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
          raise ValueError(
              'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
              ' ``config.feat_extract_norm=\'layer\''     )

     __A		:					int						=			WavaVecaForPreTraining(__snake_case     )

     __A		:					Optional[int]						=			DataCollatorForWavaVecaPretraining(model=__snake_case					,				feature_extractor=__snake_case     )

     __A		:					Dict						=			WavaVecaPreTrainer(
         model=__snake_case					,				data_collator=__snake_case					,				args=__snake_case					,				train_dataset=vectorized_datasets['train']					,				eval_dataset=vectorized_datasets['validation']					,				tokenizer=__snake_case					,				max_gumbel_temp=model_args.max_gumbel_temperature					,				min_gumbel_temp=model_args.min_gumbel_temperature					,				gumbel_temp_decay=model_args.gumbel_temperature_decay					,				)
     trainer.train()


if __name__ == "__main__":
  main()






import torch
from torch import nn



class 					__snake_case	(  nn.Module    ):


    '''simple docstring'''







    def __init__(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_=1   ,				A_=False     ):





        '''simple docstring'''

        super().__init__()

        SCREAMING_SNAKE_CASE__											=			n_token
        SCREAMING_SNAKE_CASE__											=			d_embed
        SCREAMING_SNAKE_CASE__											=			d_proj

        SCREAMING_SNAKE_CASE__											=			cutoffs + [n_token]
        SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
        SCREAMING_SNAKE_CASE__											=			div_val

        SCREAMING_SNAKE_CASE__											=			self.cutoffs[0]
        SCREAMING_SNAKE_CASE__											=			len(self.cutoffs     ) - 1
        SCREAMING_SNAKE_CASE__											=			self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters   ,				self.d_embed     )     )
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters     )     )

        SCREAMING_SNAKE_CASE__											=			nn.ModuleList()
        SCREAMING_SNAKE_CASE__											=			nn.ParameterList()

        if div_val == 1:
            for i in range(len(self.cutoffs     )     ):
                if d_proj != d_embed:
                    self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )
                else:
                    self.out_projs.append(A_     )

            self.out_layers.append(nn.Linear(A_   ,				A_     )     )
        else:
            for i in range(len(self.cutoffs     )     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                SCREAMING_SNAKE_CASE__											=			d_embed // (div_val**i)

                self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )

                self.out_layers.append(nn.Linear(A_   ,				r_idx - l_idx     )     )

        SCREAMING_SNAKE_CASE__											=			keep_order



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        if proj is None:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
        else:
            # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				proj.t().contiguous()     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
            # else:
            #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
            #     if bias is not None:
            #         logit = logit + bias

        return logit



    def  lowercase_					(       self   ,				A_   ,				A_=None   ,				A_=False     ):





        '''simple docstring'''

        if labels is not None:
            # Shift so that tokens < n predict n
            SCREAMING_SNAKE_CASE__											=			hidden[..., :-1, :].contiguous()
            SCREAMING_SNAKE_CASE__											=			labels[..., 1:].contiguous()
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )
            SCREAMING_SNAKE_CASE__											=			labels.view(-1     )
            if hidden.size(0     ) != labels.size(0     ):
                raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''     )
        else:
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            if labels is not None:
                SCREAMING_SNAKE_CASE__											=			labels != -1_00
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )
                SCREAMING_SNAKE_CASE__											=			(
                    -nn.functional.log_softmax(A_   ,				dim=-1     )[mask].gather(1   ,				labels[mask].unsqueeze(1     )     ).squeeze(1     )
                )
            else:
                SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]

            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            if labels is None:
                SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            else:
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )

            SCREAMING_SNAKE_CASE__											=			0
            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if labels is not None:
                    SCREAMING_SNAKE_CASE__											=			(labels >= l_idx) & (labels < r_idx)
                    SCREAMING_SNAKE_CASE__											=			mask_i.nonzero().squeeze()

                    if indices_i.numel() == 0:
                        continue

                    SCREAMING_SNAKE_CASE__											=			labels.index_select(0   ,				A_     ) - l_idx
                    SCREAMING_SNAKE_CASE__											=			head_logprob.index_select(0   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			hidden.index_select(0   ,				A_     )
                else:
                    SCREAMING_SNAKE_CASE__											=			hidden

                if i == 0:
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i.gather(1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )
                    SCREAMING_SNAKE_CASE__											=			self.cutoffs[0] + i - 1  # No probability for the head cluster
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                            1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                        SCREAMING_SNAKE_CASE__											=			logprob_i

                if labels is not None:
                    if (hasattr(self   ,				'''keep_order'''     ) and self.keep_order) or keep_order:
                        out.index_copy_(0   ,				A_   ,				-logprob_i     )
                    else:
                        out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                    offset += logprob_i.size(0     )

        return out




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            return nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )

            SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, -i] + tail_logprob_i
                    SCREAMING_SNAKE_CASE__											=			logprob_i

            return out
