





import io
import math
from typing import Dict, Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import convert_to_rgb, normalize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    get_image_size,
    infer_channel_dimension_format,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_vision_available, logging
from ...utils.import_utils import requires_backends


if is_vision_available():
  import textwrap

  from PIL import Image, ImageDraw, ImageFont

if is_torch_available():
  import torch

  from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
  _A					:				List[Any]				=			False


_A					:				List[Any]				=			logging.get_logger(__name__)
_A					:				str				=			"""ybelkada/fonts"""


def 							__snake_case							(    )				->						Dict:
    if is_torch_available() and not is_torch_greater_or_equal_than_1_11:
        raise ImportError(
            f'''You are using torch=={torch.__version__}, but torch>=1.11.0 is required to use '''
            '''Pix2StructImageProcessor. Please upgrade torch.'''	)


def 							__snake_case							(    lowerCAmelCase_ ,							lowerCAmelCase_ ,							lowerCAmelCase_	)				->						Any:
    requires_backends(lowerCAmelCase_ ,							['''torch''']	)
    _check_torch_version()

    SCREAMING_SNAKE_CASE__											=			image_tensor.unsqueeze(0	)
    SCREAMING_SNAKE_CASE__											=			torch.nn.functional.unfold(lowerCAmelCase_ ,							(patch_height, patch_width) ,							stride=(patch_height, patch_width)	)
    SCREAMING_SNAKE_CASE__											=			patches.reshape(image_tensor.size(0	) ,							image_tensor.size(1	) ,							lowerCAmelCase_ ,							lowerCAmelCase_ ,							-1	)
    SCREAMING_SNAKE_CASE__											=			patches.permute(0 ,							4 ,							2 ,							3 ,							1	).reshape(
        image_tensor.size(2	) // patch_height ,							image_tensor.size(3	) // patch_width ,							image_tensor.size(1	) * patch_height * patch_width ,							)
    return patches.unsqueeze(0	)


def 							__snake_case							(    lowerCAmelCase_ ,							lowerCAmelCase_ = 3_6 ,							lowerCAmelCase_ = "black" ,							lowerCAmelCase_ = "white" ,							lowerCAmelCase_ = 5 ,							lowerCAmelCase_ = 5 ,							lowerCAmelCase_ = 5 ,							lowerCAmelCase_ = 5 ,							lowerCAmelCase_ = None ,							lowerCAmelCase_ = None ,							)				->						Image.Image:
    requires_backends(lowerCAmelCase_ ,							'''vision'''	)
    # Add new lines so that each line is no more than 80 characters.

    SCREAMING_SNAKE_CASE__											=			textwrap.TextWrapper(width=8_0	)
    SCREAMING_SNAKE_CASE__											=			wrapper.wrap(text=lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			'''\n'''.join(lowerCAmelCase_	)

    if font_bytes is not None and font_path is None:
        SCREAMING_SNAKE_CASE__											=			io.BytesIO(lowerCAmelCase_	)
    elif font_path is not None:
        SCREAMING_SNAKE_CASE__											=			font_path
    else:
        SCREAMING_SNAKE_CASE__											=			hf_hub_download(lowerCAmelCase_ ,							'''Arial.TTF'''	)
    SCREAMING_SNAKE_CASE__											=			ImageFont.truetype(lowerCAmelCase_ ,							encoding='''UTF-8''' ,							size=lowerCAmelCase_	)

    # Use a temporary canvas to determine the width and height in pixels when
    # rendering the text.
    SCREAMING_SNAKE_CASE__											=			ImageDraw.Draw(Image.new('''RGB''' ,							(1, 1) ,							lowerCAmelCase_	)	)
    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			temp_draw.textbbox((0, 0) ,							lowerCAmelCase_ ,							lowerCAmelCase_	)

    # Create the actual image with a bit of padding around the text.
    SCREAMING_SNAKE_CASE__											=			text_width + left_padding + right_padding
    SCREAMING_SNAKE_CASE__											=			text_height + top_padding + bottom_padding
    SCREAMING_SNAKE_CASE__											=			Image.new('''RGB''' ,							(image_width, image_height) ,							lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			ImageDraw.Draw(lowerCAmelCase_	)
    draw.text(xy=(left_padding, top_padding) ,							text=lowerCAmelCase_ ,							fill=lowerCAmelCase_ ,							font=lowerCAmelCase_	)
    return image


def 							__snake_case							(    lowerCAmelCase_ ,							lowerCAmelCase_ ,							**lowerCAmelCase_	)				->						int:
    requires_backends(lowerCAmelCase_ ,							'''vision'''	)

    # Convert to PIL image if necessary
    SCREAMING_SNAKE_CASE__											=			to_pil_image(lowerCAmelCase_	)

    SCREAMING_SNAKE_CASE__											=			render_text(lowerCAmelCase_ ,							**lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			max(header_image.width ,							image.width	)

    SCREAMING_SNAKE_CASE__											=			int(image.height * (new_width / image.width)	)
    SCREAMING_SNAKE_CASE__											=			int(header_image.height * (new_width / header_image.width)	)

    SCREAMING_SNAKE_CASE__											=			Image.new('''RGB''' ,							(new_width, new_height + new_header_height) ,							'''white'''	)
    new_image.paste(header_image.resize((new_width, new_header_height)	) ,							(0, 0)	)
    new_image.paste(image.resize((new_width, new_height)	) ,							(0, new_header_height)	)

    # Convert back to the original framework if necessary
    SCREAMING_SNAKE_CASE__											=			to_numpy_array(lowerCAmelCase_	)

    if infer_channel_dimension_format(lowerCAmelCase_	) == ChannelDimension.LAST:
        SCREAMING_SNAKE_CASE__											=			to_channel_dimension_format(lowerCAmelCase_ ,							ChannelDimension.LAST	)

    return new_image



class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    lowerCamelCase__    :		Optional[Any]				=	["""flattened_patches"""]



    def __init__(       self   ,				A_ = True   ,				A_ = True   ,				A_ = None   ,				A_ = 20_48   ,				A_ = False   ,				**A_   ,				):





        '''simple docstring'''

        super().__init__(**A_     )
        SCREAMING_SNAKE_CASE__											=			patch_size if patch_size is not None else {'''height''': 16, '''width''': 16}
        SCREAMING_SNAKE_CASE__											=			do_normalize
        SCREAMING_SNAKE_CASE__											=			do_convert_rgb
        SCREAMING_SNAKE_CASE__											=			max_patches
        SCREAMING_SNAKE_CASE__											=			is_vqa



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				**A_     ):





        '''simple docstring'''

        requires_backends(self.extract_flattened_patches   ,				'''torch'''     )
        _check_torch_version()

        # convert to torch
        SCREAMING_SNAKE_CASE__											=			to_channel_dimension_format(A_   ,				ChannelDimension.FIRST     )
        SCREAMING_SNAKE_CASE__											=			torch.from_numpy(A_     )

        SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			patch_size['''height'''], patch_size['''width''']
        SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			get_image_size(A_     )

        # maximize scale s.t.
        SCREAMING_SNAKE_CASE__											=			math.sqrt(max_patches * (patch_height / image_height) * (patch_width / image_width)     )
        SCREAMING_SNAKE_CASE__											=			max(min(math.floor(scale * image_height / patch_height     )   ,				A_     )   ,				1     )
        SCREAMING_SNAKE_CASE__											=			max(min(math.floor(scale * image_width / patch_width     )   ,				A_     )   ,				1     )
        SCREAMING_SNAKE_CASE__											=			max(num_feasible_rows * patch_height   ,				1     )
        SCREAMING_SNAKE_CASE__											=			max(num_feasible_cols * patch_width   ,				1     )

        SCREAMING_SNAKE_CASE__											=			torch.nn.functional.interpolate(
            image.unsqueeze(0     )   ,				size=(resized_height, resized_width)   ,				mode='''bilinear'''   ,				align_corners=A_   ,				antialias=A_   ,				).squeeze(0     )

        # [1, rows, columns, patch_height * patch_width * image_channels]
        SCREAMING_SNAKE_CASE__											=			torch_extract_patches(A_   ,				A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			patches.shape
        SCREAMING_SNAKE_CASE__											=			patches_shape[1]
        SCREAMING_SNAKE_CASE__											=			patches_shape[2]
        SCREAMING_SNAKE_CASE__											=			patches_shape[3]

        # [rows * columns, patch_height * patch_width * image_channels]
        SCREAMING_SNAKE_CASE__											=			patches.reshape([rows * columns, depth]     )

        # [rows * columns, 1]
        SCREAMING_SNAKE_CASE__											=			torch.arange(A_     ).reshape([rows, 1]     ).repeat(1   ,				A_     ).reshape([rows * columns, 1]     )
        SCREAMING_SNAKE_CASE__											=			torch.arange(A_     ).reshape([1, columns]     ).repeat(A_   ,				1     ).reshape([rows * columns, 1]     )

        # Offset by 1 so the ids do not contain zeros, which represent padding.
        row_ids += 1
        col_ids += 1

        # Prepare additional patch features.
        # [rows * columns, 1]
        SCREAMING_SNAKE_CASE__											=			row_ids.to(torch.floataa     )
        SCREAMING_SNAKE_CASE__											=			col_ids.to(torch.floataa     )

        # [rows * columns, 2 + patch_height * patch_width * image_channels]
        SCREAMING_SNAKE_CASE__											=			torch.cat([row_ids, col_ids, patches]   ,				-1     )

        # [max_patches, 2 + patch_height * patch_width * image_channels]
        SCREAMING_SNAKE_CASE__											=			torch.nn.functional.pad(A_   ,				[0, 0, 0, max_patches - (rows * columns)]     ).float()

        SCREAMING_SNAKE_CASE__											=			to_numpy_array(A_     )

        return result



    def  lowercase_					(       self   ,				A_   ,				A_ = None   ,				**A_     ):





        '''simple docstring'''

        if image.dtype == np.uinta:
            SCREAMING_SNAKE_CASE__											=			image.astype(np.floataa     )

        # take mean across the whole `image`
        SCREAMING_SNAKE_CASE__											=			np.mean(A_     )
        SCREAMING_SNAKE_CASE__											=			np.std(A_     )
        SCREAMING_SNAKE_CASE__											=			max(A_   ,				1.0 / math.sqrt(np.prod(image.shape     )     )     )

        return normalize(A_   ,				mean=A_   ,				std=A_   ,				**A_     )




    def  lowercase_					(       self   ,				A_   ,				A_ = None   ,				A_ = None   ,				A_ = None   ,				A_ = None   ,				A_ = None   ,				A_ = None   ,				A_ = ChannelDimension.FIRST   ,				**A_   ,				):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			do_normalize if do_normalize is not None else self.do_normalize
        SCREAMING_SNAKE_CASE__											=			do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
        SCREAMING_SNAKE_CASE__											=			patch_size if patch_size is not None else self.patch_size
        SCREAMING_SNAKE_CASE__											=			max_patches if max_patches is not None else self.max_patches
        SCREAMING_SNAKE_CASE__											=			self.is_vqa

        if kwargs.get('''data_format'''   ,				A_     ) is not None:
            raise ValueError('''data_format is not an accepted input as the outputs are '''     )

        SCREAMING_SNAKE_CASE__											=			make_list_of_images(A_     )

        if not valid_images(A_     ):
            raise ValueError(
                '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                '''torch.Tensor, tf.Tensor or jax.ndarray.'''     )

        # PIL RGBA images are converted to RGB
        if do_convert_rgb:
            SCREAMING_SNAKE_CASE__											=			[convert_to_rgb(A_     ) for image in images]

        # All transformations expect numpy arrays.
        SCREAMING_SNAKE_CASE__											=			[to_numpy_array(A_     ) for image in images]

        if is_vqa:
            if header_text is None:
                raise ValueError('''A header text must be provided for VQA models.'''     )
            SCREAMING_SNAKE_CASE__											=			kwargs.pop('''font_bytes'''   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			kwargs.pop('''font_path'''   ,				A_     )

            if isinstance(A_   ,				A_     ):
                SCREAMING_SNAKE_CASE__											=			[header_text] * len(A_     )

            SCREAMING_SNAKE_CASE__											=			[
                render_header(A_   ,				header_text[i]   ,				font_bytes=A_   ,				font_path=A_     )
                for i, image in enumerate(A_     )
            ]

        if do_normalize:
            SCREAMING_SNAKE_CASE__											=			[self.normalize(image=A_     ) for image in images]

        # convert to torch tensor and permute
        SCREAMING_SNAKE_CASE__											=			[
            self.extract_flattened_patches(image=A_   ,				max_patches=A_   ,				patch_size=A_     )
            for image in images
        ]

        # create attention mask in numpy
        SCREAMING_SNAKE_CASE__											=			[(image.sum(axis=-1     ) != 0).astype(np.floataa     ) for image in images]

        SCREAMING_SNAKE_CASE__											=			BatchFeature(
            data={'''flattened_patches''': images, '''attention_mask''': attention_masks}   ,				tensor_type=A_     )

        return encoded_outputs







import torch
from torch import nn



class 					__snake_case	(  nn.Module    ):


    '''simple docstring'''







    def __init__(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_=1   ,				A_=False     ):





        '''simple docstring'''

        super().__init__()

        SCREAMING_SNAKE_CASE__											=			n_token
        SCREAMING_SNAKE_CASE__											=			d_embed
        SCREAMING_SNAKE_CASE__											=			d_proj

        SCREAMING_SNAKE_CASE__											=			cutoffs + [n_token]
        SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
        SCREAMING_SNAKE_CASE__											=			div_val

        SCREAMING_SNAKE_CASE__											=			self.cutoffs[0]
        SCREAMING_SNAKE_CASE__											=			len(self.cutoffs     ) - 1
        SCREAMING_SNAKE_CASE__											=			self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters   ,				self.d_embed     )     )
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters     )     )

        SCREAMING_SNAKE_CASE__											=			nn.ModuleList()
        SCREAMING_SNAKE_CASE__											=			nn.ParameterList()

        if div_val == 1:
            for i in range(len(self.cutoffs     )     ):
                if d_proj != d_embed:
                    self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )
                else:
                    self.out_projs.append(A_     )

            self.out_layers.append(nn.Linear(A_   ,				A_     )     )
        else:
            for i in range(len(self.cutoffs     )     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                SCREAMING_SNAKE_CASE__											=			d_embed // (div_val**i)

                self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )

                self.out_layers.append(nn.Linear(A_   ,				r_idx - l_idx     )     )

        SCREAMING_SNAKE_CASE__											=			keep_order



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        if proj is None:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
        else:
            # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				proj.t().contiguous()     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
            # else:
            #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
            #     if bias is not None:
            #         logit = logit + bias

        return logit



    def  lowercase_					(       self   ,				A_   ,				A_=None   ,				A_=False     ):





        '''simple docstring'''

        if labels is not None:
            # Shift so that tokens < n predict n
            SCREAMING_SNAKE_CASE__											=			hidden[..., :-1, :].contiguous()
            SCREAMING_SNAKE_CASE__											=			labels[..., 1:].contiguous()
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )
            SCREAMING_SNAKE_CASE__											=			labels.view(-1     )
            if hidden.size(0     ) != labels.size(0     ):
                raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''     )
        else:
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            if labels is not None:
                SCREAMING_SNAKE_CASE__											=			labels != -1_00
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )
                SCREAMING_SNAKE_CASE__											=			(
                    -nn.functional.log_softmax(A_   ,				dim=-1     )[mask].gather(1   ,				labels[mask].unsqueeze(1     )     ).squeeze(1     )
                )
            else:
                SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]

            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            if labels is None:
                SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            else:
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )

            SCREAMING_SNAKE_CASE__											=			0
            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if labels is not None:
                    SCREAMING_SNAKE_CASE__											=			(labels >= l_idx) & (labels < r_idx)
                    SCREAMING_SNAKE_CASE__											=			mask_i.nonzero().squeeze()

                    if indices_i.numel() == 0:
                        continue

                    SCREAMING_SNAKE_CASE__											=			labels.index_select(0   ,				A_     ) - l_idx
                    SCREAMING_SNAKE_CASE__											=			head_logprob.index_select(0   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			hidden.index_select(0   ,				A_     )
                else:
                    SCREAMING_SNAKE_CASE__											=			hidden

                if i == 0:
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i.gather(1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )
                    SCREAMING_SNAKE_CASE__											=			self.cutoffs[0] + i - 1  # No probability for the head cluster
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                            1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                        SCREAMING_SNAKE_CASE__											=			logprob_i

                if labels is not None:
                    if (hasattr(self   ,				'''keep_order'''     ) and self.keep_order) or keep_order:
                        out.index_copy_(0   ,				A_   ,				-logprob_i     )
                    else:
                        out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                    offset += logprob_i.size(0     )

        return out




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            return nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )

            SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, -i] + tail_logprob_i
                    SCREAMING_SNAKE_CASE__											=			logprob_i

            return out
