





import tempfile
import unittest

import numpy as np

from diffusers import (
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionPipeline,
    PNDMScheduler,
)
from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
  import onnxruntime as ort



class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Optional[Any]				=	"""hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline"""



    def  lowercase_					(       self   ,				A_=0     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			np.random.RandomState(A_     )
        SCREAMING_SNAKE_CASE__											=			{
            '''prompt''': '''A painting of a squirrel eating a burger''',
            '''generator''': generator,
            '''num_inference_steps''': 2,
            '''guidance_scale''': 7.5,
            '''output_type''': '''numpy''',
        }
        return inputs



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.65072, 0.58492, 0.48219, 0.55521, 0.53180, 0.55939, 0.50697, 0.39800, 0.46455]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        SCREAMING_SNAKE_CASE__											=			PNDMScheduler.from_config(pipe.scheduler.config   ,				skip_prk_steps=A_     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.65863, 0.59425, 0.49326, 0.56313, 0.53875, 0.56627, 0.51065, 0.39777, 0.46330]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        SCREAMING_SNAKE_CASE__											=			LMSDiscreteScheduler.from_config(pipe.scheduler.config     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.53755, 0.60786, 0.47402, 0.49488, 0.51869, 0.49819, 0.47985, 0.38957, 0.44279]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        SCREAMING_SNAKE_CASE__											=			EulerDiscreteScheduler.from_config(pipe.scheduler.config     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.53755, 0.60786, 0.47402, 0.49488, 0.51869, 0.49819, 0.47985, 0.38957, 0.44279]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        SCREAMING_SNAKE_CASE__											=			EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.53817, 0.60812, 0.47384, 0.49530, 0.51894, 0.49814, 0.47984, 0.38958, 0.44271]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        SCREAMING_SNAKE_CASE__											=			DPMSolverMultistepScheduler.from_config(pipe.scheduler.config     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     ).images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 1_28, 1_28, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.53895, 0.60808, 0.47933, 0.49608, 0.51886, 0.49950, 0.48053, 0.38957, 0.44200]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			3 * [inputs['''prompt''']]

        # forward
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     )
        SCREAMING_SNAKE_CASE__											=			output.images[0, -3:, -3:, -1]

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			3 * [inputs.pop('''prompt'''     )]

        SCREAMING_SNAKE_CASE__											=			pipe.tokenizer(
            A_   ,				padding='''max_length'''   ,				max_length=pipe.tokenizer.model_max_length   ,				truncation=A_   ,				return_tensors='''np'''   ,				)
        SCREAMING_SNAKE_CASE__											=			text_inputs['''input_ids''']

        SCREAMING_SNAKE_CASE__											=			pipe.text_encoder(input_ids=text_inputs.astype(np.intaa     )     )[0]

        SCREAMING_SNAKE_CASE__											=			prompt_embeds

        # forward
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     )
        SCREAMING_SNAKE_CASE__											=			output.images[0, -3:, -3:, -1]

        assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()     ).max() < 1E-4




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint   ,				provider='''CPUExecutionProvider'''     )
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			3 * ['''this is a negative prompt''']
        SCREAMING_SNAKE_CASE__											=			negative_prompt
        SCREAMING_SNAKE_CASE__											=			3 * [inputs['''prompt''']]

        # forward
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     )
        SCREAMING_SNAKE_CASE__											=			output.images[0, -3:, -3:, -1]

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_inputs()
        SCREAMING_SNAKE_CASE__											=			3 * [inputs.pop('''prompt'''     )]

        SCREAMING_SNAKE_CASE__											=			[]
        for p in [prompt, negative_prompt]:
            SCREAMING_SNAKE_CASE__											=			pipe.tokenizer(
                A_   ,				padding='''max_length'''   ,				max_length=pipe.tokenizer.model_max_length   ,				truncation=A_   ,				return_tensors='''np'''   ,				)
            SCREAMING_SNAKE_CASE__											=			text_inputs['''input_ids''']

            embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.intaa     )     )[0]     )

        SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			embeds

        # forward
        SCREAMING_SNAKE_CASE__											=			pipe(**A_     )
        SCREAMING_SNAKE_CASE__											=			output.images[0, -3:, -3:, -1]

        assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()     ).max() < 1E-4





@nightly
@require_onnxruntime
@require_torch_gpu
class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    @property
    def  lowercase_					(       self     ):





        '''simple docstring'''

        return (
            "CUDAExecutionProvider",
            {
                "gpu_mem_limit": "15000000000",  # 15GB
                "arena_extend_strategy": "kSameAsRequested",
            },
        )



    @property
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			ort.SessionOptions()
        SCREAMING_SNAKE_CASE__											=			False
        return options



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(
            '''CompVis/stable-diffusion-v1-4'''   ,				revision='''onnx'''   ,				safety_checker=A_   ,				feature_extractor=A_   ,				provider=self.gpu_provider   ,				sess_options=self.gpu_options   ,				)
        sd_pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			'''A painting of a squirrel eating a burger'''
        np.random.seed(0     )
        SCREAMING_SNAKE_CASE__											=			sd_pipe([prompt]   ,				guidance_scale=6.0   ,				num_inference_steps=10   ,				output_type='''np'''     )
        SCREAMING_SNAKE_CASE__											=			output.images

        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_12, 5_12, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.0452, 0.0390, 0.0087, 0.0350, 0.0617, 0.0364, 0.0544, 0.0523, 0.0720]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-3



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DDIMScheduler.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				subfolder='''scheduler'''   ,				revision='''onnx'''     )
        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				revision='''onnx'''   ,				scheduler=A_   ,				safety_checker=A_   ,				feature_extractor=A_   ,				provider=self.gpu_provider   ,				sess_options=self.gpu_options   ,				)
        sd_pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			'''open neural network exchange'''
        SCREAMING_SNAKE_CASE__											=			np.random.RandomState(0     )
        SCREAMING_SNAKE_CASE__											=			sd_pipe([prompt]   ,				guidance_scale=7.5   ,				num_inference_steps=10   ,				generator=A_   ,				output_type='''np'''     )
        SCREAMING_SNAKE_CASE__											=			output.images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_12, 5_12, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.2867, 0.1974, 0.1481, 0.7294, 0.7251, 0.6667, 0.4194, 0.5642, 0.6486]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-3



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			LMSDiscreteScheduler.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				subfolder='''scheduler'''   ,				revision='''onnx'''     )
        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				revision='''onnx'''   ,				scheduler=A_   ,				safety_checker=A_   ,				feature_extractor=A_   ,				provider=self.gpu_provider   ,				sess_options=self.gpu_options   ,				)
        sd_pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			'''open neural network exchange'''
        SCREAMING_SNAKE_CASE__											=			np.random.RandomState(0     )
        SCREAMING_SNAKE_CASE__											=			sd_pipe([prompt]   ,				guidance_scale=7.5   ,				num_inference_steps=10   ,				generator=A_   ,				output_type='''np'''     )
        SCREAMING_SNAKE_CASE__											=			output.images
        SCREAMING_SNAKE_CASE__											=			image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_12, 5_12, 3)
        SCREAMING_SNAKE_CASE__											=			np.array([0.2306, 0.1959, 0.1593, 0.6549, 0.6394, 0.5408, 0.5065, 0.6010, 0.6161]     )

        assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-3



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			0

        def test_callback_fn(A_   ,				A_   ,				A_     ) -> None:
            SCREAMING_SNAKE_CASE__											=			True
            nonlocal number_of_steps
            number_of_steps += 1
            if step == 0:
                assert latents.shape == (1, 4, 64, 64)
                SCREAMING_SNAKE_CASE__											=			latents[0, -3:, -3:, -1]
                SCREAMING_SNAKE_CASE__											=			np.array(
                    [-0.6772, -0.3835, -1.2456, 0.1905, -1.0974, 0.6967, -1.9353, 0.0178, 1.0167]     )

                assert np.abs(latents_slice.flatten() - expected_slice     ).max() < 1E-3
            elif step == 5:
                assert latents.shape == (1, 4, 64, 64)
                SCREAMING_SNAKE_CASE__											=			latents[0, -3:, -3:, -1]
                SCREAMING_SNAKE_CASE__											=			np.array(
                    [-0.3351, 0.2241, -0.1837, -0.2325, -0.6577, 0.3393, -0.0241, 0.5899, 1.3875]     )

                assert np.abs(latents_slice.flatten() - expected_slice     ).max() < 1E-3

        SCREAMING_SNAKE_CASE__											=			False

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				revision='''onnx'''   ,				safety_checker=A_   ,				feature_extractor=A_   ,				provider=self.gpu_provider   ,				sess_options=self.gpu_options   ,				)
        pipe.set_progress_bar_config(disable=A_     )

        SCREAMING_SNAKE_CASE__											=			'''Andromeda galaxy in a bottle'''

        SCREAMING_SNAKE_CASE__											=			np.random.RandomState(0     )
        pipe(
            prompt=A_   ,				num_inference_steps=5   ,				guidance_scale=7.5   ,				generator=A_   ,				callback=A_   ,				callback_steps=1   ,				)
        assert test_callback_fn.has_been_called
        assert number_of_steps == 6




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(
            '''runwayml/stable-diffusion-v1-5'''   ,				revision='''onnx'''   ,				safety_checker=A_   ,				feature_extractor=A_   ,				provider=self.gpu_provider   ,				sess_options=self.gpu_options   ,				)
        assert isinstance(A_   ,				A_     )
        assert pipe.safety_checker is None

        SCREAMING_SNAKE_CASE__											=			pipe('''example prompt'''   ,				num_inference_steps=2     ).images[0]
        assert image is not None

        # check that there's no error when saving a pipeline with one of the models being None
        with tempfile.TemporaryDirectory() as tmpdirname:
            pipe.save_pretrained(A_     )
            SCREAMING_SNAKE_CASE__											=			OnnxStableDiffusionPipeline.from_pretrained(A_     )

        # sanity check that the pipeline still works
        assert pipe.safety_checker is None
        SCREAMING_SNAKE_CASE__											=			pipe('''example prompt'''   ,				num_inference_steps=2     ).images[0]
        assert image is not None







import torch
from torch import nn



class 					__snake_case	(  nn.Module    ):


    '''simple docstring'''







    def __init__(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_=1   ,				A_=False     ):





        '''simple docstring'''

        super().__init__()

        SCREAMING_SNAKE_CASE__											=			n_token
        SCREAMING_SNAKE_CASE__											=			d_embed
        SCREAMING_SNAKE_CASE__											=			d_proj

        SCREAMING_SNAKE_CASE__											=			cutoffs + [n_token]
        SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
        SCREAMING_SNAKE_CASE__											=			div_val

        SCREAMING_SNAKE_CASE__											=			self.cutoffs[0]
        SCREAMING_SNAKE_CASE__											=			len(self.cutoffs     ) - 1
        SCREAMING_SNAKE_CASE__											=			self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters   ,				self.d_embed     )     )
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters     )     )

        SCREAMING_SNAKE_CASE__											=			nn.ModuleList()
        SCREAMING_SNAKE_CASE__											=			nn.ParameterList()

        if div_val == 1:
            for i in range(len(self.cutoffs     )     ):
                if d_proj != d_embed:
                    self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )
                else:
                    self.out_projs.append(A_     )

            self.out_layers.append(nn.Linear(A_   ,				A_     )     )
        else:
            for i in range(len(self.cutoffs     )     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                SCREAMING_SNAKE_CASE__											=			d_embed // (div_val**i)

                self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )

                self.out_layers.append(nn.Linear(A_   ,				r_idx - l_idx     )     )

        SCREAMING_SNAKE_CASE__											=			keep_order



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        if proj is None:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
        else:
            # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				proj.t().contiguous()     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
            # else:
            #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
            #     if bias is not None:
            #         logit = logit + bias

        return logit



    def  lowercase_					(       self   ,				A_   ,				A_=None   ,				A_=False     ):





        '''simple docstring'''

        if labels is not None:
            # Shift so that tokens < n predict n
            SCREAMING_SNAKE_CASE__											=			hidden[..., :-1, :].contiguous()
            SCREAMING_SNAKE_CASE__											=			labels[..., 1:].contiguous()
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )
            SCREAMING_SNAKE_CASE__											=			labels.view(-1     )
            if hidden.size(0     ) != labels.size(0     ):
                raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''     )
        else:
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            if labels is not None:
                SCREAMING_SNAKE_CASE__											=			labels != -1_00
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )
                SCREAMING_SNAKE_CASE__											=			(
                    -nn.functional.log_softmax(A_   ,				dim=-1     )[mask].gather(1   ,				labels[mask].unsqueeze(1     )     ).squeeze(1     )
                )
            else:
                SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]

            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            if labels is None:
                SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            else:
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )

            SCREAMING_SNAKE_CASE__											=			0
            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if labels is not None:
                    SCREAMING_SNAKE_CASE__											=			(labels >= l_idx) & (labels < r_idx)
                    SCREAMING_SNAKE_CASE__											=			mask_i.nonzero().squeeze()

                    if indices_i.numel() == 0:
                        continue

                    SCREAMING_SNAKE_CASE__											=			labels.index_select(0   ,				A_     ) - l_idx
                    SCREAMING_SNAKE_CASE__											=			head_logprob.index_select(0   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			hidden.index_select(0   ,				A_     )
                else:
                    SCREAMING_SNAKE_CASE__											=			hidden

                if i == 0:
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i.gather(1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )
                    SCREAMING_SNAKE_CASE__											=			self.cutoffs[0] + i - 1  # No probability for the head cluster
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                            1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                        SCREAMING_SNAKE_CASE__											=			logprob_i

                if labels is not None:
                    if (hasattr(self   ,				'''keep_order'''     ) and self.keep_order) or keep_order:
                        out.index_copy_(0   ,				A_   ,				-logprob_i     )
                    else:
                        out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                    offset += logprob_i.size(0     )

        return out




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            return nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )

            SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, -i] + tail_logprob_i
                    SCREAMING_SNAKE_CASE__											=			logprob_i

            return out
