


'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowercase			(							_UpperCAmelCase			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       BertTokenizer
		_a									=       BertTokenizerFast
		_a									=       True
		_a									=       True
		_a									=       filter_non_english




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :Dict							      =				[
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				'''UNwant\u00E9d,running'''
							UpperCamelCase__     :int							      =				'''unwanted, running'''
							return input_text, output_text




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.tokenizer_class(self.vocab_file							)

							UpperCamelCase__     :int							      =				tokenizer.tokenize('''UNwant\u00E9d,running'''							)
							self.assertListEqual(__UpperCamelCase					,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase							)					,     [9, 6, 7, 12, 10, 11]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							if not self.test_rust_tokenizer:
												return

							UpperCamelCase__     :Optional[Any]							      =				self.get_tokenizer()
							UpperCamelCase__     :str							      =				self.get_rust_tokenizer()

							UpperCamelCase__     :int							      =				'''UNwant\u00E9d,running'''

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(__UpperCamelCase							)
							UpperCamelCase__     :Union[str, Any]							      =				rust_tokenizer.tokenize(__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

							UpperCamelCase__     :List[Any]							      =				tokenizer.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
							UpperCamelCase__     :List[str]							      =				rust_tokenizer.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

							UpperCamelCase__     :List[Any]							      =				self.get_rust_tokenizer()
							UpperCamelCase__     :Dict							      =				tokenizer.encode(__UpperCamelCase							)
							UpperCamelCase__     :Dict							      =				rust_tokenizer.encode(__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

							# With lower casing
							UpperCamelCase__     :int							      =				self.get_tokenizer(do_lower_case=__UpperCamelCase							)
							UpperCamelCase__     :Dict							      =				self.get_rust_tokenizer(do_lower_case=__UpperCamelCase							)

							UpperCamelCase__     :List[Any]							      =				'''UNwant\u00E9d,running'''

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(__UpperCamelCase							)
							UpperCamelCase__     :Tuple							      =				rust_tokenizer.tokenize(__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
							UpperCamelCase__     :Optional[Any]							      =				rust_tokenizer.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

							UpperCamelCase__     :List[Any]							      =				self.get_rust_tokenizer()
							UpperCamelCase__     :Optional[Any]							      =				tokenizer.encode(__UpperCamelCase							)
							UpperCamelCase__     :Union[str, Any]							      =				rust_tokenizer.encode(__UpperCamelCase							)
							self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)					,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				BasicTokenizer(do_lower_case=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				BasicTokenizer(do_lower_case=__UpperCamelCase					,     strip_accents=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''h\u00E9llo''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				BasicTokenizer(do_lower_case=__UpperCamelCase					,     strip_accents=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[int]							      =				BasicTokenizer(do_lower_case=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				BasicTokenizer(do_lower_case=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				BasicTokenizer(do_lower_case=__UpperCamelCase					,     strip_accents=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				BasicTokenizer(do_lower_case=__UpperCamelCase					,     strip_accents=__UpperCamelCase							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				BasicTokenizer(do_lower_case=__UpperCamelCase					,     never_split=['''[UNK]''']							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				BasicTokenizer()
							UpperCamelCase__     :Optional[int]							      =				'''a\n\'ll !!to?\'d of, can\'t.'''
							UpperCamelCase__     :int							      =				['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
							self.assertListEqual(tokenizer.tokenize(__UpperCamelCase							)					,     __UpperCamelCase							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__     :Optional[Any]							      =				{}
							for i, token in enumerate(__UpperCamelCase							):
												UpperCamelCase__     :List[str]							      =				i
							UpperCamelCase__     :List[Any]							      =				WordpieceTokenizer(vocab=__UpperCamelCase					,     unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							)					,     []							)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)					,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)					,     ['''[UNK]''', '''runn''', '''##ing''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''							)							)
							self.assertTrue(_is_whitespace('''\t'''							)							)
							self.assertTrue(_is_whitespace('''\r'''							)							)
							self.assertTrue(_is_whitespace('''\n'''							)							)
							self.assertTrue(_is_whitespace('''\u00A0'''							)							)

							self.assertFalse(_is_whitespace('''A'''							)							)
							self.assertFalse(_is_whitespace('''-'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''							)							)

							self.assertFalse(_is_control('''A'''							)							)
							self.assertFalse(_is_control(''' '''							)							)
							self.assertFalse(_is_control('''\t'''							)							)
							self.assertFalse(_is_control('''\r'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''							)							)
							self.assertTrue(_is_punctuation('''$'''							)							)
							self.assertTrue(_is_punctuation('''`'''							)							)
							self.assertTrue(_is_punctuation('''.'''							)							)

							self.assertFalse(_is_punctuation('''A'''							)							)
							self.assertFalse(_is_punctuation(''' '''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.get_tokenizer()
							UpperCamelCase__     :List[Any]							      =				self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__UpperCamelCase							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__UpperCamelCase							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)




		@slow
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				self.tokenizer_class.from_pretrained('''bert-base-uncased'''							)

							UpperCamelCase__     :int							      =				tokenizer.encode('''sequence builders'''					,     add_special_tokens=__UpperCamelCase							)
							UpperCamelCase__     :Any							      =				tokenizer.encode('''multi-sequence build'''					,     add_special_tokens=__UpperCamelCase							)

							UpperCamelCase__     :List[str]							      =				tokenizer.build_inputs_with_special_tokens(__UpperCamelCase							)
							UpperCamelCase__     :Any							      =				tokenizer.build_inputs_with_special_tokens(__UpperCamelCase					,     __UpperCamelCase							)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :List[str]							      =				self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,     **__UpperCamelCase							)

																	UpperCamelCase__     :int							      =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__     :Tuple							      =				tokenizer_r.encode_plus(
																	    __UpperCamelCase					,     return_attention_mask=__UpperCamelCase					,     return_token_type_ids=__UpperCamelCase					,     return_offsets_mapping=__UpperCamelCase					,     add_special_tokens=__UpperCamelCase					,     )

																	UpperCamelCase__     :Optional[Any]							      =				tokenizer_r.do_lower_case if hasattr(__UpperCamelCase					,     '''do_lower_case'''							) else False
																	UpperCamelCase__     :List[Any]							      =				(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																	self.assertEqual([e[0] for e in expected_results]					,     tokens['''offset_mapping''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				['''的''', '''人''', '''有''']
							UpperCamelCase__     :Optional[Any]							      =				''''''.join(__UpperCamelCase							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Any							      =				True
																	UpperCamelCase__     :Tuple							      =				self.tokenizer_class.from_pretrained(__UpperCamelCase					,     **__UpperCamelCase							)
																	UpperCamelCase__     :Optional[int]							      =				self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,     **__UpperCamelCase							)

																	UpperCamelCase__     :List[Any]							      =				tokenizer_p.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer_r.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)

																	UpperCamelCase__     :List[Any]							      =				tokenizer_r.convert_ids_to_tokens(__UpperCamelCase							)
																	UpperCamelCase__     :List[str]							      =				tokenizer_p.convert_ids_to_tokens(__UpperCamelCase							)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)
																	self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)

																	UpperCamelCase__     :Optional[Any]							      =				False
																	UpperCamelCase__     :Dict							      =				self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,     **__UpperCamelCase							)
																	UpperCamelCase__     :str							      =				self.tokenizer_class.from_pretrained(__UpperCamelCase					,     **__UpperCamelCase							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)
																	UpperCamelCase__     :Any							      =				tokenizer_p.encode(__UpperCamelCase					,     add_special_tokens=__UpperCamelCase							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(__UpperCamelCase							)
																	UpperCamelCase__     :Tuple							      =				tokenizer_p.convert_ids_to_tokens(__UpperCamelCase							)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__     :Optional[int]							      =				[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__UpperCamelCase							)
																	]
																	self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)
																	self.assertListEqual(__UpperCamelCase					,     __UpperCamelCase							)



'''simple docstring'''
import torch
from torch import nn
class      lowercase			(							nn.Module      ):

		"""simple docstring"""

		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_=1					,     UpperCamelCase_=False							):

							'''simple docstring'''


							super().__init__()

							UpperCamelCase__     :Dict							      =				n_token
							UpperCamelCase__     :List[Any]							      =				d_embed
							UpperCamelCase__     :Dict							      =				d_proj

							UpperCamelCase__     :Dict							      =				cutoffs + [n_token]
							UpperCamelCase__     :Union[str, Any]							      =				[0] + self.cutoffs
							UpperCamelCase__     :Any							      =				div_val

							UpperCamelCase__     :int							      =				self.cutoffs[0]
							UpperCamelCase__     :List[Any]							      =				len(self.cutoffs							) - 1
							UpperCamelCase__     :List[Any]							      =				self.shortlist_size + self.n_clusters

							if self.n_clusters > 0:
												UpperCamelCase__     :Any							      =				nn.Parameter(torch.zeros(self.n_clusters					,     self.d_embed							)							)
												UpperCamelCase__     :Optional[Any]							      =				nn.Parameter(torch.zeros(self.n_clusters							)							)

							UpperCamelCase__     :Union[str, Any]							      =				nn.ModuleList()
							UpperCamelCase__     :str							      =				nn.ParameterList()

							if div_val == 1:
												for i in range(len(self.cutoffs							)							):
																	if d_proj != d_embed:
																						self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase_					,     UpperCamelCase_							)							)							)
																	else:
																						self.out_projs.append(UpperCamelCase_							)

												self.out_layers.append(nn.Linear(UpperCamelCase_					,     UpperCamelCase_							)							)
							else:
												for i in range(len(self.cutoffs							)							):
																	UpperCamelCase__      ,       UpperCamelCase__     :List[str]							      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																	UpperCamelCase__     :Dict							      =				d_embed // (div_val**i)

																	self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase_					,     UpperCamelCase_							)							)							)

																	self.out_layers.append(nn.Linear(UpperCamelCase_					,     r_idx - l_idx							)							)

							UpperCamelCase__     :Tuple							      =				keep_order




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

							'''simple docstring'''


							if proj is None:
												UpperCamelCase__     :List[str]							      =				nn.functional.linear(UpperCamelCase_					,     UpperCamelCase_					,     bias=UpperCamelCase_							)
							else:
												# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
												UpperCamelCase__     :Any							      =				nn.functional.linear(UpperCamelCase_					,     proj.t().contiguous()							)
												UpperCamelCase__     :Union[str, Any]							      =				nn.functional.linear(UpperCamelCase_					,     UpperCamelCase_					,     bias=UpperCamelCase_							)
												# else:
												#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
												#     if bias is not None:
												#         logit = logit + bias

							return logit




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_=None					,     UpperCamelCase_=False							):

							'''simple docstring'''


							if labels is not None:
												# Shift so that tokens < n predict n
												UpperCamelCase__     :Optional[Any]							      =				hidden[..., :-1, :].contiguous()
												UpperCamelCase__     :Optional[Any]							      =				labels[..., 1:].contiguous()
												UpperCamelCase__     :Optional[Any]							      =				hidden.view(-1					,     hidden.size(-1							)							)
												UpperCamelCase__     :str							      =				labels.view(-1							)
												if hidden.size(0							) != labels.size(0							):
																	raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''							)
							else:
												UpperCamelCase__     :int							      =				hidden.view(-1					,     hidden.size(-1							)							)

							if self.n_clusters == 0:
												UpperCamelCase__     :Optional[int]							      =				self._compute_logit(UpperCamelCase_					,     self.out_layers[0].weight					,     self.out_layers[0].bias					,     self.out_projs[0]							)
												if labels is not None:
																	UpperCamelCase__     :int							      =				labels != -100
																	UpperCamelCase__     :List[Any]							      =				torch.zeros_like(UpperCamelCase_					,     dtype=hidden.dtype					,     device=hidden.device							)
																	UpperCamelCase__     :str							      =				(
																	    -nn.functional.log_softmax(UpperCamelCase_					,     dim=-1							)[mask].gather(1					,     labels[mask].unsqueeze(1							)							).squeeze(1							)
																	)
												else:
																	UpperCamelCase__     :Dict							      =				nn.functional.log_softmax(UpperCamelCase_					,     dim=-1							)
							else:
												# construct weights and biases
												UpperCamelCase__      ,       UpperCamelCase__     :Dict							      =				[], []
												for i in range(len(self.cutoffs							)							):
																	if self.div_val == 1:
																						UpperCamelCase__      ,       UpperCamelCase__     :Union[str, Any]							      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						UpperCamelCase__     :str							      =				self.out_layers[0].weight[l_idx:r_idx]
																						UpperCamelCase__     :Optional[int]							      =				self.out_layers[0].bias[l_idx:r_idx]
																	else:
																						UpperCamelCase__     :Optional[Any]							      =				self.out_layers[i].weight
																						UpperCamelCase__     :Optional[int]							      =				self.out_layers[i].bias

																	if i == 0:
																						UpperCamelCase__     :str							      =				torch.cat([weight_i, self.cluster_weight]					,     dim=0							)
																						UpperCamelCase__     :List[str]							      =				torch.cat([bias_i, self.cluster_bias]					,     dim=0							)

																	weights.append(UpperCamelCase_							)
																	biases.append(UpperCamelCase_							)

												UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Optional[Any]							      =				weights[0], biases[0], self.out_projs[0]

												UpperCamelCase__     :str							      =				self._compute_logit(UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)
												UpperCamelCase__     :List[Any]							      =				nn.functional.log_softmax(UpperCamelCase_					,     dim=1							)

												if labels is None:
																	UpperCamelCase__     :Union[str, Any]							      =				hidden.new_empty((head_logit.size(0							), self.n_token)							)
												else:
																	UpperCamelCase__     :Any							      =				torch.zeros_like(UpperCamelCase_					,     dtype=hidden.dtype					,     device=hidden.device							)

												UpperCamelCase__     :Any							      =				0
												UpperCamelCase__     :str							      =				[0] + self.cutoffs
												for i in range(len(UpperCamelCase_							) - 1							):
																	UpperCamelCase__      ,       UpperCamelCase__     :Optional[int]							      =				cutoff_values[i], cutoff_values[i + 1]

																	if labels is not None:
																						UpperCamelCase__     :Any							      =				(labels >= l_idx) & (labels < r_idx)
																						UpperCamelCase__     :int							      =				mask_i.nonzero().squeeze()

																						if indices_i.numel() == 0:
																											continue

																						UpperCamelCase__     :Tuple							      =				labels.index_select(0					,     UpperCamelCase_							) - l_idx
																						UpperCamelCase__     :str							      =				head_logprob.index_select(0					,     UpperCamelCase_							)
																						UpperCamelCase__     :int							      =				hidden.index_select(0					,     UpperCamelCase_							)
																	else:
																						UpperCamelCase__     :Dict							      =				hidden

																	if i == 0:
																						if labels is not None:
																											UpperCamelCase__     :Any							      =				head_logprob_i.gather(1					,     target_i[:, None]							).squeeze(1							)
																						else:
																											UpperCamelCase__     :List[str]							      =				head_logprob[:, : self.cutoffs[0]]
																	else:
																						UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Dict							      =				weights[i], biases[i], self.out_projs[i]

																						UpperCamelCase__     :str							      =				self._compute_logit(UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)
																						UpperCamelCase__     :Union[str, Any]							      =				nn.functional.log_softmax(UpperCamelCase_					,     dim=1							)
																						UpperCamelCase__     :Any							      =				self.cutoffs[0] + i - 1  # No probability for the head cluster
																						if labels is not None:
																											UpperCamelCase__     :List[Any]							      =				head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																											    1					,     target_i[:, None]							).squeeze(1							)
																						else:
																											UpperCamelCase__     :List[str]							      =				head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																											UpperCamelCase__     :Dict							      =				logprob_i

																	if labels is not None:
																						if (hasattr(self					,     '''keep_order'''							) and self.keep_order) or keep_order:
																											out.index_copy_(0					,     UpperCamelCase_					,     -logprob_i							)
																						else:
																											out[offset : offset + logprob_i.size(0							)].copy_(-logprob_i							)
																						offset += logprob_i.size(0							)

							return out




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if self.n_clusters == 0:
												UpperCamelCase__     :Optional[Any]							      =				self._compute_logit(UpperCamelCase_					,     self.out_layers[0].weight					,     self.out_layers[0].bias					,     self.out_projs[0]							)
												return nn.functional.log_softmax(UpperCamelCase_					,     dim=-1							)
							else:
												# construct weights and biases
												UpperCamelCase__      ,       UpperCamelCase__     :List[Any]							      =				[], []
												for i in range(len(self.cutoffs							)							):
																	if self.div_val == 1:
																						UpperCamelCase__      ,       UpperCamelCase__     :Tuple							      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						UpperCamelCase__     :Union[str, Any]							      =				self.out_layers[0].weight[l_idx:r_idx]
																						UpperCamelCase__     :str							      =				self.out_layers[0].bias[l_idx:r_idx]
																	else:
																						UpperCamelCase__     :List[Any]							      =				self.out_layers[i].weight
																						UpperCamelCase__     :List[Any]							      =				self.out_layers[i].bias

																	if i == 0:
																						UpperCamelCase__     :Dict							      =				torch.cat([weight_i, self.cluster_weight]					,     dim=0							)
																						UpperCamelCase__     :Optional[Any]							      =				torch.cat([bias_i, self.cluster_bias]					,     dim=0							)

																	weights.append(UpperCamelCase_							)
																	biases.append(UpperCamelCase_							)

												UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :str							      =				weights[0], biases[0], self.out_projs[0]
												UpperCamelCase__     :Optional[int]							      =				self._compute_logit(UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)

												UpperCamelCase__     :int							      =				hidden.new_empty((head_logit.size(0							), self.n_token)							)
												UpperCamelCase__     :Union[str, Any]							      =				nn.functional.log_softmax(UpperCamelCase_					,     dim=1							)

												UpperCamelCase__     :int							      =				[0] + self.cutoffs
												for i in range(len(UpperCamelCase_							) - 1							):
																	UpperCamelCase__      ,       UpperCamelCase__     :Union[str, Any]							      =				cutoff_values[i], cutoff_values[i + 1]

																	if i == 0:
																						UpperCamelCase__     :str							      =				head_logprob[:, : self.cutoffs[0]]
																	else:
																						UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Dict							      =				weights[i], biases[i], self.out_projs[i]

																						UpperCamelCase__     :List[str]							      =				self._compute_logit(UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							)
																						UpperCamelCase__     :Tuple							      =				nn.functional.log_softmax(UpperCamelCase_					,     dim=1							)

																						UpperCamelCase__     :Optional[Any]							      =				head_logprob[:, -i] + tail_logprob_i
																						UpperCamelCase__     :Union[str, Any]							      =				logprob_i

												return out