
"""simple docstring"""





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


_a								=						logging.get_logger(__name__)

_a								=						{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

_a								=						{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

_a								=						{
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}



def  lowerCamelCase__						(     __snake_case,	__snake_case      )				->						Union[str, Any]:


					"""simple docstring"""





					with open(__snake_case,	'''r''',	encoding='''utf-8'''      ) as f:
										_UpperCamelCase									=    json.loads(f.read()      )

					_UpperCamelCase									=    collections.OrderedDict()
					_UpperCamelCase									=    collections.OrderedDict()
					_UpperCamelCase									=    collections.OrderedDict()
					with open(__snake_case,	'''r''',	encoding='''utf-8'''      ) as f:
										_UpperCamelCase									=    f.readlines()
					_UpperCamelCase									=    [[t.rstrip('''\n'''      )] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''      ).split(''','''      ) for t in token]
					for idx, b in enumerate(__snake_case      ):
										_UpperCamelCase									=    b
										_UpperCamelCase									=    idx
										for wd in b:
															_UpperCamelCase									=    idx

					return vocab, raw_vocab, ids_to_tokens, emoji






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = VOCAB_FILES_NAMES
		lowercase__         = PRETRAINED_VOCAB_FILES_MAP
		lowercase__         = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase__         = ['input_ids', 'attention_mask']



		def __init__(   self							,						__a							,						__a							,						__a="<|endoftext|>"							,						__a="<|endoftext|>"							,						__a="<|startoftext|>"							,						__a="<|endoftext|>"							,						__a=False							,						**__a							,						)			->			Optional[Any]:






							'''simple docstring'''
							super().__init__(
							    unk_token=__a							,						pad_token=__a							,						bos_token=__a							,						eos_token=__a							,						do_clean_text=__a							,						**__a							,						)
							if not os.path.isfile(__a):
												raise ValueError(
												    F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
												    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
							if not os.path.isfile(__a):
												raise ValueError(
												    F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
												    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
							_UpperCamelCase									=    do_clean_text
							_UpperCamelCase					,    _UpperCamelCase					,    _UpperCamelCase					,    _UpperCamelCase									=    load_vocab_and_emoji(__a							,						__a)
							_UpperCamelCase									=    SubWordJapaneseTokenizer(
							    vocab=self.vocab							,						ids_to_tokens=self.ids_to_tokens							,						emoji=self.emoji)



		@property
		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
							return len(self.raw_vocab)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return dict(self.raw_vocab							,						**self.added_tokens_encoder)



		def 		UpperCAmelCase				(   self							,						__a)			->			Optional[Any]:






							'''simple docstring'''
							return self.subword_tokenizer.tokenize(__a							,						clean=self.do_clean_text)



		def 		UpperCAmelCase				(   self							,						__a)			->			Optional[Any]:






							'''simple docstring'''
							return self.vocab.get(__a							,						self.vocab.get(self.unk_token))



		def 		UpperCAmelCase				(   self							,						__a)			->			str:






							'''simple docstring'''
							return self.subword_tokenizer.convert_id_to_token(__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    ''''''.join(__a).strip()
							return out_string



		def 		UpperCAmelCase				(   self							,						__a)			->			List[int]:






							'''simple docstring'''
							_UpperCamelCase									=    []
							for is_user, text in conversation.iter_texts():
												input_ids.extend(self.encode(__a							,						add_special_tokens=__a) + [self.eos_token_id])

							if len(__a) > self.model_max_length:
												_UpperCamelCase									=    input_ids[-self.model_max_length :]
							return input_ids




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			Tuple[str]:






							'''simple docstring'''
							_UpperCamelCase									=    0
							if os.path.isdir(__a):
												_UpperCamelCase									=    os.path.join(
												    __a							,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
												_UpperCamelCase									=    os.path.join(
												    __a							,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file'''])
							else:
												_UpperCamelCase									=    (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
												)
												_UpperCamelCase									=    (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
												)
							with open(__a							,						'''w'''							,						encoding='''utf-8''') as writer:
												for token_index, token in self.ids_to_tokens.items():
																	if index != token_index:
																						logger.warning(
																						    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																						    ''' Please check that the vocabulary is not corrupted!''')
																						_UpperCamelCase									=    token_index
																	writer.write(''','''.join(__a) + '''\n''')
																	index += 1
							with open(__a							,						'''w'''							,						encoding='''utf-8''') as writer:
												json.dump(self.emoji							,						__a)
							return vocab_file, emoji_file






class        _UpperCAmelCase(       lowerCamelCase      ):



		def __init__(   self							,						__a							,						__a							,						__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    vocab  # same as swe
							_UpperCamelCase									=    ids_to_tokens  # same as bpe
							_UpperCamelCase									=    emoji
							_UpperCamelCase									=    np.max([len(__a) for w in self.vocab.keys()])
							_UpperCamelCase									=    re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)''')
							_UpperCamelCase									=    re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*''')
							_UpperCamelCase									=    re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}''')
							_UpperCamelCase									=    re.compile(
							    R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
							_UpperCamelCase									=    re.compile(
							    R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
							_UpperCamelCase									=    re.compile(
							    R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*''')
							_UpperCamelCase									=    '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
							_UpperCamelCase									=    '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
							_UpperCamelCase									=    str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks})



		def __len__(   self)			->			Optional[Any]:






							'''simple docstring'''
							return len(self.ids_to_tokens)



		def 		UpperCAmelCase				(   self							,						__a)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    self.content_repattera.sub('''<URL>'''							,						__a)
							_UpperCamelCase									=    self.content_repattera.sub('''<EMAIL>'''							,						__a)
							_UpperCamelCase									=    self.content_repattera.sub('''<TEL>'''							,						__a)
							_UpperCamelCase									=    self.content_repattera.sub('''<DATE>'''							,						__a)
							_UpperCamelCase									=    self.content_repattera.sub('''<DATE>'''							,						__a)
							_UpperCamelCase									=    self.content_repattera.sub('''<PRICE>'''							,						__a)
							_UpperCamelCase									=    content.translate(self.content_transa)
							while "<BLOCK><BLOCK>" in content:
												_UpperCamelCase									=    content.replace('''<BLOCK><BLOCK>'''							,						'''<BLOCK>''')
							return content



		def 		UpperCAmelCase				(   self							,						__a							,						__a=False)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    text.replace(''' '''							,						'''<SP>''')
							_UpperCamelCase									=    text.replace('''　'''							,						'''<SP>''')
							_UpperCamelCase									=    text.replace('''\r\n'''							,						'''<BR>''')
							_UpperCamelCase									=    text.replace('''\n'''							,						'''<BR>''')
							_UpperCamelCase									=    text.replace('''\r'''							,						'''<BR>''')
							_UpperCamelCase									=    text.replace('''\t'''							,						'''<TAB>''')
							_UpperCamelCase									=    text.replace('''—'''							,						'''ー''')
							_UpperCamelCase									=    text.replace('''−'''							,						'''ー''')
							for k, v in self.emoji["emoji"].items():
												if k in text:
																	_UpperCamelCase									=    text.replace(__a							,						__a)
							if clean:
												_UpperCamelCase									=    self.clean_text(__a)

							def check_simbol(__a):
												_UpperCamelCase									=    x.encode()
												if len(__a) == 1 and len(__a) == 2:
																	_UpperCamelCase									=    (int(e[0]) << 8) + int(e[1])
																	if (
																	    (c >= 0XC2A1 and c <= 0XC2BF)
																	    or (c >= 0XC780 and c <= 0XC783)
																	    or (c >= 0XCAB9 and c <= 0XCBBF)
																	    or (c >= 0XCC80 and c <= 0XCDA2)
																	):
																						return True
												return False

							def checkuae(__a):
												_UpperCamelCase									=    x.encode()
												if len(__a) == 1 and len(__a) == 3:
																	_UpperCamelCase									=    (int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
																	if c >= 0XE28080 and c <= 0XE2B07F:
																						return True
												return False

							_UpperCamelCase									=    0
							_UpperCamelCase									=    []
							while pos < len(__a):
												_UpperCamelCase									=    min(len(__a)							,						pos + self.maxlen + 1) if text[pos] == '''<''' else pos + 3
												_UpperCamelCase									=    []  # (token_id, token, pos)
												for e in range(__a							,						__a							,						-1):
																	_UpperCamelCase									=    text[pos:e]
																	if wd in self.vocab:
																						if wd[0] == "<" and len(__a) > 2:
																											_UpperCamelCase									=    [(self.vocab[wd], wd, e)]
																											break
																						else:
																											candidates.append((self.vocab[wd], wd, e))
												if len(__a) > 0:
																	# the smallest token_id is adopted
																	_UpperCamelCase					,    _UpperCamelCase					,    _UpperCamelCase									=    sorted(__a							,						key=lambda __a: x[0])[0]
																	result.append(__a)
																	_UpperCamelCase									=    e
												else:
																	_UpperCamelCase									=    pos + 1
																	_UpperCamelCase									=    text[pos:end]
																	if check_simbol(__a):
																						result.append('''<KIGOU>''')
																	elif checkuae(__a):
																						result.append('''<U2000U2BFF>''')
																	else:
																						for i in wd.encode('''utf-8'''):
																											result.append('''<|byte%d|>''' % i)
																	_UpperCamelCase									=    end
							return result




		def 		UpperCAmelCase				(   self							,						__a							,						__a="\n")			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    []
							_UpperCamelCase									=    []
							_UpperCamelCase									=    self.ids_to_tokens[index][0]
							if word[:6] == "<|byte" and word[-2:] == "|>":
												byte_tokens.append(int(word[6:-2]))
							else:
												if len(__a) > 0:
																	words.append(bytearray(__a).decode('''utf-8'''							,						errors='''replace'''))
																	_UpperCamelCase									=    []
												if word[:7] == "<|emoji" and word[-2:] == "|>":
																	words.append(self.emoji['''emoji_inv'''][word])
												elif word == "<SP>":
																	words.append(''' ''')
												elif word == "<BR>":
																	words.append(__a)
												elif word == "<TAB>":
																	words.append('''\t''')
												elif word == "<BLOCK>":
																	words.append('''▀''')
												elif word == "<KIGOU>":
																	words.append('''ǀ''')
												elif word == "<U2000U2BFF>":
																	words.append('''‖''')
												else:
																	words.append(__a)
							if len(__a) > 0:
												words.append(bytearray(__a).decode('''utf-8'''							,						errors='''replace'''))
							_UpperCamelCase									=    ''''''.join(__a)
							return text








"""simple docstring"""





import copy
from collections import OrderedDict
from typing import Dict, Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


_a								=						logging.get_logger(__name__)

_a								=						{
    """facebook/detr-resnet-50""": """https://huggingface.co/facebook/detr-resnet-50/resolve/main/config.json""",
    # See all DETR models at https://huggingface.co/models?filter=detr
}






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = 'detr'
		lowercase__         = ['past_key_values']
		lowercase__         = {
		    'hidden_size': 'd_model',
		    'num_attention_heads': 'encoder_attention_heads',
		}



		def __init__(   self							,						__a=True							,						__a=None							,						__a=3							,						__a=1_00							,						__a=6							,						__a=20_48							,						__a=8							,						__a=6							,						__a=20_48							,						__a=8							,						__a=0.0							,						__a=0.0							,						__a=True							,						__a="relu"							,						__a=2_56							,						__a=0.1							,						__a=0.0							,						__a=0.0							,						__a=0.02							,						__a=1.0							,						__a=False							,						__a="sine"							,						__a="resnet50"							,						__a=True							,						__a=False							,						__a=1							,						__a=5							,						__a=2							,						__a=1							,						__a=1							,						__a=5							,						__a=2							,						__a=0.1							,						**__a							,						)			->			int:






							'''simple docstring'''
							if backbone_config is not None and use_timm_backbone:
												raise ValueError('''You can\'t specify both `backbone_config` and `use_timm_backbone`.''')

							if not use_timm_backbone:
												if backbone_config is None:
																	logger.info('''`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.''')
																	_UpperCamelCase									=    CONFIG_MAPPING['''resnet'''](out_features=['''stage4'''])
												elif isinstance(__a							,						__a):
																	_UpperCamelCase									=    backbone_config.get('''model_type''')
																	_UpperCamelCase									=    CONFIG_MAPPING[backbone_model_type]
																	_UpperCamelCase									=    config_class.from_dict(__a)
												# set timm attributes to None
												_UpperCamelCase					,    _UpperCamelCase					,    _UpperCamelCase									=    None, None, None

							_UpperCamelCase									=    use_timm_backbone
							_UpperCamelCase									=    backbone_config
							_UpperCamelCase									=    num_channels
							_UpperCamelCase									=    num_queries
							_UpperCamelCase									=    d_model
							_UpperCamelCase									=    encoder_ffn_dim
							_UpperCamelCase									=    encoder_layers
							_UpperCamelCase									=    encoder_attention_heads
							_UpperCamelCase									=    decoder_ffn_dim
							_UpperCamelCase									=    decoder_layers
							_UpperCamelCase									=    decoder_attention_heads
							_UpperCamelCase									=    dropout
							_UpperCamelCase									=    attention_dropout
							_UpperCamelCase									=    activation_dropout
							_UpperCamelCase									=    activation_function
							_UpperCamelCase									=    init_std
							_UpperCamelCase									=    init_xavier_std
							_UpperCamelCase									=    encoder_layerdrop
							_UpperCamelCase									=    decoder_layerdrop
							_UpperCamelCase									=    encoder_layers
							_UpperCamelCase									=    auxiliary_loss
							_UpperCamelCase									=    position_embedding_type
							_UpperCamelCase									=    backbone
							_UpperCamelCase									=    use_pretrained_backbone
							_UpperCamelCase									=    dilation
							# Hungarian matcher
							_UpperCamelCase									=    class_cost
							_UpperCamelCase									=    bbox_cost
							_UpperCamelCase									=    giou_cost
							# Loss coefficients
							_UpperCamelCase									=    mask_loss_coefficient
							_UpperCamelCase									=    dice_loss_coefficient
							_UpperCamelCase									=    bbox_loss_coefficient
							_UpperCamelCase									=    giou_loss_coefficient
							_UpperCamelCase									=    eos_coefficient
							super().__init__(is_encoder_decoder=__a							,						**__a)



		@property
		def 		UpperCAmelCase				(   self)			->			int:






							'''simple docstring'''
							return self.encoder_attention_heads



		@property
		def 		UpperCAmelCase				(   self)			->			int:






							'''simple docstring'''
							return self.d_model



		@classmethod
		def 		UpperCAmelCase				(   cls							,						__a							,						**__a)			->			int:






							'''simple docstring'''
							return cls(backbone_config=__a							,						**__a)




		def 		UpperCAmelCase				(   self)			->			Dict[str, any]:






							'''simple docstring'''
							_UpperCamelCase									=    copy.deepcopy(self.__dict__)
							if output["backbone_config"] is not None:
												_UpperCamelCase									=    self.backbone_config.to_dict()
							_UpperCamelCase									=    self.__class__.model_type
							return output






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = version.parse('1.11'      )



		@property
		def 		UpperCAmelCase				(   self)			->			Mapping[str, Mapping[int, str]]:






							'''simple docstring'''
							return OrderedDict(
							    [
							        ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
							        ('''pixel_mask''', {0: '''batch'''}),
							    ])



		@property
		def 		UpperCAmelCase				(   self)			->			float:






							'''simple docstring'''
							return 1e-5




		@property
		def 		UpperCAmelCase				(   self)			->			int:






							'''simple docstring'''
							return 12






