


"""simple docstring"""



import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


lowercase_							=							logging.get_logger(__name__)

lowercase_							=							{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

lowercase_							=							{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

lowercase_							=							{
    'abeja/gpt-neox-japanese-2.7b': 20_48,
}
def      UpperCAmelCase     (       _lowercase :   Tuple			,   _lowercase :   Tuple	)						-> Optional[int]:


	"""simple docstring"""




	with open(_lowercase			,   '''r'''			,   encoding='''utf-8'''	) as f:
		lowerCAmelCase_      							=			json.loads(f.read()	)

	lowerCAmelCase_      							=			collections.OrderedDict()
	lowerCAmelCase_      							=			collections.OrderedDict()
	lowerCAmelCase_      							=			collections.OrderedDict()
	with open(_lowercase			,   '''r'''			,   encoding='''utf-8'''	) as f:
		lowerCAmelCase_      							=			f.readlines()
	lowerCAmelCase_      							=			[[t.rstrip('''\n'''	)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''	).split(''','''	) for t in token]
	for idx, b in enumerate(_lowercase	):
		lowerCAmelCase_      							=			b
		lowerCAmelCase_      							=			idx
		for wd in b:
			lowerCAmelCase_      							=			idx

	return vocab, raw_vocab, ids_to_tokens, emoji







class    __a						( __snake_case							):
				lowerCamelCase     :       Dict			       =VOCAB_FILES_NAMES
				lowerCamelCase     :       str			       =PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase     :       Tuple			       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase     :       List[Any]			       =['input_ids', 'attention_mask']







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase="<|endoftext|>"      ,	UpperCAmelCase="<|endoftext|>"      ,	UpperCAmelCase="<|startoftext|>"      ,	UpperCAmelCase="<|endoftext|>"      ,	UpperCAmelCase=False      ,	**UpperCAmelCase      ,	):

					'''simple docstring'''






					super().__init__(
					    unk_token=UpperCAmelCase      ,	pad_token=UpperCAmelCase      ,	bos_token=UpperCAmelCase      ,	eos_token=UpperCAmelCase      ,	do_clean_text=UpperCAmelCase      ,	**UpperCAmelCase      ,	)
					if not os.path.isfile(UpperCAmelCase   ):
						raise ValueError(
						    F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
						    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''   )
					if not os.path.isfile(UpperCAmelCase   ):
						raise ValueError(
						    F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
						    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''   )
					lowerCAmelCase_      							=			do_clean_text
					lowerCAmelCase_							,	lowerCAmelCase_							,	lowerCAmelCase_							,	lowerCAmelCase_      							=			load_vocab_and_emoji(UpperCAmelCase      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			SubWordJapaneseTokenizer(
					    vocab=self.vocab      ,	ids_to_tokens=self.ids_to_tokens      ,	emoji=self.emoji   )







				@property
				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					return len(self.raw_vocab   )







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					return dict(self.raw_vocab      ,	**self.added_tokens_encoder   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					return self.subword_tokenizer.tokenize(UpperCAmelCase      ,	clean=self.do_clean_text   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					return self.vocab.get(UpperCAmelCase      ,	self.vocab.get(self.unk_token   )   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					return self.subword_tokenizer.convert_id_to_token(UpperCAmelCase   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			''''''.join(UpperCAmelCase   ).strip()
					return out_string







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			[]
					for is_user, text in conversation.iter_texts():
						input_ids.extend(self.encode(UpperCAmelCase      ,	add_special_tokens=UpperCAmelCase   ) + [self.eos_token_id]   )

					if len(UpperCAmelCase   ) > self.model_max_length:
						lowerCAmelCase_      							=			input_ids[-self.model_max_length :]
					return input_ids







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			0
					if os.path.isdir(UpperCAmelCase   ):
						lowerCAmelCase_      							=			os.path.join(
						    UpperCAmelCase      ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
						lowerCAmelCase_      							=			os.path.join(
						    UpperCAmelCase      ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']   )
					else:
						lowerCAmelCase_      							=			(
						    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
						)
						lowerCAmelCase_      							=			(
						    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
						)
					with open(UpperCAmelCase      ,	'''w'''      ,	encoding='''utf-8'''   ) as writer:
						for token_index, token in self.ids_to_tokens.items():
							if index != token_index:
								logger.warning(
								    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
								    ''' Please check that the vocabulary is not corrupted!'''   )
								lowerCAmelCase_      							=			token_index
							writer.write(''','''.join(UpperCAmelCase   ) + '''\n'''   )
							index += 1
					with open(UpperCAmelCase      ,	'''w'''      ,	encoding='''utf-8'''   ) as writer:
						json.dump(self.emoji      ,	UpperCAmelCase   )
					return vocab_file, emoji_file







class    __a						( __snake_case							):







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			vocab  # same as swe
					lowerCAmelCase_      							=			ids_to_tokens  # same as bpe
					lowerCAmelCase_      							=			emoji
					lowerCAmelCase_      							=			np.max([len(UpperCAmelCase   ) for w in self.vocab.keys()]   )
					lowerCAmelCase_      							=			re.compile(r'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''   )
					lowerCAmelCase_      							=			re.compile(r'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''   )
					lowerCAmelCase_      							=			re.compile(r'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''   )
					lowerCAmelCase_      							=			re.compile(
					    r'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''   )
					lowerCAmelCase_      							=			re.compile(
					    r'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''   )
					lowerCAmelCase_      							=			re.compile(
					    r'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''   )
					lowerCAmelCase_      							=			'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
					lowerCAmelCase_      							=			'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
					lowerCAmelCase_      							=			str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}   )







				def __len__( self   ):

					'''simple docstring'''






					return len(self.ids_to_tokens   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.content_repattera.sub('''<URL>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			self.content_repattera.sub('''<EMAIL>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			self.content_repattera.sub('''<TEL>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			self.content_repattera.sub('''<DATE>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			self.content_repattera.sub('''<DATE>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			self.content_repattera.sub('''<PRICE>'''      ,	UpperCAmelCase   )
					lowerCAmelCase_      							=			content.translate(self.content_transa   )
					while "<BLOCK><BLOCK>" in content:
						lowerCAmelCase_      							=			content.replace('''<BLOCK><BLOCK>'''      ,	'''<BLOCK>'''   )
					return content







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase=False   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			text.replace(''' '''      ,	'''<SP>'''   )
					lowerCAmelCase_      							=			text.replace('''　'''      ,	'''<SP>'''   )
					lowerCAmelCase_      							=			text.replace('''\r\n'''      ,	'''<BR>'''   )
					lowerCAmelCase_      							=			text.replace('''\n'''      ,	'''<BR>'''   )
					lowerCAmelCase_      							=			text.replace('''\r'''      ,	'''<BR>'''   )
					lowerCAmelCase_      							=			text.replace('''\t'''      ,	'''<TAB>'''   )
					lowerCAmelCase_      							=			text.replace('''—'''      ,	'''ー'''   )
					lowerCAmelCase_      							=			text.replace('''−'''      ,	'''ー'''   )
					for k, v in self.emoji["emoji"].items():
						if k in text:
							lowerCAmelCase_      							=			text.replace(UpperCAmelCase      ,	UpperCAmelCase   )
					if clean:
						lowerCAmelCase_      							=			self.clean_text(UpperCAmelCase   )

					def check_simbol(UpperCAmelCase   ):
						lowerCAmelCase_      							=			x.encode()
						if len(UpperCAmelCase   ) == 1 and len(UpperCAmelCase   ) == 2:
							lowerCAmelCase_      							=			(int(e[0]   ) << 8) + int(e[1]   )
							if (
							    (c >= 0xc_2a1 and c <= 0xc_2bf)
							    or (c >= 0xc_780 and c <= 0xc_783)
							    or (c >= 0xc_ab9 and c <= 0xc_bbf)
							    or (c >= 0xc_c80 and c <= 0xc_da2)
							):
								return True
						return False

					def checkuae(UpperCAmelCase   ):
						lowerCAmelCase_      							=			x.encode()
						if len(UpperCAmelCase   ) == 1 and len(UpperCAmelCase   ) == 3:
							lowerCAmelCase_      							=			(int(e[0]   ) << 16) + (int(e[1]   ) << 8) + int(e[2]   )
							if c >= 0xe28_080 and c <= 0xe2b_07f:
								return True
						return False

					lowerCAmelCase_      							=			0
					lowerCAmelCase_      							=			[]
					while pos < len(UpperCAmelCase   ):
						lowerCAmelCase_      							=			min(len(UpperCAmelCase   )      ,	pos + self.maxlen + 1   ) if text[pos] == '''<''' else pos + 3
						lowerCAmelCase_      							=			[]  # (token_id, token, pos)
						for e in range(UpperCAmelCase      ,	UpperCAmelCase      ,	-1   ):
							lowerCAmelCase_      							=			text[pos:e]
							if wd in self.vocab:
								if wd[0] == "<" and len(UpperCAmelCase   ) > 2:
									lowerCAmelCase_      							=			[(self.vocab[wd], wd, e)]
									break
								else:
									candidates.append((self.vocab[wd], wd, e)   )
						if len(UpperCAmelCase   ) > 0:
							# the smallest token_id is adopted
							lowerCAmelCase_							,	lowerCAmelCase_							,	lowerCAmelCase_      							=			sorted(UpperCAmelCase      ,	key=lambda UpperCAmelCase   : x[0]   )[0]
							result.append(UpperCAmelCase   )
							lowerCAmelCase_      							=			e
						else:
							lowerCAmelCase_      							=			pos + 1
							lowerCAmelCase_      							=			text[pos:end]
							if check_simbol(UpperCAmelCase   ):
								result.append('''<KIGOU>'''   )
							elif checkuae(UpperCAmelCase   ):
								result.append('''<U2000U2BFF>'''   )
							else:
								for i in wd.encode('''utf-8'''   ):
									result.append('''<|byte%d|>''' % i   )
							lowerCAmelCase_      							=			end
					return result







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase="\n"   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			[]
					lowerCAmelCase_      							=			[]
					lowerCAmelCase_      							=			self.ids_to_tokens[index][0]
					if word[:6] == "<|byte" and word[-2:] == "|>":
						byte_tokens.append(int(word[6:-2]   )   )
					else:
						if len(UpperCAmelCase   ) > 0:
							words.append(bytearray(UpperCAmelCase   ).decode('''utf-8'''      ,	errors='''replace'''   )   )
							lowerCAmelCase_      							=			[]
						if word[:7] == "<|emoji" and word[-2:] == "|>":
							words.append(self.emoji['''emoji_inv'''][word]   )
						elif word == "<SP>":
							words.append(''' '''   )
						elif word == "<BR>":
							words.append(UpperCAmelCase   )
						elif word == "<TAB>":
							words.append('''\t'''   )
						elif word == "<BLOCK>":
							words.append('''▀'''   )
						elif word == "<KIGOU>":
							words.append('''ǀ'''   )
						elif word == "<U2000U2BFF>":
							words.append('''‖'''   )
						else:
							words.append(UpperCAmelCase   )
					if len(UpperCAmelCase   ) > 0:
						words.append(bytearray(UpperCAmelCase   ).decode('''utf-8'''      ,	errors='''replace'''   )   )
					lowerCAmelCase_      							=			''''''.join(UpperCAmelCase   )
					return text



"""simple docstring"""



from typing import Optional, Tuple

import jax
import jax.numpy as jnp
from flax import linen as nn
from flax.core.frozen_dict import FrozenDict
from transformers import CLIPConfig, FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPVisionModule
def      UpperCAmelCase     (       _lowercase :   str			,   _lowercase :   int			,   _lowercase :   Optional[int]=1E-12	)						-> List[str]:


	"""simple docstring"""




	lowerCAmelCase_      							=			jnp.divide(emb_a.T			,   jnp.clip(jnp.linalg.norm(_lowercase			,   axis=1	)			,   a_min=_lowercase	)	).T
	lowerCAmelCase_      							=			jnp.divide(emb_a.T			,   jnp.clip(jnp.linalg.norm(_lowercase			,   axis=1	)			,   a_min=_lowercase	)	).T
	return jnp.matmul(_lowercase			,   norm_emb_a.T	)







class    __a						( nn.Module							):
				lowerCamelCase     :       CLIPConfig
				lowerCamelCase     :       jnp.dtype			       =jnp.floataa







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			FlaxCLIPVisionModule(self.config.vision_config   )
					lowerCAmelCase_      							=			nn.Dense(self.config.projection_dim      ,	use_bias=UpperCAmelCase      ,	dtype=self.dtype   )

					lowerCAmelCase_      							=			self.param('''concept_embeds'''      ,	jax.nn.initializers.ones      ,	(17, self.config.projection_dim)   )
					lowerCAmelCase_      							=			self.param(
					    '''special_care_embeds'''      ,	jax.nn.initializers.ones      ,	(3, self.config.projection_dim)   )

					lowerCAmelCase_      							=			self.param('''concept_embeds_weights'''      ,	jax.nn.initializers.ones      ,	(17,)   )
					lowerCAmelCase_      							=			self.param('''special_care_embeds_weights'''      ,	jax.nn.initializers.ones      ,	(3,)   )







				def __call__( self      ,	UpperCAmelCase   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			self.vision_model(UpperCAmelCase   )[1]
					lowerCAmelCase_      							=			self.visual_projection(UpperCAmelCase   )

					lowerCAmelCase_      							=			jax_cosine_distance(UpperCAmelCase      ,	self.special_care_embeds   )
					lowerCAmelCase_      							=			jax_cosine_distance(UpperCAmelCase      ,	self.concept_embeds   )

					# increase this value to create a stronger `nfsw` filter
					# at the cost of increasing the possibility of filtering benign image inputs
					lowerCAmelCase_      							=			0.0

					lowerCAmelCase_      							=			special_cos_dist - self.special_care_embeds_weights[None, :] + adjustment
					lowerCAmelCase_      							=			jnp.round(UpperCAmelCase      ,	3   )
					lowerCAmelCase_      							=			jnp.any(special_scores > 0      ,	axis=1      ,	keepdims=UpperCAmelCase   )
					# Use a lower threshold if an image has any special care concept
					lowerCAmelCase_      							=			is_special_care * 0.0_1

					lowerCAmelCase_      							=			cos_dist - self.concept_embeds_weights[None, :] + special_adjustment
					lowerCAmelCase_      							=			jnp.round(UpperCAmelCase      ,	3   )
					lowerCAmelCase_      							=			jnp.any(concept_scores > 0      ,	axis=1   )

					return has_nsfw_concepts







class    __a						( __snake_case							):
				lowerCamelCase     :       Optional[Any]			       =CLIPConfig
				lowerCamelCase     :       List[str]			       ='clip_input'
				lowerCamelCase     :       int			       =FlaxStableDiffusionSafetyCheckerModule







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None      ,	UpperCAmelCase = 0      ,	UpperCAmelCase = jnp.floataa      ,	UpperCAmelCase = True      ,	**UpperCAmelCase      ,	):

					'''simple docstring'''






					if input_shape is None:
						lowerCAmelCase_      							=			(1, 224, 224, 3)
					lowerCAmelCase_      							=			self.module_class(config=UpperCAmelCase      ,	dtype=UpperCAmelCase      ,	**UpperCAmelCase   )
					super().__init__(UpperCAmelCase      ,	UpperCAmelCase      ,	input_shape=UpperCAmelCase      ,	seed=UpperCAmelCase      ,	dtype=UpperCAmelCase      ,	_do_init=_do_init   )







				def      lowerCamelCase_     ( self      ,	UpperCAmelCase      ,	UpperCAmelCase      ,	UpperCAmelCase = None   ):

					'''simple docstring'''






					lowerCAmelCase_      							=			jax.random.normal(UpperCAmelCase      ,	UpperCAmelCase   )

					lowerCAmelCase_							,	lowerCAmelCase_      							=			jax.random.split(UpperCAmelCase   )
					lowerCAmelCase_      							=			{'''params''': params_rng, '''dropout''': dropout_rng}

					lowerCAmelCase_      							=			self.module.init(UpperCAmelCase      ,	UpperCAmelCase   )['''params''']

					return random_params







				def __call__( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None      ,	):

					'''simple docstring'''






					lowerCAmelCase_      							=			jnp.transpose(UpperCAmelCase      ,	(0, 2, 3, 1)   )

					return self.module.apply(
					    {'''params''': params or self.params}      ,	jnp.array(UpperCAmelCase      ,	dtype=jnp.floataa   )      ,	rngs={}      ,	)