




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


_lowerCamelCase					:     str					  =  logging.get_logger(__name__)

_lowerCamelCase					:     int					  =  {'vocab_file': 'vocab.txt'}

_lowerCamelCase					:     Dict					  =  {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

_lowerCamelCase					:     Tuple					  =  {
    'tau/splinter-base': 5_1_2,
    'tau/splinter-base-qass': 5_1_2,
    'tau/splinter-large': 5_1_2,
    'tau/splinter-large-qass': 5_1_2,
}

_lowerCamelCase					:     Union[str, Any]					  =  {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}
class 				__snake_case     (_A						):
				lowerCAmelCase__			=   VOCAB_FILES_NAMES
				lowerCAmelCase__			=   PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__			=   PRETRAINED_INIT_CONFIGURATION
				lowerCAmelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__			=   SplinterTokenizer



				def __init__(					self					:						Optional[Any]						,		_UpperCAmelCase					:						Optional[Any]=None						,		_UpperCAmelCase					:						int=None						,		_UpperCAmelCase					:						List[Any]=True						,		_UpperCAmelCase					:						Tuple="[UNK]"						,		_UpperCAmelCase					:						List[Any]="[SEP]"						,		_UpperCAmelCase					:						Any="[PAD]"						,		_UpperCAmelCase					:						Dict="[CLS]"						,		_UpperCAmelCase					:						Tuple="[MASK]"						,		_UpperCAmelCase					:						Tuple="[QUESTION]"						,		_UpperCAmelCase					:						List[Any]=True						,		_UpperCAmelCase					:						Optional[Any]=None						,		**_UpperCAmelCase					:						Union[str, Any]						,		)     ->			Tuple:





								'''simple docstring'''




								super().__init__(
								    __SCREAMING_SNAKE_CASE						,		tokenizer_file=__SCREAMING_SNAKE_CASE						,		do_lower_case=__SCREAMING_SNAKE_CASE						,		unk_token=__SCREAMING_SNAKE_CASE						,		sep_token=__SCREAMING_SNAKE_CASE						,		pad_token=__SCREAMING_SNAKE_CASE						,		cls_token=__SCREAMING_SNAKE_CASE						,		mask_token=__SCREAMING_SNAKE_CASE						,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE						,		strip_accents=__SCREAMING_SNAKE_CASE						,		additional_special_tokens=(question_token,)						,		**__SCREAMING_SNAKE_CASE						,		)

								_lowerCAmelCase			:			Union[str, Any]        =					json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
								if (
								    pre_tok_state.get("""lowercase"""						,		__SCREAMING_SNAKE_CASE					) != do_lower_case
								    or pre_tok_state.get("""strip_accents"""						,		__SCREAMING_SNAKE_CASE					) != strip_accents
								):
												_lowerCAmelCase			:			Optional[int]        =					getattr(__SCREAMING_SNAKE_CASE						,		pre_tok_state.pop("""type"""					)					)
												_lowerCAmelCase			:			Any        =					do_lower_case
												_lowerCAmelCase			:			Optional[int]        =					strip_accents
												_lowerCAmelCase			:			Dict        =					pre_tok_class(**__SCREAMING_SNAKE_CASE					)

								_lowerCAmelCase			:			List[str]        =					do_lower_case



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]					)     ->			Any:





								'''simple docstring'''




								return self.convert_tokens_to_ids(self.question_token					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			str:





								'''simple docstring'''




								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								_lowerCAmelCase			:			str        =					[self.cls_token_id]
								_lowerCAmelCase			:			Dict        =					[self.sep_token_id]
								_lowerCAmelCase			:			Optional[Any]        =					[self.question_token_id] + [self.convert_tokens_to_ids("""."""					)]
								if self.padding_side == "right":
												# Input is question-then-context
												return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
								else:
												# Input is context-then-question
												return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					[self.sep_token_id]
								_lowerCAmelCase			:			Any        =					[self.cls_token_id]
								_lowerCAmelCase			:			Union[str, Any]        =					[self.question_token_id] + [self.convert_tokens_to_ids("""."""					)]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep					) * [0]

								if self.padding_side == "right":
												# Input is question-then-context
												return len(cls + token_ids_a + question_suffix + sep					) * [0] + len(token_ids_a + sep					) * [1]
								else:
												# Input is context-then-question
												return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + question_suffix + sep					) * [1]



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[str] = None					)     ->			Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					self._tokenizer.model.save(__SCREAMING_SNAKE_CASE						,		name=__SCREAMING_SNAKE_CASE					)
								return tuple(__SCREAMING_SNAKE_CASE					)




"""simple docstring"""




import argparse
import collections
import importlib.util
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
__A     :    List[str]    =		'src/diffusers'
__A     :    Optional[int]    =		'docs/source/en'
__A     :    Union[str, Any]    =		'.'
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					with open(lowercase__			,							"r"			,							encoding="utf-8"			,							newline="\n"      ) as f:
										A			=					f.readlines()
					# Find the start prompt.
					A			=					0
					while not lines[start_index].startswith(lowercase__      ):
										start_index += 1
					start_index += 1

					A			=					start_index
					while not lines[end_index].startswith(lowercase__      ):
										end_index += 1
					end_index -= 1

					while len(lines[start_index]      ) <= 1:
										start_index += 1
					while len(lines[end_index]      ) <= 1:
										end_index -= 1
					end_index += 1
					return "".join(lines[start_index:end_index]      ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
__A     :    Any    =		'Model|Encoder|Decoder|ForConditionalGeneration'
# Regexes that match TF/Flax/PT model names.
__A     :    Union[str, Any]    =		re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
__A     :    int    =		re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__A     :    Any    =		re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the diffusers module imported is the one in the repo.
__A     :    Optional[int]    =		importlib.util.spec_from_file_location(
    'diffusers',
    os.path.join(TRANSFORMERS_PATH, '__init__.py'),
    submodule_search_locations=[TRANSFORMERS_PATH],
)
__A     :    Tuple    =		spec.loader.load_module()
def    __SCREAMING_SNAKE_CASE						(						lowercase__      ):
					"""simple docstring"""
					A			=					re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"			,							lowercase__      )
					return [m.group(0      ) for m in matches]
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__      ):
					"""simple docstring"""
					A			=					2 if text == "✅" or text == "❌" else len(lowercase__      )
					A			=					(width - text_length) // 2
					A			=					width - text_length - left_indent
					return " " * left_indent + text + " " * right_indent
def    __SCREAMING_SNAKE_CASE						(						):
					"""simple docstring"""
					A			=					diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
					A			=					{
					    name: config_mapping_names[code]
					    for code, name in diffusers_module.MODEL_NAMES_MAPPING.items()
					    if code in config_mapping_names
					}
					A			=					{name: config.replace("ConfigMixin"			,							""      ) for name, config in model_name_to_config.items()}

					# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
					A			=					collections.defaultdict(lowercase__      )
					A			=					collections.defaultdict(lowercase__      )
					A			=					collections.defaultdict(lowercase__      )
					A			=					collections.defaultdict(lowercase__      )
					A			=					collections.defaultdict(lowercase__      )

					# Let's lookup through all diffusers object (once).
					for attr_name in dir(lowercase__      ):
										A			=					None
										if attr_name.endswith("Tokenizer"      ):
															A			=					slow_tokenizers
															A			=					attr_name[:-9]
										elif attr_name.endswith("TokenizerFast"      ):
															A			=					fast_tokenizers
															A			=					attr_name[:-13]
										elif _re_tf_models.match(lowercase__      ) is not None:
															A			=					tf_models
															A			=					_re_tf_models.match(lowercase__      ).groups()[0]
										elif _re_flax_models.match(lowercase__      ) is not None:
															A			=					flax_models
															A			=					_re_flax_models.match(lowercase__      ).groups()[0]
										elif _re_pt_models.match(lowercase__      ) is not None:
															A			=					pt_models
															A			=					_re_pt_models.match(lowercase__      ).groups()[0]

										if lookup_dict is not None:
															while len(lowercase__      ) > 0:
																				if attr_name in model_name_to_prefix.values():
																									A			=					True
																									break
																				# Try again after removing the last word in the name
																				A			=					"".join(camel_case_split(lowercase__      )[:-1]      )

    # Let's build that table!
					A			=					list(model_name_to_config.keys()      )
					model_names.sort(key=str.lower      )
					A			=					["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
					# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
					A			=					[len(lowercase__      ) + 2 for c in columns]
					A			=					max([len(lowercase__      ) for name in model_names]      ) + 2

					# Build the table per se
					A			=					"|" + "|".join([_center_text(lowercase__			,							lowercase__      ) for c, w in zip(lowercase__			,							lowercase__      )]      ) + "|\n"
					# Use ":-----:" format to center-aligned table cell texts
					table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]      ) + "|\n"

					A			=					{True: "✅", False: "❌"}
					for name in model_names:
										A			=					model_name_to_prefix[name]
										A			=					[
										    name,
										    check[slow_tokenizers[prefix]],
										    check[fast_tokenizers[prefix]],
										    check[pt_models[prefix]],
										    check[tf_models[prefix]],
										    check[flax_models[prefix]],
										]
										table += "|" + "|".join([_center_text(lowercase__			,							lowercase__      ) for l, w in zip(lowercase__			,							lowercase__      )]      ) + "|\n"
					return table
def    __SCREAMING_SNAKE_CASE						(						lowercase__=False      ):
					"""simple docstring"""
					A			,    A			,    A			,    A			=					_find_text_in_file(
					    filename=os.path.join(lowercase__			,							"index.mdx"      )			,							start_prompt="<!--This table is updated automatically from the auto modules"			,							end_prompt="<!-- End table-->"			,							)
					A			=					get_model_table_from_auto_modules()

					if current_table != new_table:
										if overwrite:
															with open(os.path.join(lowercase__			,							"index.mdx"      )			,							"w"			,							encoding="utf-8"			,							newline="\n"      ) as f:
																				f.writelines(lines[:start_index] + [new_table] + lines[end_index:]      )
										else:
															raise ValueError(
															    "The model table in the `index.mdx` has not been updated. Run `make fix-copies` to fix this."      )


if __name__ == "__main__":
	__A     :    Any    =		argparse.ArgumentParser()
	parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
	__A     :    str    =		parser.parse_args()

	check_model_table(args.fix_and_overwrite)





