





'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase_      							=     logging.get_logger(__name__)

UpperCAmelCase_      							=     {"""vocab_file""": """spiece.model"""}

UpperCAmelCase_      							=     {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

UpperCAmelCase_      							=     {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
UpperCAmelCase_      							=     0
UpperCAmelCase_      							=     1
UpperCAmelCase_      							=     2
UpperCAmelCase_      							=     3
UpperCAmelCase_      							=     4
class 			UpperCamelCase_		(  lowercase__					):
					lowerCAmelCase_							=	VOCAB_FILES_NAMES
					lowerCAmelCase_							=	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=	'''left'''




					def __init__(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=False				,						lowerCAmelCase_=True				,						lowerCAmelCase_=False				,						lowerCAmelCase_="<s>"				,						lowerCAmelCase_="</s>"				,						lowerCAmelCase_="<unk>"				,						lowerCAmelCase_="<sep>"				,						lowerCAmelCase_="<pad>"				,						lowerCAmelCase_="<cls>"				,						lowerCAmelCase_="<mask>"				,						lowerCAmelCase_=["<eop>", "<eod>"]				,						lowerCAmelCase_ = None				,						**lowerCAmelCase_				,						)		->							int:
										_snake_case									=				AddedToken(UpperCAmelCase__				,						lstrip=UpperCAmelCase__				,						rstrip=UpperCAmelCase__  ) if isinstance(UpperCAmelCase__				,						UpperCAmelCase__  ) else mask_token

										_snake_case									=				{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=UpperCAmelCase__				,						remove_space=UpperCAmelCase__				,						keep_accents=UpperCAmelCase__				,						bos_token=UpperCAmelCase__				,						eos_token=UpperCAmelCase__				,						unk_token=UpperCAmelCase__				,						sep_token=UpperCAmelCase__				,						pad_token=UpperCAmelCase__				,						cls_token=UpperCAmelCase__				,						mask_token=UpperCAmelCase__				,						additional_special_tokens=UpperCAmelCase__				,						sp_model_kwargs=self.sp_model_kwargs				,						**UpperCAmelCase__				,						)

										_snake_case									=				3

										_snake_case									=				do_lower_case
										_snake_case									=				remove_space
										_snake_case									=				keep_accents
										_snake_case									=				vocab_file

										_snake_case									=				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(UpperCAmelCase__  )




					@property
					def 	lowerCAmelCase			(		self  )		->							Optional[Any]:
										return len(self.sp_model  )




					def 	lowerCAmelCase			(		self  )		->							Union[str, Any]:
										_snake_case									=				{self.convert_ids_to_tokens(UpperCAmelCase__  ): i for i in range(self.vocab_size  )}
										vocab.update(self.added_tokens_encoder  )
										return vocab




					def __getstate__(		self  )		->							List[Any]:
										_snake_case									=				self.__dict__.copy()
										_snake_case									=				None
										return state




					def __setstate__(		self				,						lowerCAmelCase_  )		->							Any:
										_snake_case									=				d

										# for backward compatibility
										if not hasattr(self				,						'sp_model_kwargs'  ):
															_snake_case									=				{}

										_snake_case									=				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(self.vocab_file  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Any:
										if self.remove_space:
															_snake_case									=				''' '''.join(inputs.strip().split()  )
										else:
															_snake_case									=				inputs
										_snake_case									=				outputs.replace('``'				,						'"'  ).replace('\'\''				,						'"'  )

										if not self.keep_accents:
															_snake_case									=				unicodedata.normalize('NFKD'				,						UpperCAmelCase__  )
															_snake_case									=				''''''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__  )]  )
										if self.do_lower_case:
															_snake_case									=				outputs.lower()

										return outputs




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Tuple:
										_snake_case									=				self.preprocess_text(UpperCAmelCase__  )
										_snake_case									=				self.sp_model.encode(UpperCAmelCase__				,						out_type=UpperCAmelCase__  )
										_snake_case									=				[]
										for piece in pieces:
															if len(UpperCAmelCase__  ) > 1 and piece[-1] == str(','  ) and piece[-2].isdigit():
																				_snake_case									=				self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__				,						''  )  )
																				if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																									if len(cur_pieces[0]  ) == 1:
																														_snake_case									=				cur_pieces[1:]
																									else:
																														_snake_case									=				cur_pieces[0][1:]
																				cur_pieces.append(piece[-1]  )
																				new_pieces.extend(UpperCAmelCase__  )
															else:
																				new_pieces.append(UpperCAmelCase__  )

										return new_pieces




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Union[str, Any]:
										return self.sp_model.PieceToId(UpperCAmelCase__  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							int:
										return self.sp_model.IdToPiece(UpperCAmelCase__  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							List[Any]:
										_snake_case									=				''''''.join(UpperCAmelCase__  ).replace(UpperCAmelCase__				,						' '  ).strip()
										return out_string




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = False				,						lowerCAmelCase_ = None				,						lowerCAmelCase_ = True				,						**lowerCAmelCase_				,						)		->							Optional[Any]:
										_snake_case									=				kwargs.pop('use_source_tokenizer'				,						UpperCAmelCase__  )

										_snake_case									=				self.convert_ids_to_tokens(UpperCAmelCase__				,						skip_special_tokens=UpperCAmelCase__  )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_snake_case									=				[]
										_snake_case									=				[]
										for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																				continue
															if token in self.added_tokens_encoder:
																				if current_sub_text:
																									sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase__  )  )
																									_snake_case									=				[]
																				sub_texts.append(UpperCAmelCase__  )
															else:
																				current_sub_text.append(UpperCAmelCase__  )
										if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase__  )  )

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										_snake_case									=				''''''.join(UpperCAmelCase__  )

										_snake_case									=				(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
															_snake_case									=				self.clean_up_tokenization(UpperCAmelCase__  )
															return clean_text
										else:
															return text




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Optional[Any]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[self.cls_token_id]
										if token_ids_a is None:
															return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None				,						lowerCAmelCase_ = False  )		->							List[Any]:

										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCAmelCase__				,						token_ids_a=UpperCAmelCase__				,						already_has_special_tokens=UpperCAmelCase__  )

										if token_ids_a is not None:
															return ([0] * len(UpperCAmelCase__  )) + [1] + ([0] * len(UpperCAmelCase__  )) + [1, 1]
										return ([0] * len(UpperCAmelCase__  )) + [1, 1]




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							List[Any]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[2]

										if token_ids_a is None:
															return len(token_ids_a + sep  ) * [0] + cls_segment_id
										return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							str:
										if not os.path.isdir(UpperCAmelCase__  ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
															return
										_snake_case									=				os.path.join(
										    UpperCAmelCase__				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

										if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase__  ) and os.path.isfile(self.vocab_file  ):
															copyfile(self.vocab_file				,						UpperCAmelCase__  )
										elif not os.path.isfile(self.vocab_file  ):
															with open(UpperCAmelCase__				,						'wb'  ) as fi:
																				_snake_case									=				self.sp_model.serialized_model_proto()
																				fi.write(UpperCAmelCase__  )

										return (out_vocab_file,)










import argparse
from collections import defaultdict

import yaml


UpperCAmelCase_      							=     """docs/source/en/_toctree.yml"""





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]						)			-> str:



					'''simple docstring'''




					_snake_case									=				defaultdict(UpperCamelCase__						)
					_snake_case									=				[]
					_snake_case									=				[]
					for doc in doc_list:
										if "local" in doc:
															counts[doc["local"]] += 1

										if doc["title"].lower() == "overview":
															overview_doc.append({'local': doc['local'], 'title': doc['title']}						)
										else:
															new_doc_list.append(UpperCamelCase__						)

					_snake_case									=				new_doc_list
					_snake_case									=				[key for key, value in counts.items() if value > 1]

					_snake_case									=				[]
					for duplicate_key in duplicates:
										_snake_case									=				list({doc['title'] for doc in doc_list if doc['local'] == duplicate_key}						)
										if len(UpperCamelCase__						) > 1:
															raise ValueError(
															    F'''{duplicate_key} is present several times in the documentation table of content at '''
															    '`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '
															    'others.'						)
										# Only add this once
										new_doc.append({'local': duplicate_key, 'title': titles[0]}						)

					# Add none duplicate-keys
					new_doc.extend([doc for doc in doc_list if 'local' not in counts or counts[doc['local']] == 1]						)
					_snake_case									=				sorted(UpperCamelCase__       ,    key=lambda UpperCamelCase__						: s["title"].lower()						)

					# "overview" gets special treatment and is always first
					if len(UpperCamelCase__						) > 1:
										raise ValueError('{doc_list} has two \'overview\' docs which is not allowed.'						)

					overview_doc.extend(UpperCamelCase__						)

					# Sort
					return overview_doc





def 						lowerCamelCase__     (     UpperCamelCase__			:       Dict=False						)			-> Optional[int]:



					'''simple docstring'''




					with open(UpperCamelCase__       ,    encoding='utf-8'						) as f:
										_snake_case									=				yaml.safe_load(f.read()						)

					# Get to the API doc
					_snake_case									=				0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					_snake_case									=				content[api_idx]['sections']

					# Then to the model doc
					_snake_case									=				0
					while api_doc[scheduler_idx]["title"] != "Schedulers":
										scheduler_idx += 1

					_snake_case									=				api_doc[scheduler_idx]['sections']
					_snake_case									=				clean_doc_toc(UpperCamelCase__						)

					_snake_case									=				False
					if new_scheduler_doc != scheduler_doc:
										_snake_case									=				True
										if overwrite:
															_snake_case									=				new_scheduler_doc

					if diff:
										if overwrite:
															_snake_case									=				api_doc
															with open(UpperCamelCase__       ,    'w'       ,    encoding='utf-8'						) as f:
																				f.write(yaml.dump(UpperCamelCase__       ,    allow_unicode=UpperCamelCase__						)						)
										else:
															raise ValueError(
															    'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       Tuple=False						)			-> List[Any]:



					'''simple docstring'''




					with open(UpperCamelCase__       ,    encoding='utf-8'						) as f:
										_snake_case									=				yaml.safe_load(f.read()						)

					# Get to the API doc
					_snake_case									=				0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					_snake_case									=				content[api_idx]['sections']

					# Then to the model doc
					_snake_case									=				0
					while api_doc[pipeline_idx]["title"] != "Pipelines":
										pipeline_idx += 1

					_snake_case									=				False
					_snake_case									=				api_doc[pipeline_idx]['sections']
					_snake_case									=				[]

					# sort sub pipeline docs
					for pipeline_doc in pipeline_docs:
										if "section" in pipeline_doc:
															_snake_case									=				pipeline_doc['section']
															_snake_case									=				clean_doc_toc(UpperCamelCase__						)
															if overwrite:
																				_snake_case									=				new_sub_pipeline_doc
										new_pipeline_docs.append(UpperCamelCase__						)

					# sort overall pipeline doc
					_snake_case									=				clean_doc_toc(UpperCamelCase__						)

					if new_pipeline_docs != pipeline_docs:
										_snake_case									=				True
										if overwrite:
															_snake_case									=				new_pipeline_docs

					if diff:
										if overwrite:
															_snake_case									=				api_doc
															with open(UpperCamelCase__       ,    'w'       ,    encoding='utf-8'						) as f:
																				f.write(yaml.dump(UpperCamelCase__       ,    allow_unicode=UpperCamelCase__						)						)
										else:
															raise ValueError(
															    'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'						)


if __name__ == "__main__":
					UpperCAmelCase_      							=     argparse.ArgumentParser()
					parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
					UpperCAmelCase_      							=     parser.parse_args()

					check_scheduler_doc(args.fix_and_overwrite)
					check_pipeline_doc(args.fix_and_overwrite)



