

"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlnet import XLNetTokenizer
else:
							__UpperCamelCase    : str				 =						None


__UpperCamelCase    : List[str]				 =						logging.get_logger(__name__)

__UpperCamelCase    : List[Any]				 =						{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCamelCase    : Optional[Any]				 =						{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

__UpperCamelCase    : Dict				 =						{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

__UpperCamelCase    : int				 =						'''▁'''

# Segments (not really needed)
__UpperCamelCase    : List[str]				 =						0
__UpperCamelCase    : Dict				 =						1
__UpperCamelCase    : Tuple				 =						2
__UpperCamelCase    : Optional[int]				 =						3
__UpperCamelCase    : Optional[int]				 =						4







class 							SCREAMING_SNAKE_CASE   (						a_			):






					"""simple docstring"""



					lowercase__												=			VOCAB_FILES_NAMES
					lowercase__												=			PRETRAINED_VOCAB_FILES_MAP
					lowercase__												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__												=			"left"
					lowercase__												=			XLNetTokenizer







					def __init__(			self     :							Dict  ,lowercase_     :							Dict=None  ,lowercase_     :							Dict=None  ,lowercase_     :							Optional[int]=False  ,lowercase_     :							Tuple=True  ,lowercase_     :							Tuple=False  ,lowercase_     :							Union[str, Any]="<s>"  ,lowercase_     :							str="</s>"  ,lowercase_     :							str="<unk>"  ,lowercase_     :							int="<sep>"  ,lowercase_     :							Dict="<pad>"  ,lowercase_     :							int="<cls>"  ,lowercase_     :							Union[str, Any]="<mask>"  ,lowercase_     :							Optional[Any]=["<eop>", "<eod>"]  ,**lowercase_     :							List[str]  ,):
						# Mask token behave like a normal word, i.e. include the space before it
						lowerCAmelCase__      :		Tuple    							=      AddedToken(lowercase_  ,lstrip=lowercase_  ,rstrip=lowercase_						) if isinstance(lowercase_  ,lowercase_						) else mask_token

						super().__init__(
						    vocab_file=lowercase_  ,tokenizer_file=lowercase_  ,do_lower_case=lowercase_  ,remove_space=lowercase_  ,keep_accents=lowercase_  ,bos_token=lowercase_  ,eos_token=lowercase_  ,unk_token=lowercase_  ,sep_token=lowercase_  ,pad_token=lowercase_  ,cls_token=lowercase_  ,mask_token=lowercase_  ,additional_special_tokens=lowercase_  ,**lowercase_  ,)

						lowerCAmelCase__      :		Tuple    							=      3
						lowerCAmelCase__      :		str    							=      do_lower_case
						lowerCAmelCase__      :		List[str]    							=      remove_space
						lowerCAmelCase__      :		str    							=      keep_accents
						lowerCAmelCase__      :		List[str]    							=      vocab_file
						lowerCAmelCase__      :		Tuple    							=      False if not self.vocab_file else True







					def 			__lowerCAmelCase					(			self     :							Tuple  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		Optional[Any]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Union[str, Any]    							=      [self.cls_token_id]
						if token_ids_a is None:
							return token_ids_a + sep + cls
						return token_ids_a + sep + token_ids_a + sep + cls







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		List[Any]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Dict    							=      [2]

						if token_ids_a is None:
							return len(token_ids_a + sep						) * [0] + cls_segment_id
						return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







					def 			__lowerCAmelCase					(			self     :							List[str]  ,lowercase_     :							str  ,lowercase_     :							Optional[str] = None						):
						if not self.can_save_slow_tokenizer:
							raise ValueError(
							    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
							    '''tokenizer.'''						)

						if not os.path.isdir(lowercase_						):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
							return
						lowerCAmelCase__      :		Tuple    							=      os.path.join(
						    lowercase_  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

						if os.path.abspath(self.vocab_file						) != os.path.abspath(lowercase_						):
							copyfile(self.vocab_file  ,lowercase_						)

						return (out_vocab_file,)







"""simple docstring"""





from collections import deque
from math import floor
from random import random
from time import time







class 							SCREAMING_SNAKE_CASE   :






					"""simple docstring"""



					def __init__(			self     :							Union[str, Any]						):
						lowerCAmelCase__      :		List[str]    							=      {}







					def 			__lowerCAmelCase					(			self     :							Optional[Any]  ,lowercase_     :							Optional[Any]  ,lowercase_     :							str  ,lowercase_     :							List[str]=1						):
						if self.graph.get(lowercase_						):
							if self.graph[u].count([w, v]						) == 0:
								self.graph[u].append([w, v]						)
						else:
							lowerCAmelCase__      :		Optional[int]    							=      [[w, v]]
						if not self.graph.get(lowercase_						):
							lowerCAmelCase__      :		Union[str, Any]    							=      []







					def 			__lowerCAmelCase					(			self     :							List[Any]						):
						return list(self.graph						)







					def 			__lowerCAmelCase					(			self     :							Dict  ,lowercase_     :							Tuple  ,lowercase_     :							Tuple						):
						if self.graph.get(lowercase_						):
							for _ in self.graph[u]:
								if _[1] == v:
									self.graph[u].remove(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							int  ,lowercase_     :							Any=-2  ,lowercase_     :							List[Any]=-1						):
						if s == d:
							return []
						lowerCAmelCase__      :		int    							=      []
						lowerCAmelCase__      :		List[Any]    							=      []
						if s == -2:
							lowerCAmelCase__      :		int    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		int    							=      s

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		List[Any]    							=      s
								for node in self.graph[s]:
									if visited.count(node[1]						) < 1:
										if node[1] == d:
											visited.append(lowercase_						)
											return visited
										else:
											stack.append(node[1]						)
											visited.append(node[1]						)
											lowerCAmelCase__      :		Optional[int]    							=      node[1]
											break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		str    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		Union[str, Any]    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return visited







					def 			__lowerCAmelCase					(			self     :							Dict  ,lowercase_     :							Optional[Any]=-1						):
						if c == -1:
							lowerCAmelCase__      :		Any    							=      floor(random() * 1_0_0_0_0						) + 1_0
						for i in range(lowercase_						):
							# every vertex has max 100 edges
							for _ in range(floor(random() * 1_0_2						) + 1						):
								lowerCAmelCase__      :		Any    							=      floor(random() * c						) + 1
								if n != i:
									self.add_pair(lowercase_  ,lowercase_  ,1						)







					def 			__lowerCAmelCase					(			self     :							Optional[Any]  ,lowercase_     :							Union[str, Any]=-2						):
						lowerCAmelCase__      :		Tuple    							=      deque()
						lowerCAmelCase__      :		Tuple    							=      []
						if s == -2:
							lowerCAmelCase__      :		Any    							=      list(self.graph						)[0]
						d.append(lowercase_						)
						visited.append(lowercase_						)
						while d:
							lowerCAmelCase__      :		Dict    							=      d.popleft()
							if len(self.graph[s]						) != 0:
								for node in self.graph[s]:
									if visited.count(node[1]						) < 1:
										d.append(node[1]						)
										visited.append(node[1]						)
						return visited







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]  ,lowercase_     :							Optional[Any]						):
						lowerCAmelCase__      :		Optional[Any]    							=      0
						for x in self.graph:
							for y in self.graph[x]:
								if y[1] == u:
									count += 1
						return count







					def 			__lowerCAmelCase					(			self     :							str  ,lowercase_     :							Optional[int]						):
						return len(self.graph[u]						)







					def 			__lowerCAmelCase					(			self     :							List[str]  ,lowercase_     :							List[str]=-2						):
						lowerCAmelCase__      :		Union[str, Any]    							=      []
						lowerCAmelCase__      :		Any    							=      []
						if s == -2:
							lowerCAmelCase__      :		Any    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		Tuple    							=      s
						lowerCAmelCase__      :		Optional[int]    							=      []

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		Optional[Any]    							=      s
								for node in self.graph[s]:
									if visited.count(node[1]						) < 1:
										stack.append(node[1]						)
										visited.append(node[1]						)
										lowerCAmelCase__      :		Dict    							=      node[1]
										break

            # check if all the children are visited
							if s == ss:
								sorted_nodes.append(stack.pop()						)
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		Union[str, Any]    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		int    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return sorted_nodes







					def 			__lowerCAmelCase					(			self     :							List[Any]						):
						lowerCAmelCase__      :		str    							=      []
						lowerCAmelCase__      :		List[str]    							=      []
						lowerCAmelCase__      :		int    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		List[Any]    							=      -2
						lowerCAmelCase__      :		str    							=      []
						lowerCAmelCase__      :		Optional[Any]    							=      s
						lowerCAmelCase__      :		Union[str, Any]    							=      False
						lowerCAmelCase__      :		Optional[int]    							=      set()

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		List[str]    							=      s
								for node in self.graph[s]:
									if (
									    visited.count(node[1]						) > 0
									    and node[1] != parent
									    and indirect_parents.count(node[1]						) > 0
									    and not on_the_way_back
									):
										lowerCAmelCase__      :		List[Any]    							=      len(lowercase_						) - 1
										while len_stack >= 0:
											if stack[len_stack] == node[1]:
												anticipating_nodes.add(node[1]						)
												break
											else:
												anticipating_nodes.add(stack[len_stack]						)
												len_stack -= 1
									if visited.count(node[1]						) < 1:
										stack.append(node[1]						)
										visited.append(node[1]						)
										lowerCAmelCase__      :		Optional[int]    							=      node[1]
										break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								lowerCAmelCase__      :		str    							=      True
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		Tuple    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		int    							=      False
								indirect_parents.append(lowercase_						)
								lowerCAmelCase__      :		str    							=      s
								lowerCAmelCase__      :		Tuple    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return list(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							int						):
						lowerCAmelCase__      :		Tuple    							=      []
						lowerCAmelCase__      :		Tuple    							=      []
						lowerCAmelCase__      :		Any    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		List[str]    							=      -2
						lowerCAmelCase__      :		int    							=      []
						lowerCAmelCase__      :		int    							=      s
						lowerCAmelCase__      :		List[Any]    							=      False
						lowerCAmelCase__      :		List[str]    							=      set()

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		Optional[int]    							=      s
								for node in self.graph[s]:
									if (
									    visited.count(node[1]						) > 0
									    and node[1] != parent
									    and indirect_parents.count(node[1]						) > 0
									    and not on_the_way_back
									):
										lowerCAmelCase__      :		List[Any]    							=      len(lowercase_						) - 1
										while len_stack_minus_one >= 0:
											if stack[len_stack_minus_one] == node[1]:
												anticipating_nodes.add(node[1]						)
												break
											else:
												return True
									if visited.count(node[1]						) < 1:
										stack.append(node[1]						)
										visited.append(node[1]						)
										lowerCAmelCase__      :		Optional[Any]    							=      node[1]
										break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								lowerCAmelCase__      :		List[str]    							=      True
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		Dict    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		Union[str, Any]    							=      False
								indirect_parents.append(lowercase_						)
								lowerCAmelCase__      :		Tuple    							=      s
								lowerCAmelCase__      :		Union[str, Any]    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return False







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							List[Any]=-2  ,lowercase_     :							Union[str, Any]=-1						):
						lowerCAmelCase__      :		Optional[int]    							=      time()
						self.dfs(lowercase_  ,lowercase_						)
						lowerCAmelCase__      :		Any    							=      time()
						return end - begin







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							Dict=-2						):
						lowerCAmelCase__      :		List[str]    							=      time()
						self.bfs(lowercase_						)
						lowerCAmelCase__      :		List[Any]    							=      time()
						return end - begin







class 							SCREAMING_SNAKE_CASE   :






					"""simple docstring"""



					def __init__(			self     :							Union[str, Any]						):
						lowerCAmelCase__      :		Tuple    							=      {}







					def 			__lowerCAmelCase					(			self     :							List[Any]  ,lowercase_     :							Optional[int]  ,lowercase_     :							Dict  ,lowercase_     :							Any=1						):
						# check if the u exists
						if self.graph.get(lowercase_						):
							# if there already is a edge
							if self.graph[u].count([w, v]						) == 0:
								self.graph[u].append([w, v]						)
						else:
							# if u does not exist
							lowerCAmelCase__      :		List[Any]    							=      [[w, v]]
						# add the other way
						if self.graph.get(lowercase_						):
							# if there already is a edge
							if self.graph[v].count([w, u]						) == 0:
								self.graph[v].append([w, u]						)
						else:
							# if u does not exist
							lowerCAmelCase__      :		int    							=      [[w, u]]







					def 			__lowerCAmelCase					(			self     :							Dict  ,lowercase_     :							Union[str, Any]  ,lowercase_     :							Dict						):
						if self.graph.get(lowercase_						):
							for _ in self.graph[u]:
								if _[1] == v:
									self.graph[u].remove(lowercase_						)
        # the other way round
						if self.graph.get(lowercase_						):
							for _ in self.graph[v]:
								if _[1] == u:
									self.graph[v].remove(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							int  ,lowercase_     :							Optional[int]=-2  ,lowercase_     :							str=-1						):
						if s == d:
							return []
						lowerCAmelCase__      :		Optional[Any]    							=      []
						lowerCAmelCase__      :		Any    							=      []
						if s == -2:
							lowerCAmelCase__      :		Tuple    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		int    							=      s

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		Optional[int]    							=      s
								for node in self.graph[s]:
									if visited.count(node[1]						) < 1:
										if node[1] == d:
											visited.append(lowercase_						)
											return visited
										else:
											stack.append(node[1]						)
											visited.append(node[1]						)
											lowerCAmelCase__      :		List[str]    							=      node[1]
											break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		str    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		Dict    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return visited







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]  ,lowercase_     :							List[Any]=-1						):
						if c == -1:
							lowerCAmelCase__      :		str    							=      floor(random() * 1_0_0_0_0						) + 1_0
						for i in range(lowercase_						):
							# every vertex has max 100 edges
							for _ in range(floor(random() * 1_0_2						) + 1						):
								lowerCAmelCase__      :		List[str]    							=      floor(random() * c						) + 1
								if n != i:
									self.add_pair(lowercase_  ,lowercase_  ,1						)







					def 			__lowerCAmelCase					(			self     :							Optional[Any]  ,lowercase_     :							List[Any]=-2						):
						lowerCAmelCase__      :		Tuple    							=      deque()
						lowerCAmelCase__      :		List[Any]    							=      []
						if s == -2:
							lowerCAmelCase__      :		Tuple    							=      list(self.graph						)[0]
						d.append(lowercase_						)
						visited.append(lowercase_						)
						while d:
							lowerCAmelCase__      :		Any    							=      d.popleft()
							if len(self.graph[s]						) != 0:
								for node in self.graph[s]:
									if visited.count(node[1]						) < 1:
										d.append(node[1]						)
										visited.append(node[1]						)
						return visited







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							Any						):
						return len(self.graph[u]						)







					def 			__lowerCAmelCase					(			self     :							Any						):
						lowerCAmelCase__      :		List[Any]    							=      []
						lowerCAmelCase__      :		int    							=      []
						lowerCAmelCase__      :		List[Any]    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		Optional[int]    							=      -2
						lowerCAmelCase__      :		List[str]    							=      []
						lowerCAmelCase__      :		Tuple    							=      s
						lowerCAmelCase__      :		Tuple    							=      False
						lowerCAmelCase__      :		Union[str, Any]    							=      set()

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		Union[str, Any]    							=      s
								for node in self.graph[s]:
									if (
									    visited.count(node[1]						) > 0
									    and node[1] != parent
									    and indirect_parents.count(node[1]						) > 0
									    and not on_the_way_back
									):
										lowerCAmelCase__      :		Tuple    							=      len(lowercase_						) - 1
										while len_stack >= 0:
											if stack[len_stack] == node[1]:
												anticipating_nodes.add(node[1]						)
												break
											else:
												anticipating_nodes.add(stack[len_stack]						)
												len_stack -= 1
									if visited.count(node[1]						) < 1:
										stack.append(node[1]						)
										visited.append(node[1]						)
										lowerCAmelCase__      :		Optional[int]    							=      node[1]
										break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								lowerCAmelCase__      :		Optional[int]    							=      True
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		str    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		Tuple    							=      False
								indirect_parents.append(lowercase_						)
								lowerCAmelCase__      :		Optional[int]    							=      s
								lowerCAmelCase__      :		Optional[int]    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return list(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							Optional[Any]						):
						lowerCAmelCase__      :		Optional[Any]    							=      []
						lowerCAmelCase__      :		Union[str, Any]    							=      []
						lowerCAmelCase__      :		Tuple    							=      list(self.graph						)[0]
						stack.append(lowercase_						)
						visited.append(lowercase_						)
						lowerCAmelCase__      :		Dict    							=      -2
						lowerCAmelCase__      :		Union[str, Any]    							=      []
						lowerCAmelCase__      :		Optional[Any]    							=      s
						lowerCAmelCase__      :		Any    							=      False
						lowerCAmelCase__      :		Optional[int]    							=      set()

						while True:
							# check if there is any non isolated nodes
							if len(self.graph[s]						) != 0:
								lowerCAmelCase__      :		str    							=      s
								for node in self.graph[s]:
									if (
									    visited.count(node[1]						) > 0
									    and node[1] != parent
									    and indirect_parents.count(node[1]						) > 0
									    and not on_the_way_back
									):
										lowerCAmelCase__      :		str    							=      len(lowercase_						) - 1
										while len_stack_minus_one >= 0:
											if stack[len_stack_minus_one] == node[1]:
												anticipating_nodes.add(node[1]						)
												break
											else:
												return True
									if visited.count(node[1]						) < 1:
										stack.append(node[1]						)
										visited.append(node[1]						)
										lowerCAmelCase__      :		List[str]    							=      node[1]
										break

            # check if all the children are visited
							if s == ss:
								stack.pop()
								lowerCAmelCase__      :		List[Any]    							=      True
								if len(lowercase_						) != 0:
									lowerCAmelCase__      :		List[str]    							=      stack[len(lowercase_						) - 1]
							else:
								lowerCAmelCase__      :		List[str]    							=      False
								indirect_parents.append(lowercase_						)
								lowerCAmelCase__      :		Optional[int]    							=      s
								lowerCAmelCase__      :		List[Any]    							=      ss

							# check if se have reached the starting point
							if len(lowercase_						) == 0:
								return False







					def 			__lowerCAmelCase					(			self     :							str						):
						return list(self.graph						)







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							Optional[int]=-2  ,lowercase_     :							Any=-1						):
						lowerCAmelCase__      :		Dict    							=      time()
						self.dfs(lowercase_  ,lowercase_						)
						lowerCAmelCase__      :		List[str]    							=      time()
						return end - begin







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]  ,lowercase_     :							int=-2						):
						lowerCAmelCase__      :		Dict    							=      time()
						self.bfs(lowercase_						)
						lowerCAmelCase__      :		Any    							=      time()
						return end - begin




