





"""simple docstring"""





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase							=      logging.get_logger(__name__)

_UpperCamelCase							=      {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCamelCase							=      {
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

_UpperCamelCase							=      {
    'ctrl': 256,
}

_UpperCamelCase							=      {
    'Pregnancy': 16_8629,
    'Christianity': 7675,
    'Explain': 10_6423,
    'Fitness': 6_3440,
    'Saving': 6_3163,
    'Ask': 2_7171,
    'Ass': 9_5985,
    'Joke': 16_3509,
    'Questions': 4_5622,
    'Thoughts': 4_9605,
    'Retail': 5_2342,
    'Feminism': 16_4338,
    'Writing': 1_1992,
    'Atheism': 19_2263,
    'Netflix': 4_8616,
    'Computing': 3_9639,
    'Opinion': 4_3213,
    'Alone': 4_4967,
    'Funny': 5_8917,
    'Gaming': 4_0358,
    'Human': 4088,
    'India': 1331,
    'Joker': 7_7138,
    'Diet': 3_6206,
    'Legal': 1_1859,
    'Norman': 4939,
    'Tip': 7_2689,
    'Weight': 5_2343,
    'Movies': 4_6273,
    'Running': 2_3425,
    'Science': 2090,
    'Horror': 3_7793,
    'Confession': 6_0572,
    'Finance': 1_2250,
    'Politics': 1_6360,
    'Scary': 19_1985,
    'Support': 1_2654,
    'Technologies': 3_2516,
    'Teenage': 6_6160,
    'Event': 3_2769,
    'Learned': 6_7460,
    'Notion': 18_2770,
    'Wikipedia': 3_7583,
    'Books': 6665,
    'Extract': 7_6050,
    'Confessions': 10_2701,
    'Conspiracy': 7_5932,
    'Links': 6_3674,
    'Narcissus': 15_0425,
    'Relationship': 5_4766,
    'Relationships': 13_4796,
    'Reviews': 4_1671,
    'News': 4256,
    'Translation': 2_6820,
    'multilingual': 12_8406,
}



def 				lowerCAmelCase_			(      SCREAMING_SNAKE_CASE       :							str   ):


							'''simple docstring'''







							__lowerCamelCase      :      str         =set()
							__lowerCamelCase      :      Optional[Any]         =word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)   )
														__lowerCamelCase      :      Tuple         =char

							__lowerCamelCase      :      int         =set(SCREAMING_SNAKE_CASE   )
							return pairs
class 		SCREAMING_SNAKE_CASE_							(    snake_case__   ):
				"""simple docstring"""
				__snake_case   :       Tuple	 =		VOCAB_FILES_NAMES
				__snake_case   :       List[str]	 =		PRETRAINED_VOCAB_FILES_MAP
				__snake_case   :       Optional[int]	 =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__snake_case   :       str	 =		CONTROL_CODES




				def __init__(     self  :Union[str, Any]						,    __lowercase  :Union[str, Any]						,    __lowercase  :Optional[int]						,    __lowercase  :List[str]="<unk>"						,    **__lowercase  :str			):
											super().__init__(unk_token=__lowercase						,    **__lowercase			)

											with open(__lowercase						,    encoding='''utf-8'''			) as vocab_handle:
																		__lowerCamelCase      :      Dict         =json.load(__lowercase			)
											__lowerCamelCase      :      Optional[Any]         ={v: k for k, v in self.encoder.items()}
											with open(__lowercase						,    encoding='''utf-8'''			) as merges_handle:
																		__lowerCamelCase      :      str         =merges_handle.read().split('''\n'''			)[1:-1]
											__lowerCamelCase      :      str         =[tuple(merge.split()			) for merge in merges]
											__lowerCamelCase      :      Union[str, Any]         =dict(zip(__lowercase						,    range(len(__lowercase			)			)			)			)
											__lowerCamelCase      :      Union[str, Any]         ={}




				@property
				def 					__lowercase  (     self  :List[str]			):
											return len(self.encoder			)




				def 					__lowercase  (     self  :str			):
											return dict(self.encoder						,    **self.added_tokens_encoder			)




				def 					__lowercase  (     self  :Optional[int]						,    __lowercase  :int			):
											if token in self.cache:
																		return self.cache[token]
											__lowerCamelCase      :      Optional[Any]         =tuple(__lowercase			)
											__lowerCamelCase      :      Optional[Any]         =tuple(list(word[:-1]			) + [word[-1] + '''</w>''']			)
											__lowerCamelCase      :      Union[str, Any]         =get_pairs(__lowercase			)

											if not pairs:
																		return token

											while True:
																		__lowerCamelCase      :      List[str]         =min(__lowercase						,    key=lambda __lowercase			: self.bpe_ranks.get(__lowercase						,    float('''inf'''			)			)			)
																		if bigram not in self.bpe_ranks:
																									break
																		__lowerCamelCase    ,						__lowerCamelCase      :      Tuple         =bigram
																		__lowerCamelCase      :      Optional[Any]         =[]
																		__lowerCamelCase      :      Optional[Any]         =0
																		while i < len(__lowercase			):
																									try:
																																__lowerCamelCase      :      Dict         =word.index(__lowercase						,    __lowercase			)
																									except ValueError:
																																new_word.extend(word[i:]			)
																																break
																									else:
																																new_word.extend(word[i:j]			)
																																__lowerCamelCase      :      List[str]         =j

																									if word[i] == first and i < len(__lowercase			) - 1 and word[i + 1] == second:
																																new_word.append(first + second			)
																																i += 2
																									else:
																																new_word.append(word[i]			)
																																i += 1
																		__lowerCamelCase      :      List[Any]         =tuple(__lowercase			)
																		__lowerCamelCase      :      List[Any]         =new_word
																		if len(__lowercase			) == 1:
																									break
																		else:
																									__lowerCamelCase      :      Optional[int]         =get_pairs(__lowercase			)
											__lowerCamelCase      :      int         ='''@@ '''.join(__lowercase			)
											__lowerCamelCase      :      Tuple         =word[:-4]
											__lowerCamelCase      :      Optional[Any]         =word
											return word




				def 					__lowercase  (     self  :Optional[int]						,    __lowercase  :Dict			):
											__lowerCamelCase      :      str         =[]

											__lowerCamelCase      :      Tuple         =re.findall(r'''\S+\n?'''						,    __lowercase			)

											for token in words:
																		split_tokens.extend(list(self.bpe(__lowercase			).split(''' '''			)			)			)
											return split_tokens




				def 					__lowercase  (     self  :List[Any]						,    __lowercase  :Union[str, Any]			):
											return self.encoder.get(__lowercase						,    self.encoder.get(self.unk_token			)			)




				def 					__lowercase  (     self  :Union[str, Any]						,    __lowercase  :Union[str, Any]			):
											return self.decoder.get(__lowercase						,    self.unk_token			)




				def 					__lowercase  (     self  :List[Any]						,    __lowercase  :Optional[int]			):
											__lowerCamelCase      :      Optional[Any]         =''' '''.join(__lowercase			).replace('''@@ '''						,    ''''''			).strip()
											return out_string




				def 					__lowercase  (     self  :str						,    __lowercase  :str						,    __lowercase  :Optional[str] = None			):
											if not os.path.isdir(__lowercase			):
																		logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
																		return
											__lowerCamelCase      :      Union[str, Any]         =os.path.join(
											    __lowercase						,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
											__lowerCamelCase      :      Optional[Any]         =os.path.join(
											    __lowercase						,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

											with open(__lowercase						,    '''w'''						,    encoding='''utf-8'''			) as f:
																		f.write(json.dumps(self.encoder						,    indent=2						,    sort_keys=__lowercase						,    ensure_ascii=__lowercase			) + '''\n'''			)

											__lowerCamelCase      :      List[Any]         =0
											with open(__lowercase						,    '''w'''						,    encoding='''utf-8'''			) as writer:
																		writer.write('''#version: 0.2\n'''			)
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,    key=lambda __lowercase			: kv[1]			):
																									if index != token_index:
																																logger.warning(
																																    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																																    ''' Please check that the tokenizer is not corrupted!'''			)
																																__lowerCamelCase      :      List[str]         =token_index
																									writer.write(''' '''.join(__lowercase			) + '''\n'''			)
																									index += 1

											return vocab_file, merge_file

				# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
				#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
				#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
				#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
				#     return ''.join(tokens_generated_so_far)











"""simple docstring"""





from typing import Any
class 		SCREAMING_SNAKE_CASE_							:
				"""simple docstring"""




				def __init__(     self  :Tuple						,    __lowercase  :Any			):
											__lowerCamelCase      :      str         =data
											__lowerCamelCase      :      Any         =None




				def __repr__(     self  :Optional[Any]			):
											return f'Node({self.data})'

class 		SCREAMING_SNAKE_CASE_							:
				"""simple docstring"""




				def __init__(     self  :Optional[int]			):
											__lowerCamelCase      :      Union[str, Any]         =None




				def __iter__(     self  :Optional[int]			):
											__lowerCamelCase      :      Optional[Any]         =self.head
											while node:
																		yield node.data
																		__lowerCamelCase      :      str         =node.next




				def __len__(     self  :Any			):
											return sum(1 for _ in self			)




				def __repr__(     self  :Any			):
											return "->".join([str(__lowercase			) for item in self]			)




				def __getitem__(     self  :Dict						,    __lowercase  :int			):
											if not 0 <= index < len(self			):
																		raise ValueError('''list index out of range.'''			)
											for i, node in enumerate(self			):
																		if i == index:
																									return node
											return None




				def __setitem__(     self  :Dict						,    __lowercase  :int						,    __lowercase  :Any			):
											if not 0 <= index < len(self			):
																		raise ValueError('''list index out of range.'''			)
											__lowerCamelCase      :      Optional[Any]         =self.head
											for _ in range(__lowercase			):
																		__lowerCamelCase      :      List[str]         =current.next
											__lowerCamelCase      :      Optional[int]         =data




				def 					__lowercase  (     self  :Tuple						,    __lowercase  :Any			):
											self.insert_nth(len(self			)						,    __lowercase			)




				def 					__lowercase  (     self  :int						,    __lowercase  :Any			):
											self.insert_nth(0						,    __lowercase			)




				def 					__lowercase  (     self  :Dict						,    __lowercase  :int						,    __lowercase  :Any			):
											if not 0 <= index <= len(self			):
																		raise IndexError('''list index out of range'''			)
											__lowerCamelCase      :      List[Any]         =Node(__lowercase			)
											if self.head is None:
																		__lowerCamelCase      :      List[Any]         =new_node
											elif index == 0:
																		__lowerCamelCase      :      Optional[Any]         =self.head  # link new_node to head
																		__lowerCamelCase      :      Optional[Any]         =new_node
											else:
																		__lowerCamelCase      :      str         =self.head
																		for _ in range(index - 1			):
																									__lowerCamelCase      :      Any         =temp.next
																		__lowerCamelCase      :      List[Any]         =temp.next
																		__lowerCamelCase      :      List[Any]         =new_node




				def 					__lowercase  (     self  :Tuple			):  # print every node data
											print(self			)




				def 					__lowercase  (     self  :List[str]			):
											return self.delete_nth(0			)




				def 					__lowercase  (     self  :Any			):  # delete from tail
											return self.delete_nth(len(self			) - 1			)




				def 					__lowercase  (     self  :Dict						,    __lowercase  :int = 0			):
											if not 0 <= index <= len(self			) - 1:  # test if index is valid
																		raise IndexError('''List index out of range.'''			)
											__lowerCamelCase      :      List[Any]         =self.head  # default first node
											if index == 0:
																		__lowerCamelCase      :      int         =self.head.next
											else:
																		__lowerCamelCase      :      int         =self.head
																		for _ in range(index - 1			):
																									__lowerCamelCase      :      str         =temp.next
																		__lowerCamelCase      :      int         =temp.next
																		__lowerCamelCase      :      Optional[Any]         =temp.next.next
											return delete_node.data




				def 					__lowercase  (     self  :Any			):
											return self.head is None




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Any         =None
											__lowerCamelCase      :      Optional[int]         =self.head

											while current:
																		# Store the current node's next node.
																		__lowerCamelCase      :      Any         =current.next
																		# Make the current node's next point backwards
																		__lowerCamelCase      :      Optional[Any]         =prev
																		# Make the previous node be the current node
																		__lowerCamelCase      :      List[str]         =current
																		# Make the current node the next node (to progress iteration)
																		__lowerCamelCase      :      Tuple         =next_node
											# Return prev in order to put the head at the end
											__lowerCamelCase      :      List[str]         =prev



def 				lowerCAmelCase_			(      ):


							'''simple docstring'''







							__lowerCamelCase      :      Tuple         =LinkedList()
							assert linked_list.is_empty() is True
							assert str(SCREAMING_SNAKE_CASE   ) == ""

							try:
														linked_list.delete_head()
														raise AssertionError  # This should not happen.
							except IndexError:
														assert True  # This should happen.

							try:
														linked_list.delete_tail()
														raise AssertionError  # This should not happen.
							except IndexError:
														assert True  # This should happen.

							for i in range(10   ):
														assert len(SCREAMING_SNAKE_CASE   ) == i
														linked_list.insert_nth(SCREAMING_SNAKE_CASE    ,						i + 1   )
							assert str(SCREAMING_SNAKE_CASE   ) == "->".join(str(SCREAMING_SNAKE_CASE   ) for i in range(1    ,						11   )   )

							linked_list.insert_head(0   )
							linked_list.insert_tail(11   )
							assert str(SCREAMING_SNAKE_CASE   ) == "->".join(str(SCREAMING_SNAKE_CASE   ) for i in range(0    ,						12   )   )

							assert linked_list.delete_head() == 0
							assert linked_list.delete_nth(9   ) == 10
							assert linked_list.delete_tail() == 11
							assert len(SCREAMING_SNAKE_CASE   ) == 9
							assert str(SCREAMING_SNAKE_CASE   ) == "->".join(str(SCREAMING_SNAKE_CASE   ) for i in range(1    ,						10   )   )

							assert all(linked_list[i] == i + 1 for i in range(0    ,						9   )   ) is True

							for i in range(0    ,						9   ):
														__lowerCamelCase      :      List[Any]         =-i
							assert all(linked_list[i] == -i for i in range(0    ,						9   )   ) is True

							linked_list.reverse()
							assert str(SCREAMING_SNAKE_CASE   ) == "->".join(str(SCREAMING_SNAKE_CASE   ) for i in range(-8    ,						1   )   )



def 				lowerCAmelCase_			(      ):


							'''simple docstring'''







							__lowerCamelCase      :      Any         =[
							    -9,
							    100,
							    Node(77345112   ),
							    '''dlrow olleH''',
							    7,
							    5555,
							    0,
							    -192.55_555,
							    '''Hello, world!''',
							    77.9,
							    Node(10   ),
							    None,
							    None,
							    12.20,
							]
							__lowerCamelCase      :      str         =LinkedList()

							for i in test_input:
														linked_list.insert_tail(SCREAMING_SNAKE_CASE   )

							# Check if it's empty or not
							assert linked_list.is_empty() is False
							assert (
							    str(SCREAMING_SNAKE_CASE   ) == "-9->100->Node(77345112)->dlrow olleH->7->5555->0->"
							    "-192.55555->Hello, world!->77.9->Node(10)->None->None->12.2"
							)

							# Delete the head
							__lowerCamelCase      :      str         =linked_list.delete_head()
							assert result == -9
							assert (
							    str(SCREAMING_SNAKE_CASE   ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
							    "Hello, world!->77.9->Node(10)->None->None->12.2"
							)

							# Delete the tail
							__lowerCamelCase      :      Union[str, Any]         =linked_list.delete_tail()
							assert result == 12.2
							assert (
							    str(SCREAMING_SNAKE_CASE   ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
							    "Hello, world!->77.9->Node(10)->None->None"
							)

							# Delete a node in specific location in linked list
							__lowerCamelCase      :      Union[str, Any]         =linked_list.delete_nth(10   )
							assert result is None
							assert (
							    str(SCREAMING_SNAKE_CASE   ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
							    "Hello, world!->77.9->Node(10)->None"
							)

							# Add a Node instance to its head
							linked_list.insert_head(Node('''Hello again, world!'''   )   )
							assert (
							    str(SCREAMING_SNAKE_CASE   )
							    == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
							    "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None"
							)

							# Add None to its tail
							linked_list.insert_tail(SCREAMING_SNAKE_CASE   )
							assert (
							    str(SCREAMING_SNAKE_CASE   )
							    == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
							    "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None->None"
							)

							# Reverse the linked list
							linked_list.reverse()
							assert (
							    str(SCREAMING_SNAKE_CASE   )
							    == "None->None->Node(10)->77.9->Hello, world!->-192.55555->0->5555->"
							    "7->dlrow olleH->Node(77345112)->100->Node(Hello again, world!)"
							)



def 				lowerCAmelCase_			(      ):


							'''simple docstring'''







							from doctest import testmod

							testmod()

							__lowerCamelCase      :      Optional[Any]         =LinkedList()
							linked_list.insert_head(input('''Inserting 1st at head '''   ).strip()   )
							linked_list.insert_head(input('''Inserting 2nd at head '''   ).strip()   )
							print('''\nPrint list:'''   )
							linked_list.print_list()
							linked_list.insert_tail(input('''\nInserting 1st at tail '''   ).strip()   )
							linked_list.insert_tail(input('''Inserting 2nd at tail '''   ).strip()   )
							print('''\nPrint list:'''   )
							linked_list.print_list()
							print('''\nDelete head'''   )
							linked_list.delete_head()
							print('''Delete tail'''   )
							linked_list.delete_tail()
							print('''\nPrint list:'''   )
							linked_list.print_list()
							print('''\nReverse linked list'''   )
							linked_list.reverse()
							print('''\nPrint list:'''   )
							linked_list.print_list()
							print('''\nString representation of linked list:'''   )
							print(SCREAMING_SNAKE_CASE   )
							print('''\nReading/changing Node data using indexing:'''   )
							print(F'Element at Position 1: {linked_list[1]}'   )
							__lowerCamelCase      :      Any         =input('''Enter New Value: '''   ).strip()
							print('''New list:'''   )
							print(SCREAMING_SNAKE_CASE   )
							print(F'length of linked_list is : {len(SCREAMING_SNAKE_CASE   )}'   )


if __name__ == "__main__":
							main()




