





'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a   	=    logging.get_logger(__name__)

a   	=    {'''vocab_file''': '''spiece.model'''}

a   	=    {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

a   	=    {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
a   	=    0
a   	=    1
a   	=    2
a   	=    3
a   	=    4







class __a			( a__      ):
   __UpperCamelCase :			Optional[int]						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			Optional[Any]						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			str						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase :			Optional[Any]						   =						'left'





   def __init__(       self					:      str					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      str=False					,lowerCamelCase					:      Any=True					,lowerCamelCase					:      Union[str, Any]=False					,lowerCamelCase					:      Optional[int]="<s>"					,lowerCamelCase					:      int="</s>"					,lowerCamelCase					:      List[str]="<unk>"					,lowerCamelCase					:      int="<sep>"					,lowerCamelCase					:      List[Any]="<pad>"					,lowerCamelCase					:      Optional[int]="<cls>"					,lowerCamelCase					:      List[Any]="<mask>"					,lowerCamelCase					:      Optional[int]=["<eop>", "<eod>"]					,lowerCamelCase					:      Dict = None					,**lowerCamelCase					:      str					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							AddedToken(_snake_case					,lstrip=_snake_case					,rstrip=_snake_case     ) if isinstance(_snake_case					,_snake_case     ) else mask_token

      __SCREAMING_SNAKE_CASE   				=							{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=_snake_case					,remove_space=_snake_case					,keep_accents=_snake_case					,bos_token=_snake_case					,eos_token=_snake_case					,unk_token=_snake_case					,sep_token=_snake_case					,pad_token=_snake_case					,cls_token=_snake_case					,mask_token=_snake_case					,additional_special_tokens=_snake_case					,sp_model_kwargs=self.sp_model_kwargs					,**_snake_case					,)

      __SCREAMING_SNAKE_CASE   				=							3

      __SCREAMING_SNAKE_CASE   				=							do_lower_case
      __SCREAMING_SNAKE_CASE   				=							remove_space
      __SCREAMING_SNAKE_CASE   				=							keep_accents
      __SCREAMING_SNAKE_CASE   				=							vocab_file

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(_snake_case     )





   @property
   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      return len(self.sp_model     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							{self.convert_ids_to_tokens(_snake_case     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab





   def __getstate__(       self					:      Union[str, Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.__dict__.copy()
      __SCREAMING_SNAKE_CASE   				=							None
      return state





   def __setstate__(       self					:      Union[str, Any]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							d

      # for backward compatibility
      if not hasattr(self					,"""sp_model_kwargs"""     ):
         __SCREAMING_SNAKE_CASE   				=							{}

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )





   def   UpperCAmelCase__  (       self					:      int					,lowerCamelCase					:      Tuple     ):

      '''simple docstring'''

      if self.remove_space:
         __SCREAMING_SNAKE_CASE   				=							""" """.join(inputs.strip().split()     )
      else:
         __SCREAMING_SNAKE_CASE   				=							inputs
      __SCREAMING_SNAKE_CASE   				=							outputs.replace("""``"""					,"""\""""     ).replace("""\'\'"""					,"""\""""     )

      if not self.keep_accents:
         __SCREAMING_SNAKE_CASE   				=							unicodedata.normalize("""NFKD"""					,_snake_case     )
         __SCREAMING_SNAKE_CASE   				=							"""""".join([c for c in outputs if not unicodedata.combining(_snake_case     )]     )
      if self.do_lower_case:
         __SCREAMING_SNAKE_CASE   				=							outputs.lower()

      return outputs





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      Optional[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.preprocess_text(_snake_case     )
      __SCREAMING_SNAKE_CASE   				=							self.sp_model.encode(_snake_case					,out_type=_snake_case     )
      __SCREAMING_SNAKE_CASE   				=							[]
      for piece in pieces:
         if len(_snake_case     ) > 1 and piece[-1] == str(""","""     ) and piece[-2].isdigit():
            __SCREAMING_SNAKE_CASE   				=							self.sp_model.EncodeAsPieces(piece[:-1].replace(_snake_case					,""""""     )     )
            if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
               if len(cur_pieces[0]     ) == 1:
                  __SCREAMING_SNAKE_CASE   				=							cur_pieces[1:]
               else:
                  __SCREAMING_SNAKE_CASE   				=							cur_pieces[0][1:]
            cur_pieces.append(piece[-1]     )
            new_pieces.extend(_snake_case     )
         else:
            new_pieces.append(_snake_case     )

      return new_pieces





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      List[Any]     ):

      '''simple docstring'''

      return self.sp_model.PieceToId(_snake_case     )





   def   UpperCAmelCase__  (       self					:      Tuple					,lowerCamelCase					:      Dict     ):

      '''simple docstring'''

      return self.sp_model.IdToPiece(_snake_case     )





   def   UpperCAmelCase__  (       self					:      Any					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""""".join(_snake_case     ).replace(_snake_case					,""" """     ).strip()
      return out_string





   def   UpperCAmelCase__  (       self					:      int					,lowerCamelCase					:      Tuple					,lowerCamelCase					:      Dict = False					,lowerCamelCase					:      Union[str, Any] = None					,lowerCamelCase					:      Optional[Any] = True					,**lowerCamelCase					:      int					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							kwargs.pop("""use_source_tokenizer"""					,_snake_case     )

      __SCREAMING_SNAKE_CASE   				=							self.convert_ids_to_tokens(_snake_case					,skip_special_tokens=_snake_case     )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							[]
      for token in filtered_tokens:
         if skip_special_tokens and token in self.all_special_ids:
            continue
         if token in self.added_tokens_encoder:
            if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(_snake_case     )     )
               __SCREAMING_SNAKE_CASE   				=							[]
            sub_texts.append(_snake_case     )
         else:
            current_sub_text.append(_snake_case     )
      if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(_snake_case     )     )

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      __SCREAMING_SNAKE_CASE   				=							"""""".join(_snake_case     )

      __SCREAMING_SNAKE_CASE   				=							(
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
         __SCREAMING_SNAKE_CASE   				=							self.clean_up_tokenization(_snake_case     )
         return clean_text
      else:
         return text





   def   UpperCAmelCase__  (       self					:      Any					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      Optional[int] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[self.sep_token_id]
      __SCREAMING_SNAKE_CASE   				=							[self.cls_token_id]
      if token_ids_a is None:
         return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      Optional[int] = None					,lowerCamelCase					:      Tuple = False     ):

      '''simple docstring'''

      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=_snake_case					,token_ids_a=_snake_case					,already_has_special_tokens=_snake_case     )

      if token_ids_a is not None:
         return ([0] * len(_snake_case     )) + [1] + ([0] * len(_snake_case     )) + [1, 1]
      return ([0] * len(_snake_case     )) + [1, 1]





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      Dict					,lowerCamelCase					:      Union[str, Any] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[self.sep_token_id]
      __SCREAMING_SNAKE_CASE   				=							[2]

      if token_ids_a is None:
         return len(token_ids_a + sep     ) * [0] + cls_segment_id
      return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      Optional[int] = None     ):

      '''simple docstring'''

      if not os.path.isdir(_snake_case     ):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
         return
      __SCREAMING_SNAKE_CASE   				=							os.path.join(
          _snake_case					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(_snake_case     ) and os.path.isfile(self.vocab_file     ):
         copyfile(self.vocab_file					,_snake_case     )
      elif not os.path.isfile(self.vocab_file     ):
         with open(_snake_case					,"""wb"""     ) as fi:
            __SCREAMING_SNAKE_CASE   				=							self.sp_model.serialized_model_proto()
            fi.write(_snake_case     )

      return (out_vocab_file,)












"""simple docstring"""
from typing import Any







class       a							:



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= data
  lowerCAmelCase    				= None



 def __repr__(     self     ):







  """simple docstring"""






  return F'Node({self.data})'







class       a							:



 def __init__(     self     ):







  """simple docstring"""






  lowerCAmelCase    				= None



 def __iter__(     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.head
  while node:
   yield node.data
   lowerCAmelCase    				= node.next



 def __len__(     self     ):







  """simple docstring"""






  return sum(1 for _ in self     )



 def __repr__(     self     ):







  """simple docstring"""






  return "->".join([str(_snake_case     ) for item in self]     )



 def __getitem__(     self				,       _snake_case     ):







  """simple docstring"""






  if not 0 <= index < len(self     ):
   raise ValueError('list index out of range.'     )
  for i, node in enumerate(self     ):
   if i == index:
    return node
  return None



 def __setitem__(     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  if not 0 <= index < len(self     ):
   raise ValueError('list index out of range.'     )
  lowerCAmelCase    				= self.head
  for _ in range(_snake_case     ):
   lowerCAmelCase    				= current.next
  lowerCAmelCase    				= data



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  self.insert_nth(len(self     )				,       _snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  self.insert_nth(0				,       _snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  if not 0 <= index <= len(self     ):
   raise IndexError('list index out of range'     )
  lowerCAmelCase    				= Node(_snake_case     )
  if self.head is None:
   lowerCAmelCase    				= new_node
  elif index == 0:
   lowerCAmelCase    				= self.head  # link new_node to head
   lowerCAmelCase    				= new_node
  else:
   lowerCAmelCase    				= self.head
   for _ in range(index - 1     ):
    lowerCAmelCase    				= temp.next
   lowerCAmelCase    				= temp.next
   lowerCAmelCase    				= new_node



 def    UpperCamelCase__  (     self     ):  # print every node data







  """simple docstring"""






  print(self     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self.delete_nth(0     )



 def    UpperCamelCase__  (     self     ):  # delete from tail







  """simple docstring"""






  return self.delete_nth(len(self     ) - 1     )



 def    UpperCamelCase__  (     self				,       _snake_case = 0     ):







  """simple docstring"""






  if not 0 <= index <= len(self     ) - 1:  # test if index is valid
   raise IndexError('List index out of range.'     )
  lowerCAmelCase    				= self.head  # default first node
  if index == 0:
   lowerCAmelCase    				= self.head.next
  else:
   lowerCAmelCase    				= self.head
   for _ in range(index - 1     ):
    lowerCAmelCase    				= temp.next
   lowerCAmelCase    				= temp.next
   lowerCAmelCase    				= temp.next.next
  return delete_node.data



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self.head is None



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= None
  lowerCAmelCase    				= self.head

  while current:
   # Store the current node's next node.
   lowerCAmelCase    				= current.next
   # Make the current node's next point backwards
   lowerCAmelCase    				= prev
   # Make the previous node be the current node
   lowerCAmelCase    				= current
   # Make the current node the next node (to progress iteration)
   lowerCAmelCase    				= next_node
  # Return prev in order to put the head at the end
  lowerCAmelCase    				= prev


def 		_SCREAMING_SNAKE_CASE			():
 lowerCAmelCase    				= LinkedList()
 assert linked_list.is_empty() is True
 assert str(_UpperCAmelCase			) == ""

 try:
  linked_list.delete_head()
  raise AssertionError  # This should not happen.
 except IndexError:
  assert True  # This should happen.

 try:
  linked_list.delete_tail()
  raise AssertionError  # This should not happen.
 except IndexError:
  assert True  # This should happen.

 for i in range(10			):
  assert len(_UpperCAmelCase			) == i
  linked_list.insert_nth(_UpperCAmelCase  ,     i + 1			)
 assert str(_UpperCAmelCase			) == "->".join(str(_UpperCAmelCase			) for i in range(1  ,     11			)			)

 linked_list.insert_head(0			)
 linked_list.insert_tail(11			)
 assert str(_UpperCAmelCase			) == "->".join(str(_UpperCAmelCase			) for i in range(0  ,     12			)			)

 assert linked_list.delete_head() == 0
 assert linked_list.delete_nth(9			) == 10
 assert linked_list.delete_tail() == 11
 assert len(_UpperCAmelCase			) == 9
 assert str(_UpperCAmelCase			) == "->".join(str(_UpperCAmelCase			) for i in range(1  ,     10			)			)

 assert all(linked_list[i] == i + 1 for i in range(0  ,     9			)			) is True

 for i in range(0  ,     9			):
  lowerCAmelCase    				= -i
 assert all(linked_list[i] == -i for i in range(0  ,     9			)			) is True

 linked_list.reverse()
 assert str(_UpperCAmelCase			) == "->".join(str(_UpperCAmelCase			) for i in range(-8  ,     1			)			)

def 		_SCREAMING_SNAKE_CASE			():
 lowerCAmelCase    				= [
     -9,
     100,
     Node(7734_5112			),
     'dlrow olleH',
     7,
     5555,
     0,
     -192.5_5555,
     'Hello, world!',
     77.9,
     Node(10			),
     None,
     None,
     12.20,
 ]
 lowerCAmelCase    				= LinkedList()

 for i in test_input:
  linked_list.insert_tail(_UpperCAmelCase			)

 # Check if it's empty or not
 assert linked_list.is_empty() is False
 assert (
     str(_UpperCAmelCase			) == "-9->100->Node(77345112)->dlrow olleH->7->5555->0->"
     "-192.55555->Hello, world!->77.9->Node(10)->None->None->12.2"
 )

 # Delete the head
 lowerCAmelCase    				= linked_list.delete_head()
 assert result == -9
 assert (
     str(_UpperCAmelCase			) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
     "Hello, world!->77.9->Node(10)->None->None->12.2"
 )

 # Delete the tail
 lowerCAmelCase    				= linked_list.delete_tail()
 assert result == 12.2
 assert (
     str(_UpperCAmelCase			) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
     "Hello, world!->77.9->Node(10)->None->None"
 )

 # Delete a node in specific location in linked list
 lowerCAmelCase    				= linked_list.delete_nth(10			)
 assert result is None
 assert (
     str(_UpperCAmelCase			) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
     "Hello, world!->77.9->Node(10)->None"
 )

 # Add a Node instance to its head
 linked_list.insert_head(Node('Hello again, world!'			)			)
 assert (
     str(_UpperCAmelCase			)
     == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
     "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None"
 )

 # Add None to its tail
 linked_list.insert_tail(_UpperCAmelCase			)
 assert (
     str(_UpperCAmelCase			)
     == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
     "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None->None"
 )

 # Reverse the linked list
 linked_list.reverse()
 assert (
     str(_UpperCAmelCase			)
     == "None->None->Node(10)->77.9->Hello, world!->-192.55555->0->5555->"
     "7->dlrow olleH->Node(77345112)->100->Node(Hello again, world!)"
 )

def 		_SCREAMING_SNAKE_CASE			():
 from doctest import testmod

 testmod()

 lowerCAmelCase    				= LinkedList()
 linked_list.insert_head(input('Inserting 1st at head '			).strip()			)
 linked_list.insert_head(input('Inserting 2nd at head '			).strip()			)
 print('\nPrint list:'			)
 linked_list.print_list()
 linked_list.insert_tail(input('\nInserting 1st at tail '			).strip()			)
 linked_list.insert_tail(input('Inserting 2nd at tail '			).strip()			)
 print('\nPrint list:'			)
 linked_list.print_list()
 print('\nDelete head'			)
 linked_list.delete_head()
 print('Delete tail'			)
 linked_list.delete_tail()
 print('\nPrint list:'			)
 linked_list.print_list()
 print('\nReverse linked list'			)
 linked_list.reverse()
 print('\nPrint list:'			)
 linked_list.print_list()
 print('\nString representation of linked list:'			)
 print(_UpperCAmelCase			)
 print('\nReading/changing Node data using indexing:'			)
 print(F'Element at Position 1: {linked_list[1]}'			)
 lowerCAmelCase    				= input('Enter New Value: '			).strip()
 print('New list:'			)
 print(_UpperCAmelCase			)
 print(F'length of linked_list is : {len(_UpperCAmelCase			)}'			)


if __name__ == "__main__":
       main()





