


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


snake_case_						:			Optional[Any]													=							logging.get_logger(__name__)

snake_case_						:			Union[str, Any]													=							{'''vocab_file''': '''spiece.model'''}

snake_case_						:			List[Any]													=							{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

snake_case_						:			Union[str, Any]													=							{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
snake_case_						:			str													=							0
snake_case_						:			Tuple													=							1
snake_case_						:			List[str]													=							2
snake_case_						:			Optional[Any]													=							3
snake_case_						:			Any													=							4

class 						A__							(		UpperCamelCase__  ):
      UpperCAmelCase             =						VOCAB_FILES_NAMES
      UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase             =						"left"




      def __init__(       self					:      int				,    _a					:      Any				,    _a					:      Dict=False				,    _a					:      str=True				,    _a					:      List[Any]=False				,    _a					:      int="<s>"				,    _a					:      Union[str, Any]="</s>"				,    _a					:      str="<unk>"				,    _a					:      str="<sep>"				,    _a					:      List[Any]="<pad>"				,    _a					:      Tuple="<cls>"				,    _a					:      Union[str, Any]="<mask>"				,    _a					:      Union[str, Any]=["<eop>", "<eod>"]				,    _a					:      Optional[Dict[str, Any]] = None				,    **_a					:      Tuple				,    )							->     None:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =AddedToken(_a				,    lstrip=_a				,    rstrip=_a  ) if isinstance(_a				,    _a  ) else mask_token

          _SCREAMING_SNAKE_CASE			  ={} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=_a				,    remove_space=_a				,    keep_accents=_a				,    bos_token=_a				,    eos_token=_a				,    unk_token=_a				,    sep_token=_a				,    pad_token=_a				,    cls_token=_a				,    mask_token=_a				,    additional_special_tokens=_a				,    sp_model_kwargs=self.sp_model_kwargs				,    **_a				,    )

          _SCREAMING_SNAKE_CASE			  =3

          _SCREAMING_SNAKE_CASE			  =do_lower_case
          _SCREAMING_SNAKE_CASE			  =remove_space
          _SCREAMING_SNAKE_CASE			  =keep_accents
          _SCREAMING_SNAKE_CASE			  =vocab_file

          _SCREAMING_SNAKE_CASE			  =spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(_a  )




      @property
      def 							__UpperCamelCase				(       self					:      str  )							->     Any:


          """simple docstring"""





          return len(self.sp_model  )




      def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Optional[Any]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  ={self.convert_ids_to_tokens(_a  ): i for i in range(self.vocab_size  )}
          vocab.update(self.added_tokens_encoder  )
          return vocab




      def __getstate__(       self					:      List[Any]  )							->     List[str]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =self.__dict__.copy()
          _SCREAMING_SNAKE_CASE			  =None
          return state




      def __setstate__(       self					:      Union[str, Any]				,    _a					:      str  )							->     Optional[Any]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =d

          # for backward compatibility
          if not hasattr(self				,    '''sp_model_kwargs'''  ):
              _SCREAMING_SNAKE_CASE			  ={}

          _SCREAMING_SNAKE_CASE			  =spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(self.vocab_file  )




      def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      int  )							->     Optional[Any]:


          """simple docstring"""





          if self.remove_space:
              _SCREAMING_SNAKE_CASE			  =''' '''.join(inputs.strip().split()  )
          else:
              _SCREAMING_SNAKE_CASE			  =inputs
          _SCREAMING_SNAKE_CASE			  =outputs.replace('''``'''				,    '''"'''  ).replace('''\'\''''				,    '''"'''  )

          if not self.keep_accents:
              _SCREAMING_SNAKE_CASE			  =unicodedata.normalize('''NFKD'''				,    _a  )
              _SCREAMING_SNAKE_CASE			  =''''''.join([c for c in outputs if not unicodedata.combining(_a  )]  )
          if self.do_lower_case:
              _SCREAMING_SNAKE_CASE			  =outputs.lower()

          return outputs




      def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      str  )							->     List[str]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =self.preprocess_text(_a  )
          _SCREAMING_SNAKE_CASE			  =self.sp_model.encode(_a				,    out_type=_a  )
          _SCREAMING_SNAKE_CASE			  =[]
          for piece in pieces:
              if len(_a  ) > 1 and piece[-1] == str(''','''  ) and piece[-2].isdigit():
                  _SCREAMING_SNAKE_CASE			  =self.sp_model.EncodeAsPieces(piece[:-1].replace(_a				,    ''''''  )  )
                  if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]  ) == 1:
                          _SCREAMING_SNAKE_CASE			  =cur_pieces[1:]
                      else:
                          _SCREAMING_SNAKE_CASE			  =cur_pieces[0][1:]
                  cur_pieces.append(piece[-1]  )
                  new_pieces.extend(_a  )
              else:
                  new_pieces.append(_a  )

          return new_pieces




      def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      Any  )							->     List[str]:


          """simple docstring"""





          return self.sp_model.PieceToId(_a  )




      def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      str  )							->     Dict:


          """simple docstring"""





          return self.sp_model.IdToPiece(_a  )




      def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Union[str, Any]  )							->     Union[str, Any]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =''''''.join(_a  ).replace(_a				,    ''' '''  ).strip()
          return out_string




      def 							__UpperCamelCase				(       self					:      str				,    _a					:      List[int]				,    _a					:      bool = False				,    _a					:      bool = None				,    _a					:      bool = True				,    **_a					:      Any				,    )							->     str:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =kwargs.pop('''use_source_tokenizer'''				,    _a  )

          _SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(_a				,    skip_special_tokens=_a  )

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          _SCREAMING_SNAKE_CASE			  =[]
          _SCREAMING_SNAKE_CASE			  =[]
          for token in filtered_tokens:
              if skip_special_tokens and token in self.all_special_ids:
                  continue
              if token in self.added_tokens_encoder:
                  if current_sub_text:
                      sub_texts.append(self.convert_tokens_to_string(_a  )  )
                      _SCREAMING_SNAKE_CASE			  =[]
                  sub_texts.append(_a  )
              else:
                  current_sub_text.append(_a  )
          if current_sub_text:
              sub_texts.append(self.convert_tokens_to_string(_a  )  )

          # Mimic the behavior of the Rust tokenizer:
          # By default, there are no spaces between special tokens
          _SCREAMING_SNAKE_CASE			  =''''''.join(_a  )

          _SCREAMING_SNAKE_CASE			  =(
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
              _SCREAMING_SNAKE_CASE			  =self.clean_up_tokenization(_a  )
              return clean_text
          else:
              return text




      def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =[self.sep_token_id]
          _SCREAMING_SNAKE_CASE			  =[self.cls_token_id]
          if token_ids_a is None:
              return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls




      def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None				,    _a					:      bool = False  )							->     List[int]:


          """simple docstring"""





          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_a				,    token_ids_a=_a				,    already_has_special_tokens=_a  )

          if token_ids_a is not None:
              return ([0] * len(_a  )) + [1] + ([0] * len(_a  )) + [1, 1]
          return ([0] * len(_a  )) + [1, 1]




      def 							__UpperCamelCase				(       self					:      str				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


          """simple docstring"""





          _SCREAMING_SNAKE_CASE			  =[self.sep_token_id]
          _SCREAMING_SNAKE_CASE			  =[2]

          if token_ids_a is None:
              return len(token_ids_a + sep  ) * [0] + cls_segment_id
          return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id




      def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


          """simple docstring"""





          if not os.path.isdir(_a  ):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"  )
              return
          _SCREAMING_SNAKE_CASE			  =os.path.join(
              _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

          if os.path.abspath(self.vocab_file  ) != os.path.abspath(_a  ) and os.path.isfile(self.vocab_file  ):
              copyfile(self.vocab_file				,    _a  )
          elif not os.path.isfile(self.vocab_file  ):
              with open(_a				,    '''wb'''  ) as fi:
                  _SCREAMING_SNAKE_CASE			  =self.sp_model.serialized_model_proto()
                  fi.write(_a  )

          return (out_vocab_file,)



import string
from math import logaa







def 						lowerCamelCase(   a__	,a__):
				_SCREAMING_SNAKE_CASE			  =document.translate(
				    str.maketrans(''''''	,''''''	,string.punctuation)).replace('''\n'''	,'''''')
				_SCREAMING_SNAKE_CASE			  =document_without_punctuation.split(''' ''')  # word tokenization
				return len([word for word in tokenize_document if word.lower() == term.lower()])







def 						lowerCamelCase(   a__	,a__):
				_SCREAMING_SNAKE_CASE			  =corpus.lower().translate(
				    str.maketrans(''''''	,''''''	,string.punctuation))  # strip all punctuation and replace it with ''
				_SCREAMING_SNAKE_CASE			  =corpus_without_punctuation.split('''\n''')
				_SCREAMING_SNAKE_CASE			  =term.lower()
				return (len([doc for doc in docs if term in doc]), len(a__))







def 						lowerCamelCase(   a__	,a__	,a__=False):
				if smoothing:
								if n == 0:
												raise ValueError('''log10(0) is undefined.''')
								return round(1 + logaa(n / (1 + df))	,3)

				if df == 0:
								raise ZeroDivisionError('''df must be > 0''')
				elif n == 0:
								raise ValueError('''log10(0) is undefined.''')
				return round(logaa(n / df)	,3)







def 						lowerCamelCase(   a__	,a__):
				return round(tf * idf	,3)