


'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase				:  str    		=      logging.get_logger(__name__)

lowerCamelCase				:  Tuple    		=      {'vocab_file': 'sentencepiece.model'}

lowerCamelCase				:  str    		=      {
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

lowerCamelCase				:  Dict    		=      {
    'google/rembert': 256,
}
class     __lowerCAmelCase		(lowercase_	):





      '''simple docstring'''






      lowerCAmelCase__   :	Any       =				VOCAB_FILES_NAMES
      lowerCAmelCase__   :	List[str]       =				PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__   :	int       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



      def __init__(self       :       Tuple  ,		UpperCamelCase       :       Optional[int]  ,		UpperCamelCase       :       Union[str, Any]=False  ,		UpperCamelCase       :       Any=True  ,		UpperCamelCase       :       str=True  ,		UpperCamelCase       :       List[Any]="[CLS]"  ,		UpperCamelCase       :       List[Any]="[SEP]"  ,		UpperCamelCase       :       List[Any]="[UNK]"  ,		UpperCamelCase       :       Optional[Any]="[SEP]"  ,		UpperCamelCase       :       str="[PAD]"  ,		UpperCamelCase       :       List[str]="[CLS]"  ,		UpperCamelCase       :       str="[MASK]"  ,		**UpperCamelCase       :       Optional[Any]  ,		):

             '''simple docstring'''







             super().__init__(
                 do_lower_case=UpperCamelCase  ,		remove_space=UpperCamelCase  ,		keep_accents=UpperCamelCase  ,		bos_token=UpperCamelCase  ,		eos_token=UpperCamelCase  ,		unk_token=UpperCamelCase  ,		sep_token=UpperCamelCase  ,		pad_token=UpperCamelCase  ,		cls_token=UpperCamelCase  ,		mask_token=UpperCamelCase  ,		**UpperCamelCase  ,		)

             lowercase__      =					do_lower_case
             lowercase__      =					remove_space
             lowercase__      =					keep_accents
             lowercase__      =					vocab_file

             lowercase__      =					spm.SentencePieceProcessor()
             self.sp_model.Load(UpperCamelCase					)



      @property
      def  UpperCamelCase__						(self       :       Union[str, Any]					):

             '''simple docstring'''







             return len(self.sp_model					)



      def  UpperCamelCase__						(self       :       Optional[Any]					):

             '''simple docstring'''







             lowercase__      =					{self.convert_ids_to_tokens(UpperCamelCase					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab



      def __getstate__(self       :       List[str]					):

             '''simple docstring'''







             lowercase__      =					self.__dict__.copy()
             lowercase__      =					None
             return state



      def __setstate__(self       :       Optional[int]  ,		UpperCamelCase       :       int					):

             '''simple docstring'''







             lowercase__      =					d
             lowercase__      =					spm.SentencePieceProcessor()
             self.sp_model.Load(self.vocab_file					)



      def  UpperCamelCase__						(self       :       Dict  ,		UpperCamelCase       :       int  ,		UpperCamelCase       :       Union[str, Any]=False					):

             '''simple docstring'''







             lowercase__      =					self.sp_model.EncodeAsPieces(UpperCamelCase					)
             return pieces



      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       Optional[int]					):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Union[str, Any]  ,		UpperCamelCase       :       List[str]					):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       int  ,		UpperCamelCase       :       Optional[Any]					):

             '''simple docstring'''







             lowercase__      =					self.sp_model.decode_pieces(UpperCamelCase					)
             return out_string



      def  UpperCamelCase__						(self       :       List[Any]  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[self.cls_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep



      def  UpperCamelCase__						(self       :       Any  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None  ,		UpperCamelCase       :       bool = False					):

             '''simple docstring'''







             if already_has_special_tokens:
                    if token_ids_a is not None:
                           raise ValueError(
                               '''You should not supply a second sequence if the provided sequence of '''
                               '''ids is already formatted with special tokens for the model.'''					)
                    return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

             if token_ids_a is not None:
                    return [1] + ([0] * len(UpperCamelCase					)) + [1] + ([0] * len(UpperCamelCase					)) + [1]
             return [1] + ([0] * len(UpperCamelCase					)) + [1]



      def  UpperCamelCase__						(self       :       List[str]  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep					) * [0]
             return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       str  ,		UpperCamelCase       :       Optional[str] = None					):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase					):
                    logger.error('''Vocabulary path ({}) should be a directory'''.format(UpperCamelCase					)					)
                    return
             lowercase__      =					os.path.join(
                 UpperCamelCase  ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase					):
                    copyfile(self.vocab_file  ,		UpperCamelCase					)

             return (out_vocab_file,)








'''simple docstring'''

from __future__ import annotations

from collections import deque
class     __lowerCAmelCase		:





      '''simple docstring'''






      def __init__(self       :       str  ,		UpperCamelCase       :       list[str]					):

             '''simple docstring'''







             lowercase__      =					[]
             self.adlist.append(
                 {'''value''': '''''', '''next_states''': [], '''fail_state''': 0, '''output''': []}					)

             for keyword in keywords:
                    self.add_keyword(UpperCamelCase					)
             self.set_fail_transitions()



      def  UpperCamelCase__						(self       :       Optional[int]  ,		UpperCamelCase       :       int  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             for state in self.adlist[current_state]["next_states"]:
                    if char == self.adlist[state]["value"]:
                           return state
             return None



      def  UpperCamelCase__						(self       :       Optional[Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					0
             for character in keyword:
                    lowercase__      =					self.find_next_state(UpperCamelCase  ,		UpperCamelCase					)
                    if next_state is None:
                           self.adlist.append(
                               {
                                   '''value''': character,
                                   '''next_states''': [],
                                   '''fail_state''': 0,
                                   '''output''': [],
                               }					)
                           self.adlist[current_state]["next_states"].append(len(self.adlist					) - 1					)
                           lowercase__      =					len(self.adlist					) - 1
                    else:
                           lowercase__      =					next_state
             self.adlist[current_state]["output"].append(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       List[str]					):

             '''simple docstring'''







             lowercase__      =					deque()
             for node in self.adlist[0]["next_states"]:
                    q.append(UpperCamelCase					)
                    lowercase__      =					0
             while q:
                    lowercase__      =					q.popleft()
                    for child in self.adlist[r]["next_states"]:
                           q.append(UpperCamelCase					)
                           lowercase__      =					self.adlist[r]['''fail_state''']
                           while (
                               self.find_next_state(UpperCamelCase  ,		self.adlist[child]['''value''']					) is None
                               and state != 0
                           ):
                                  lowercase__      =					self.adlist[state]['''fail_state''']
                           lowercase__      =					self.find_next_state(
                               UpperCamelCase  ,		self.adlist[child]['''value''']					)
                           if self.adlist[child]["fail_state"] is None:
                                  lowercase__      =					0
                           lowercase__      =					(
                               self.adlist[child]['''output''']
                               + self.adlist[self.adlist[child]['''fail_state''']]['''output''']
                           )






      def  UpperCamelCase__						(self       :       Optional[int]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					{}  # returns a dict with keywords and list of its occurrences
             lowercase__      =					0
             for i in range(len(UpperCamelCase					)					):
                    while (
                        self.find_next_state(UpperCamelCase  ,		string[i]					) is None
                        and current_state != 0
                    ):
                           lowercase__      =					self.adlist[current_state]['''fail_state''']
                    lowercase__      =					self.find_next_state(UpperCamelCase  ,		string[i]					)
                    if next_state is None:
                           lowercase__      =					0
                    else:
                           lowercase__      =					next_state
                           for key in self.adlist[current_state]["output"]:
                                  if key not in result:
                                         lowercase__      =					[]
                                  result[key].append(i - len(UpperCamelCase					) + 1					)
             return result


if __name__ == "__main__":
 import doctest

 doctest.testmod()




