






from abc import ABC, abstractmethod
from typing import List, Optional




class       snake_case__ (				lowercase_):

       '''simple docstring'''



       def __init__(				self     )			->   Optional[int]:






            '''simple docstring'''





            self.test()



       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            __snake_case      :int 							=    0
            __snake_case      :Tuple 							=    False
            while not completed:
                 if counter == 1:
                      self.reset()
                 __snake_case      :Optional[int] 							=    self.advance()
                 if not self.does_advance(a__     ):
                      raise Exception(
                          """Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."""     )

                 __snake_case							,			__snake_case							,			__snake_case      :int 							=    self.update(a__     )
                 counter += 1

                 if counter > 1_00_00:
                      raise Exception("""update() does not fulfill the constraint."""     )

            if self.remaining() != 0:
                 raise Exception("""Custom Constraint is not defined correctly."""     )



       @abstractmethod
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



       @abstractmethod
       def        __lowercase    (				self      ,    a__     )			->   Any:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



       @abstractmethod
       def        __lowercase    (				self      ,    a__     )			->   Optional[Any]:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



       @abstractmethod
       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



       @abstractmethod
       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



       @abstractmethod
       def        __lowercase    (				self      ,    a__=False     )			->   Tuple:






            '''simple docstring'''





            raise NotImplementedError(
                F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )




class       snake_case__ (				lowercase_):

       '''simple docstring'''



       def __init__(				self      ,    a__     )			->   List[str]:






            '''simple docstring'''





            super(a__      ,    self     ).__init__()

            if not isinstance(a__      ,    a__     ) or len(a__     ) == 0:
                 raise ValueError(F'''`token_ids` has to be a non-empty list, but is {token_ids}.'''     )
            if any((not isinstance(a__      ,    a__     ) or token_id < 0) for token_id in token_ids     ):
                 raise ValueError(F'''Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'''     )

            __snake_case      :Any 							=    token_ids

            __snake_case      :Optional[int] 							=    len(self.token_ids     )
            __snake_case      :int 							=    -1  # the index of the currently fulfilled step
            __snake_case      :Tuple 							=    False



       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            if self.completed:
                 return None
            return self.token_ids[self.fulfilled_idx + 1]



       def        __lowercase    (				self      ,    a__     )			->   Any:






            '''simple docstring'''





            if not isinstance(a__      ,    a__     ):
                 raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(a__     )}'''     )

            if self.completed:
                 return False

            return token_id == self.token_ids[self.fulfilled_idx + 1]



       def        __lowercase    (				self      ,    a__     )			->   Union[str, Any]:






            '''simple docstring'''





            if not isinstance(a__      ,    a__     ):
                 raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(a__     )}'''     )

            __snake_case      :int 							=    False
            __snake_case      :Dict 							=    False
            __snake_case      :Any 							=    False

            if self.does_advance(a__     ):
                 self.fulfilled_idx += 1
                 __snake_case      :Tuple 							=    True
                 if self.fulfilled_idx == (self.seqlen - 1):
                      __snake_case      :str 							=    True
                 __snake_case      :Optional[Any] 							=    completed
            else:
                 # failed to make progress.
                 __snake_case      :Tuple 							=    True
                 self.reset()
            return stepped, completed, reset



       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :str 							=    False
            __snake_case      :List[str] 							=    0



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            return self.seqlen - (self.fulfilled_idx + 1)



       def        __lowercase    (				self      ,    a__=False     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    PhrasalConstraint(self.token_ids     )

            if stateful:
                 __snake_case      :Tuple 							=    self.seqlen
                 __snake_case      :str 							=    self.fulfilled_idx
                 __snake_case      :Optional[int] 							=    self.completed

            return new_constraint




class       snake_case__ :

       '''simple docstring'''



       def __init__(				self      ,    a__      ,    a__=True     )			->   Dict:






            '''simple docstring'''





            __snake_case      :Tuple 							=    max([len(a__     ) for one in nested_token_ids]     )

            __snake_case      :List[Any] 							=    {}
            for token_ids in nested_token_ids:
                 __snake_case      :List[str] 							=    root
                 for tidx, token_id in enumerate(a__     ):
                      if token_id not in level:
                           __snake_case      :str 							=    {}

                      __snake_case      :Optional[Any] 							=    level[token_id]

            if no_subsets and self.has_subsets(a__      ,    a__     ):
                 raise ValueError(
                     """Each list in `nested_token_ids` can't be a complete subset of another list, but is"""
                     F''' {nested_token_ids}.'''     )

            __snake_case      :int 							=    root



       def        __lowercase    (				self      ,    a__     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    self.trie

            for current_token in current_seq:
                 __snake_case      :int 							=    start[current_token]

            __snake_case      :Union[str, Any] 							=    list(start.keys()     )

            return next_tokens



       def        __lowercase    (				self      ,    a__     )			->   Dict:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    self.next_tokens(a__     )

            return len(a__     ) == 0



       def        __lowercase    (				self      ,    a__     )			->   int:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    list(root.values()     )
            if len(a__     ) == 0:
                 return 1
            else:
                 return sum([self.count_leaves(a__     ) for nn in next_nodes]     )



       def        __lowercase    (				self      ,    a__      ,    a__     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :int 							=    self.count_leaves(a__     )
            return len(a__     ) != leaf_count




class       snake_case__ (				lowercase_):

       '''simple docstring'''



       def __init__(				self      ,    a__     )			->   List[Any]:






            '''simple docstring'''





            super(a__      ,    self     ).__init__()

            if not isinstance(a__      ,    a__     ) or len(a__     ) == 0:
                 raise ValueError(F'''`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'''     )
            if any(not isinstance(a__      ,    a__     ) for token_ids in nested_token_ids     ):
                 raise ValueError(F'''`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'''     )
            if any(
                any((not isinstance(a__      ,    a__     ) or token_id < 0) for token_id in token_ids     )
                for token_ids in nested_token_ids     ):
                 raise ValueError(
                     F'''Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'''     )

            __snake_case      :List[Any] 							=    DisjunctiveTrie(a__     )
            __snake_case      :str 							=    nested_token_ids

            __snake_case      :Any 							=    self.trie.max_height
            __snake_case      :Optional[Any] 							=    []
            __snake_case      :Tuple 							=    False



       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            __snake_case      :str 							=    self.trie.next_tokens(self.current_seq     )

            if len(a__     ) == 0:
                 return None
            else:
                 return token_list



       def        __lowercase    (				self      ,    a__     )			->   Any:






            '''simple docstring'''





            if not isinstance(a__      ,    a__     ):
                 raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(a__     )}'''     )

            __snake_case      :Dict 							=    self.trie.next_tokens(self.current_seq     )

            return token_id in next_tokens



       def        __lowercase    (				self      ,    a__     )			->   int:






            '''simple docstring'''





            if not isinstance(a__      ,    a__     ):
                 raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(a__     )}'''     )

            __snake_case      :Optional[int] 							=    False
            __snake_case      :List[str] 							=    False
            __snake_case      :Tuple 							=    False

            if self.does_advance(a__     ):
                 self.current_seq.append(a__     )
                 __snake_case      :Dict 							=    True
            else:
                 __snake_case      :Tuple 							=    True
                 self.reset()

            __snake_case      :str 							=    self.trie.reached_leaf(self.current_seq     )
            __snake_case      :Optional[Any] 							=    completed

            return stepped, completed, reset



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Tuple 							=    False
            __snake_case      :Optional[Any] 							=    []



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            if self.completed:
                 # since this can be completed without reaching max height
                 return 0
            else:
                 return self.seqlen - len(self.current_seq     )



       def        __lowercase    (				self      ,    a__=False     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    DisjunctiveConstraint(self.token_ids     )

            if stateful:
                 __snake_case      :Optional[Any] 							=    self.seqlen
                 __snake_case      :int 							=    self.current_seq
                 __snake_case      :Tuple 							=    self.completed

            return new_constraint




class       snake_case__ :

       '''simple docstring'''



       def __init__(				self      ,    a__     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    constraints

            # max # of steps required to fulfill a given constraint
            __snake_case      :List[str] 							=    max([c.seqlen for c in constraints]     )
            __snake_case      :Union[str, Any] 							=    len(a__     )
            __snake_case      :str 							=    False

            self.init_state()



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            __snake_case      :List[str] 							=    []
            __snake_case      :List[str] 							=    None
            __snake_case      :List[str] 							=    [constraint.copy(stateful=a__     ) for constraint in self.constraints]



       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    0
            if self.inprogress_constraint:
                 # extra points for having a constraint mid-fulfilled
                 add += self.max_seqlen - self.inprogress_constraint.remaining()

            return (len(self.complete_constraints     ) * self.max_seqlen) + add



       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    []
            if self.inprogress_constraint is None:
                 for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
                      __snake_case      :Optional[Any] 							=    constraint.advance()
                      if isinstance(a__      ,    a__     ):
                           token_list.append(a__     )
                      elif isinstance(a__      ,    a__     ):
                           token_list.extend(a__     )
            else:
                 __snake_case      :str 							=    self.inprogress_constraint.advance()
                 if isinstance(a__      ,    a__     ):
                      token_list.append(a__     )
                 elif isinstance(a__      ,    a__     ):
                      token_list.extend(a__     )

            if len(a__     ) == 0:
                 return None
            else:
                 return token_list



       def        __lowercase    (				self      ,    a__     )			->   Optional[int]:






            '''simple docstring'''





            self.init_state()

            if token_ids is not None:
                 for token in token_ids:
                      # completes or steps **one** constraint
                      __snake_case							,			__snake_case      :Optional[Any] 							=    self.add(a__     )

                      # the entire list of constraints are fulfilled
                      if self.completed:
                           break



       def        __lowercase    (				self      ,    a__     )			->   List[Any]:






            '''simple docstring'''





            if not isinstance(a__      ,    a__     ):
                 raise ValueError(F'''`token_id` should be an `int`, but is `{token_id}`.'''     )

            __snake_case							,			__snake_case      :Any 							=    False, False

            if self.completed:
                 __snake_case      :List[Any] 							=    True
                 __snake_case      :Optional[int] 							=    False
                 return complete, stepped

            if self.inprogress_constraint is not None:
                 # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
                 # job, simply update the state

                 __snake_case							,			__snake_case							,			__snake_case      :int 							=    self.inprogress_constraint.update(a__     )
                 if reset:
                      # 1. If the next token breaks the progress, then we must restart.
                      #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

                      #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
                      #     constraint, not the full list of constraints.

                      self.pending_constraints.append(self.inprogress_constraint.copy(stateful=a__     )     )
                      __snake_case      :int 							=    None

                 if complete:
                      # 2. If the next token completes the constraint, move it to completed list, set
                      #     inprogress to None. If there are no pending constraints either, then this full list of constraints
                      #     is complete.

                      self.complete_constraints.append(self.inprogress_constraint     )
                      __snake_case      :List[str] 							=    None

                      if len(self.pending_constraints     ) == 0:
                           # we're done!
                           __snake_case      :str 							=    True

            else:
                 # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
                 # of constraints?

                 for cidx, pending_constraint in enumerate(self.pending_constraints     ):
                      if pending_constraint.does_advance(a__     ):
                           __snake_case							,			__snake_case							,			__snake_case      :Optional[int] 							=    pending_constraint.update(a__     )

                           if not stepped:
                                raise Exception(
                                    """`constraint.update(token_id)` is not yielding incremental progress, """
                                    """even though `constraint.does_advance(token_id)` is true."""     )

                           if complete:
                                self.complete_constraints.append(a__     )
                                __snake_case      :List[str] 							=    None

                           if not complete and stepped:
                                __snake_case      :Optional[int] 							=    pending_constraint

                           if complete or stepped:
                                # If we made any progress at all, then it's at least not a "pending constraint".

                                __snake_case      :Any 							=    (
                                    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
                                )

                                if len(self.pending_constraints     ) == 0 and self.inprogress_constraint is None:
                                     # If there's no longer any pending after this and no inprogress either, then we must be
                                     # complete.

                                     __snake_case      :Dict 							=    True

                                break  # prevent accidentally stepping through multiple constraints with just one token.

            return complete, stepped



       def        __lowercase    (				self      ,    a__=True     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    ConstraintListState(self.constraints     )  # we actually never though self.constraints objects
            # throughout this process. So it's at initialization state.

            if stateful:
                 __snake_case      :str 							=    [
                     constraint.copy(stateful=a__     ) for constraint in self.complete_constraints
                 ]
                 if self.inprogress_constraint is not None:
                      __snake_case      :Tuple 							=    self.inprogress_constraint.copy(stateful=a__     )
                 __snake_case      :Optional[int] 							=    [constraint.copy() for constraint in self.pending_constraints]

            return new_state










import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__       		=  logging.get_logger(__name__)

lowerCamelCase__       		=  {"""vocab_file""": """sentencepiece.model"""}

lowerCamelCase__       		=  {
    """vocab_file""": {
        """google/rembert""": """https://huggingface.co/google/rembert/resolve/main/sentencepiece.model""",
    },
}

lowerCamelCase__       		=  {
    """google/rembert""": 256,
}




class       snake_case__ (				lowercase_):

       '''simple docstring'''
       lowerCamelCase				:					str								=			VOCAB_FILES_NAMES
       lowerCamelCase				:					Tuple								=			PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase				:					Optional[int]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



       def __init__(				self      ,    a__      ,    a__=False      ,    a__=True      ,    a__=True      ,    a__="[CLS]"      ,    a__="[SEP]"      ,    a__="[UNK]"      ,    a__="[SEP]"      ,    a__="[PAD]"      ,    a__="[CLS]"      ,    a__="[MASK]"      ,    **a__      ,    )			->   List[str]:






            '''simple docstring'''





            super().__init__(
                do_lower_case=a__      ,    remove_space=a__      ,    keep_accents=a__      ,    bos_token=a__      ,    eos_token=a__      ,    unk_token=a__      ,    sep_token=a__      ,    pad_token=a__      ,    cls_token=a__      ,    mask_token=a__      ,    **a__      ,    )

            __snake_case      :List[Any] 							=    do_lower_case
            __snake_case      :Dict 							=    remove_space
            __snake_case      :int 							=    keep_accents
            __snake_case      :Optional[int] 							=    vocab_file

            __snake_case      :int 							=    spm.SentencePieceProcessor()
            self.sp_model.Load(a__     )



       @property
       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            return len(self.sp_model     )



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :int 							=    {self.convert_ids_to_tokens(a__     ): i for i in range(self.vocab_size     )}
            vocab.update(self.added_tokens_encoder     )
            return vocab



       def __getstate__(				self     )			->   str:






            '''simple docstring'''





            __snake_case      :str 							=    self.__dict__.copy()
            __snake_case      :Dict 							=    None
            return state



       def __setstate__(				self      ,    a__     )			->   str:






            '''simple docstring'''





            __snake_case      :Dict 							=    d
            __snake_case      :Dict 							=    spm.SentencePieceProcessor()
            self.sp_model.Load(self.vocab_file     )



       def        __lowercase    (				self      ,    a__      ,    a__=False     )			->   Tuple:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    self.sp_model.EncodeAsPieces(a__     )
            return pieces



       def        __lowercase    (				self      ,    a__     )			->   str:






            '''simple docstring'''





            return self.sp_model.PieceToId(a__     )



       def        __lowercase    (				self      ,    a__     )			->   Any:






            '''simple docstring'''





            return self.sp_model.IdToPiece(a__     )



       def        __lowercase    (				self      ,    a__     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    self.sp_model.decode_pieces(a__     )
            return out_string



       def        __lowercase    (				self      ,    a__      ,    a__ = None     )			->   List[int]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    [self.sep_token_id]
            __snake_case      :List[Any] 							=    [self.cls_token_id]
            if token_ids_a is None:
                 return cls + token_ids_a + sep
            return cls + token_ids_a + sep + token_ids_a + sep



       def        __lowercase    (				self      ,    a__      ,    a__ = None      ,    a__ = False     )			->   List[int]:






            '''simple docstring'''





            if already_has_special_tokens:
                 if token_ids_a is not None:
                      raise ValueError(
                          """You should not supply a second sequence if the provided sequence of """
                          """ids is already formatted with special tokens for the model."""     )
                 return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

            if token_ids_a is not None:
                 return [1] + ([0] * len(a__     )) + [1] + ([0] * len(a__     )) + [1]
            return [1] + ([0] * len(a__     )) + [1]



       def        __lowercase    (				self      ,    a__      ,    a__ = None     )			->   List[int]:






            '''simple docstring'''





            __snake_case      :Any 							=    [self.sep_token_id]
            __snake_case      :Any 							=    [self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep     ) * [0]
            return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



       def        __lowercase    (				self      ,    a__      ,    a__ = None     )			->   Tuple[str]:






            '''simple docstring'''





            if not os.path.isdir(a__     ):
                 logger.error("""Vocabulary path ({}) should be a directory""".format(a__     )     )
                 return
            __snake_case      :Optional[int] 							=    os.path.join(
                a__      ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

            if os.path.abspath(self.vocab_file     ) != os.path.abspath(a__     ):
                 copyfile(self.vocab_file      ,    a__     )

            return (out_vocab_file,)


