



"""simple docstring"""






from abc import ABC, abstractmethod
from typing import List, Optional



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	List[str]				):



     """simple docstring"""
     # test for the above condition
     self.test()



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      0
     _snake_case   	=      False
     while not completed:
       if counter == 1:
         self.reset()
       _snake_case   	=      self.advance()
       if not self.does_advance(__lowerCamelCase				):
         raise Exception(
             '''Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.'''				)

       _snake_case	,     _snake_case	,     _snake_case   	=      self.update(__lowerCamelCase				)
       counter += 1

       if counter > 1_0_0_0_0:
         raise Exception('''update() does not fulfill the constraint.'''				)

     if self.remaining() != 0:
       raise Exception('''Custom Constraint is not defined correctly.'''				)



   @abstractmethod
   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)



   @abstractmethod
   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)



   @abstractmethod
   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)



   @abstractmethod
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)



   @abstractmethod
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)





   @abstractmethod
   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	Optional[Any]=False				):



     """simple docstring"""
     raise NotImplementedError(
         f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""				)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	Dict							,       __lowerCamelCase    :	List[int]				):



     """simple docstring"""
     super(__lowerCamelCase							,       self				).__init__()

     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				) or len(__lowerCamelCase				) == 0:
       raise ValueError(f"""`token_ids` has to be a non-empty list, but is {token_ids}."""				)
     if any((not isinstance(__lowerCamelCase							,       __lowerCamelCase				) or token_id < 0) for token_id in token_ids				):
       raise ValueError(f"""Each list in `token_ids` has to be a list of positive integers, but is {token_ids}."""				)

     _snake_case   	=      token_ids

     _snake_case   	=      len(self.token_ids				)
     _snake_case   	=      -1  # the index of the currently fulfilled step
     _snake_case   	=      False



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     if self.completed:
       return None
     return self.token_ids[self.fulfilled_idx + 1]



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(f"""`token_id` has to be an `int`, but is {token_id} of type {type(__lowerCamelCase				)}"""				)

     if self.completed:
       return False

     return token_id == self.token_ids[self.fulfilled_idx + 1]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(f"""`token_id` has to be an `int`, but is {token_id} of type {type(__lowerCamelCase				)}"""				)

     _snake_case   	=      False
     _snake_case   	=      False
     _snake_case   	=      False

     if self.does_advance(__lowerCamelCase				):
       self.fulfilled_idx += 1
       _snake_case   	=      True
       if self.fulfilled_idx == (self.seqlen - 1):
         _snake_case   	=      True
       _snake_case   	=      completed
     else:
       # failed to make progress.
       _snake_case   	=      True
       self.reset()
     return stepped, completed, reset



   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      False
     _snake_case   	=      0



   def    __UpperCAmelCase    (       self    :	Dict				):



     """simple docstring"""
     return self.seqlen - (self.fulfilled_idx + 1)





   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	str=False				):



     """simple docstring"""
     _snake_case   	=      PhrasalConstraint(self.token_ids				)

     if stateful:
       _snake_case   	=      self.seqlen
       _snake_case   	=      self.fulfilled_idx
       _snake_case   	=      self.completed

     return new_constraint



class     UpperCAmelCase    :



   def __init__(       self    :	Optional[Any]							,       __lowerCamelCase    :	List[List[int]]							,       __lowerCamelCase    :	Optional[Any]=True				):



     """simple docstring"""
     _snake_case   	=      max([len(__lowerCamelCase				) for one in nested_token_ids]				)

     _snake_case   	=      {}
     for token_ids in nested_token_ids:
       _snake_case   	=      root
       for tidx, token_id in enumerate(__lowerCamelCase				):
         if token_id not in level:
           _snake_case   	=      {}

         _snake_case   	=      level[token_id]

     if no_subsets and self.has_subsets(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(
           '''Each list in `nested_token_ids` can\'t be a complete subset of another list, but is'''
           f""" {nested_token_ids}."""				)

     _snake_case   	=      root



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	Dict				):



     """simple docstring"""
     _snake_case   	=      self.trie

     for current_token in current_seq:
       _snake_case   	=      start[current_token]

     _snake_case   	=      list(start.keys()				)

     return next_tokens



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      self.next_tokens(__lowerCamelCase				)

     return len(__lowerCamelCase				) == 0



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     _snake_case   	=      list(root.values()				)
     if len(__lowerCamelCase				) == 0:
       return 1
     else:
       return sum([self.count_leaves(__lowerCamelCase				) for nn in next_nodes]				)





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      self.count_leaves(__lowerCamelCase				)
     return len(__lowerCamelCase				) != leaf_count



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	int							,       __lowerCamelCase    :	List[List[int]]				):



     """simple docstring"""
     super(__lowerCamelCase							,       self				).__init__()

     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				) or len(__lowerCamelCase				) == 0:
       raise ValueError(f"""`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}."""				)
     if any(not isinstance(__lowerCamelCase							,       __lowerCamelCase				) for token_ids in nested_token_ids				):
       raise ValueError(f"""`nested_token_ids` has to be a list of lists, but is {nested_token_ids}."""				)
     if any(
         any((not isinstance(__lowerCamelCase							,       __lowerCamelCase				) or token_id < 0) for token_id in token_ids				)
         for token_ids in nested_token_ids				):
       raise ValueError(
           f"""Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}."""				)

     _snake_case   	=      DisjunctiveTrie(__lowerCamelCase				)
     _snake_case   	=      nested_token_ids

     _snake_case   	=      self.trie.max_height
     _snake_case   	=      []
     _snake_case   	=      False



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      self.trie.next_tokens(self.current_seq				)

     if len(__lowerCamelCase				) == 0:
       return None
     else:
       return token_list



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(f"""`token_id` is supposed to be type `int`, but is {token_id} of type {type(__lowerCamelCase				)}"""				)

     _snake_case   	=      self.trie.next_tokens(self.current_seq				)

     return token_id in next_tokens



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(f"""`token_id` is supposed to be type `int`, but is {token_id} of type {type(__lowerCamelCase				)}"""				)

     _snake_case   	=      False
     _snake_case   	=      False
     _snake_case   	=      False

     if self.does_advance(__lowerCamelCase				):
       self.current_seq.append(__lowerCamelCase				)
       _snake_case   	=      True
     else:
       _snake_case   	=      True
       self.reset()

     _snake_case   	=      self.trie.reached_leaf(self.current_seq				)
     _snake_case   	=      completed

     return stepped, completed, reset



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      False
     _snake_case   	=      []



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     if self.completed:
       # since this can be completed without reaching max height
       return 0
     else:
       return self.seqlen - len(self.current_seq				)





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	Optional[int]=False				):



     """simple docstring"""
     _snake_case   	=      DisjunctiveConstraint(self.token_ids				)

     if stateful:
       _snake_case   	=      self.seqlen
       _snake_case   	=      self.current_seq
       _snake_case   	=      self.completed

     return new_constraint



class     UpperCAmelCase    :



   def __init__(       self    :	Optional[int]							,       __lowerCamelCase    :	List[Constraint]				):



     """simple docstring"""
     _snake_case   	=      constraints

     # max # of steps required to fulfill a given constraint
     _snake_case   	=      max([c.seqlen for c in constraints]				)
     _snake_case   	=      len(__lowerCamelCase				)
     _snake_case   	=      False

     self.init_state()



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      []
     _snake_case   	=      None
     _snake_case   	=      [constraint.copy(stateful=__lowerCamelCase				) for constraint in self.constraints]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      0
     if self.inprogress_constraint:
       # extra points for having a constraint mid-fulfilled
       add += self.max_seqlen - self.inprogress_constraint.remaining()

     return (len(self.complete_constraints				) * self.max_seqlen) + add



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      []
     if self.inprogress_constraint is None:
       for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
         _snake_case   	=      constraint.advance()
         if isinstance(__lowerCamelCase							,       __lowerCamelCase				):
           token_list.append(__lowerCamelCase				)
         elif isinstance(__lowerCamelCase							,       __lowerCamelCase				):
           token_list.extend(__lowerCamelCase				)
     else:
       _snake_case   	=      self.inprogress_constraint.advance()
       if isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         token_list.append(__lowerCamelCase				)
       elif isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         token_list.extend(__lowerCamelCase				)

     if len(__lowerCamelCase				) == 0:
       return None
     else:
       return token_list



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Optional[List[int]]				):



     """simple docstring"""
     self.init_state()

     if token_ids is not None:
       for token in token_ids:
         # completes or steps **one** constraint
         _snake_case	,     _snake_case   	=      self.add(__lowerCamelCase				)

         # the entire list of constraints are fulfilled
         if self.completed:
           break



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
       raise ValueError(f"""`token_id` should be an `int`, but is `{token_id}`."""				)

     _snake_case	,     _snake_case   	=      False, False

     if self.completed:
       _snake_case   	=      True
       _snake_case   	=      False
       return complete, stepped

     if self.inprogress_constraint is not None:
       # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
       # job, simply update the state

       _snake_case	,     _snake_case	,     _snake_case   	=      self.inprogress_constraint.update(__lowerCamelCase				)
       if reset:
         # 1. If the next token breaks the progress, then we must restart.
         #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

         #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
         #     constraint, not the full list of constraints.

         self.pending_constraints.append(self.inprogress_constraint.copy(stateful=__lowerCamelCase				)				)
         _snake_case   	=      None

       if complete:
         # 2. If the next token completes the constraint, move it to completed list, set
         #     inprogress to None. If there are no pending constraints either, then this full list of constraints
         #     is complete.

         self.complete_constraints.append(self.inprogress_constraint				)
         _snake_case   	=      None

         if len(self.pending_constraints				) == 0:
           # we're done!
           _snake_case   	=      True

     else:
       # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
       # of constraints?

       for cidx, pending_constraint in enumerate(self.pending_constraints				):
         if pending_constraint.does_advance(__lowerCamelCase				):
           _snake_case	,     _snake_case	,     _snake_case   	=      pending_constraint.update(__lowerCamelCase				)

           if not stepped:
             raise Exception(
                 '''`constraint.update(token_id)` is not yielding incremental progress, '''
                 '''even though `constraint.does_advance(token_id)` is true.'''				)

           if complete:
             self.complete_constraints.append(__lowerCamelCase				)
             _snake_case   	=      None

           if not complete and stepped:
             _snake_case   	=      pending_constraint

           if complete or stepped:
             # If we made any progress at all, then it's at least not a "pending constraint".

             _snake_case   	=      (
                 self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
             )

             if len(self.pending_constraints				) == 0 and self.inprogress_constraint is None:
               # If there's no longer any pending after this and no inprogress either, then we must be
               # complete.

               _snake_case   	=      True

             break  # prevent accidentally stepping through multiple constraints with just one token.

     return complete, stepped





   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[str, Any]=True				):



     """simple docstring"""
     _snake_case   	=      ConstraintListState(self.constraints				)  # we actually never though self.constraints objects
     # throughout this process. So it's at initialization state.

     if stateful:
       _snake_case   	=      [
           constraint.copy(stateful=__lowerCamelCase				) for constraint in self.complete_constraints
       ]
       if self.inprogress_constraint is not None:
         _snake_case   	=      self.inprogress_constraint.copy(stateful=__lowerCamelCase				)
       _snake_case   	=      [constraint.copy() for constraint in self.pending_constraints]

     return new_state










"""simple docstring"""






import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
     import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				{'''vocab_file''': '''vocab.txt'''}

snake_case   					=				{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

snake_case   					=				{
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}




def 			snake_case		(		lowerCAmelCase_	)						->				int:
  _snake_case   	=      collections.OrderedDict()
  with open(lowerCAmelCase_  ,						'''r'''  ,						encoding='''utf-8'''	) as reader:
    _snake_case   	=      reader.readlines()
  for index, token in enumerate(lowerCAmelCase_	):
    _snake_case   	=      token.rstrip('''\n'''	)
    _snake_case   	=      index
  return vocab



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	Union[str, Any]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	List[str]="<unk>"							,       __lowerCamelCase    :	Tuple=2_0_0				):



     """simple docstring"""
     _snake_case   	=      vocab
     _snake_case   	=      unk_token
     _snake_case   	=      max_input_chars_per_word





   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      list(__lowerCamelCase				)
     if len(__lowerCamelCase				) > self.max_input_chars_per_word:
       return [self.unk_token]

     _snake_case   	=      0
     _snake_case   	=      []
     while start < len(__lowerCamelCase				):
       _snake_case   	=      len(__lowerCamelCase				)
       _snake_case   	=      None
       while start < end:
         _snake_case   	=      ''''''.join(chars[start:end]				)
         if substr in self.vocab:
           _snake_case   	=      substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token				)
         start += 1
       else:
         sub_tokens.append(__lowerCamelCase				)
         _snake_case   	=      end

     return sub_tokens



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	List[str]           =  VOCAB_FILES_NAMES
   A__		:	str           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	str           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Union[str, Any]           =  ['''input_ids''', '''attention_mask''']
   A__		:	Optional[int]           =  False



   def __init__(       self    :	Dict							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	str="<d>"							,       __lowerCamelCase    :	Tuple="</d>"							,       __lowerCamelCase    :	Tuple="<s>"							,       __lowerCamelCase    :	int="</s>"							,       __lowerCamelCase    :	List[str]="<pad>"							,       __lowerCamelCase    :	int="<unk>"							,       __lowerCamelCase    :	int="</n>"							,       __lowerCamelCase    :	Tuple="</_>"							,       __lowerCamelCase    :	Optional[Any]="left"							,       **__lowerCamelCase    :	str							,       ):



     """simple docstring"""
     requires_backends(self							,       ['''jieba''']				)
     super().__init__(
         bod_token=__lowerCamelCase							,       eod_token=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       line_token=__lowerCamelCase							,       space_token=__lowerCamelCase							,       padding_side=__lowerCamelCase							,       **__lowerCamelCase							,       )
     _snake_case   	=      bod_token
     _snake_case   	=      eod_token
     _snake_case   	=      load_vocab(__lowerCamelCase				)
     _snake_case   	=      self.encoder[space_token]
     _snake_case   	=      self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     _snake_case   	=      {v: k for k, v in self.encoder.items()}

     _snake_case   	=      WordpieceTokenizer(vocab=self.encoder							,       unk_token=self.unk_token				)



   @property
   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     return self.encoder[self.bod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return self.encoder[self.eod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     return self.encoder["\n"]



   @property
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     return len(self.encoder				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return dict(self.encoder							,       **self.added_tokens_encoder				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      []
     for x in jieba.cut(__lowerCamelCase							,       cut_all=__lowerCamelCase				):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowerCamelCase				)				)
     return output_tokens



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Union[str, Any]							,       **__lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      [i for i in token_ids if i >= 0]
     _snake_case   	=      [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(__lowerCamelCase							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     return token in self.encoder



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     return "".join(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.encoder.get(__lowerCamelCase							,       self.encoder.get(self.unk_token				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.decoder.get(__lowerCamelCase							,       self.unk_token				)



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if os.path.isdir(__lowerCamelCase				):
       _snake_case   	=      os.path.join(
           __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
     else:
       _snake_case   	=      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
     _snake_case   	=      0
     if " " in self.encoder:
       _snake_case   	=      self.encoder[''' ''']
       del self.encoder[" "]
     if "\n" in self.encoder:
       _snake_case   	=      self.encoder['''\n''']
       del self.encoder["\n"]
     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               ''' Please check that the vocabulary is not corrupted!'''				)
           _snake_case   	=      token_index
         writer.write(token + '''\n'''				)
         index += 1
     return (vocab_file,)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	List[int] = None				):



     """simple docstring"""
     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is not None:
       return [1] + ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				))
     return [1] + ([0] * len(__lowerCamelCase				))





