
'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_			:			Dict			   =     logging.get_logger(__name__)
lowerCAmelCase_			:			Optional[Any]			   =     {'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_			:			Tuple			   =     {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
}

lowerCAmelCase_			:			Union[str, Any]			   =     {
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

lowerCAmelCase_			:			List[str]			   =     '''▁'''







class 	__lowerCAmelCase    (  __a				):
     snake_case   :    List[str]							      =      VOCAB_FILES_NAMES
     snake_case   :    int							      =      PRETRAINED_VOCAB_FILES_MAP
     snake_case   :    Dict							      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     snake_case   :    int							      =      ["""input_ids""", """token_type_ids"""]



     def __init__(self						,	lowerCAmelCase__						,	lowerCAmelCase__=False						,	lowerCAmelCase__=True						,	lowerCAmelCase__=True						,	lowerCAmelCase__="<unk>"						,	lowerCAmelCase__="[SEP]"						,	lowerCAmelCase__="<pad>"						,	lowerCAmelCase__="[CLS]"						,	lowerCAmelCase__="[MASK]"						,	lowerCAmelCase__ = None						,	**lowerCAmelCase__						,	):
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.
          _UpperCAmelCase					:  str         =    (
              AddedToken(lowerCAmelCase__						,	lstrip=lowerCAmelCase__						,	rstrip=lowerCAmelCase__						,	normalized=lowerCAmelCase__	)
              if isinstance(lowerCAmelCase__						,	lowerCAmelCase__	)
              else mask_token
          )

          _UpperCAmelCase					:  Tuple         =    {} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=lowerCAmelCase__						,	remove_space=lowerCAmelCase__						,	keep_accents=lowerCAmelCase__						,	unk_token=lowerCAmelCase__						,	sep_token=lowerCAmelCase__						,	pad_token=lowerCAmelCase__						,	cls_token=lowerCAmelCase__						,	mask_token=lowerCAmelCase__						,	sp_model_kwargs=self.sp_model_kwargs						,	**lowerCAmelCase__						,	)

          _UpperCAmelCase					:  int         =    do_lower_case
          _UpperCAmelCase					:  Optional[int]         =    remove_space
          _UpperCAmelCase					:  Optional[Any]         =    keep_accents
          _UpperCAmelCase					:  List[str]         =    vocab_file

          _UpperCAmelCase					:  Any         =    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(lowerCAmelCase__	)



     @property
     def     snake_case_ (self	):
          return len(self.sp_model	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  List[str]         =    {self.convert_ids_to_tokens(lowerCAmelCase__	): i for i in range(self.vocab_size	)}
          vocab.update(self.added_tokens_encoder	)
          return vocab



     def __getstate__(self	):
          _UpperCAmelCase					:  Any         =    self.__dict__.copy()
          _UpperCAmelCase					:  str         =    None
          return state



     def __setstate__(self						,	lowerCAmelCase__	):
          _UpperCAmelCase					:  Union[str, Any]         =    d

          # for backward compatibility
          if not hasattr(self						,	"""sp_model_kwargs"""	):
               _UpperCAmelCase					:  Tuple         =    {}

          _UpperCAmelCase					:  Any         =    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(self.vocab_file	)



     def     snake_case_ (self						,	lowerCAmelCase__	):
          if self.remove_space:
               _UpperCAmelCase					:  Tuple         =    """ """.join(inputs.strip().split()	)
          else:
               _UpperCAmelCase					:  int         =    inputs
          _UpperCAmelCase					:  Optional[int]         =    outputs.replace("""``"""						,	"""\""""	).replace("""''"""						,	"""\""""	)

          if not self.keep_accents:
               _UpperCAmelCase					:  Any         =    unicodedata.normalize("""NFKD"""						,	lowerCAmelCase__	)
               _UpperCAmelCase					:  str         =    """""".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__	)]	)
          if self.do_lower_case:
               _UpperCAmelCase					:  Dict         =    outputs.lower()

          return outputs



     def     snake_case_ (self						,	lowerCAmelCase__	):
          _UpperCAmelCase					:  Tuple         =    self.preprocess_text(lowerCAmelCase__	)
          _UpperCAmelCase					:  List[str]         =    self.sp_model.encode(lowerCAmelCase__						,	out_type=lowerCAmelCase__	)
          _UpperCAmelCase					:  Union[str, Any]         =    []
          for piece in pieces:
               if len(lowerCAmelCase__	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
                    _UpperCAmelCase					:  Optional[Any]         =    self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__						,	""""""	)	)
                    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]	) == 1:
                              _UpperCAmelCase					:  Dict         =    cur_pieces[1:]
                         else:
                              _UpperCAmelCase					:  Tuple         =    cur_pieces[0][1:]
                    cur_pieces.append(piece[-1]	)
                    new_pieces.extend(lowerCAmelCase__	)
               else:
                    new_pieces.append(lowerCAmelCase__	)

          return new_pieces



     def     snake_case_ (self						,	lowerCAmelCase__	):
          return self.sp_model.PieceToId(lowerCAmelCase__	)



     def     snake_case_ (self						,	lowerCAmelCase__	):
          return self.sp_model.IdToPiece(lowerCAmelCase__	)



     def     snake_case_ (self						,	lowerCAmelCase__	):
          _UpperCAmelCase					:  Optional[int]         =    []
          _UpperCAmelCase					:  Dict         =    """"""
          _UpperCAmelCase					:  Dict         =    False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(lowerCAmelCase__	) + token
                    _UpperCAmelCase					:  str         =    True
                    _UpperCAmelCase					:  List[Any]         =    []
               else:
                    current_sub_tokens.append(lowerCAmelCase__	)
                    _UpperCAmelCase					:  Union[str, Any]         =    False
          out_string += self.sp_model.decode(lowerCAmelCase__	)
          return out_string.strip()



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__ = False						,	lowerCAmelCase__ = None						,	lowerCAmelCase__ = True						,	**lowerCAmelCase__						,	):
          _UpperCAmelCase					:  List[Any]         =    kwargs.pop("""use_source_tokenizer"""						,	lowerCAmelCase__	)

          _UpperCAmelCase					:  Optional[int]         =    self.convert_ids_to_tokens(lowerCAmelCase__						,	skip_special_tokens=lowerCAmelCase__	)

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          _UpperCAmelCase					:  Optional[Any]         =    []
          _UpperCAmelCase					:  Optional[int]         =    []
          for token in filtered_tokens:
               if skip_special_tokens and token in self.all_special_ids:
                    continue
               if token in self.added_tokens_encoder:
                    if current_sub_text:
                         sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__	)	)
                         _UpperCAmelCase					:  int         =    []
                    sub_texts.append(lowerCAmelCase__	)
               else:
                    current_sub_text.append(lowerCAmelCase__	)
          if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__	)	)

          # Mimic the behavior of the Rust tokenizer:
          # No space after <unk>
          if spaces_between_special_tokens:
               _UpperCAmelCase					:  Union[str, Any]         =    re.sub(r"""(<unk>) """						,	r"""\1"""						,	""" """.join(lowerCAmelCase__	)	)
          else:
               _UpperCAmelCase					:  Optional[Any]         =    """""".join(lowerCAmelCase__	)

          _UpperCAmelCase					:  Optional[int]         =    (
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
               _UpperCAmelCase					:  int         =    self.clean_up_tokenization(lowerCAmelCase__	)
               return clean_text
          else:
               return text



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__ = None	):
          _UpperCAmelCase					:  Optional[Any]         =    [self.sep_token_id]
          _UpperCAmelCase					:  Any         =    [self.cls_token_id]
          if token_ids_a is None:
               return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__ = None						,	lowerCAmelCase__ = False	):

          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase__						,	token_ids_a=lowerCAmelCase__						,	already_has_special_tokens=lowerCAmelCase__	)

          if token_ids_a is not None:
               return [1] + ([0] * len(lowerCAmelCase__	)) + [1] + ([0] * len(lowerCAmelCase__	)) + [1]
          return [1] + ([0] * len(lowerCAmelCase__	)) + [1]



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__ = None	):
          _UpperCAmelCase					:  Dict         =    [self.sep_token_id]
          _UpperCAmelCase					:  Union[str, Any]         =    [self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep	) * [0]
          return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__ = None	):
          if not os.path.isdir(lowerCAmelCase__	):
               logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
               return
          _UpperCAmelCase					:  Dict         =    os.path.join(
              lowerCAmelCase__						,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

          if os.path.abspath(self.vocab_file	) != os.path.abspath(lowerCAmelCase__	) and os.path.isfile(self.vocab_file	):
               copyfile(self.vocab_file						,	lowerCAmelCase__	)
          elif not os.path.isfile(self.vocab_file	):
               with open(lowerCAmelCase__						,	"""wb"""	) as fi:
                    _UpperCAmelCase					:  List[str]         =    self.sp_model.serialized_model_proto()
                    fi.write(lowerCAmelCase__	)

          return (out_vocab_file,)









"""simple docstring"""


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


lowerCamelCase__      =      {
    "configuration_utils": ["GenerationConfig"],
    "streamers": ["TextIteratorStreamer", "TextStreamer"],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "Constraint",
      "ConstraintListState",
      "DisjunctiveConstraint",
      "PhrasalConstraint",
  ]
  lowerCamelCase__      =      [
      "BeamHypotheses",
      "BeamScorer",
      "BeamSearchScorer",
      "ConstrainedBeamSearchScorer",
  ]
  lowerCamelCase__      =      [
      "EpsilonLogitsWarper",
      "EtaLogitsWarper",
      "ForcedBOSTokenLogitsProcessor",
      "ForcedEOSTokenLogitsProcessor",
      "HammingDiversityLogitsProcessor",
      "InfNanRemoveLogitsProcessor",
      "LogitsProcessor",
      "LogitsProcessorList",
      "LogitsWarper",
      "MinLengthLogitsProcessor",
      "MinNewTokensLengthLogitsProcessor",
      "NoBadWordsLogitsProcessor",
      "NoRepeatNGramLogitsProcessor",
      "PrefixConstrainedLogitsProcessor",
      "RepetitionPenaltyLogitsProcessor",
      "SequenceBiasLogitsProcessor",
      "EncoderRepetitionPenaltyLogitsProcessor",
      "TemperatureLogitsWarper",
      "TopKLogitsWarper",
      "TopPLogitsWarper",
      "TypicalLogitsWarper",
      "EncoderNoRepeatNGramLogitsProcessor",
      "ExponentialDecayLengthPenalty",
      "LogitNormalization",
  ]
  lowerCamelCase__      =      [
      "MaxNewTokensCriteria",
      "MaxLengthCriteria",
      "MaxTimeCriteria",
      "StoppingCriteria",
      "StoppingCriteriaList",
      "validate_stopping_criteria",
  ]
  lowerCamelCase__      =      [
      "GenerationMixin",
      "top_k_top_p_filtering",
      "GreedySearchEncoderDecoderOutput",
      "GreedySearchDecoderOnlyOutput",
      "SampleEncoderDecoderOutput",
      "SampleDecoderOnlyOutput",
      "BeamSearchEncoderDecoderOutput",
      "BeamSearchDecoderOnlyOutput",
      "BeamSampleEncoderDecoderOutput",
      "BeamSampleDecoderOnlyOutput",
      "ContrastiveSearchEncoderDecoderOutput",
      "ContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "TFForcedBOSTokenLogitsProcessor",
      "TFForcedEOSTokenLogitsProcessor",
      "TFLogitsProcessor",
      "TFLogitsProcessorList",
      "TFLogitsWarper",
      "TFMinLengthLogitsProcessor",
      "TFNoBadWordsLogitsProcessor",
      "TFNoRepeatNGramLogitsProcessor",
      "TFRepetitionPenaltyLogitsProcessor",
      "TFTemperatureLogitsWarper",
      "TFTopKLogitsWarper",
      "TFTopPLogitsWarper",
      "TFForceTokensLogitsProcessor",
      "TFSuppressTokensAtBeginLogitsProcessor",
      "TFSuppressTokensLogitsProcessor",
  ]
  lowerCamelCase__      =      [
      "TFGenerationMixin",
      "tf_top_k_top_p_filtering",
      "TFGreedySearchDecoderOnlyOutput",
      "TFGreedySearchEncoderDecoderOutput",
      "TFSampleEncoderDecoderOutput",
      "TFSampleDecoderOnlyOutput",
      "TFBeamSearchEncoderDecoderOutput",
      "TFBeamSearchDecoderOnlyOutput",
      "TFBeamSampleEncoderDecoderOutput",
      "TFBeamSampleDecoderOnlyOutput",
      "TFContrastiveSearchEncoderDecoderOutput",
      "TFContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "FlaxForcedBOSTokenLogitsProcessor",
      "FlaxForcedEOSTokenLogitsProcessor",
      "FlaxLogitsProcessor",
      "FlaxLogitsProcessorList",
      "FlaxLogitsWarper",
      "FlaxMinLengthLogitsProcessor",
      "FlaxTemperatureLogitsWarper",
      "FlaxTopKLogitsWarper",
      "FlaxTopPLogitsWarper",
  ]
  lowerCamelCase__      =      [
      "FlaxGenerationMixin",
      "FlaxGreedySearchOutput",
      "FlaxSampleOutput",
      "FlaxBeamSearchOutput",
  ]

if TYPE_CHECKING:
  from .configuration_utils import GenerationConfig
  from .streamers import TextIteratorStreamer, TextStreamer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
    from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
    from .logits_process import (
        EncoderNoRepeatNGramLogitsProcessor,
        EncoderRepetitionPenaltyLogitsProcessor,
        EpsilonLogitsWarper,
        EtaLogitsWarper,
        ExponentialDecayLengthPenalty,
        ForcedBOSTokenLogitsProcessor,
        ForcedEOSTokenLogitsProcessor,
        HammingDiversityLogitsProcessor,
        InfNanRemoveLogitsProcessor,
        LogitNormalization,
        LogitsProcessor,
        LogitsProcessorList,
        LogitsWarper,
        MinLengthLogitsProcessor,
        MinNewTokensLengthLogitsProcessor,
        NoBadWordsLogitsProcessor,
        NoRepeatNGramLogitsProcessor,
        PrefixConstrainedLogitsProcessor,
        RepetitionPenaltyLogitsProcessor,
        SequenceBiasLogitsProcessor,
        TemperatureLogitsWarper,
        TopKLogitsWarper,
        TopPLogitsWarper,
        TypicalLogitsWarper,
    )
    from .stopping_criteria import (
        MaxLengthCriteria,
        MaxNewTokensCriteria,
        MaxTimeCriteria,
        StoppingCriteria,
        StoppingCriteriaList,
        validate_stopping_criteria,
    )
    from .utils import (
        BeamSampleDecoderOnlyOutput,
        BeamSampleEncoderDecoderOutput,
        BeamSearchDecoderOnlyOutput,
        BeamSearchEncoderDecoderOutput,
        ContrastiveSearchDecoderOnlyOutput,
        ContrastiveSearchEncoderDecoderOutput,
        GenerationMixin,
        GreedySearchDecoderOnlyOutput,
        GreedySearchEncoderDecoderOutput,
        SampleDecoderOnlyOutput,
        SampleEncoderDecoderOutput,
        top_k_top_p_filtering,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tf_logits_process import (
        TFForcedBOSTokenLogitsProcessor,
        TFForcedEOSTokenLogitsProcessor,
        TFForceTokensLogitsProcessor,
        TFLogitsProcessor,
        TFLogitsProcessorList,
        TFLogitsWarper,
        TFMinLengthLogitsProcessor,
        TFNoBadWordsLogitsProcessor,
        TFNoRepeatNGramLogitsProcessor,
        TFRepetitionPenaltyLogitsProcessor,
        TFSuppressTokensAtBeginLogitsProcessor,
        TFSuppressTokensLogitsProcessor,
        TFTemperatureLogitsWarper,
        TFTopKLogitsWarper,
        TFTopPLogitsWarper,
    )
    from .tf_utils import (
        TFBeamSampleDecoderOnlyOutput,
        TFBeamSampleEncoderDecoderOutput,
        TFBeamSearchDecoderOnlyOutput,
        TFBeamSearchEncoderDecoderOutput,
        TFContrastiveSearchDecoderOnlyOutput,
        TFContrastiveSearchEncoderDecoderOutput,
        TFGenerationMixin,
        TFGreedySearchDecoderOnlyOutput,
        TFGreedySearchEncoderDecoderOutput,
        TFSampleDecoderOnlyOutput,
        TFSampleEncoderDecoderOutput,
        tf_top_k_top_p_filtering,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .flax_logits_process import (
        FlaxForcedBOSTokenLogitsProcessor,
        FlaxForcedEOSTokenLogitsProcessor,
        FlaxLogitsProcessor,
        FlaxLogitsProcessorList,
        FlaxLogitsWarper,
        FlaxMinLengthLogitsProcessor,
        FlaxTemperatureLogitsWarper,
        FlaxTopKLogitsWarper,
        FlaxTopPLogitsWarper,
    )
    from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
  import sys

  lowerCamelCase__      =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
