





"""simple docstring"""



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_			=      logging.get_logger(__name__)

a_			=      """▁"""

a_			=      {"""vocab_file""": """sentencepiece.bpe.model"""}

a_			=      {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

a_			=      {
    """facebook/xglm-564M""": 2048,
}






class 		__snake_case (		SCREAMING_SNAKE_CASE__					):





 """simple docstring"""


 _lowerCamelCase				=					VOCAB_FILES_NAMES
 _lowerCamelCase				=					PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase				=					["""input_ids""", """attention_mask"""]

 def __init__(   self ,						__lowerCamelCase ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="</s>" ,						__lowerCamelCase="<s>" ,						__lowerCamelCase="<unk>" ,						__lowerCamelCase="<pad>" ,						__lowerCamelCase = None ,						**__lowerCamelCase ,						):



     '''simple docstring'''




     __A   :						Optional[int]									=					{} if sp_model_kwargs is None else sp_model_kwargs

     # Compatibility with the original tokenizer
     __A   :						Dict									=					7
     __A   :						Any									=					[F"""<madeupword{i}>""" for i in range(self.num_madeup_words  )]

     __A   :						Tuple									=					kwargs.get('''additional_special_tokens''' ,						[]  )
     kwargs["additional_special_tokens"] += [
         word for word in madeup_words if word not in kwargs["additional_special_tokens"]
     ]

     super().__init__(
         bos_token=__lowerCamelCase ,						eos_token=__lowerCamelCase ,						unk_token=__lowerCamelCase ,						sep_token=__lowerCamelCase ,						cls_token=__lowerCamelCase ,						pad_token=__lowerCamelCase ,						sp_model_kwargs=self.sp_model_kwargs ,						**__lowerCamelCase ,						)

     __A   :						Union[str, Any]									=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
     self.sp_model.Load(str(__lowerCamelCase  )  )
     __A   :						Tuple									=					vocab_file

     # Original fairseq vocab and spm vocab must be "aligned":
     # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
     # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
     # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
     # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

     # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
     __A   :						Dict									=					1

     # Mimic fairseq token-to-id alignment for the first 4 token
     __A   :						int									=					{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

     __A   :						Optional[int]									=					len(self.sp_model  )
     __A   :						Dict									=					{F"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words  )}
     self.fairseq_tokens_to_ids.update(__lowerCamelCase  )

     __A   :						Dict									=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}

 def __getstate__(   self  ):



     '''simple docstring'''




     __A   :						Optional[Any]									=					self.__dict__.copy()
     __A   :						Tuple									=					None
     __A   :						List[str]									=					self.sp_model.serialized_model_proto()
     return state

 def __setstate__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						List[Any]									=					d

     # for backward compatibility
     if not hasattr(self ,						'''sp_model_kwargs'''  ):
         __A   :						Union[str, Any]									=					{}

     __A   :						Optional[int]									=					spm.SentencePieceProcessor(**self.sp_model_kwargs  )
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     if token_ids_a is None:
         return [self.sep_token_id] + token_ids_a
     __A   :						str									=					[self.sep_token_id]
     return sep + token_ids_a + sep + sep + token_ids_a

 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None ,						__lowerCamelCase = False  ):



     '''simple docstring'''




     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=__lowerCamelCase ,						token_ids_a=__lowerCamelCase ,						already_has_special_tokens=__lowerCamelCase  )

     if token_ids_a is None:
         return [1] + ([0] * len(__lowerCamelCase  ))
     return [1] + ([0] * len(__lowerCamelCase  )) + [1, 1] + ([0] * len(__lowerCamelCase  ))

 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A   :						int									=					[self.sep_token_id]

     if token_ids_a is None:
         return len(sep + token_ids_a  ) * [0]
     return len(sep + token_ids_a + sep + sep + token_ids_a  ) * [0]

 @property
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     return len(self.sp_model  ) + self.fairseq_offset + self.num_madeup_words

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						List[Any]									=					{self.convert_ids_to_tokens(__lowerCamelCase  ): i for i in range(self.vocab_size  )}
     vocab.update(self.added_tokens_encoder  )
     return vocab

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     return self.sp_model.encode(__lowerCamelCase ,						out_type=__lowerCamelCase  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
     __A   :						Union[str, Any]									=					self.sp_model.PieceToId(__lowerCamelCase  )

     # Need to return unknown token if the SP model returned 0
     return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     if index in self.fairseq_ids_to_tokens:
         return self.fairseq_ids_to_tokens[index]
     return self.sp_model.IdToPiece(index - self.fairseq_offset  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						Union[str, Any]									=					''''''.join(__lowerCamelCase  ).replace(__lowerCamelCase ,						''' '''  ).strip()
     return out_string







 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     if not os.path.isdir(__lowerCamelCase  ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
         return
     __A   :						Union[str, Any]									=					os.path.join(
         __lowerCamelCase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

     if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file ,						__lowerCamelCase  )
     elif not os.path.isfile(self.vocab_file  ):
         with open(__lowerCamelCase ,						'''wb'''  ) as fi:
             __A   :						Any									=					self.sp_model.serialized_model_proto()
             fi.write(__lowerCamelCase  )

     return (out_vocab_file,)











"""simple docstring"""



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


a_			=      {
    """configuration_vivit""": ["""VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """VivitConfig"""],
}
try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_			=      ["""VivitImageProcessor"""]


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_			=      [
           """VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """VivitModel""",
           """VivitPreTrainedModel""",
           """VivitForVideoClassification""",
       ]


if TYPE_CHECKING:
       from .configuration_vivit import VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, VivitConfig

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .image_processing_vivit import VivitImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_vivit import (
                  VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  VivitForVideoClassification,
                  VivitModel,
                  VivitPreTrainedModel,
              )


else:
       import sys

       a_			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




