






"""simple docstring"""


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase       :					Optional[int]									=		logging.get_logger(__name__)

_UpperCamelCase       :					Union[str, Any]									=		{'vocab_file': 'spiece.model'}

_UpperCamelCase       :					Optional[Any]									=		{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_UpperCamelCase       :					Union[str, Any]									=		{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}






class 		snake_case						(						UpperCAmelCase ):
    __magic_name__					  =  VOCAB_FILES_NAMES
    __magic_name__					  =  PRETRAINED_VOCAB_FILES_MAP
    __magic_name__					  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __magic_name__					  =  ['''input_ids''', '''attention_mask''']
    __magic_name__					  =  []





    def __init__(							self			:      Tuple       ,	A			:      Optional[Any]       ,	A			:      Optional[int]="<unk>"       ,	A			:      Optional[int]="<s>"       ,	A			:      int="</s>"       ,	A			:      Optional[Any]="<pad>"       ,	A			:      Optional[Any]="[SEP]"       ,	A			:      Optional[int]="[MASK]"       ,	A			:      Optional[int]="[CLS]"       ,	A			:      Optional[Dict[str, Any]] = None       ,	**A			:      str       ,	):


      '''simple docstring'''




      a  :    Dict       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else bos_token
      a  :    Optional[Any]       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else eos_token
      a  :    List[str]       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else unk_token
      a  :    str       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else pad_token
      a  :    int       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else cls_token
      a  :    Union[str, Any]       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      a  :    str       =     AddedToken(A       ,	lstrip=A       ,	rstrip=A    ) if isinstance(A       ,	A    ) else mask_token

      a  :    Union[str, Any]       =     {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=A       ,	eos_token=A       ,	unk_token=A       ,	pad_token=A       ,	sep_token=A       ,	mask_token=A       ,	cls_token=A       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**A       ,	)

      a  :    Tuple       =     vocab_file

      a  :    int       =     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
      self.sp_model.Load(A    )





    @property
    def        lowerCamelCase__   (							self			:      Optional[Any]    ):


      '''simple docstring'''




      return self.sp_model.get_piece_size()





    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      a  :    Union[str, Any]       =     {self.convert_ids_to_tokens(A    ): i for i in range(self.vocab_size    )}
      vocab.update(self.added_tokens_encoder    )
      return vocab





    def __getstate__(							self			:      Union[str, Any]    ):


      '''simple docstring'''




      a  :    Dict       =     self.__dict__.copy()
      a  :    Union[str, Any]       =     None
      return state





    def __setstate__(							self			:      Any       ,	A			:      List[str]    ):


      '''simple docstring'''




      a  :    Union[str, Any]       =     d

      # for backward compatibility
      if not hasattr(self       ,	'sp_model_kwargs'    ):
        a  :    Tuple       =     {}

      a  :    List[Any]       =     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
      self.sp_model.Load(self.vocab_file    )





    def        lowerCamelCase__   (							self			:      List[str]       ,	A			:      str    ):


      '''simple docstring'''




      return self.sp_model.encode(A       ,	out_type=A    )





    def        lowerCamelCase__   (							self			:      int       ,	A			:      List[str]    ):


      '''simple docstring'''




      return self.sp_model.piece_to_id(A    )





    def        lowerCamelCase__   (							self			:      Optional[int]       ,	A			:      Union[str, Any]    ):


      '''simple docstring'''




      a  :    Optional[int]       =     self.sp_model.IdToPiece(A    )
      return token





    def        lowerCamelCase__   (							self			:      str       ,	A			:      List[str]    ):


      '''simple docstring'''




      a  :    List[str]       =     []
      a  :    Dict       =     ''
      a  :    Tuple       =     False
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          if not prev_is_special:
            out_string += " "
          out_string += self.sp_model.decode(A    ) + token
          a  :    str       =     True
          a  :    List[str]       =     []
        else:
          current_sub_tokens.append(A    )
          a  :    int       =     False
      out_string += self.sp_model.decode(A    )
      return out_string.strip()





    def        lowerCamelCase__   (							self			:      Dict       ,	A			:      List[int]       ,	A			:      bool = False       ,	A			:      bool = None       ,	A			:      bool = True       ,	**A			:      Tuple       ,	):


      '''simple docstring'''




      a  :    Union[str, Any]       =     kwargs.pop('use_source_tokenizer'       ,	A    )

      a  :    Any       =     self.convert_ids_to_tokens(A       ,	skip_special_tokens=A    )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      a  :    Optional[int]       =     []
      a  :    List[str]       =     []
      for token in filtered_tokens:
        if skip_special_tokens and token in self.all_special_ids:
          continue
        if token in self.added_tokens_encoder:
          if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(A    )    )
            a  :    List[Any]       =     []
          sub_texts.append(A    )
        else:
          current_sub_text.append(A    )
      if current_sub_text:
        sub_texts.append(self.convert_tokens_to_string(A    )    )

      # Mimic the behavior of the Rust tokenizer:
      # No space before [MASK] and [SEP]
      if spaces_between_special_tokens:
        a  :    Dict       =     re.sub(R' (\[(MASK|SEP)\])'       ,	R'\1'       ,	' '.join(A    )    )
      else:
        a  :    Optional[int]       =     ''.join(A    )

      a  :    Tuple       =     (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
        a  :    List[str]       =     self.clean_up_tokenization(A    )
        return clean_text
      else:
        return text





    def        lowerCamelCase__   (							self			:      Union[str, Any]       ,	A			:      str       ,	A			:      Optional[str] = None    ):


      '''simple docstring'''




      if not os.path.isdir(A    ):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
        return
      a  :    Union[str, Any]       =     os.path.join(
          A       ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

      if os.path.abspath(self.vocab_file    ) != os.path.abspath(A    ) and os.path.isfile(self.vocab_file    ):
        copyfile(self.vocab_file       ,	A    )
      elif not os.path.isfile(self.vocab_file    ):
        with open(A       ,	'wb'    ) as fi:
          a  :    Optional[int]       =     self.sp_model.serialized_model_proto()
          fi.write(A    )

      return (out_vocab_file,)





    def        lowerCamelCase__   (							self			:      int       ,	A			:      List[int]       ,	A			:      Optional[List[int]] = None    ):


      '''simple docstring'''




      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      a  :    Any       =     [self.cls_token_id]
      a  :    List[str]       =     [self.sep_token_id]
      return cls + token_ids_a + sep + token_ids_a + sep





    def        lowerCamelCase__   (							self			:      Union[str, Any]       ,	A			:      List[int]       ,	A			:      Optional[List[int]] = None       ,	A			:      bool = False    ):


      '''simple docstring'''




      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=A       ,	token_ids_a=A       ,	already_has_special_tokens=A    )

      if token_ids_a is None:
        return [1] + ([0] * len(A    )) + [1]
      return [1] + ([0] * len(A    )) + [1] + ([0] * len(A    )) + [1]





    def        lowerCamelCase__   (							self			:      str       ,	A			:      List[int]       ,	A			:      Optional[List[int]] = None    ):


      '''simple docstring'''




      a  :    int       =     [self.sep_token_id]
      a  :    Optional[Any]       =     [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]












"""simple docstring"""


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


_UpperCamelCase       :					int									=		'3'

print('Python version:', sys.version)

print('OS platform:', platform.platform())
print('OS architecture:', platform.machine())

try:
   import torch

   print('Torch version:', torch.__version__)
   print('Cuda available:', torch.cuda.is_available())
   print('Cuda version:', torch.version.cuda)
   print('CuDNN version:', torch.backends.cudnn.version())
   print('Number of GPUs available:', torch.cuda.device_count())
except ImportError:
   print('Torch version:', None)

try:
   import transformers

   print('transformers version:', transformers.__version__)
except ImportError:
   print('transformers version:', None)




