
"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     '▁'

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'sentencepiece.bpe.model'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      ['''input_ids''', '''attention_mask''']



     def __init__(							self :List[str]      ,  _lowerCamelCase :List[Any]      ,  _lowerCamelCase :List[str]="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :List[Any]="</s>"      ,  _lowerCamelCase :str="<s>"      ,  _lowerCamelCase :Dict="<unk>"      ,  _lowerCamelCase :List[str]="<pad>"      ,  _lowerCamelCase :Any="<mask>"      ,  _lowerCamelCase :Optional[Dict[str, Any]] = None      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					Any   	=AddedToken(_lowerCamelCase      ,  lstrip=_lowerCamelCase      ,  rstrip=_lowerCamelCase   ) if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ) else mask_token

      UpperCamelCase_     :					str   	={} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  sp_model_kwargs=self.sp_model_kwargs      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					int   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(str(_lowerCamelCase   )   )
      UpperCamelCase_     :					Optional[Any]   	=vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      UpperCamelCase_     :					Optional[int]   	={'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      UpperCamelCase_     :					Optional[int]   	=1

      UpperCamelCase_     :					Any   	=len(self.sp_model   ) + self.fairseq_offset
      UpperCamelCase_     :					List[str]   	={v: k for k, v in self.fairseq_tokens_to_ids.items()}



     def __getstate__(							self :Optional[int]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Any   	=self.__dict__.copy()
      UpperCamelCase_     :					int   	=None
      UpperCamelCase_     :					Optional[int]   	=self.sp_model.serialized_model_proto()
      return state



     def __setstate__(							self :List[str]      ,  _lowerCamelCase :Dict   ):







      '''simple docstring'''



      UpperCamelCase_     :					Dict   	=d

      # for backward compatibility
      if not hasattr(self      ,  'sp_model_kwargs'   ):
       UpperCamelCase_     :					int   	={}

      UpperCamelCase_     :					Union[str, Any]   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None      ,  _lowerCamelCase :bool = False   ):







      '''simple docstring'''



      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowerCamelCase      ,  token_ids_a=_lowerCamelCase      ,  already_has_special_tokens=_lowerCamelCase   )

      if token_ids_a is None:
       return [1] + ([0] * len(_lowerCamelCase   )) + [1]
      return [1] + ([0] * len(_lowerCamelCase   )) + [1, 1] + ([0] * len(_lowerCamelCase   )) + [1]



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=[self.sep_token_id]
      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



     @property
     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      return len(self.sp_model   ) + self.fairseq_offset + 1  # Add the <mask> token



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	={self.convert_ids_to_tokens(_lowerCamelCase   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab



     def  lowerCamelCase_						(							self :Any      ,  _lowerCamelCase :str   ):







      '''simple docstring'''



      return self.sp_model.encode(_lowerCamelCase      ,  out_type=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :Tuple   ):







      '''simple docstring'''



      if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
      UpperCamelCase_     :					List[Any]   	=self.sp_model.PieceToId(_lowerCamelCase   )

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset   )



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Any   	=''.join(_lowerCamelCase   ).replace(_lowerCamelCase      ,  ' '   ).strip()
      return out_string




     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      if not os.path.isdir(_lowerCamelCase   ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
       return
      UpperCamelCase_     :					Any   	=os.path.join(
          _lowerCamelCase      ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
       copyfile(self.vocab_file      ,  _lowerCamelCase   )
      elif not os.path.isfile(self.vocab_file   ):
       with open(_lowerCamelCase      ,  'wb'   ) as fi:
        UpperCamelCase_     :					Any   	=self.sp_model.serialized_model_proto()
        fi.write(_lowerCamelCase   )

      return (out_vocab_file,)


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      RoFormerTokenizer



     def __init__(							self :Tuple      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :Dict="[UNK]"      ,  _lowerCamelCase :List[str]="[SEP]"      ,  _lowerCamelCase :str="[PAD]"      ,  _lowerCamelCase :Optional[Any]="[CLS]"      ,  _lowerCamelCase :Optional[int]="[MASK]"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Tuple=None      ,  **_lowerCamelCase :Any      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					Any   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          pre_tok_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or pre_tok_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
      ):
       UpperCamelCase_     :					Optional[Any]   	=getattr(_lowerCamelCase      ,  pre_tok_state.pop('type'   )   )
       UpperCamelCase_     :					Tuple   	=do_lower_case
       UpperCamelCase_     :					Union[str, Any]   	=strip_accents
       UpperCamelCase_     :					Union[str, Any]   	=pre_tok_class(**_lowerCamelCase   )

      UpperCamelCase_     :					List[str]   	=do_lower_case



     def __getstate__(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.__dict__.copy()
      UpperCamelCase_     :					Union[str, Any]   	=BertPreTokenizer()
      return state



     def __setstate__(							self :str      ,  _lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=d
      UpperCamelCase_     :					Optional[int]   	=self.__dict__['_tokenizer'].get_vocab()
      UpperCamelCase_     :					Any   	=PreTokenizer.custom(JiebaPreTokenizer(_lowerCamelCase   )   )



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Optional[Any]      ,  _lowerCamelCase :Any=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )




     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Optional[Any]=None      ,  _lowerCamelCase :int=False      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=BertPreTokenizer()
      return super().save_pretrained(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  **_lowerCamelCase   )
