





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


A__:    Optional[Any]  	=	logging.get_logger(__name__)

A__:    int  	=	{'''tokenizer_file''': '''tokenizer.json'''}

A__:    Union[str, Any]  	=	{
    '''tokenizer_file''': {
        '''bigscience/tokenizer''': '''https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json''',
        '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json''',
        '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json''',
        '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json''',
        '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json''',
    },
}




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      UpperCamelCase__  		=       VOCAB_FILES_NAMES
      UpperCamelCase__  		=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__  		=       ["""input_ids""", """attention_mask"""]
      UpperCamelCase__  		=       None






      def __init__(		self:			Union[str, Any]	,    __lowerCamelCase:			str=None	,    __lowerCamelCase:			Dict=None	,    __lowerCamelCase:			int=None	,    __lowerCamelCase:			Tuple="<unk>"	,    __lowerCamelCase:			int="<s>"	,    __lowerCamelCase:			List[str]="</s>"	,    __lowerCamelCase:			str="<pad>"	,    __lowerCamelCase:			int=False	,    __lowerCamelCase:			int=False	,    **__lowerCamelCase:			Union[str, Any]	,    ):
          '''simple docstring'''


          super().__init__(
              __lowerCamelCase	,    __lowerCamelCase	,    tokenizer_file=__lowerCamelCase	,    unk_token=__lowerCamelCase	,    bos_token=__lowerCamelCase	,    eos_token=__lowerCamelCase	,    pad_token=__lowerCamelCase	,    add_prefix_space=__lowerCamelCase	,    clean_up_tokenization_spaces=__lowerCamelCase	,    **__lowerCamelCase	,    )
          UpperCamelCase__:	Dict					=     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
          if pre_tok_state.get("add_prefix_space"	,    __lowerCamelCase					) != add_prefix_space:
              UpperCamelCase__:	str					=     getattr(__lowerCamelCase	,    pre_tok_state.pop("type"					)					)
              UpperCamelCase__:	List[Any]					=     add_prefix_space
              UpperCamelCase__:	Optional[int]					=     pre_tok_class(**__lowerCamelCase					)

          UpperCamelCase__:	Optional[Any]					=     add_prefix_space






      def 			UpperCAmelCase_       (		self:			Optional[Any]	,    *__lowerCamelCase:			Tuple	,    **__lowerCamelCase:			Optional[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     kwargs.get("is_split_into_words"	,    __lowerCamelCase					)
          if not (self.add_prefix_space or not is_split_into_words):
              raise Exception(
                  F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                  " pretokenized inputs."					)

          return super()._batch_encode_plus(*__lowerCamelCase	,    **__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			Tuple	,    *__lowerCamelCase:			List[Any]	,    **__lowerCamelCase:			Union[str, Any]					):
          '''simple docstring'''


          UpperCamelCase__:	Any					=     kwargs.get("is_split_into_words"	,    __lowerCamelCase					)

          if not (self.add_prefix_space or not is_split_into_words):
              raise Exception(
                  F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                  " pretokenized inputs."					)

          return super()._encode_plus(*__lowerCamelCase	,    **__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			Union[str, Any]	,    __lowerCamelCase:			str	,    __lowerCamelCase:			Optional[str] = None					):
          '''simple docstring'''


          UpperCamelCase__:	int					=     self._tokenizer.model.save(__lowerCamelCase	,    name=__lowerCamelCase					)
          return tuple(__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			Optional[int]	,    __lowerCamelCase:			"Conversation"					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     []
          for is_user, text in conversation.iter_texts():
              input_ids.extend(self.encode(__lowerCamelCase	,    add_special_tokens=__lowerCamelCase					) + [self.eos_token_id]					)

          if len(__lowerCamelCase					) > self.model_max_length:
              UpperCamelCase__:	str					=     input_ids[-self.model_max_length :]
          return input_ids








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A__:    List[Any]  	=	logging.get_logger(__name__)

A__:    str  	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__:    List[Any]  	=	{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

A__:    str  	=	{
    '''junnyu/roformer_chinese_small''': 1536,
    '''junnyu/roformer_chinese_base''': 1536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


A__:    int  	=	{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      UpperCamelCase__  		=       VOCAB_FILES_NAMES
      UpperCamelCase__  		=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__  		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__  		=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__  		=       RoFormerTokenizer






      def __init__(		self:			int	,    __lowerCamelCase:			Optional[Any]=None	,    __lowerCamelCase:			Any=None	,    __lowerCamelCase:			str=True	,    __lowerCamelCase:			Any="[UNK]"	,    __lowerCamelCase:			int="[SEP]"	,    __lowerCamelCase:			Optional[int]="[PAD]"	,    __lowerCamelCase:			Optional[int]="[CLS]"	,    __lowerCamelCase:			Tuple="[MASK]"	,    __lowerCamelCase:			List[str]=True	,    __lowerCamelCase:			List[Any]=None	,    **__lowerCamelCase:			Dict	,    ):
          '''simple docstring'''


          super().__init__(
              __lowerCamelCase	,    tokenizer_file=__lowerCamelCase	,    do_lower_case=__lowerCamelCase	,    unk_token=__lowerCamelCase	,    sep_token=__lowerCamelCase	,    pad_token=__lowerCamelCase	,    cls_token=__lowerCamelCase	,    mask_token=__lowerCamelCase	,    tokenize_chinese_chars=__lowerCamelCase	,    strip_accents=__lowerCamelCase	,    **__lowerCamelCase	,    )

          UpperCamelCase__:	int					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
          if (
              pre_tok_state.get("lowercase"	,    __lowerCamelCase					) != do_lower_case
              or pre_tok_state.get("strip_accents"	,    __lowerCamelCase					) != strip_accents
          ):
              UpperCamelCase__:	int					=     getattr(__lowerCamelCase	,    pre_tok_state.pop("type"					)					)
              UpperCamelCase__:	Any					=     do_lower_case
              UpperCamelCase__:	Optional[int]					=     strip_accents
              UpperCamelCase__:	Any					=     pre_tok_class(**__lowerCamelCase					)

          UpperCamelCase__:	Tuple					=     do_lower_case






      def __getstate__(		self:			Optional[int]					):
          '''simple docstring'''


          UpperCamelCase__:	List[Any]					=     self.__dict__.copy()
          UpperCamelCase__:	Dict					=     BertPreTokenizer()
          return state






      def __setstate__(		self:			Dict	,    __lowerCamelCase:			Optional[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	str					=     d
          UpperCamelCase__:	List[Any]					=     self.__dict__["_tokenizer"].get_vocab()
          UpperCamelCase__:	str					=     PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase					)					)






      def 			UpperCAmelCase_       (		self:			int	,    __lowerCamelCase:			Optional[int]	,    __lowerCamelCase:			Dict=None					):
          '''simple docstring'''


          UpperCamelCase__:	int					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output






      def 			UpperCAmelCase_       (		self:			Optional[int]	,    __lowerCamelCase:			List[int]	,    __lowerCamelCase:			Optional[List[int]] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Tuple					=     [self.sep_token_id]
          UpperCamelCase__:	str					=     [self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






      def 			UpperCAmelCase_       (		self:			Tuple	,    __lowerCamelCase:			str	,    __lowerCamelCase:			Optional[str] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Dict					=     self._tokenizer.model.save(__lowerCamelCase	,    name=__lowerCamelCase					)
          return tuple(__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			Dict	,    __lowerCamelCase:			Any	,    __lowerCamelCase:			Any=None	,    __lowerCamelCase:			Dict=None	,    __lowerCamelCase:			List[str]=False	,    **__lowerCamelCase:			int	,    ):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     BertPreTokenizer()
          return super().save_pretrained(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    **__lowerCamelCase					)

