
"""simple docstring"""



import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch





class 				a			(					unittest.TestCase      ):
      UpperCamelCase							:   Optional[Any]           =     JukeboxTokenizer
      UpperCamelCase							:   int           =     {
          """artist""": """Zac Brown Band""",
          """genres""": """Country""",
          """lyrics""": """I met a traveller from an antique land,
        Who said \"Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        """,
      }



      @require_torch
      def  __snake_case							(   self      ):
          import torch

          UpperCAmelCase__	:      str     							=					JukeboxTokenizer.from_pretrained('openai/jukebox-1b-lyrics'      )
          UpperCAmelCase__	:      Dict     							=					tokenizer(**self.metas      )['input_ids']
          # fmt: off
          UpperCAmelCase__	:      Optional[Any]     							=					[
              torch.tensor([[
                  0, 0, 0, 7_169, 507, 9, 76, 39, 31, 46, 76, 27,
                  76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
                  44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
                  47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
                  76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
                  30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
                  27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
                  45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
                  41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
                  76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
                  76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
                  64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
                  30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
                  27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
                  34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
                  27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
                  41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
                  44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
                  76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
                  32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
                  40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
                  45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
                  31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
                  45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
                  34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
                  31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
                  40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
                  38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
                  76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
                  41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
                  27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
                  46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
                  76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
                  41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
                  46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
                  41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
                  40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
                  27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
                  76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
                  76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
                  41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
                  76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
                  27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
                  34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
                  44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
                  40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
                  78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
                  46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
                  38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
                  40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
                  27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                  20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
                  76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
                  76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
                  76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
                  76, 76]]      ),
              torch.tensor([[0, 0, 0, 1_069, 11]]      ),
              torch.tensor([[0, 0, 0, 1_069, 11]]      ),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0]     ,	EXPECTED_OUTPUT[0]      )      )
          self.assertTrue(torch.allclose(tokens[1]     ,	EXPECTED_OUTPUT[1]      )      )
          self.assertTrue(torch.allclose(tokens[2]     ,	EXPECTED_OUTPUT[2]      )      )



      @require_torch
      def  __snake_case							(   self      ):
          import torch

          UpperCAmelCase__	:      Optional[int]     							=					JukeboxTokenizer.from_pretrained('openai/jukebox-5b-lyrics'      )
          UpperCAmelCase__	:      List[str]     							=					tokenizer(**self.metas      )['input_ids']
          # fmt: off
          UpperCAmelCase__	:      Union[str, Any]     							=					[
              torch.tensor([[
                  0, 0, 0, 1_069, 11, -1, -1, -1, -1, 9, 77, 39,
                  31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
                  31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
                  40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
                  79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
                  77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
                  27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
                  37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
                  32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
                  77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
                  77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
                  46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
                  77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
                  77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
                  77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
                  77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
                  64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
                  40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
                  40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
                  38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
                  31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
                  41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
                  46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
                  41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
                  31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
                  31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
                  44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
                  31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
                  38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
                  40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
                  27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
                  31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
                  34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
                  31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                  1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
                  31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
                  45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
                  31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
                  15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
                  11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
                  45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
                  41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
                  44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
                  46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
                  27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
                  35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
                  77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
                  31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
                  77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
                  41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
                  77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
                  40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
                  77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
                  77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
                  27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
                  77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
                  77, 77, 77, 77, 77, 77]]      ),
              torch.tensor([[0, 0, 0, 1_069, 11, -1, -1, -1, -1]]      ),
              torch.tensor([[0, 0, 0, 1_069, 11, -1, -1, -1, -1]]      ),
          ]
          # fmt: on
          self.assertTrue(torch.allclose(tokens[0]     ,	EXPECTED_OUTPUT[0]      )      )
          self.assertTrue(torch.allclose(tokens[1]     ,	EXPECTED_OUTPUT[1]      )      )
          self.assertTrue(torch.allclose(tokens[2]     ,	EXPECTED_OUTPUT[2]      )      )



"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase__          =	logging.get_logger(__name__)

UpperCamelCase__          =	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__          =	{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

UpperCamelCase__          =	{
    'junnyu/roformer_chinese_small': 15_36,
    'junnyu/roformer_chinese_base': 15_36,
    'junnyu/roformer_chinese_char_small': 5_12,
    'junnyu/roformer_chinese_char_base': 5_12,
    'junnyu/roformer_small_discriminator': 1_28,
    'junnyu/roformer_small_generator': 1_28,
}


UpperCamelCase__          =	{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 				a			(					lowercase      ):
      UpperCamelCase							:   int           =     VOCAB_FILES_NAMES
      UpperCamelCase							:   Dict           =     PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase							:   Union[str, Any]           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase							:   Tuple           =     PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase							:   Optional[int]           =     RoFormerTokenizer



      def __init__(   self     ,	UpperCamelCase_=None     ,	UpperCamelCase_=None     ,	UpperCamelCase_=True     ,	UpperCamelCase_="[UNK]"     ,	UpperCamelCase_="[SEP]"     ,	UpperCamelCase_="[PAD]"     ,	UpperCamelCase_="[CLS]"     ,	UpperCamelCase_="[MASK]"     ,	UpperCamelCase_=True     ,	UpperCamelCase_=None     ,	**UpperCamelCase_     ,	):
          super().__init__(
              UpperCamelCase_     ,	tokenizer_file=UpperCamelCase_     ,	do_lower_case=UpperCamelCase_     ,	unk_token=UpperCamelCase_     ,	sep_token=UpperCamelCase_     ,	pad_token=UpperCamelCase_     ,	cls_token=UpperCamelCase_     ,	mask_token=UpperCamelCase_     ,	tokenize_chinese_chars=UpperCamelCase_     ,	strip_accents=UpperCamelCase_     ,	**UpperCamelCase_     ,	)

          UpperCAmelCase__	:      Union[str, Any]     							=					json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
          if (
              pre_tok_state.get('lowercase'     ,	UpperCamelCase_      ) != do_lower_case
              or pre_tok_state.get('strip_accents'     ,	UpperCamelCase_      ) != strip_accents
          ):
              UpperCAmelCase__	:      Any     							=					getattr(UpperCamelCase_     ,	pre_tok_state.pop('type'      )      )
              UpperCAmelCase__	:      str     							=					do_lower_case
              UpperCAmelCase__	:      Union[str, Any]     							=					strip_accents
              UpperCAmelCase__	:      Dict     							=					pre_tok_class(**UpperCamelCase_      )

          UpperCAmelCase__	:      Union[str, Any]     							=					do_lower_case



      def __getstate__(   self      ):
          UpperCAmelCase__	:      int     							=					self.__dict__.copy()
          UpperCAmelCase__	:      int     							=					BertPreTokenizer()
          return state



      def __setstate__(   self     ,	UpperCamelCase_      ):
          UpperCAmelCase__	:      Union[str, Any]     							=					d
          UpperCAmelCase__	:      List[str]     							=					self.__dict__['_tokenizer'].get_vocab()
          UpperCAmelCase__	:      List[Any]     							=					PreTokenizer.custom(JiebaPreTokenizer(UpperCamelCase_      )      )



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_=None      ):
          UpperCAmelCase__	:      List[Any]     							=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_ = None      ):
          UpperCAmelCase__	:      int     							=					[self.sep_token_id]
          UpperCAmelCase__	:      Dict     							=					[self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep      ) * [0]
          return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_ = None      ):
          UpperCAmelCase__	:      Any     							=					self._tokenizer.model.save(UpperCamelCase_     ,	name=UpperCamelCase_      )
          return tuple(UpperCamelCase_      )



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_=None     ,	UpperCamelCase_=None     ,	UpperCamelCase_=False     ,	**UpperCamelCase_     ,	):
          UpperCAmelCase__	:      int     							=					BertPreTokenizer()
          return super().save_pretrained(UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	**UpperCamelCase_      )

