




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 	A	(						_lowerCAmelCase	,  unittest.TestCase						):
     """simple docstring"""

     _snake_case:		str   =RealmTokenizer
     _snake_case:		Tuple   =RealmTokenizerFast
     _snake_case:		Optional[Any]   =True
     _snake_case:		str   =True
     _snake_case:		str   =filter_non_english




     def        __snake_case      (			self     :      int	) ->				Union[str, Any]:
           super().setUp()

           lowerCamelCase__					     = [
               '[UNK]',
               '[CLS]',
               '[SEP]',
               '[PAD]',
               '[MASK]',
               'want',
               '##want',
               '##ed',
               'wa',
               'un',
               'runn',
               '##ing',
               ',',
               'low',
               'lowest',
           ]
           lowerCamelCase__					     = os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['vocab_file']	)
           with open(self.vocab_file		,     'w'		,     encoding='utf-8'	) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)




     def        __snake_case      (			self     :      Any		,     A__     :      Optional[int]	) ->				List[Any]:
           lowerCamelCase__					     = 'UNwant\u00E9d,running'
           lowerCamelCase__					     = 'unwanted, running'
           return input_text, output_text




     def        __snake_case      (			self     :      str	) ->				List[str]:
           lowerCamelCase__					     = self.tokenizer_class(self.vocab_file	)

           lowerCamelCase__					     = tokenizer.tokenize('UNwant\u00E9d,running'	)
           self.assertListEqual(A__		,     ['un', '##want', '##ed', ',', 'runn', '##ing']	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A__	)		,     [9, 6, 7, 1_2, 1_0, 1_1]	)




     def        __snake_case      (			self     :      int	) ->				List[Any]:
           if not self.test_rust_tokenizer:
                 return

           lowerCamelCase__					     = self.get_tokenizer()
           lowerCamelCase__					     = self.get_rust_tokenizer()

           lowerCamelCase__					     = 'UNwant\u00E9d,running'

           lowerCamelCase__					     = tokenizer.tokenize(A__	)
           lowerCamelCase__					     = rust_tokenizer.tokenize(A__	)
           self.assertListEqual(A__		,     A__	)

           lowerCamelCase__					     = tokenizer.encode(A__		,     add_special_tokens=A__	)
           lowerCamelCase__					     = rust_tokenizer.encode(A__		,     add_special_tokens=A__	)
           self.assertListEqual(A__		,     A__	)

           lowerCamelCase__					     = self.get_rust_tokenizer()
           lowerCamelCase__					     = tokenizer.encode(A__	)
           lowerCamelCase__					     = rust_tokenizer.encode(A__	)
           self.assertListEqual(A__		,     A__	)

           # With lower casing
           lowerCamelCase__					     = self.get_tokenizer(do_lower_case=A__	)
           lowerCamelCase__					     = self.get_rust_tokenizer(do_lower_case=A__	)

           lowerCamelCase__					     = 'UNwant\u00E9d,running'

           lowerCamelCase__					     = tokenizer.tokenize(A__	)
           lowerCamelCase__					     = rust_tokenizer.tokenize(A__	)
           self.assertListEqual(A__		,     A__	)

           lowerCamelCase__					     = tokenizer.encode(A__		,     add_special_tokens=A__	)
           lowerCamelCase__					     = rust_tokenizer.encode(A__		,     add_special_tokens=A__	)
           self.assertListEqual(A__		,     A__	)

           lowerCamelCase__					     = self.get_rust_tokenizer()
           lowerCamelCase__					     = tokenizer.encode(A__	)
           lowerCamelCase__					     = rust_tokenizer.encode(A__	)
           self.assertListEqual(A__		,     A__	)




     def        __snake_case      (			self     :      List[str]	) ->				Optional[Any]:
           lowerCamelCase__					     = BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'	)		,     ['ah', '\u535A', '\u63A8', 'zz']	)




     def        __snake_case      (			self     :      Optional[int]	) ->				List[Any]:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)		,     ['hello', '!', 'how', 'are', 'you', '?']	)
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,     ['hello']	)




     def        __snake_case      (			self     :      Dict	) ->				int:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__		,     strip_accents=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,     ['hällo', '!', 'how', 'are', 'you', '?']	)
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,     ['h\u00E9llo']	)




     def        __snake_case      (			self     :      Union[str, Any]	) ->				Optional[int]:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__		,     strip_accents=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,     ['hallo', '!', 'how', 'are', 'you', '?']	)
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,     ['hello']	)




     def        __snake_case      (			self     :      int	) ->				List[Any]:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,     ['hallo', '!', 'how', 'are', 'you', '?']	)
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)		,     ['hello']	)




     def        __snake_case      (			self     :      Dict	) ->				Any:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)		,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']	)




     def        __snake_case      (			self     :      str	) ->				Tuple:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__		,     strip_accents=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']	)




     def        __snake_case      (			self     :      Optional[int]	) ->				List[str]:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__		,     strip_accents=A__	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)		,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']	)




     def        __snake_case      (			self     :      List[Any]	) ->				Union[str, Any]:
           lowerCamelCase__					     = BasicTokenizer(do_lower_case=A__		,     never_split=['[UNK]']	)

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'	)		,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']	)




     def        __snake_case      (			self     :      Optional[int]	) ->				Optional[Any]:
           lowerCamelCase__					     = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

           lowerCamelCase__					     = {}
           for i, token in enumerate(A__	):
                 lowerCamelCase__					     = i
           lowerCamelCase__					     = WordpieceTokenizer(vocab=A__		,     unk_token='[UNK]'	)

           self.assertListEqual(tokenizer.tokenize(''	)		,     []	)

           self.assertListEqual(tokenizer.tokenize('unwanted running'	)		,     ['un', '##want', '##ed', 'runn', '##ing']	)

           self.assertListEqual(tokenizer.tokenize('unwantedX running'	)		,     ['[UNK]', 'runn', '##ing']	)




     def        __snake_case      (			self     :      Optional[int]	) ->				List[str]:
           self.assertTrue(_is_whitespace(' '	)	)
           self.assertTrue(_is_whitespace('\t'	)	)
           self.assertTrue(_is_whitespace('\r'	)	)
           self.assertTrue(_is_whitespace('\n'	)	)
           self.assertTrue(_is_whitespace('\u00A0'	)	)

           self.assertFalse(_is_whitespace('A'	)	)
           self.assertFalse(_is_whitespace('-'	)	)




     def        __snake_case      (			self     :      int	) ->				Any:
           self.assertTrue(_is_control('\u0005'	)	)

           self.assertFalse(_is_control('A'	)	)
           self.assertFalse(_is_control(' '	)	)
           self.assertFalse(_is_control('\t'	)	)
           self.assertFalse(_is_control('\r'	)	)




     def        __snake_case      (			self     :      str	) ->				Any:
           self.assertTrue(_is_punctuation('-'	)	)
           self.assertTrue(_is_punctuation('$'	)	)
           self.assertTrue(_is_punctuation('`'	)	)
           self.assertTrue(_is_punctuation('.'	)	)

           self.assertFalse(_is_punctuation('A'	)	)
           self.assertFalse(_is_punctuation(' '	)	)




     def        __snake_case      (			self     :      Dict	) ->				Dict:
           lowerCamelCase__					     = self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(A__	) for t in ['Test', '\xad', 'test']]		,     [['[UNK]'], [], ['[UNK]']]	)

           if self.test_rust_tokenizer:
                 lowerCamelCase__					     = self.get_rust_tokenizer()
                 self.assertListEqual(
                     [rust_tokenizer.tokenize(A__	) for t in ['Test', '\xad', 'test']]		,     [['[UNK]'], [], ['[UNK]']]	)




     @slow
     def        __snake_case      (			self     :      List[str]	) ->				Dict:
           lowerCamelCase__					     = self.tokenizer_class.from_pretrained('bert-base-uncased'	)

           lowerCamelCase__					     = tokenizer.encode('sequence builders'		,     add_special_tokens=A__	)
           lowerCamelCase__					     = tokenizer.encode('multi-sequence build'		,     add_special_tokens=A__	)

           lowerCamelCase__					     = tokenizer.build_inputs_with_special_tokens(A__	)
           lowerCamelCase__					     = tokenizer.build_inputs_with_special_tokens(A__		,     A__	)

           assert encoded_sentence == [1_0_1] + text + [1_0_2]
           assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




     def        __snake_case      (			self     :      Any	) ->				Optional[Any]:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       lowerCamelCase__					     = self.rust_tokenizer_class.from_pretrained(A__		,     **A__	)

                       lowerCamelCase__					     = F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                       lowerCamelCase__					     = tokenizer_r.encode_plus(
                           A__		,     return_attention_mask=A__		,     return_token_type_ids=A__		,     return_offsets_mapping=A__		,     add_special_tokens=A__		,     )

                       lowerCamelCase__					     = tokenizer_r.do_lower_case if hasattr(A__		,     'do_lower_case'	) else False
                       lowerCamelCase__					     = (
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'A'),
                               ((1, 2), ','),
                               ((3, 5), 'na'),
                               ((5, 6), '##ï'),
                               ((6, 8), '##ve'),
                               ((9, 1_5), tokenizer_r.mask_token),
                               ((1_6, 2_1), 'Allen'),
                               ((2_1, 2_3), '##NL'),
                               ((2_3, 2_4), '##P'),
                               ((2_5, 3_3), 'sentence'),
                               ((3_3, 3_4), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'a'),
                               ((1, 2), ','),
                               ((3, 8), 'naive'),
                               ((9, 1_5), tokenizer_r.mask_token),
                               ((1_6, 2_1), 'allen'),
                               ((2_1, 2_3), '##nl'),
                               ((2_3, 2_4), '##p'),
                               ((2_5, 3_3), 'sentence'),
                               ((3_3, 3_4), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']	)	)
                       self.assertEqual([e[0] for e in expected_results]		,     tokens['offset_mapping']	)




     @slow
     def        __snake_case      (			self     :      Any	) ->				Optional[int]:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       lowerCamelCase__					     = self.rust_tokenizer_class.from_pretrained(A__		,     **A__	)
                       lowerCamelCase__					     = self.tokenizer_class.from_pretrained(A__		,     **A__	)
                       lowerCamelCase__					     = [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                       lowerCamelCase__					     = tokenizer_r.batch_encode_candidates(A__		,     max_length=1_0		,     return_tensors='np'	)
                       lowerCamelCase__					     = tokenizer_p.batch_encode_candidates(A__		,     max_length=1_0		,     return_tensors='np'	)

                       lowerCamelCase__					     = (2, 2, 1_0)

                       self.assertEqual(encoded_sentence_r['input_ids'].shape		,     A__	)
                       self.assertEqual(encoded_sentence_r['attention_mask'].shape		,     A__	)
                       self.assertEqual(encoded_sentence_r['token_type_ids'].shape		,     A__	)

                       self.assertEqual(encoded_sentence_p['input_ids'].shape		,     A__	)
                       self.assertEqual(encoded_sentence_p['attention_mask'].shape		,     A__	)
                       self.assertEqual(encoded_sentence_p['token_type_ids'].shape		,     A__	)





import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


__SCREAMING_SNAKE_CASE			:      Union[str, Any]      			=get_logger(__name__)

def 	lowerCAmelCase_	(     lowerCamelCase:				str   , lowerCamelCase:				List[dict]   , lowerCamelCase:				list   , lowerCamelCase:				list   , lowerCamelCase:				int   , lowerCamelCase:				GradScaler = None   , )    ->	Any:


      '''simple docstring'''




      for i, state in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Model weights saved in {output_model_file}"""  )
      # Optimizer states
      for i, opt in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = opt.state_dict()
            lowerCamelCase__					     = f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Optimizer state saved in {output_optimizer_file}"""  )
      # Scheduler states
      for i, scheduler in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = scheduler.state_dict()
            lowerCamelCase__					     = f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Scheduler state saved in {output_scheduler_file}"""  )
      # GradScaler state
      if scaler is not None:
            lowerCamelCase__					     = scaler.state_dict()
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            torch.save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Gradient scaler state saved in {output_scaler_file}"""  )
      # Random number generator states
      lowerCamelCase__					     = {}
      lowerCamelCase__					     = f"""{RNG_STATE_NAME}_{process_index}.pkl"""
      lowerCamelCase__					     = random.getstate()
      lowerCamelCase__					     = np.random.get_state()
      lowerCamelCase__					     = torch.get_rng_state()
      if is_xpu_available():
            lowerCamelCase__					     = torch.xpu.get_rng_state_all()
      else:
            lowerCamelCase__					     = torch.cuda.get_rng_state_all()
      if is_tpu_available():
            lowerCamelCase__					     = xm.get_rng_state()
      lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
      torch.save(lowerCamelCase   , lowerCamelCase  )
      logger.info(f"""Random states saved in {output_states_file}"""  )
      return output_dir

def 	lowerCAmelCase_	(     lowerCamelCase:				int   , lowerCamelCase:				Dict   , lowerCamelCase:				Union[str, Any]   , lowerCamelCase:				Any   , lowerCamelCase:				Any   , lowerCamelCase:				Any=None   , lowerCamelCase:				Dict=None   , **lowerCamelCase:				Optional[Any]   , )    ->	Any:


      '''simple docstring'''




      if map_location not in [None, "cpu", "on_device"]:
            raise TypeError(
                'Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'  )
      if map_location is None:
            lowerCamelCase__					     = 'cpu'
      elif map_location == "on_device":
            lowerCamelCase__					     = PartialState().device
      # Model states
      for i, model in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            models[i].load_state_dict(torch.load(lowerCamelCase   , map_location=lowerCamelCase  )   , **lowerCamelCase  )
      logger.info('All model weights loaded successfully'  )

      # Optimizer states
      for i, opt in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            lowerCamelCase__					     = torch.load(lowerCamelCase   , map_location=lowerCamelCase  )
            optimizers[i].load_state_dict(lowerCamelCase  )
      logger.info('All optimizer states loaded successfully'  )

      # Scheduler states
      for i, scheduler in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            scheduler.load_state_dict(torch.load(lowerCamelCase  )  )
      logger.info('All scheduler states loaded successfully'  )

      # GradScaler state
      if scaler is not None:
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            scaler.load_state_dict(torch.load(lowerCamelCase  )  )
            logger.info('GradScaler state loaded successfully'  )

      # Random states
      try:
            lowerCamelCase__					     = torch.load(os.path.join(lowerCamelCase   , f"""{RNG_STATE_NAME}_{process_index}.pkl"""  )  )
            random.setstate(states['random_state']  )
            np.random.set_state(states['numpy_random_seed']  )
            torch.set_rng_state(states['torch_manual_seed']  )
            if is_xpu_available():
                  torch.xpu.set_rng_state_all(states['torch_xpu_manual_seed']  )
            else:
                  torch.cuda.set_rng_state_all(states['torch_cuda_manual_seed']  )
            if is_tpu_available():
                  xm.set_rng_state(states['xm_seed']  )
            logger.info('All random states loaded successfully'  )
      except Exception:
            logger.info('Could not load random states'  )

def 	lowerCAmelCase_	(     lowerCamelCase:				Dict   , lowerCamelCase:				Any   , lowerCamelCase:				int = 0  )    ->	Union[str, Any]:


      '''simple docstring'''




      lowerCamelCase__					     = Path(lowerCamelCase  ) / f"""custom_checkpoint_{index}.pkl"""
      logger.info(f"""Saving the state of {get_pretty_name(lowerCamelCase  )} to {save_location}"""  )
      torch.save(obj.state_dict()   , lowerCamelCase  )


def 	lowerCAmelCase_	(     lowerCamelCase:				int   , lowerCamelCase:				Optional[int]   , lowerCamelCase:				int = 0  )    ->	Any:


      '''simple docstring'''




      lowerCamelCase__					     = f"""{path}/custom_checkpoint_{index}.pkl"""
      logger.info(f"""Loading the state of {get_pretty_name(lowerCamelCase  )} from {load_location}"""  )
      obj.load_state_dict(torch.load(lowerCamelCase   , map_location='cpu'  )  )