




'''simple docstring'''



import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
      import tensorflow as tf
      from tensorflow.python.eager import context
      from tensorflow.python.framework import ops

      from transformers import GradientAccumulator, create_optimizer






@require_tf
class 							_a				(			unittest.TestCase ):





       '''simple docstring'''






       def        UpperCamelCase_							(      self,							A,							A,							A  ):







             '''simple docstring'''







             self.assertEqual(len(A  ),							len(A  )  )
             for a, b in zip(A,							A  ):
                   self.assertAlmostEqual(A,							A,							delta=A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						GradientAccumulator()
             accumulator([tf.constant([1.0, 2.0]  )]  )
             accumulator([tf.constant([-2.0, 1.0]  )]  )
             accumulator([tf.constant([-1.0, 2.0]  )]  )
             with self.assertRaises(A  ):
                   accumulator([tf.constant([1.0, 1.0]  ), tf.constant([2.0, 2.0]  )]  )
             self.assertEqual(accumulator.step,							3  )
             self.assertEqual(len(accumulator.gradients  ),							1  )
             self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(),							[-2.0, 5.0],							tol=1E-2  )
             accumulator.reset()
             self.assertEqual(accumulator.step,							0  )
             self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(),							[0.0, 0.0],							tol=1E-2  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						None
             ops.enable_eager_execution_internal()
             SCREAMING_SNAKE_CASE   :      str    					=						tf.config.list_physical_devices('CPU'  )
             if len(A  ) == 1:
                   tf.config.set_logical_device_configuration(
                       physical_devices[0],							[tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tf.config.list_logical_devices(device_type='CPU'  )
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tf.distribute.MirroredStrategy(devices=devices[:2]  )

             with strategy.scope():
                   SCREAMING_SNAKE_CASE   :      Optional[int]    					=						GradientAccumulator()
                   SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tf.Variable([4.0, 3.0]  )
                   SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      str    					=						create_optimizer(5E-5,							10,							5  )
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						tf.Variable([0.0, 0.0],							trainable=A  )

             def accumulate_on_replica(A  ):
                   accumulator([gradient]  )

             def apply_on_replica():
                   optimizer.apply_gradients(list(zip(accumulator.gradients,							[variable]  )  )  )

             @tf.function
             def accumulate(A,							A  ):
                   with strategy.scope():
                         SCREAMING_SNAKE_CASE   :      Any    					=						strategy.experimental_local_results(A  )
                         local_variables[0].assign(A  )
                         local_variables[1].assign(A  )
                         strategy.run(A,							args=(gradient_placeholder,)  )

             @tf.function
             def apply_grad():
                   with strategy.scope():
                         strategy.run(A  )

             def _check_local_values(A,							A  ):
                   SCREAMING_SNAKE_CASE   :      str    					=						strategy.experimental_local_results(accumulator._gradients[0]  )
                   self.assertListAlmostEqual(values[0].value(),							A,							tol=1E-2  )
                   self.assertListAlmostEqual(values[1].value(),							A,							tol=1E-2  )

             accumulate([1.0, 2.0],							[-1.0, 1.0]  )
             accumulate([3.0, -1.0],							[-1.0, -1.0]  )
             accumulate([-2.0, 2.0],							[3.0, -2.0]  )
             self.assertEqual(accumulator.step,							3  )
             _check_local_values([2.0, 3.0],							[1.0, -2.0]  )
             apply_grad()
             self.assertListAlmostEqual(variable.value(),							[4.0, 3.0],							tol=1E-2  )
             accumulator.reset()
             self.assertEqual(accumulator.step,							0  )
             _check_local_values([0.0, 0.0],							[0.0, 0.0]  )








'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCamelCase_    	=				logging.get_logger(__name__)

UpperCamelCase_    	=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase_    	=				{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase_    	=				{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


UpperCamelCase_    	=				{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}






class 							_a				(			SCREAMING_SNAKE_CASE ):





       '''simple docstring'''






       A						:				List[Any]        =						VOCAB_FILES_NAMES
       A						:				Dict        =						PRETRAINED_VOCAB_FILES_MAP
       A						:				Optional[int]        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A						:				Optional[Any]        =						PRETRAINED_INIT_CONFIGURATION
       A						:				Optional[int]        =						['''input_ids''', '''attention_mask''']
       A						:				List[Any]        =						DistilBertTokenizer







       def __init__(      self,							A=None,							A=None,							A=True,							A="[UNK]",							A="[SEP]",							A="[PAD]",							A="[CLS]",							A="[MASK]",							A=True,							A=None,							**A,							):







             '''simple docstring'''







             super().__init__(
                 A,							tokenizer_file=A,							do_lower_case=A,							unk_token=A,							sep_token=A,							pad_token=A,							cls_token=A,							mask_token=A,							tokenize_chinese_chars=A,							strip_accents=A,							**A,							)

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
             if (
                 normalizer_state.get('lowercase',							A  ) != do_lower_case
                 or normalizer_state.get('strip_accents',							A  ) != strip_accents
                 or normalizer_state.get('handle_chinese_chars',							A  ) != tokenize_chinese_chars
             ):
                   SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						getattr(A,							normalizer_state.pop('type'  )  )
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						do_lower_case
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						strip_accents
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenize_chinese_chars
                   SCREAMING_SNAKE_CASE   :      Dict    					=						normalizer_class(**A  )

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						do_lower_case







       def        UpperCamelCase_							(      self,							A,							A=None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						[self.sep_token_id]
             SCREAMING_SNAKE_CASE   :      str    					=						[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls + token_ids_a + sep  ) * [0]
             return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						self._tokenizer.model.save(A,							name=A  )
             return tuple(A  )


