
"""simple docstring"""
from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


lowercase_       :						Dict     					=	logging.get_logger(__name__)





def    _lowerCAmelCase    (     lowerCamelCase__							:							Union[tf.Tensor, np.ndarray]				)  ->    List[int]:
  if isinstance(lowerCamelCase__,						np.ndarray				):
    return list(tensor.shape				)

  _SCREAMING_SNAKE_CASE   :				int							    = tf.shape(lowerCamelCase__				)

  if tensor.shape == tf.TensorShape(lowerCamelCase__				):
    return dynamic

  _SCREAMING_SNAKE_CASE   :				List[str]							    = tensor.shape.as_list()

  return [dynamic[i] if s is None else s for i, s in enumerate(lowerCamelCase__				)]





def    _lowerCAmelCase    (     lowerCamelCase__							:							tf.Tensor,						lowerCamelCase__							:							Optional[int] = None,						lowerCamelCase__							:							Optional[str] = None				)  ->    tf.Tensor:
  return tf.nn.softmax(logits=logits + 1E-9,						axis=lowerCamelCase__,						name=lowerCamelCase__				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							Optional[int],						lowerCamelCase__							:							Optional[int],						lowerCamelCase__							:							Optional[int],						lowerCamelCase__							:							int=1E-5,						lowerCamelCase__							:							Union[str, Any]=-1				)  ->    Optional[int]:
  # This is a very simplified functional layernorm, designed to duplicate
  # the functionality of PyTorch nn.functional.layer_norm when this is needed to port
  # models in Transformers.

  if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(lowerCamelCase__,						lowerCamelCase__				):
    raise NotImplementedError("Only 1D weight and bias tensors are supported for now, with only a single axis."				)

  # Get mean and variance on the axis to be normalized
  _SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE   :				Optional[Any]							    = tf.nn.moments(lowerCamelCase__,						axes=[axis],						keepdims=lowerCamelCase__				)

  if axis != -1:
    # Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
    # on every dimension except axis
    _SCREAMING_SNAKE_CASE   :				List[Any]							    = [1] * inputs.shape.rank
    _SCREAMING_SNAKE_CASE   :				int							    = shape_list(lowerCamelCase__				)[axis]
    _SCREAMING_SNAKE_CASE   :				Tuple							    = tf.reshape(lowerCamelCase__,						lowerCamelCase__				)
    _SCREAMING_SNAKE_CASE   :				Dict							    = tf.reshape(lowerCamelCase__,						lowerCamelCase__				)

  # Compute layer normalization using the batch_normalization
  # function.
  _SCREAMING_SNAKE_CASE   :				Tuple							    = tf.nn.batch_normalization(
      lowerCamelCase__,						lowerCamelCase__,						lowerCamelCase__,						offset=lowerCamelCase__,						scale=lowerCamelCase__,						variance_epsilon=lowerCamelCase__,						)
  return outputs





def    _lowerCAmelCase    (     lowerCamelCase__							:							Optional[Any],						lowerCamelCase__							:							Optional[int]=0,						lowerCamelCase__							:							List[str]=-1				)  ->    Union[str, Any]:
  # Replicates the behavior of torch.flatten in TF

  # If end_dim or start_dim is negative, count them from the end
  if end_dim < 0:
    end_dim += input.shape.rank
  if start_dim < 0:
    start_dim += input.shape.rank

  if start_dim == end_dim:
    return input

  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = tf.shape(lowerCamelCase__				)
  _SCREAMING_SNAKE_CASE   :				Dict							    = tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]				)
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]],						axis=0				)
  return tf.reshape(lowerCamelCase__,						lowerCamelCase__				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							tf.Tensor				)  ->    tf.Tensor:
  if not isinstance(lowerCamelCase__,						tf.Tensor				):
    _SCREAMING_SNAKE_CASE   :				Tuple							    = tf.convert_to_tensor(lowerCamelCase__				)  # Catches stray NumPy inputs
  if encoder_attention_mask.shape.rank == 3:
    _SCREAMING_SNAKE_CASE   :				Dict							    = encoder_attention_mask[:, None, :, :]
  if encoder_attention_mask.shape.rank == 2:
    _SCREAMING_SNAKE_CASE   :				List[Any]							    = encoder_attention_mask[:, None, None, :]
  # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
  # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
  # /transformer/transformer_layers.py#L270
  # encoder_extended_attention_mask = (encoder_extended_attention_mask ==
  # encoder_extended_attention_mask.transpose(-1, -2))
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = (
      tf.cast(1,						encoder_attention_mask.dtype				) - encoder_extended_attention_mask
  ) * encoder_extended_attention_mask.dtype.min

  return encoder_extended_attention_mask





def    _lowerCAmelCase    (     lowerCamelCase__							:							tf.Tensor,						lowerCamelCase__							:							int,						lowerCamelCase__							:							str = "input_ids"				)  ->    None:
  tf.debugging.assert_less(
      lowerCamelCase__,						tf.cast(lowerCamelCase__,						dtype=tensor.dtype				),						message=(
          f'''The maximum value of {tensor_name} ({tf.math.reduce_max(lowerCamelCase__				)}) must be smaller than the embedding '''
          f'''layer\'s input dimension ({embed_dim}). The likely cause is some problem at tokenization time.'''
      ),						)





def    _lowerCAmelCase    (     lowerCamelCase__							:							Any,						lowerCamelCase__							:							int,						lowerCamelCase__							:							List[str]				)  ->    Union[str, Any]:
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = 6_4_5_1_2
  # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
  # because in that case even chunking the array would not make the saving
  # possible.
  _SCREAMING_SNAKE_CASE   :				int							    = [x for x in data if len(lowerCamelCase__				) > HDF5_OBJECT_HEADER_LIMIT]

  # Expecting this to never be true.
  if bad_attributes:
    raise RuntimeError(
        "The following attributes cannot be saved to HDF5 file because "
        f'''they are larger than {HDF5_OBJECT_HEADER_LIMIT} '''
        f'''bytes: {bad_attributes}'''				)

  _SCREAMING_SNAKE_CASE   :				List[Any]							    = np.asarray(lowerCamelCase__				)

  _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = 1
  _SCREAMING_SNAKE_CASE   :				Optional[int]							    = np.array_split(lowerCamelCase__,						lowerCamelCase__				)

  # This will never loop forever thanks to the test above.
  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data				):
    num_chunks += 1
    _SCREAMING_SNAKE_CASE   :				int							    = np.array_split(lowerCamelCase__,						lowerCamelCase__				)

  if num_chunks > 1:
    for chunk_id, chunk_data in enumerate(lowerCamelCase__				):
      _SCREAMING_SNAKE_CASE   :				Tuple							    = chunk_data
  else:
    _SCREAMING_SNAKE_CASE   :				int							    = data





def    _lowerCAmelCase    (     lowerCamelCase__							:							int,						lowerCamelCase__							:							List[str]				)  ->    List[str]:
  if name in group.attrs:
    _SCREAMING_SNAKE_CASE   :				Optional[int]							    = [n.decode("utf8"				) if hasattr(lowerCamelCase__,						"decode"				) else n for n in group.attrs[name]]
  else:
    _SCREAMING_SNAKE_CASE   :				List[str]							    = []
    _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = 0
    while "%s%d" % (name, chunk_id) in group.attrs:
      data.extend(
          [n.decode("utf8"				) if hasattr(lowerCamelCase__,						"decode"				) else n for n in group.attrs["%s%d" % (name, chunk_id)]]				)
      chunk_id += 1
  return data





def    _lowerCAmelCase    (     lowerCamelCase__							:							Union[str, Any]				)  ->    List[str]:

  def _expand_single_ad_tensor(lowerCamelCase__							:							int				):
    if isinstance(lowerCamelCase__,						tf.Tensor				) and t.shape.rank == 1:
      return tf.expand_dims(lowerCamelCase__,						axis=-1				)
    return t

  return tf.nest.map_structure(_expand_single_ad_tensor,						lowerCamelCase__				)





"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE      ,       unittest.TestCase  ):
     A__      =			None
     A__      =			BloomTokenizerFast
     A__      =			BloomTokenizerFast
     A__      =			True
     A__      =			False
     A__      =			"""tokenizer_file"""
     A__      =			{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       super().setUp()
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = BloomTokenizerFast.from_pretrained("bigscience/tokenizer"				)
       tokenizer.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       kwargs.update(self.special_tokens_map				)
       return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,     **snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = self.get_rust_tokenizer()

       _SCREAMING_SNAKE_CASE   :				Tuple							    = ["The quick brown fox</s>", "jumps over the lazy dog</s>"]
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = [[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tokenizer.batch_encode_plus(snake_case__				)["input_ids"]
       self.assertListEqual(snake_case__  ,     snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = tokenizer.batch_decode(snake_case__				)
       self.assertListEqual(snake_case__  ,     snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__=6				):




       """simple docstring"""


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
           _SCREAMING_SNAKE_CASE   :				Tuple							    = self.rust_tokenizer_class.from_pretrained(snake_case__  ,     **snake_case__				)
           # tokenizer_r.pad_token = None # Hotfixing padding = None
           # Simple input
           _SCREAMING_SNAKE_CASE   :				str							    = "This is a simple input"
           _SCREAMING_SNAKE_CASE   :				Tuple							    = ["This is a simple input 1", "This is a simple input 2"]
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = ("This is a simple input", "This is a pair")
           _SCREAMING_SNAKE_CASE   :				List[str]							    = [
               ("This is a simple input 1", "This is a simple input 2"),
               ("This is a simple pair 1", "This is a simple pair 2"),
           ]

           # Simple input tests
           try:
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode_plus(snake_case__  ,     max_length=snake_case__				)

             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
           except ValueError:
             self.fail("Bloom Tokenizer should be able to deal with padding"				)

           _SCREAMING_SNAKE_CASE   :				Tuple							    = None  # Hotfixing padding = None
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.get_rust_tokenizer()
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = load_dataset("xnli"  ,     "all_languages"  ,     split="test"  ,     streaming=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = next(iter(snake_case__				)				)["premise"]  # pick up one data
       _SCREAMING_SNAKE_CASE   :				int							    = list(sample_data.values()				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = list(map(tokenizer.encode  ,     snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				int							    = [tokenizer.decode(snake_case__  ,     clean_up_tokenization_spaces=snake_case__				) for x in output_tokens]
       self.assertListEqual(snake_case__  ,     snake_case__				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map				)  ,     1				)
       self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()				)[0]				)  ,     1				)



