






'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_fnet import FNetTokenizer
else:
 __A							:							Union[str, Any]  							=		None

__A							:							str  							=		logging.get_logger(__name__)
__A							:							str  							=		{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__A							:							Dict  							=		{
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json""",
    },
}

__A							:							List[Any]  							=		{
    """google/fnet-base""": 5_12,
    """google/fnet-large""": 5_12,
}

__A							:							List[Any]  							=		"""▁"""
class 				lowercase       (							_lowerCamelCase      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							VOCAB_FILES_NAMES
    lowerCAmelCase__      				=							PRETRAINED_VOCAB_FILES_MAP
    lowerCAmelCase__      				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCAmelCase__      				=							["input_ids", "token_type_ids"]
    lowerCAmelCase__      				=							FNetTokenizer

    def __init__(					self     :     Optional[int]				,     __lowerCamelCase     :     Any=None				,     __lowerCamelCase     :     Optional[Any]=None				,     __lowerCamelCase     :     str=False				,     __lowerCamelCase     :     Any=True				,     __lowerCamelCase     :     Optional[Any]=True				,     __lowerCamelCase     :     Optional[int]="<unk>"				,     __lowerCamelCase     :     str="[SEP]"				,     __lowerCamelCase     :     Optional[int]="<pad>"				,     __lowerCamelCase     :     int="[CLS]"				,     __lowerCamelCase     :     List[Any]="[MASK]"				,     **__lowerCamelCase     :     Any				,     )							->							Optional[Any]:

     '''simple docstring'''




     lowerCamelCase__  =     (
         AddedToken(__lowerCamelCase				,     lstrip=__lowerCamelCase				,     rstrip=__lowerCamelCase				,     normalized=__lowerCamelCase     )
         if isinstance(__lowerCamelCase				,     __lowerCamelCase     )
         else mask_token
     )

     super().__init__(
         __lowerCamelCase				,     tokenizer_file=__lowerCamelCase				,     do_lower_case=__lowerCamelCase				,     remove_space=__lowerCamelCase				,     keep_accents=__lowerCamelCase				,     unk_token=__lowerCamelCase				,     sep_token=__lowerCamelCase				,     pad_token=__lowerCamelCase				,     cls_token=__lowerCamelCase				,     mask_token=__lowerCamelCase				,     **__lowerCamelCase				,     )

     lowerCamelCase__  =     do_lower_case
     lowerCamelCase__  =     remove_space
     lowerCamelCase__  =     keep_accents
     lowerCamelCase__  =     vocab_file
     lowerCamelCase__  =     False if not self.vocab_file else True

    def    a__				(					self     :     Dict				,     __lowerCamelCase     :     List[int]				,     __lowerCamelCase     :     Optional[List[int]] = None     )							->							List[int]:

     '''simple docstring'''




     lowerCamelCase__  =     [self.sep_token_id]
     lowerCamelCase__  =     [self.cls_token_id]
     if token_ids_a is None:
      return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep

    def    a__				(					self     :     Optional[int]				,     __lowerCamelCase     :     List[int]				,     __lowerCamelCase     :     Optional[List[int]] = None     )							->							List[int]:

     '''simple docstring'''




     lowerCamelCase__  =     [self.sep_token_id]
     lowerCamelCase__  =     [self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep     ) * [0]
     return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




    def    a__				(					self     :     Optional[int]				,     __lowerCamelCase     :     str				,     __lowerCamelCase     :     Optional[str] = None     )							->							Tuple[str]:

     '''simple docstring'''




     if not os.path.isdir(__lowerCamelCase     ):
      logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
      return
     lowerCamelCase__  =     os.path.join(
         __lowerCamelCase				,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

     if os.path.abspath(self.vocab_file     ) != os.path.abspath(__lowerCamelCase     ):
      copyfile(self.vocab_file				,     __lowerCamelCase     )

     return (out_vocab_file,)









'''simple docstring'''





import tensorflow as tf

from ...tf_utils import shape_list
class 				lowercase       (							tf.keras.layers.Layer      ):


    '''simple docstring'''

    def __init__(					self     :     Union[str, Any]				,     __lowerCamelCase     :     str				,     __lowerCamelCase     :     int				,     __lowerCamelCase     :     Dict				,     __lowerCamelCase     :     Union[str, Any]				,     __lowerCamelCase     :     int=1				,     __lowerCamelCase     :     Tuple=False				,     **__lowerCamelCase     :     Dict     )							->							str:

     '''simple docstring'''




     super().__init__(**__lowerCamelCase     )

     lowerCamelCase__  =     vocab_size
     lowerCamelCase__  =     d_embed
     lowerCamelCase__  =     d_proj

     lowerCamelCase__  =     cutoffs + [vocab_size]
     lowerCamelCase__  =     [0] + self.cutoffs
     lowerCamelCase__  =     div_val

     lowerCamelCase__  =     self.cutoffs[0]
     lowerCamelCase__  =     len(self.cutoffs     ) - 1
     lowerCamelCase__  =     self.shortlist_size + self.n_clusters
     lowerCamelCase__  =     keep_order

     lowerCamelCase__  =     []
     lowerCamelCase__  =     []

    def    a__				(					self     :     Optional[int]				,     __lowerCamelCase     :     str     )							->							List[str]:

     '''simple docstring'''




     if self.n_clusters > 0:
      lowerCamelCase__  =     self.add_weight(
          shape=(self.n_clusters, self.d_embed)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name="cluster_weight"     )
      lowerCamelCase__  =     self.add_weight(
          shape=(self.n_clusters,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name="cluster_bias"     )

     if self.div_val == 1:
      for i in range(len(self.cutoffs     )     ):
       if self.d_proj != self.d_embed:
        lowerCamelCase__  =     self.add_weight(
            shape=(self.d_embed, self.d_proj)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_projs_._{i}'''				,     )
        self.out_projs.append(__lowerCamelCase     )
       else:
        self.out_projs.append(__lowerCamelCase     )
       lowerCamelCase__  =     self.add_weight(
           shape=(self.vocab_size, self.d_embed)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._weight'''				,     )
       lowerCamelCase__  =     self.add_weight(
           shape=(self.vocab_size,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._bias'''				,     )
       self.out_layers.append((weight, bias)     )
     else:
      for i in range(len(self.cutoffs     )     ):
       lowerCamelCase__  ,	lowerCamelCase__  =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
       lowerCamelCase__  =     self.d_embed // (self.div_val**i)

       lowerCamelCase__  =     self.add_weight(
           shape=(d_emb_i, self.d_proj)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_projs_._{i}'''     )
       self.out_projs.append(__lowerCamelCase     )
       lowerCamelCase__  =     self.add_weight(
           shape=(r_idx - l_idx, d_emb_i)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._weight'''				,     )
       lowerCamelCase__  =     self.add_weight(
           shape=(r_idx - l_idx,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._bias'''				,     )
       self.out_layers.append((weight, bias)     )
     super().build(__lowerCamelCase     )

    @staticmethod
    def    a__				(					__lowerCamelCase     :     Union[str, Any]				,     __lowerCamelCase     :     Optional[Any]				,     __lowerCamelCase     :     Tuple				,     __lowerCamelCase     :     List[str]=None     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     x
     if proj is not None:
      lowerCamelCase__  =     tf.einsum("ibd,ed->ibe"				,     __lowerCamelCase				,     __lowerCamelCase     )
     return tf.einsum("ibd,nd->ibn"				,     __lowerCamelCase				,     __lowerCamelCase     ) + b

    @staticmethod
    def    a__				(					__lowerCamelCase     :     List[str]				,     __lowerCamelCase     :     Union[str, Any]     )							->							Union[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     shape_list(__lowerCamelCase     )
     lowerCamelCase__  =     tf.range(lp_size[0]				,     dtype=target.dtype     )
     lowerCamelCase__  =     tf.stack([r, target]				,     1     )
     return tf.gather_nd(__lowerCamelCase				,     __lowerCamelCase     )




    def    a__				(					self     :     Any				,     __lowerCamelCase     :     int				,     __lowerCamelCase     :     Tuple				,     __lowerCamelCase     :     str=True				,     __lowerCamelCase     :     Tuple=False     )							->							int:

     '''simple docstring'''




     lowerCamelCase__  =     0
     if self.n_clusters == 0:
      lowerCamelCase__  =     self._logit(__lowerCamelCase				,     self.out_layers[0][0]				,     self.out_layers[0][1]				,     self.out_projs[0]     )
      if target is not None:
       lowerCamelCase__  =     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__lowerCamelCase				,     logits=__lowerCamelCase     )
      lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase				,     axis=-1     )
     else:
      lowerCamelCase__  =     shape_list(__lowerCamelCase     )
      lowerCamelCase__  =     []
      lowerCamelCase__  =     tf.zeros(hidden_sizes[:2]     )
      for i in range(len(self.cutoffs     )     ):
       lowerCamelCase__  ,	lowerCamelCase__  =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
       if target is not None:
        lowerCamelCase__  =     (target >= l_idx) & (target < r_idx)
        lowerCamelCase__  =     tf.where(__lowerCamelCase     )
        lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     ) - l_idx

       if self.div_val == 1:
        lowerCamelCase__  =     self.out_layers[0][0][l_idx:r_idx]
        lowerCamelCase__  =     self.out_layers[0][1][l_idx:r_idx]
       else:
        lowerCamelCase__  =     self.out_layers[i][0]
        lowerCamelCase__  =     self.out_layers[i][1]

       if i == 0:
        lowerCamelCase__  =     tf.concat([cur_W, self.cluster_weight]				,     0     )
        lowerCamelCase__  =     tf.concat([cur_b, self.cluster_bias]				,     0     )

        lowerCamelCase__  =     self._logit(__lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     self.out_projs[0]     )
        lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase     )
        out.append(head_logprob[..., : self.cutoffs[0]]     )
        if target is not None:
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     self._gather_logprob(__lowerCamelCase				,     __lowerCamelCase     )
       else:
        lowerCamelCase__  =     self._logit(__lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     self.out_projs[i]     )
        lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase     )
        lowerCamelCase__  =     self.cutoffs[0] + i - 1  # No probability for the head cluster
        lowerCamelCase__  =     head_logprob[..., cluster_prob_idx, None] + tail_logprob
        out.append(__lowerCamelCase     )
        if target is not None:
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     self._gather_logprob(__lowerCamelCase				,     __lowerCamelCase     )
         cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
       if target is not None:
        loss += tf.scatter_nd(__lowerCamelCase				,     -cur_logprob				,     shape_list(__lowerCamelCase     )     )
      lowerCamelCase__  =     tf.concat(__lowerCamelCase				,     axis=-1     )

     if target is not None:
      if return_mean:
       lowerCamelCase__  =     tf.reduce_mean(__lowerCamelCase     )
      # Add the training-time loss value to the layer using `self.add_loss()`.
      self.add_loss(__lowerCamelCase     )

      # Log the loss as a metric (we could log arbitrary metrics,
      # including different metrics for training and inference.
      self.add_metric(__lowerCamelCase				,     name=self.name				,     aggregation="mean" if return_mean else ""     )

     return out

