


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase__				     =		logging.get_logger(__name__)

lowercase__				     =		{'vocab_file': 'spiece.model'}

lowercase__				     =		{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

lowercase__				     =		{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
lowercase__				     =		0
lowercase__				     =		1
lowercase__				     =		2
lowercase__				     =		3
lowercase__				     =		4





class 						__SCREAMING_SNAKE_CASE					(      UpperCamelCase_ ):
     UpperCAmelCase =				VOCAB_FILES_NAMES
     UpperCAmelCase =				PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase =				'''left'''






     def __init__(			self     ,							__UpperCamelCase     ,							__UpperCamelCase=False     ,							__UpperCamelCase=True     ,							__UpperCamelCase=False     ,							__UpperCamelCase="<s>"     ,							__UpperCamelCase="</s>"     ,							__UpperCamelCase="<unk>"     ,							__UpperCamelCase="<sep>"     ,							__UpperCamelCase="<pad>"     ,							__UpperCamelCase="<cls>"     ,							__UpperCamelCase="<mask>"     ,							__UpperCamelCase=["<eop>", "<eod>"]     ,							__UpperCamelCase = None     ,							**__UpperCamelCase     ,							)       ->    Optional[Any]:
          _a						=		AddedToken(UpperCamelCase__     ,							lstrip=UpperCamelCase__     ,							rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__     ,							UpperCamelCase__		) else mask_token

          _a						=		{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=UpperCamelCase__     ,							remove_space=UpperCamelCase__     ,							keep_accents=UpperCamelCase__     ,							bos_token=UpperCamelCase__     ,							eos_token=UpperCamelCase__     ,							unk_token=UpperCamelCase__     ,							sep_token=UpperCamelCase__     ,							pad_token=UpperCamelCase__     ,							cls_token=UpperCamelCase__     ,							mask_token=UpperCamelCase__     ,							additional_special_tokens=UpperCamelCase__     ,							sp_model_kwargs=self.sp_model_kwargs     ,							**UpperCamelCase__     ,							)

          _a						=		3

          _a						=		do_lower_case
          _a						=		remove_space
          _a						=		keep_accents
          _a						=		vocab_file

          _a						=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(UpperCamelCase__		)






     @property
     def  a_					(			self		)       ->    Any:
          return len(self.sp_model		)






     def  a_					(			self		)       ->    str:
          _a						=		{self.convert_ids_to_tokens(UpperCamelCase__		): i for i in range(self.vocab_size		)}
          vocab.update(self.added_tokens_encoder		)
          return vocab






     def __getstate__(			self		)       ->    Optional[int]:
          _a						=		self.__dict__.copy()
          _a						=		None
          return state






     def __setstate__(			self     ,							__UpperCamelCase		)       ->    Union[str, Any]:
          _a						=		d

          # for backward compatibility
          if not hasattr(self     ,							"sp_model_kwargs"		):
               _a						=		{}

          _a						=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(self.vocab_file		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Union[str, Any]:





          if self.remove_space:
               _a						=		''' '''.join(inputs.strip().split()		)
          else:
               _a						=		inputs
          _a						=		outputs.replace("``"     ,							"\""		).replace("\'\'"     ,							"\""		)

          if not self.keep_accents:
               _a						=		unicodedata.normalize("NFKD"     ,							UpperCamelCase__		)
               _a						=		''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase__		)]		)
          if self.do_lower_case:
               _a						=		outputs.lower()

          return outputs






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Dict:
          _a						=		self.preprocess_text(UpperCamelCase__		)
          _a						=		self.sp_model.encode(UpperCamelCase__     ,							out_type=UpperCamelCase__		)
          _a						=		[]
          for piece in pieces:
               if len(UpperCamelCase__		) > 1 and piece[-1] == str(","		) and piece[-2].isdigit():
                    _a						=		self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase__     ,							""		)		)
                    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]		) == 1:
                              _a						=		cur_pieces[1:]
                         else:
                              _a						=		cur_pieces[0][1:]
                    cur_pieces.append(piece[-1]		)
                    new_pieces.extend(UpperCamelCase__		)
               else:
                    new_pieces.append(UpperCamelCase__		)

          return new_pieces






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Optional[Any]:
          return self.sp_model.PieceToId(UpperCamelCase__		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Union[str, Any]:
          return self.sp_model.IdToPiece(UpperCamelCase__		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Tuple:
          _a						=		''''''.join(UpperCamelCase__		).replace(UpperCamelCase__     ,							" "		).strip()
          return out_string






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = False     ,							__UpperCamelCase = None     ,							__UpperCamelCase = True     ,							**__UpperCamelCase     ,							)       ->    Any:
          _a						=		kwargs.pop("use_source_tokenizer"     ,							UpperCamelCase__		)

          _a						=		self.convert_ids_to_tokens(UpperCamelCase__     ,							skip_special_tokens=UpperCamelCase__		)

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          _a						=		[]
          _a						=		[]
          for token in filtered_tokens:
               if skip_special_tokens and token in self.all_special_ids:
                    continue
               if token in self.added_tokens_encoder:
                    if current_sub_text:
                         sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__		)		)
                         _a						=		[]
                    sub_texts.append(UpperCamelCase__		)
               else:
                    current_sub_text.append(UpperCamelCase__		)
          if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(UpperCamelCase__		)		)

          # Mimic the behavior of the Rust tokenizer:
          # By default, there are no spaces between special tokens
          _a						=		''''''.join(UpperCamelCase__		)

          _a						=		(
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
               _a						=		self.clean_up_tokenization(UpperCamelCase__		)
               return clean_text
          else:
               return text






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    List[str]:
          _a						=		[self.sep_token_id]
          _a						=		[self.cls_token_id]
          if token_ids_a is None:
               return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None     ,							__UpperCamelCase = False		)       ->    str:





          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=UpperCamelCase__     ,							token_ids_a=UpperCamelCase__     ,							already_has_special_tokens=UpperCamelCase__		)

          if token_ids_a is not None:
               return ([0] * len(UpperCamelCase__		)) + [1] + ([0] * len(UpperCamelCase__		)) + [1, 1]
          return ([0] * len(UpperCamelCase__		)) + [1, 1]






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    Tuple:
          _a						=		[self.sep_token_id]
          _a						=		[2]

          if token_ids_a is None:
               return len(token_ids_a + sep		) * [0] + cls_segment_id
          return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    Union[str, Any]:





          if not os.path.isdir(UpperCamelCase__		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          _a						=		os.path.join(
              UpperCamelCase__     ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

          if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCamelCase__		) and os.path.isfile(self.vocab_file		):
               copyfile(self.vocab_file     ,							UpperCamelCase__		)
          elif not os.path.isfile(self.vocab_file		):
               with open(UpperCamelCase__     ,							"wb"		) as fi:
                    _a						=		self.sp_model.serialized_model_proto()
                    fi.write(UpperCamelCase__		)

          return (out_vocab_file,)





'''simple docstring'''

import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
lowercase__				     =		object()

# For specifying empty leaf dict `{}`
lowercase__				     =		object()






def __UpperCamelCase   (			__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Tuple				)      ->     Union[str, Any]:







     '''simple docstring'''






     _a						=		tuple((re.compile(x + "$"				) for x in qs)				)
     for i in range(len(__lowerCamelCase				) - len(__lowerCamelCase				) + 1				):
          _a						=		[x.match(__lowerCamelCase				) for x, y in zip(__lowerCamelCase      ,  ks[i:]				)]
          if matches and all(__lowerCamelCase				):
               return True
     return False






def __UpperCamelCase   (			__lowerCamelCase		: int				)      ->     Union[str, Any]:







     '''simple docstring'''






     def replace(__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Any				):
          for rule, replacement in rules:
               if _match(__lowerCamelCase      ,  __lowerCamelCase				):
                    return replacement
          return val

     return replace






def __UpperCamelCase   (			)      ->     Tuple:







     '''simple docstring'''






     return [
         # embeddings
         (("transformer", "wpe", "embedding"), P("mp"      ,  __lowerCamelCase				)),
         (("transformer", "wte", "embedding"), P("mp"      ,  __lowerCamelCase				)),
         # atention
         (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__lowerCamelCase      ,  "mp"				)),
         (("attention", "out_proj", "kernel"), P("mp"      ,  __lowerCamelCase				)),
         (("attention", "out_proj", "bias"), None),
         # mlp
         (("mlp", "c_fc", "kernel"), P(__lowerCamelCase      ,  "mp"				)),
         (("mlp", "c_fc", "bias"), P("mp"				)),
         (("mlp", "c_proj", "kernel"), P("mp"      ,  __lowerCamelCase				)),
         (("mlp", "c_proj", "bias"), None),
         # layer norms
         ((r"ln_\d+", "bias"), None),
         ((r"\d+", r"ln_\d+", "scale"), None),
         (("ln_f", "bias"), None),
         (("ln_f", "scale"), None),
     ]






def __UpperCamelCase   (			__lowerCamelCase		: Dict				)      ->     Tuple:







     '''simple docstring'''






     _a						=		_get_partition_rules()
     _a						=		_replacement_rules(__lowerCamelCase				)
     _a						=		{k: _unmatched for k in flatten_dict(__lowerCamelCase				)}
     _a						=		{k: replace(__lowerCamelCase      ,  __lowerCamelCase				) for k, v in initd.items()}
     assert _unmatched not in result.values(), "Incomplete partition spec."
     return freeze(unflatten_dict(__lowerCamelCase				)				)

