





'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__lowerCAmelCase				:      List[Any]						      =logging.get_logger(__name__)

__lowerCAmelCase				:      int						      ={"vocab_file": "spiece.model"}

__lowerCAmelCase				:      List[str]						      ={
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

__lowerCAmelCase				:      Optional[Any]						      ={
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
__lowerCAmelCase				:      Dict						      =0
__lowerCAmelCase				:      Dict						      =1
__lowerCAmelCase				:      Tuple						      =2
__lowerCAmelCase				:      Tuple						      =3
__lowerCAmelCase				:      Tuple						      =4






class   UpperCAmelCase      (			UpperCamelCase__							):
 __lowercase  						=							VOCAB_FILES_NAMES
 __lowercase  						=							PRETRAINED_VOCAB_FILES_MAP
 __lowercase  						=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase  						=							"""left"""
 def __init__(							self :Union[str, Any]		,			lowercase_ :Optional[int]		,			lowercase_ :Optional[int]=False		,			lowercase_ :int=True		,			lowercase_ :str=False		,			lowercase_ :List[Any]="<s>"		,			lowercase_ :Tuple="</s>"		,			lowercase_ :Union[str, Any]="<unk>"		,			lowercase_ :Any="<sep>"		,			lowercase_ :str="<pad>"		,			lowercase_ :Union[str, Any]="<cls>"		,			lowercase_ :Dict="<mask>"		,			lowercase_ :int=["<eop>", "<eod>"]		,			lowercase_ :Optional[Dict[str, Any]] = None		,			**lowercase_ :Any		,			)->				None:
     # Mask token behave like a normal word, i.e. include the space before it
     A__							=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_				) if isinstance(lowercase_		,			lowercase_				) else mask_token

     A__							=     {} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=lowercase_		,			remove_space=lowercase_		,			keep_accents=lowercase_		,			bos_token=lowercase_		,			eos_token=lowercase_		,			unk_token=lowercase_		,			sep_token=lowercase_		,			pad_token=lowercase_		,			cls_token=lowercase_		,			mask_token=lowercase_		,			additional_special_tokens=lowercase_		,			sp_model_kwargs=self.sp_model_kwargs		,			**lowercase_		,			)

     A__							=     3

     A__							=     do_lower_case
     A__							=     remove_space
     A__							=     keep_accents
     A__							=     vocab_file

     A__							=     spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(lowercase_				)
 @property
 def 	UpperCAmelCase_				(							self :Dict				)->				Any:
     return len(self.sp_model				)
 def 	UpperCAmelCase_				(							self :List[str]				)->				Union[str, Any]:
     A__							=     {self.convert_ids_to_tokens(lowercase_				): i for i in range(self.vocab_size				)}
     vocab.update(self.added_tokens_encoder				)
     return vocab
 def __getstate__(							self :Optional[Any]				)->				int:
     A__							=     self.__dict__.copy()
     A__							=     None
     return state
 def __setstate__(							self :Tuple		,			lowercase_ :Optional[int]				)->				int:
     A__							=     d

     # for backward compatibility
     if not hasattr(self		,			"sp_model_kwargs"				):
         A__							=     {}

     A__							=     spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(self.vocab_file				)
 def 	UpperCAmelCase_				(							self :Optional[int]		,			lowercase_ :Tuple				)->				int:
     if self.remove_space:
         A__							=     " ".join(inputs.strip().split()				)
     else:
         A__							=     inputs
     A__							=     outputs.replace("``"		,			"\""				).replace("''"		,			"\""				)

     if not self.keep_accents:
         A__							=     unicodedata.normalize("NFKD"		,			lowercase_				)
         A__							=     "".join([c for c in outputs if not unicodedata.combining(lowercase_				)]				)
     if self.do_lower_case:
         A__							=     outputs.lower()

     return outputs
 def 	UpperCAmelCase_				(							self :Any		,			lowercase_ :str				)->				List[str]:
     A__							=     self.preprocess_text(lowercase_				)
     A__							=     self.sp_model.encode(lowercase_		,			out_type=lowercase_				)
     A__							=     []
     for piece in pieces:
         if len(lowercase_				) > 1 and piece[-1] == str(","				) and piece[-2].isdigit():
             A__							=     self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_		,			""				)				)
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                 if len(cur_pieces[0]				) == 1:
                     A__							=     cur_pieces[1:]
                 else:
                     A__							=     cur_pieces[0][1:]
             cur_pieces.append(piece[-1]				)
             new_pieces.extend(lowercase_				)
         else:
             new_pieces.append(lowercase_				)

     return new_pieces
 def 	UpperCAmelCase_				(							self :Any		,			lowercase_ :List[str]				)->				Optional[int]:
     return self.sp_model.PieceToId(lowercase_				)
 def 	UpperCAmelCase_				(							self :Union[str, Any]		,			lowercase_ :Dict				)->				List[str]:
     return self.sp_model.IdToPiece(lowercase_				)
 def 	UpperCAmelCase_				(							self :Optional[int]		,			lowercase_ :Any				)->				str:
     A__							=     "".join(lowercase_				).replace(lowercase_		,			" "				).strip()
     return out_string
 def 	UpperCAmelCase_				(							self :Optional[Any]		,			lowercase_ :List[int]		,			lowercase_ :bool = False		,			lowercase_ :bool = None		,			lowercase_ :bool = True		,			**lowercase_ :Optional[int]		,			)->				str:
     A__							=     kwargs.pop("use_source_tokenizer"		,			lowercase_				)

     A__							=     self.convert_ids_to_tokens(lowercase_		,			skip_special_tokens=lowercase_				)

     # To avoid mixing byte-level and unicode for byte-level BPT
     # we need to build string separately for added tokens and byte-level tokens
     # cf. https://github.com/huggingface/transformers/issues/1133
     A__							=     []
     A__							=     []
     for token in filtered_tokens:
         if skip_special_tokens and token in self.all_special_ids:
             continue
         if token in self.added_tokens_encoder:
             if current_sub_text:
                 sub_texts.append(self.convert_tokens_to_string(lowercase_				)				)
                 A__							=     []
             sub_texts.append(lowercase_				)
         else:
             current_sub_text.append(lowercase_				)
     if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(lowercase_				)				)

     # Mimic the behavior of the Rust tokenizer:
     # By default, there are no spaces between special tokens
     A__							=     "".join(lowercase_				)

     A__							=     (
         clean_up_tokenization_spaces
         if clean_up_tokenization_spaces is not None
         else self.clean_up_tokenization_spaces
     )
     if clean_up_tokenization_spaces:
         A__							=     self.clean_up_tokenization(lowercase_				)
         return clean_text
     else:
         return text
 def 	UpperCAmelCase_				(							self :int		,			lowercase_ :List[int]		,			lowercase_ :Optional[List[int]] = None				)->				List[int]:
     A__							=     [self.sep_token_id]
     A__							=     [self.cls_token_id]
     if token_ids_a is None:
         return token_ids_a + sep + cls
     return token_ids_a + sep + token_ids_a + sep + cls
 def 	UpperCAmelCase_				(							self :Dict		,			lowercase_ :List[int]		,			lowercase_ :Optional[List[int]] = None		,			lowercase_ :bool = False				)->				List[int]:

     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowercase_		,			token_ids_a=lowercase_		,			already_has_special_tokens=lowercase_				)

     if token_ids_a is not None:
         return ([0] * len(lowercase_				)) + [1] + ([0] * len(lowercase_				)) + [1, 1]
     return ([0] * len(lowercase_				)) + [1, 1]
 def 	UpperCAmelCase_				(							self :Tuple		,			lowercase_ :List[int]		,			lowercase_ :Optional[List[int]] = None				)->				List[int]:
     A__							=     [self.sep_token_id]
     A__							=     [2]

     if token_ids_a is None:
         return len(token_ids_a + sep				) * [0] + cls_segment_id
     return len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1] + cls_segment_id



 def 	UpperCAmelCase_				(							self :List[str]		,			lowercase_ :str		,			lowercase_ :Optional[str] = None				)->				Tuple[str]:
     if not os.path.isdir(lowercase_				):
         logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
         return
     A__							=     os.path.join(
         lowercase_		,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(lowercase_				) and os.path.isfile(self.vocab_file				):
         copyfile(self.vocab_file		,			lowercase_				)
     elif not os.path.isfile(self.vocab_file				):
         with open(lowercase_		,			"wb"				) as fi:
             A__							=     self.sp_model.serialized_model_proto()
             fi.write(lowercase_				)

     return (out_vocab_file,)










'''simple docstring'''

import os
import tempfile
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
  import torch
  from torch import nn

  from transformers import (
      Adafactor,
      AdamW,
      get_constant_schedule,
      get_constant_schedule_with_warmup,
      get_cosine_schedule_with_warmup,
      get_cosine_with_hard_restarts_schedule_with_warmup,
      get_inverse_sqrt_schedule,
      get_linear_schedule_with_warmup,
      get_polynomial_decay_schedule_with_warmup,
  )





def 		UpperCamelCase							(				_lowerCamelCase   :   Any		,			_lowerCamelCase   :   Optional[Any]=10			):
    A__							=     []
    for _ in range(_lowerCamelCase			):
        lrs.append(scheduler.get_lr()[0]			)
        scheduler.step()
    return lrs





def 		UpperCamelCase							(				_lowerCamelCase   :   Tuple		,			_lowerCamelCase   :   List[str]=10			):
    A__							=     []
    for step in range(_lowerCamelCase			):
        lrs.append(scheduler.get_lr()[0]			)
        scheduler.step()
        if step == num_steps // 2:
            with tempfile.TemporaryDirectory() as tmpdirname:
                A__							=     os.path.join(_lowerCamelCase		,			"schedule.bin"			)
                torch.save(scheduler.state_dict()		,			_lowerCamelCase			)

                A__							=     torch.load(_lowerCamelCase			)
                scheduler.load_state_dict(_lowerCamelCase			)
    return lrs






@require_torch
class   UpperCAmelCase      (			unittest.TestCase							):
 def 	UpperCAmelCase_				(							self :int		,			lowercase_ :Any		,			lowercase_ :List[Any]		,			lowercase_ :Optional[Any]				)->				int:
     self.assertEqual(len(lowercase_				)		,			len(lowercase_				)				)
     for a, b in zip(lowercase_		,			lowercase_				):
         self.assertAlmostEqual(lowercase_		,			lowercase_		,			delta=lowercase_				)
 def 	UpperCAmelCase_				(							self :Union[str, Any]				)->				Dict:
     A__							=     torch.tensor([0.1, -0.2, -0.1]		,			requires_grad=lowercase_				)
     A__							=     torch.tensor([0.4, 0.2, -0.5]				)
     A__							=     nn.MSELoss()
     # No warmup, constant schedule, no gradient clipping
     A__							=     AdamW(params=[w]		,			lr=2E-1		,			weight_decay=0.0				)
     for _ in range(1_00				):
         A__							=     criterion(lowercase_		,			lowercase_				)
         loss.backward()
         optimizer.step()
         w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
         w.grad.zero_()
     self.assertListAlmostEqual(w.tolist()		,			[0.4, 0.2, -0.5]		,			tol=1E-2				)



 def 	UpperCAmelCase_				(							self :Tuple				)->				List[str]:
     A__							=     torch.tensor([0.1, -0.2, -0.1]		,			requires_grad=lowercase_				)
     A__							=     torch.tensor([0.4, 0.2, -0.5]				)
     A__							=     nn.MSELoss()
     # No warmup, constant schedule, no gradient clipping
     A__							=     Adafactor(
         params=[w]		,			lr=1E-2		,			eps=(1E-30, 1E-3)		,			clip_threshold=1.0		,			decay_rate=-0.8		,			betaa=lowercase_		,			weight_decay=0.0		,			relative_step=lowercase_		,			scale_parameter=lowercase_		,			warmup_init=lowercase_		,			)
     for _ in range(10_00				):
         A__							=     criterion(lowercase_		,			lowercase_				)
         loss.backward()
         optimizer.step()
         w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
         w.grad.zero_()
     self.assertListAlmostEqual(w.tolist()		,			[0.4, 0.2, -0.5]		,			tol=1E-2				)






@require_torch
class   UpperCAmelCase      (			unittest.TestCase							):
 __lowercase  						=							nn.Linear(50     ,			50							) if is_torch_available() else None
 __lowercase  						=							AdamW(m.parameters()     ,			lr=1_0.0							) if is_torch_available() else None
 __lowercase  						=							10
 def 	UpperCAmelCase_				(							self :Tuple		,			lowercase_ :Any		,			lowercase_ :List[Any]		,			lowercase_ :List[Any]		,			lowercase_ :str=None				)->				Optional[int]:
     self.assertEqual(len(lowercase_				)		,			len(lowercase_				)				)
     for a, b in zip(lowercase_		,			lowercase_				):
         self.assertAlmostEqual(lowercase_		,			lowercase_		,			delta=lowercase_		,			msg=lowercase_				)



 def 	UpperCAmelCase_				(							self :Optional[Any]				)->				Any:
     A__							=     {"num_warmup_steps": 2, "num_training_steps": 10}
     # schedulers doct format
     # function: (sched_args_dict, expected_learning_rates)
     A__							=     {
         get_constant_schedule: ({}, [1_0.0] * self.num_steps),
         get_constant_schedule_with_warmup: (
             {"num_warmup_steps": 4},
             [0.0, 2.5, 5.0, 7.5, 1_0.0, 1_0.0, 1_0.0, 1_0.0, 1_0.0, 1_0.0],
         ),
         get_linear_schedule_with_warmup: (
             {**common_kwargs},
             [0.0, 5.0, 1_0.0, 8.7_5, 7.5, 6.2_5, 5.0, 3.7_5, 2.5, 1.2_5],
         ),
         get_cosine_schedule_with_warmup: (
             {**common_kwargs},
             [0.0, 5.0, 1_0.0, 9.6_1, 8.5_3, 6.9_1, 5.0, 3.0_8, 1.4_6, 0.3_8],
         ),
         get_cosine_with_hard_restarts_schedule_with_warmup: (
             {**common_kwargs, "num_cycles": 2},
             [0.0, 5.0, 1_0.0, 8.5_3, 5.0, 1.4_6, 1_0.0, 8.5_3, 5.0, 1.4_6],
         ),
         get_polynomial_decay_schedule_with_warmup: (
             {**common_kwargs, "power": 2.0, "lr_end": 1E-7},
             [0.0, 5.0, 1_0.0, 7.6_5_6, 5.6_2_5, 3.9_0_6, 2.5, 1.4_0_6, 0.6_2_5, 0.1_5_6],
         ),
         get_inverse_sqrt_schedule: (
             {"num_warmup_steps": 2},
             [0.0, 5.0, 1_0.0, 8.1_6_5, 7.0_7_1, 6.3_2_5, 5.7_7_4, 5.3_4_5, 5.0, 4.7_1_4],
         ),
     }

     for scheduler_func, data in scheds.items():
         A__,      A__							=     data

         A__							=     scheduler_func(self.optimizer		,			**lowercase_				)
         self.assertEqual(len([scheduler.get_lr()[0]]				)		,			1				)
         A__							=     unwrap_schedule(lowercase_		,			self.num_steps				)
         self.assertListAlmostEqual(
             lowercase_		,			lowercase_		,			tol=1E-2		,			msg=F"failed for {scheduler_func} in normal scheduler"		,			)

         A__							=     scheduler_func(self.optimizer		,			**lowercase_				)
         if scheduler_func.__name__ != "get_constant_schedule":
             LambdaScheduleWrapper.wrap_scheduler(lowercase_				)  # wrap to test picklability of the schedule
         A__							=     unwrap_and_save_reload_schedule(lowercase_		,			self.num_steps				)
         self.assertListEqual(lowercase_		,			lowercase_		,			msg=F"failed for {scheduler_func} in save and reload"				)






class   UpperCAmelCase      :
 def __init__(							self :str		,			lowercase_ :List[str]				)->				Tuple:
     A__							=     fn
 def __call__(							self :List[Any]		,			*lowercase_ :Dict		,			**lowercase_ :Dict				)->				Tuple:
     return self.fn(*lowercase_		,			**lowercase_				)



 @classmethod
 def 	UpperCAmelCase_				(							self :Any		,			lowercase_ :Tuple				)->				List[Any]:
     A__							=     list(map(self		,			scheduler.lr_lambdas				)				)



