




from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__					    =    logging.get_logger(__name__)

UpperCamelCase__					    =    {
    """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/config.json""",
    """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/config.json"""
    # See all FNet models at https://huggingface.co/models?filter=fnet
}
class     a__					(      snake_case__   ):
						_a							:					List[str]      =					"""fnet"""






						def __init__(     self	,    _A=3_2_0_0_0	,    _A=7_6_8	,    _A=1_2	,    _A=3_0_7_2	,    _A="gelu_new"	,    _A=0.1	,    _A=5_1_2	,    _A=4	,    _A=0.02	,    _A=1E-1_2	,    _A=False	,    _A=5_1_2	,    _A=3	,    _A=1	,    _A=2	,    **_A	,    ):




											"""simple docstring"""



											super().__init__(pad_token_id=lowercase_	,    bos_token_id=lowercase_	,    eos_token_id=lowercase_	,    **lowercase_    )

											__lowerCAmelCase            = vocab_size
											__lowerCAmelCase            = max_position_embeddings
											__lowerCAmelCase            = hidden_size
											__lowerCAmelCase            = num_hidden_layers
											__lowerCAmelCase            = intermediate_size
											__lowerCAmelCase            = hidden_act
											__lowerCAmelCase            = hidden_dropout_prob
											__lowerCAmelCase            = initializer_range
											__lowerCAmelCase            = type_vocab_size
											__lowerCAmelCase            = layer_norm_eps
											__lowerCAmelCase            = use_tpu_fourier_optimizations
											__lowerCAmelCase            = tpu_short_seq_length










import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing
class     a__					(      snake_case__   ):






						def __init__(     self	,    _A = "▁"	,    _A = True	,    _A = "<unk>"	,    _A = "</s>"	,    _A = "<pad>"	,    ):




											"""simple docstring"""



											__lowerCAmelCase            = {
											    "pad": {"id": 0, "token": pad_token},
											    "eos": {"id": 1, "token": eos_token},
											    "unk": {"id": 2, "token": unk_token},
											}

											__lowerCAmelCase            = [None] * len(self.special_tokens    )
											for token_dict in self.special_tokens.values():
																__lowerCAmelCase            = token_dict["token"]

											__lowerCAmelCase            = Tokenizer(Unigram()    )

											__lowerCAmelCase            = normalizers.Sequence(
											    [
											        normalizers.Nmt(),
											        normalizers.NFKC(),
											        normalizers.Replace(Regex(" {2,}"    )	,    " "    ),
											        normalizers.Lowercase(),
											    ]    )
											__lowerCAmelCase            = pre_tokenizers.Sequence(
											    [
											        pre_tokenizers.Metaspace(replacement=_A	,    add_prefix_space=_A    ),
											        pre_tokenizers.Digits(individual_digits=_A    ),
											        pre_tokenizers.Punctuation(),
											    ]    )
											__lowerCAmelCase            = decoders.Metaspace(replacement=_A	,    add_prefix_space=_A    )

											__lowerCAmelCase            = TemplateProcessing(
											    single=f"""$A {self.special_tokens['eos']['token']}"""	,    special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])]	,    )

											__lowerCAmelCase            = {
											    "model": "SentencePieceUnigram",
											    "replacement": replacement,
											    "add_prefix_space": add_prefix_space,
											}

											super().__init__(_A	,    _A    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = 8_0_0_0	,    _A = True	,    ):




											"""simple docstring"""



											__lowerCAmelCase            = trainers.UnigramTrainer(
											    vocab_size=_A	,    special_tokens=self.special_tokens_list	,    show_progress=_A	,    )

											if isinstance(_A	,    _A    ):
																__lowerCAmelCase            = [files]
											self._tokenizer.train(_A	,    trainer=_A    )

											self.add_unk_id()






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = 8_0_0_0	,    _A = True	,    ):




											"""simple docstring"""



											__lowerCAmelCase            = trainers.UnigramTrainer(
											    vocab_size=_A	,    special_tokens=self.special_tokens_list	,    show_progress=_A	,    )

											self._tokenizer.train_from_iterator(_A	,    trainer=_A    )

											self.add_unk_id()






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = json.loads(self._tokenizer.to_str()    )

											__lowerCAmelCase            = self.special_tokens["unk"]["id"]

											__lowerCAmelCase            = Tokenizer.from_str(json.dumps(_A    )    )




