





"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__A     =    {"configuration_fnet": ["FNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "FNetConfig"]}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __A     =    ["FNetTokenizer"]

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __A     =    ["FNetTokenizerFast"]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __A     =    [
         "FNET_PRETRAINED_MODEL_ARCHIVE_LIST",
         "FNetForMaskedLM",
         "FNetForMultipleChoice",
         "FNetForNextSentencePrediction",
         "FNetForPreTraining",
         "FNetForQuestionAnswering",
         "FNetForSequenceClassification",
         "FNetForTokenClassification",
         "FNetLayer",
         "FNetModel",
         "FNetPreTrainedModel",
     ]


if TYPE_CHECKING:
     from .configuration_fnet import FNET_PRETRAINED_CONFIG_ARCHIVE_MAP, FNetConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_fnet import FNetTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_fnet_fast import FNetTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_fnet import (
              FNET_PRETRAINED_MODEL_ARCHIVE_LIST,
              FNetForMaskedLM,
              FNetForMultipleChoice,
              FNetForNextSentencePrediction,
              FNetForPreTraining,
              FNetForQuestionAnswering,
              FNetForSequenceClassification,
              FNetForTokenClassification,
              FNetLayer,
              FNetModel,
              FNetPreTrainedModel,
          )


else:
     import sys

     __A     =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

