'''simple docstring'''


import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
 from .testing_utils import (
     floats_tensor,
     load_hf_numpy,
     load_image,
     load_numpy,
     load_pt,
     nightly,
     parse_flag_from_env,
     print_tensor_test,
     require_torch_a,
     require_torch_gpu,
     skip_mps,
     slow,
     torch_all_close,
     torch_device,
 )
 from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


snake_case__     :			Tuple      			=  get_logger(__name__)


def 					_lowerCamelCase    (      lowerCamelCase_					:							List[str]   ):



     """simple docstring"""





     if version.parse(lowerCamelCase_   ) < version.parse(lowerCamelCase_   ):
          if "dev" in min_version:
               UpperCAmelCase_							:   Tuple   =      (
                   'This example requires a source install from HuggingFace diffusers (see '
                   '`https://huggingface.co/docs/diffusers/installation#install-from-source`),'
               )
          else:
               UpperCAmelCase_							:   List[Any]   =      F'''This example requires a minimum version of {min_version},'''
          error_message += F''' but the version found is {__version__}.\n'''
          raise ImportError(lowerCamelCase_   )







'''simple docstring'''


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


snake_case__     :			Optional[int]      			=  logging.getLogger(__name__)


def 					_lowerCamelCase    (      ):



     """simple docstring"""





     UpperCAmelCase_							:   List[Any]   =      argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'   )
     parser.add_argument(
         '--dataset_name'	,     type=lowerCamelCase_	,     default='wikitext'	,     help='Name of the training. Explore datasets at: hf.co/datasets.'	,     )
     parser.add_argument(
         '--dataset_config'	,     type=lowerCamelCase_	,     default='wikitext-103-raw-v1'	,     help='Configuration name of the dataset.'   )
     parser.add_argument(
         '--batch_size'	,     type=lowerCamelCase_	,     default=1000	,     help='Batch size during training.'	,     )
     parser.add_argument(
         '--vocab_size'	,     type=lowerCamelCase_	,     default=10048	,     help='Size of the desired vocabulary.'	,     )
     parser.add_argument(
         '--limit'	,     default=lowerCamelCase_	,     type=lowerCamelCase_	,     help='Limit the number of shards (used for debugging).'	,     )
     parser.add_argument(
         '--export_to_hub'	,     action='store_true'	,     )

     UpperCAmelCase_							:   str   =      parser.parse_args()
     return args


def 					_lowerCamelCase    (      lowerCamelCase_					:							Dict   ):



     """simple docstring"""





     UpperCAmelCase_							:   List[Any]   =      datasets.load_dataset(args.dataset_name	,     args.dataset_config	,     split='train'   )

     if args.limit is not None:
          UpperCAmelCase_							:   Union[str, Any]   =      min(len(lowerCamelCase_   )	,     args.limit   )
          UpperCAmelCase_							:   int   =      dataset.select(range(lowerCamelCase_   )   )
          logger.info(F'''Limiting the dataset to {args.limit} entries.'''   )

     def batch_iterator():
          for i in range(0	,     len(lowerCamelCase_   )	,     args.batch_size   ):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     UpperCAmelCase_							:   Optional[Any]   =      Tokenizer(Unigram()   )
     UpperCAmelCase_							:   Union[str, Any]   =      normalizers.Sequence([normalizers.Replace('``'	,     '"'   ), normalizers.Replace('\'\''	,     '"'   )]   )
     UpperCAmelCase_							:   Any   =      pre_tokenizers.Metaspace()

     # Prepare the trainer.
     UpperCAmelCase_							:   Tuple   =      UnigramTrainer(
         unk_token='<unk>'	,     special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']	,     vocab_size=args.vocab_size	,     )

     logger.info('Training the tokenizer.'   )
     tokenizer.train_from_iterator(batch_iterator()	,     trainer=lowerCamelCase_   )
     logger.info('Tokenizer training complete!'   )

     UpperCAmelCase_							:   List[str]   =      tokenizer.token_to_id('[CLS]'   )
     UpperCAmelCase_							:   Optional[int]   =      tokenizer.token_to_id('[SEP]'   )
     UpperCAmelCase_							:   Optional[Any]   =      processors.TemplateProcessing(
         single='[CLS]:0 $A:0 [SEP]:0'	,     pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'	,     special_tokens=[
             ('[CLS]', cls_token_id),
             ('[SEP]', sep_token_id),
         ]	,     )
     UpperCAmelCase_							:   Any   =      decoders.Metaspace()

     if args.export_to_hub:
          logger.info('Exporting the trained tokenzier to Hub.'   )
          UpperCAmelCase_							:   str   =      AlbertTokenizerFast(tokenizer_object=lowerCamelCase_   )
          new_tokenizer.push_to_hub('unigram-tokenizer-dataset'   )


if __name__ == "__main__":
 snake_case__     :			Optional[Any]      			=  parse_args()
 main(args)






