

"""simple docstring"""
import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_A							=  logging.get_logger(__name__)


_A							=  {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}




def 					a__						(						lowerCAmelCase			,	lowerCAmelCase			,	lowerCAmelCase			,	lowerCAmelCase    )	-> int:
      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()    )}."""    )

      if tokenizer_name is None:
            UpperCAmelCase__			:							Tuple							=    TOKENIZER_CLASSES
      else:
            UpperCAmelCase__			:							Optional[int]							=    {tokenizer_name: getattr(lowerCAmelCase			,	tokenizer_name + """Fast"""    )}

      logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""    )

      for tokenizer_name in tokenizer_names:
            UpperCAmelCase__			:							Optional[Any]							=    TOKENIZER_CLASSES[tokenizer_name]

            UpperCAmelCase__			:							Optional[Any]							=    True
            if checkpoint_name is None:
                  UpperCAmelCase__			:							Tuple							=    list(tokenizer_class.max_model_input_sizes.keys()    )
            else:
                  UpperCAmelCase__			:							Optional[int]							=    [checkpoint_name]

            logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""    )

            for checkpoint in checkpoint_names:
                  logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""    )

                  # Load tokenizer
                  UpperCAmelCase__			:							Union[str, Any]							=    tokenizer_class.from_pretrained(lowerCAmelCase			,	force_download=lowerCAmelCase    )

                  # Save fast tokenizer
                  logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""    )

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        UpperCAmelCase__					,		UpperCAmelCase__			:							Dict							=    checkpoint.split("""/"""    )
                        UpperCAmelCase__			:							Dict							=    os.path.join(lowerCAmelCase			,	lowerCAmelCase    )
                  elif add_prefix:
                        UpperCAmelCase__			:							Optional[Any]							=    checkpoint
                        UpperCAmelCase__			:							Any							=    dump_path
                  else:
                        UpperCAmelCase__			:							int							=    None
                        UpperCAmelCase__			:							Tuple							=    dump_path

                  logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""    )

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()    )[0]:
                        UpperCAmelCase__			:							Optional[int]							=    list(tokenizer.pretrained_vocab_files_map.values()    )[0][checkpoint]
                        UpperCAmelCase__			:							Optional[int]							=    file_path.split(lowerCAmelCase    )[-1][0]
                        if next_char == "/":
                              UpperCAmelCase__			:							Union[str, Any]							=    os.path.join(lowerCAmelCase			,	lowerCAmelCase    )
                              UpperCAmelCase__			:							Any							=    None

                        logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""    )

                  UpperCAmelCase__			:							List[str]							=    tokenizer.save_pretrained(
                      lowerCAmelCase			,	legacy_format=lowerCAmelCase			,	filename_prefix=lowerCAmelCase    )
                  logger.info(F"""=> File names {file_names}"""    )

                  for file_name in file_names:
                        if not file_name.endswith("""tokenizer.json"""    ):
                              os.remove(lowerCAmelCase    )
                              logger.info(F"""=> removing {file_name}"""    )


if __name__ == "__main__":
    _A							=  argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
    )
    parser.add_argument(
        """--tokenizer_name""",
        default=None,
        type=str,
        help=(
            f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
            """download and convert all the checkpoints from AWS."""
        ),
    )
    parser.add_argument(
        """--checkpoint_name""",
        default=None,
        type=str,
        help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
    )
    parser.add_argument(
        """--force_download""",
        action="""store_true""",
        help="""Re-download checkpoints.""",
    )
    _A							=  parser.parse_args()

    convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




"""simple docstring"""
# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A							=  {
    """configuration_cpmant""": ["""CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CpmAntConfig"""],
    """tokenization_cpmant""": ["""CpmAntTokenizer"""],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A							=  [
        """CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """CpmAntForCausalLM""",
        """CpmAntModel""",
        """CpmAntPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
    from .tokenization_cpmant import CpmAntTokenizer

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_cpmant import (
            CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
            CpmAntForCausalLM,
            CpmAntModel,
            CpmAntPreTrainedModel,
        )


else:
    import sys

    _A							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

