






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__a   :							Dict						= logging.getLogger(__name__)



def       _SCREAMING_SNAKE_CASE    (			)    ->   Any:





    """simple docstring"""

    __A       =						argparse.ArgumentParser(
        description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""     )
    parser.add_argument(
        """--dataset_name"""      ,					type=A__      ,					default="""wikitext"""      ,					help="""Name of the training. Explore datasets at: hf.co/datasets."""      ,					)
    parser.add_argument(
        """--dataset_config"""      ,					type=A__      ,					default="""wikitext-103-raw-v1"""      ,					help="""Configuration name of the dataset."""     )
    parser.add_argument(
        """--tokenizer_name_or_path"""      ,					type=A__      ,					default="""sayakpaul/unigram-tokenizer-wikitext"""      ,					help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""      ,					)
    parser.add_argument(
        """--shard_size"""      ,					type=A__      ,					default=1_0_0_0      ,					help="""Number of entries to go in a single shard."""      ,					)
    parser.add_argument("""--split"""      ,					type=A__      ,					default="""train"""      ,					choices=["""train""", """test""", """validation"""]     )
    parser.add_argument(
        """--limit"""      ,					default=A__      ,					type=A__      ,					help="""Limit the number of shards (used for debugging)."""      ,					)
    parser.add_argument(
        """--max_length"""      ,					type=A__      ,					default=5_1_2      ,					help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
        """ sequence length that is a multiple of 8."""      ,					)
    parser.add_argument(
        """--output_dir"""      ,					default="""tf-tpu"""      ,					type=A__      ,					help="""Output directory where the TFRecord shards will be saved. If the"""
        """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
        """ shards will be directly saved to a Google Cloud Storage bucket."""      ,					)

    __A       =						parser.parse_args()
    return args



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Tuple     )    ->   str:





    """simple docstring"""

    def fn(__lowercase :      Dict     ):
        return tokenizer(examples["""text"""]     )

    return fn



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Union[str, Any]     )    ->   Union[str, Any]:





    """simple docstring"""

    __A       =						[]
    for i in range(len(tokenized_data["""input_ids"""]     )     ):
        __A       =						{
            """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]     )     ),
            """attention_mask""": tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]     )     ),
        }
        __A       =						tf.train.Features(feature=A__     )
        __A       =						tf.train.Example(features=A__     )
        __A       =						example.SerializeToString()
        records.append(A__     )
    return records



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Tuple     )    ->   Union[str, Any]:





    """simple docstring"""

    __A       =						datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split     )

    if args.limit is not None:
        __A       =						min(len(A__     )      ,					args.limit     )
        __A       =						dataset.select(range(A__     )     )
        print(f"Limiting the dataset to {args.limit} entries."     )

    __A       =						AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir     ):
            os.makedirs(args.output_dir     )
        __A       =						os.path.join(args.output_dir      ,					args.split     )
        if not os.path.exists(A__     ):
            os.makedirs(A__     )
    else:
        __A       =						os.path.join(args.output_dir      ,					args.split     )

    # Tokenize the whole dataset at once.
    __A       =						tokenize_function(A__     )
    __A       =						dataset.map(A__      ,					batched=A__      ,					num_proc=4      ,					remove_columns=["""text"""]     )

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(__lowercase :      Union[str, Any]     ):
        # Concatenate all texts.
        __A       =						{k: sum(examples[k]      ,					[]     ) for k in examples.keys()}
        __A       =						len(concatenated_examples[list(examples.keys()     )[0]]     )
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        __A       =						(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        __A       =						{
            k: [t[i : i + args.max_length] for i in range(0      ,					A__      ,					args.max_length     )]
            for k, t in concatenated_examples.items()
        }
        return result

    __A       =						dataset_tokenized.map(A__      ,					batched=A__      ,					batch_size=1_0_0_0      ,					num_proc=4     )

    __A       =						0
    __A       =						0
    for shard in range(0      ,					len(A__     )      ,					args.shard_size     ):
        __A       =						grouped_dataset[shard : shard + args.shard_size]
        __A       =						len(dataset_snapshot["""input_ids"""]     )
        __A       =						os.path.join(A__      ,					f"dataset-{shard_count}-{records_containing}.tfrecord"     )
        __A       =						get_serialized_examples(A__     )

        with tf.io.TFRecordWriter(A__     ) as out_file:
            for i in range(len(A__     )     ):
                __A       =						serialized_examples[i]
                out_file.write(A__     )
            print("""Wrote file {} containing {} records""".format(A__      ,					A__     )     )

        shard_count += 1
        total_records += records_containing

    with open(f"split-{args.split}-records-count.txt"      ,					"""w"""     ) as f:
        print(f"Total {args.split} records: {total_records}"      ,					file=A__     )


if __name__ == "__main__":
  __a   :							str						= parse_args()
  main(args)








# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
      import torch


snake_case__				:	int					       =					logging.get_logger(__name__)





@dataclass
class   SCREAMING_SNAKE_CASE_			:




       '''simple docstring'''



       def __init__(						self				:				int		,				__a				:				Optional[Any]=False		,				__a				:				Tuple=False		,				__a				:				Tuple=6.0		,				__a				:				str=None		,				__a				:				List[str]=False		,				__a				:				str=False		,				__a				:				Optional[Any]=None		,				__a				:				Dict="fp4"		,				__a				:				Optional[int]=False		,				**__a				:				Dict		,				)					->str:
          lowerCamelCase_						:   str		= load_in_abit
          lowerCamelCase_						:   Union[str, Any]		= load_in_abit
          lowerCamelCase_						:   Tuple		= llm_inta_threshold
          lowerCamelCase_						:   int		= llm_inta_skip_modules
          lowerCamelCase_						:   List[str]		= llm_inta_enable_fpaa_cpu_offload
          lowerCamelCase_						:   Optional[Any]		= llm_inta_has_fpaa_weight
          lowerCamelCase_						:   Dict		= bnb_abit_quant_type
          lowerCamelCase_						:   int		= bnb_abit_use_double_quant

          if bnb_abit_compute_dtype is None:
             lowerCamelCase_						:   int		= torch.floataa
          elif isinstance(__a		,				__a							):
             lowerCamelCase_						:   Tuple		= getattr(__a		,				__a							)
          elif isinstance(__a		,				torch.dtype							):
             lowerCamelCase_						:   List[str]		= bnb_abit_compute_dtype
          else:
             raise ValueError("""bnb_4bit_compute_dtype must be a string or a torch.dtype"""							)

          self.post_init()



       def 				_lowerCAmelCase							(						self				:				Any							)					->Union[str, Any]:
          if not isinstance(self.llm_inta_threshold		,				__a							):
             raise ValueError("""llm_int8_threshold must be a float"""							)

          if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules		,				__a							):
             raise ValueError("""llm_int8_skip_modules must be a list of strings"""							)
          if not isinstance(self.llm_inta_enable_fpaa_cpu_offload		,				__a							):
             raise ValueError("""llm_int8_enable_fp32_cpu_offload must be a boolean"""							)

          if not isinstance(self.llm_inta_has_fpaa_weight		,				__a							):
             raise ValueError("""llm_int8_has_fp16_weight must be a boolean"""							)

          if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype		,				torch.dtype							):
             raise ValueError("""bnb_4bit_compute_dtype must be torch.dtype"""							)

          if not isinstance(self.bnb_abit_quant_type		,				__a							):
             raise ValueError("""bnb_4bit_quant_type must be a string"""							)

          if not isinstance(self.bnb_abit_use_double_quant		,				__a							):
             raise ValueError("""bnb_4bit_use_double_quant must be a boolean"""							)

          if self.load_in_abit and not version.parse(importlib.metadata.version("""bitsandbytes"""							)							) >= version.parse(
              """0.39.0"""							):
             raise ValueError(
                 """4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"""							)



       def 				_lowerCAmelCase							(						self				:				Optional[int]							)					->int:
          return self.load_in_abit or self.load_in_abit



       def 				_lowerCAmelCase							(						self				:				List[str]							)					->List[Any]:
          if self.load_in_abit:
             return "llm_int8"
          elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
             return "fp4"
          elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
             return "nf4"
          else:
             return None



       @classmethod
       def 				_lowerCAmelCase							(						cls				:				int		,				__a				:				Dict		,				__a				:				str		,				**__a				:				Any							)					->Optional[Any]:
          lowerCamelCase_						:   Optional[Any]		= cls(**__a							)

          lowerCamelCase_						:   List[Any]		= []
          for key, value in kwargs.items():
             if hasattr(__a		,				__a							):
                setattr(__a		,				__a		,				__a							)
                to_remove.append(__a							)
          for key in to_remove:
             kwargs.pop(__a		,				__a							)

          if return_unused_kwargs:
             return config, kwargs
          else:
             return config



       def 				_lowerCAmelCase							(						self				:				Optional[Any]		,				__a				:				Union[str, os.PathLike]							)					->Any:
          with open(__a		,				"""w"""		,				encoding="""utf-8"""							) as writer:
             lowerCamelCase_						:   List[Any]		= self.to_dict()
             lowerCamelCase_						:   Dict		= json.dumps(__a		,				indent=2		,				sort_keys=__a							) + """\n"""

             writer.write(__a							)



       def 				_lowerCAmelCase							(						self				:				Tuple							)					->Dict[str, Any]:
          lowerCamelCase_						:   Dict		= copy.deepcopy(self.__dict__							)
          lowerCamelCase_						:   Tuple		= str(output["""bnb_4bit_compute_dtype"""]							).split("""."""							)[1]

          return output



       def __repr__(						self				:				List[str]							)					->Any:
          return F'''{self.__class__.__name__} {self.to_json_string()}'''



       def 				_lowerCAmelCase							(						self				:				Optional[int]		,				__a				:				bool = True							)					->str:
          if use_diff is True:
             lowerCamelCase_						:   Dict		= self.to_diff_dict()
          else:
             lowerCamelCase_						:   Any		= self.to_dict()
          return json.dumps(__a		,				indent=2		,				sort_keys=__a							) + "\n"




       def 				_lowerCAmelCase							(						self				:				Optional[Any]							)					->Dict[str, Any]:
          lowerCamelCase_						:   Any		= self.to_dict()

          # get the default config dict
          lowerCamelCase_						:   List[str]		= BitsAndBytesConfig().to_dict()

          lowerCamelCase_						:   str		= {}

          # only serialize values that differ from the default config
          for key, value in config_dict.items():
             if value != default_config_dict[key]:
                lowerCamelCase_						:   List[Any]		= value

          return serializable_config_dict





