import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase_			    =							logging.getLogger(__name__)




def       lowerCAmelCase_     (		)    ->				List[str]:







 '''simple docstring'''
 _UpperCamelCase:     Optional[int]							= argparse.ArgumentParser(
     description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''		)
 parser.add_argument(
     '''--dataset_name'''  ,					type=lowercase  ,					default='''wikitext'''  ,					help='''Name of the training. Explore datasets at: hf.co/datasets.'''  ,					)
 parser.add_argument(
     '''--dataset_config'''  ,					type=lowercase  ,					default='''wikitext-103-raw-v1'''  ,					help='''Configuration name of the dataset.'''		)
 parser.add_argument(
     '''--tokenizer_name_or_path'''  ,					type=lowercase  ,					default='''sayakpaul/unigram-tokenizer-wikitext'''  ,					help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''  ,					)
 parser.add_argument(
     '''--shard_size'''  ,					type=lowercase  ,					default=1_000  ,					help='''Number of entries to go in a single shard.'''  ,					)
 parser.add_argument('''--split'''  ,					type=lowercase  ,					default='''train'''  ,					choices=['''train''', '''test''', '''validation''']		)
 parser.add_argument(
     '''--limit'''  ,					default=lowercase  ,					type=lowercase  ,					help='''Limit the number of shards (used for debugging).'''  ,					)
 parser.add_argument(
     '''--max_length'''  ,					type=lowercase  ,					default=512  ,					help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
     ''' sequence length that is a multiple of 8.'''  ,					)
 parser.add_argument(
     '''--output_dir'''  ,					default='''tf-tpu'''  ,					type=lowercase  ,					help='''Output directory where the TFRecord shards will be saved. If the'''
     ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
     ''' shards will be directly saved to a Google Cloud Storage bucket.'''  ,					)

 _UpperCamelCase:     Union[str, Any]							= parser.parse_args()
 return args




def       lowerCAmelCase_     (		lowercase:							Dict		)    ->				int:







 '''simple docstring'''
 def fn(lowercase:							Tuple		):
  return tokenizer(examples['''text''']		)

 return fn




def       lowerCAmelCase_     (		lowercase:							List[str]		)    ->				List[str]:







 '''simple docstring'''
 _UpperCamelCase:     Optional[int]							= []
 for i in range(len(tokenized_data['''input_ids''']		)		):
  _UpperCamelCase:     Dict							= {
      '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]		)		),
      '''attention_mask''': tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]		)		),
  }
  _UpperCamelCase:     List[str]							= tf.train.Features(feature=lowercase		)
  _UpperCamelCase:     List[str]							= tf.train.Example(features=lowercase		)
  _UpperCamelCase:     str							= example.SerializeToString()
  records.append(lowercase		)
 return records




def       lowerCAmelCase_     (		lowercase:							Any		)    ->				Optional[int]:







 '''simple docstring'''
 _UpperCamelCase:     Tuple							= datasets.load_dataset(args.dataset_name  ,					args.dataset_config  ,					split=args.split		)

 if args.limit is not None:
  _UpperCamelCase:     int							= min(len(lowercase		)  ,					args.limit		)
  _UpperCamelCase:     Dict							= dataset.select(range(lowercase		)		)
  print(F"""Limiting the dataset to {args.limit} entries."""		)

 _UpperCamelCase:     List[str]							= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir		):
   os.makedirs(args.output_dir		)
  _UpperCamelCase:     List[str]							= os.path.join(args.output_dir  ,					args.split		)
  if not os.path.exists(lowercase		):
   os.makedirs(lowercase		)
 else:
  _UpperCamelCase:     List[str]							= os.path.join(args.output_dir  ,					args.split		)

 # Tokenize the whole dataset at once.
 _UpperCamelCase:     int							= tokenize_function(lowercase		)
 _UpperCamelCase:     Optional[Any]							= dataset.map(lowercase  ,					batched=lowercase  ,					num_proc=4  ,					remove_columns=['''text''']		)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(lowercase:							Any		):
  # Concatenate all texts.
  _UpperCamelCase:     List[Any]							= {k: sum(examples[k]  ,					[]		) for k in examples.keys()}
  _UpperCamelCase:     str							= len(concatenated_examples[list(examples.keys()		)[0]]		)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  _UpperCamelCase:     int							= (total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  _UpperCamelCase:     List[Any]							= {
      k: [t[i : i + args.max_length] for i in range(0  ,					lowercase  ,					args.max_length		)]
      for k, t in concatenated_examples.items()
  }
  return result

 _UpperCamelCase:     Union[str, Any]							= dataset_tokenized.map(lowercase  ,					batched=lowercase  ,					batch_size=1_000  ,					num_proc=4		)

 _UpperCamelCase:     Any							= 0
 _UpperCamelCase:     str							= 0
 for shard in range(0  ,					len(lowercase		)  ,					args.shard_size		):
  _UpperCamelCase:     str							= grouped_dataset[shard : shard + args.shard_size]
  _UpperCamelCase:     Any							= len(dataset_snapshot['''input_ids''']		)
  _UpperCamelCase:     Tuple							= os.path.join(lowercase  ,					F"""dataset-{shard_count}-{records_containing}.tfrecord"""		)
  _UpperCamelCase:     str							= get_serialized_examples(lowercase		)

  with tf.io.TFRecordWriter(lowercase		) as out_file:
   for i in range(len(lowercase		)		):
    _UpperCamelCase:     Optional[Any]							= serialized_examples[i]
    out_file.write(lowercase		)
   print('''Wrote file {} containing {} records'''.format(lowercase  ,					lowercase		)		)

  shard_count += 1
  total_records += records_containing

 with open(F"""split-{args.split}-records-count.txt"""  ,					'''w'''		) as f:
  print(F"""Total {args.split} records: {total_records}"""  ,					file=lowercase		)


if __name__ == "__main__":
  UpperCAmelCase_			    =							parse_args()
  main(args)
from __future__ import annotations

UpperCAmelCase_			    =							1.6_0_2_1E-1_9  # units = C




def       lowerCAmelCase_     (		lowercase:							float  ,					lowercase:							float  ,					lowercase:							float  ,					)    ->				tuple[str, float]:







 '''simple docstring'''
 if (conductivity, electron_conc, mobility).count(0		) != 1:
  raise ValueError('''You cannot supply more or less than 2 values'''		)
 elif conductivity < 0:
  raise ValueError('''Conductivity cannot be negative'''		)
 elif electron_conc < 0:
  raise ValueError('''Electron concentration cannot be negative'''		)
 elif mobility < 0:
  raise ValueError('''mobility cannot be negative'''		)
 elif conductivity == 0:
  return (
      "conductivity",
      mobility * electron_conc * ELECTRON_CHARGE,
  )
 elif electron_conc == 0:
  return (
      "electron_conc",
      conductivity / (mobility * ELECTRON_CHARGE),
  )
 else:
  return (
      "mobility",
      conductivity / (electron_conc * ELECTRON_CHARGE),
  )


if __name__ == "__main__":
  import doctest

  doctest.testmod()