"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE__:Any			=	logging.getLogger(__name__)



def 					_lowerCamelCase(					):
       __a     				=  argparse.ArgumentParser(
           description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
       parser.add_argument(
           "--dataset_name" ,  type=__lowerCAmelCase ,  default="wikitext" ,  help="Name of the training. Explore datasets at: hf.co/datasets." ,  )
       parser.add_argument(
           "--dataset_config" ,  type=__lowerCAmelCase ,  default="wikitext-103-raw-v1" ,  help="Configuration name of the dataset."							)
       parser.add_argument(
           "--tokenizer_name_or_path" ,  type=__lowerCAmelCase ,  default="sayakpaul/unigram-tokenizer-wikitext" ,  help="Tokenizer identifier. Can be a local filepath or a Hub identifier." ,  )
       parser.add_argument(
           "--shard_size" ,  type=__lowerCAmelCase ,  default=1_0_0_0 ,  help="Number of entries to go in a single shard." ,  )
       parser.add_argument("--split" ,  type=__lowerCAmelCase ,  default="train" ,  choices=["train", "test", "validation"]							)
       parser.add_argument(
           "--limit" ,  default=__lowerCAmelCase ,  type=__lowerCAmelCase ,  help="Limit the number of shards (used for debugging)." ,  )
       parser.add_argument(
           "--max_length" ,  type=__lowerCAmelCase ,  default=5_1_2 ,  help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
           " sequence length that is a multiple of 8." ,  )
       parser.add_argument(
           "--output_dir" ,  default="tf-tpu" ,  type=__lowerCAmelCase ,  help="Output directory where the TFRecord shards will be saved. If the"
           " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
           " shards will be directly saved to a Google Cloud Storage bucket." ,  )

       __a     				=  parser.parse_args()
       return args



def 					_lowerCamelCase(					a							):
       def fn(a							):
              return tokenizer(examples["text"]							)

       return fn



def 					_lowerCamelCase(					a							):
       __a     				=  []
       for i in range(len(tokenized_data["input_ids"]							)							):
              __a     				=  {
                  "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
                  "attention_mask": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
              }
              __a     				=  tf.train.Features(feature=__lowerCAmelCase							)
              __a     				=  tf.train.Example(features=__lowerCAmelCase							)
              __a     				=  example.SerializeToString()
              records.append(__lowerCAmelCase							)
       return records



def 					_lowerCamelCase(					a							):
       __a     				=  datasets.load_dataset(args.dataset_name ,  args.dataset_config ,  split=args.split							)

       if args.limit is not None:
              __a     				=  min(len(__lowerCAmelCase							) ,  args.limit							)
              __a     				=  dataset.select(range(__lowerCAmelCase							)							)
              print(F"Limiting the dataset to {args.limit} entries."							)

       __a     				=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir							):
                     os.makedirs(args.output_dir							)
              __a     				=  os.path.join(args.output_dir ,  args.split							)
              if not os.path.exists(__lowerCAmelCase							):
                     os.makedirs(__lowerCAmelCase							)
       else:
              __a     				=  os.path.join(args.output_dir ,  args.split							)

       # Tokenize the whole dataset at once.
       __a     				=  tokenize_function(__lowerCAmelCase							)
       __a     				=  dataset.map(__lowerCAmelCase ,  batched=__lowerCAmelCase ,  num_proc=4 ,  remove_columns=["text"]							)

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(a							):
              # Concatenate all texts.
              __a     				=  {k: sum(examples[k] ,  []							) for k in examples.keys()}
              __a     				=  len(concatenated_examples[list(examples.keys()							)[0]]							)
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              __a     				=  (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              __a     				=  {
                  k: [t[i : i + args.max_length] for i in range(0 ,  __lowerCAmelCase ,  args.max_length							)]
                  for k, t in concatenated_examples.items()
              }
              return result

       __a     				=  dataset_tokenized.map(__lowerCAmelCase ,  batched=__lowerCAmelCase ,  batch_size=1_0_0_0 ,  num_proc=4							)

       __a     				=  0
       __a     				=  0
       for shard in range(0 ,  len(__lowerCAmelCase							) ,  args.shard_size							):
              __a     				=  grouped_dataset[shard : shard + args.shard_size]
              __a     				=  len(dataset_snapshot["input_ids"]							)
              __a     				=  os.path.join(__lowerCAmelCase ,  F"dataset-{shard_count}-{records_containing}.tfrecord"							)
              __a     				=  get_serialized_examples(__lowerCAmelCase							)

              with tf.io.TFRecordWriter(__lowerCAmelCase							) as out_file:
                     for i in range(len(__lowerCAmelCase							)							):
                            __a     				=  serialized_examples[i]
                            out_file.write(__lowerCAmelCase							)
                     print("Wrote file {} containing {} records".format(__lowerCAmelCase ,  __lowerCAmelCase							)							)

              shard_count += 1
              total_records += records_containing

       with open(F"split-{args.split}-records-count.txt" ,  "w"							) as f:
              print(F"Total {args.split} records: {total_records}" ,  file=__lowerCAmelCase							)


if __name__ == "__main__":
   SCREAMING_SNAKE_CASE__:str			=	parse_args()
   main(args)

"""simple docstring"""



import os



def 					_lowerCamelCase(					):
       with open(os.path.dirname(a							) + "/grid.txt"							) as f:
              __a     				=  []  # noqa: E741
              for _ in range(2_0							):
                     l.append([int(a							) for x in f.readline().split()]							)

              __a     				=  0

              # right
              for i in range(2_0							):
                     for j in range(1_7							):
                            __a     				=  l[i][j] * l[i][j + 1] * l[i][j + 2] * l[i][j + 3]
                            if temp > maximum:
                                   __a     				=  temp

        # down
              for i in range(1_7							):
                     for j in range(2_0							):
                            __a     				=  l[i][j] * l[i + 1][j] * l[i + 2][j] * l[i + 3][j]
                            if temp > maximum:
                                   __a     				=  temp

        # diagonal 1
              for i in range(1_7							):
                     for j in range(1_7							):
                            __a     				=  l[i][j] * l[i + 1][j + 1] * l[i + 2][j + 2] * l[i + 3][j + 3]
                            if temp > maximum:
                                   __a     				=  temp

        # diagonal 2
              for i in range(1_7							):
                     for j in range(3 ,  2_0							):
                            __a     				=  l[i][j] * l[i + 1][j - 1] * l[i + 2][j - 2] * l[i + 3][j - 3]
                            if temp > maximum:
                                   __a     				=  temp
              return maximum


if __name__ == "__main__":
   print(solution())
