



'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__:     List[Any]										=    logging.getLogger(__name__)


def 				snake_case_  (      )   ->     int:
							UpperCAmelCase				: int											=	argparse.ArgumentParser(
							    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''						)
							parser.add_argument(
							    '''--dataset_name'''     ,   type=_lowerCAmelCase     ,   default='''wikitext'''     ,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''     ,   )
							parser.add_argument(
							    '''--dataset_config'''     ,   type=_lowerCAmelCase     ,   default='''wikitext-103-raw-v1'''     ,   help='''Configuration name of the dataset.'''						)
							parser.add_argument(
							    '''--tokenizer_name_or_path'''     ,   type=_lowerCAmelCase     ,   default='''sayakpaul/unigram-tokenizer-wikitext'''     ,   help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''     ,   )
							parser.add_argument(
							    '''--shard_size'''     ,   type=_lowerCAmelCase     ,   default=1000     ,   help='''Number of entries to go in a single shard.'''     ,   )
							parser.add_argument('''--split'''     ,   type=_lowerCAmelCase     ,   default='''train'''     ,   choices=['''train''', '''test''', '''validation''']						)
							parser.add_argument(
							    '''--limit'''     ,   default=_lowerCAmelCase     ,   type=_lowerCAmelCase     ,   help='''Limit the number of shards (used for debugging).'''     ,   )
							parser.add_argument(
							    '''--max_length'''     ,   type=_lowerCAmelCase     ,   default=512     ,   help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
							    ''' sequence length that is a multiple of 8.'''     ,   )
							parser.add_argument(
							    '''--output_dir'''     ,   default='''tf-tpu'''     ,   type=_lowerCAmelCase     ,   help='''Output directory where the TFRecord shards will be saved. If the'''
							    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
							    ''' shards will be directly saved to a Google Cloud Storage bucket.'''     ,   )

							UpperCAmelCase				: List[str]											=	parser.parse_args()
							return args


def 				snake_case_  (      _lowerCAmelCase						:		Union[str, Any]						)   ->     Optional[int]:
							def fn(_lowerCAmelCase						:		Tuple						):
														return tokenizer(examples['''text''']						)

							return fn


def 				snake_case_  (      _lowerCAmelCase						:		int						)   ->     Dict:
							UpperCAmelCase				: Optional[Any]											=	[]
							for i in range(len(tokenized_data['''input_ids''']						)						):
														UpperCAmelCase				: Optional[Any]											=	{
														    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]						)						),
														    '''attention_mask''': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]						)						),
														}
														UpperCAmelCase				: Optional[Any]											=	tf.train.Features(feature=_lowerCAmelCase						)
														UpperCAmelCase				: Union[str, Any]											=	tf.train.Example(features=_lowerCAmelCase						)
														UpperCAmelCase				: Any											=	example.SerializeToString()
														records.append(_lowerCAmelCase						)
							return records


def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     List[str]:
							UpperCAmelCase				: List[str]											=	datasets.load_dataset(args.dataset_name     ,   args.dataset_config     ,   split=args.split						)

							if args.limit is not None:
														UpperCAmelCase				: List[Any]											=	min(len(_lowerCAmelCase						)     ,   args.limit						)
														UpperCAmelCase				: Dict											=	dataset.select(range(_lowerCAmelCase						)						)
														print(f"""Limiting the dataset to {args.limit} entries."""						)

							UpperCAmelCase				: List[Any]											=	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir						):
																					os.makedirs(args.output_dir						)
														UpperCAmelCase				: List[Any]											=	os.path.join(args.output_dir     ,   args.split						)
														if not os.path.exists(_lowerCAmelCase						):
																					os.makedirs(_lowerCAmelCase						)
							else:
														UpperCAmelCase				: Tuple											=	os.path.join(args.output_dir     ,   args.split						)

							# Tokenize the whole dataset at once.
							UpperCAmelCase				: Optional[Any]											=	tokenize_function(_lowerCAmelCase						)
							UpperCAmelCase				: int											=	dataset.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   num_proc=4     ,   remove_columns=['''text''']						)

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(_lowerCAmelCase						:		List[str]						):
														# Concatenate all texts.
														UpperCAmelCase				: Optional[Any]											=	{k: sum(examples[k]     ,   []						) for k in examples.keys()}
														UpperCAmelCase				: Optional[Any]											=	len(concatenated_examples[list(examples.keys()						)[0]]						)
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														UpperCAmelCase				: Any											=	(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														UpperCAmelCase				: List[str]											=	{
														    k: [t[i : i + args.max_length] for i in range(0     ,   _lowerCAmelCase     ,   args.max_length						)]
														    for k, t in concatenated_examples.items()
														}
														return result

							UpperCAmelCase				: str											=	dataset_tokenized.map(_lowerCAmelCase     ,   batched=_lowerCAmelCase     ,   batch_size=1000     ,   num_proc=4						)

							UpperCAmelCase				: Union[str, Any]											=	0
							UpperCAmelCase				: Dict											=	0
							for shard in range(0     ,   len(_lowerCAmelCase						)     ,   args.shard_size						):
														UpperCAmelCase				: Union[str, Any]											=	grouped_dataset[shard : shard + args.shard_size]
														UpperCAmelCase				: Optional[int]											=	len(dataset_snapshot['''input_ids''']						)
														UpperCAmelCase				: Dict											=	os.path.join(_lowerCAmelCase     ,   f"""dataset-{shard_count}-{records_containing}.tfrecord"""						)
														UpperCAmelCase				: List[Any]											=	get_serialized_examples(_lowerCAmelCase						)

														with tf.io.TFRecordWriter(_lowerCAmelCase						) as out_file:
																					for i in range(len(_lowerCAmelCase						)						):
																												UpperCAmelCase				: Any											=	serialized_examples[i]
																												out_file.write(_lowerCAmelCase						)
																					print('''Wrote file {} containing {} records'''.format(_lowerCAmelCase     ,   _lowerCAmelCase						)						)

														shard_count += 1
														total_records += records_containing

							with open(f"""split-{args.split}-records-count.txt"""     ,   '''w'''						) as f:
														print(f"""Total {args.split} records: {total_records}"""     ,   file=_lowerCAmelCase						)


if __name__ == "__main__":
						UpperCamelCase__:     List[Any]										=    parse_args()
						main(args)










'''simple docstring'''






from __future__ import annotations

from collections.abc import Generator

import requests
from bsa import BeautifulSoup

UpperCamelCase__:     Union[str, Any]										=    "https://www.indeed.co.in/jobs?q=mobile+app+development&l="


def 				snake_case_  (      _lowerCAmelCase						:		str = "mumbai"						)   ->     Generator[tuple[str, str], None, None]:
							UpperCAmelCase				: Optional[int]											=	BeautifulSoup(requests.get(url + location						).content     ,   '''html.parser'''						)
							# This attribute finds out all the specifics listed in a job
							for job in soup.find_all('''div'''     ,   attrs={'''data-tn-component''': '''organicJob'''}						):
														UpperCAmelCase				: Optional[Any]											=	job.find('''a'''     ,   attrs={'''data-tn-element''': '''jobTitle'''}						).text.strip()
														UpperCAmelCase				: int											=	job.find('''span'''     ,   {'''class''': '''company'''}						).text.strip()
														yield job_title, company_name


if __name__ == "__main__":
						for i, job in enumerate(fetch_jobs("Bangalore"), 1):
												print(F"Job {i:>2} is {job[0]} at {job[1]}")





