



"""simple docstring"""






import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
     import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				{'''vocab_file''': '''vocab.txt'''}

snake_case   					=				{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

snake_case   					=				{
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}




def 			snake_case		(		lowerCAmelCase_	)						->				int:
  _snake_case   	=      collections.OrderedDict()
  with open(lowerCAmelCase_  ,						'''r'''  ,						encoding='''utf-8'''	) as reader:
    _snake_case   	=      reader.readlines()
  for index, token in enumerate(lowerCAmelCase_	):
    _snake_case   	=      token.rstrip('''\n'''	)
    _snake_case   	=      index
  return vocab



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	Union[str, Any]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	List[str]="<unk>"							,       __lowerCamelCase    :	Tuple=2_0_0				):



     """simple docstring"""
     _snake_case   	=      vocab
     _snake_case   	=      unk_token
     _snake_case   	=      max_input_chars_per_word





   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      list(__lowerCamelCase				)
     if len(__lowerCamelCase				) > self.max_input_chars_per_word:
       return [self.unk_token]

     _snake_case   	=      0
     _snake_case   	=      []
     while start < len(__lowerCamelCase				):
       _snake_case   	=      len(__lowerCamelCase				)
       _snake_case   	=      None
       while start < end:
         _snake_case   	=      ''''''.join(chars[start:end]				)
         if substr in self.vocab:
           _snake_case   	=      substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token				)
         start += 1
       else:
         sub_tokens.append(__lowerCamelCase				)
         _snake_case   	=      end

     return sub_tokens



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	List[str]           =  VOCAB_FILES_NAMES
   A__		:	str           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	str           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Union[str, Any]           =  ['''input_ids''', '''attention_mask''']
   A__		:	Optional[int]           =  False



   def __init__(       self    :	Dict							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	str="<d>"							,       __lowerCamelCase    :	Tuple="</d>"							,       __lowerCamelCase    :	Tuple="<s>"							,       __lowerCamelCase    :	int="</s>"							,       __lowerCamelCase    :	List[str]="<pad>"							,       __lowerCamelCase    :	int="<unk>"							,       __lowerCamelCase    :	int="</n>"							,       __lowerCamelCase    :	Tuple="</_>"							,       __lowerCamelCase    :	Optional[Any]="left"							,       **__lowerCamelCase    :	str							,       ):



     """simple docstring"""
     requires_backends(self							,       ['''jieba''']				)
     super().__init__(
         bod_token=__lowerCamelCase							,       eod_token=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       line_token=__lowerCamelCase							,       space_token=__lowerCamelCase							,       padding_side=__lowerCamelCase							,       **__lowerCamelCase							,       )
     _snake_case   	=      bod_token
     _snake_case   	=      eod_token
     _snake_case   	=      load_vocab(__lowerCamelCase				)
     _snake_case   	=      self.encoder[space_token]
     _snake_case   	=      self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     _snake_case   	=      {v: k for k, v in self.encoder.items()}

     _snake_case   	=      WordpieceTokenizer(vocab=self.encoder							,       unk_token=self.unk_token				)



   @property
   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     return self.encoder[self.bod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return self.encoder[self.eod_token]



   @property
   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     return self.encoder["\n"]



   @property
   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     return len(self.encoder				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     return dict(self.encoder							,       **self.added_tokens_encoder				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      []
     for x in jieba.cut(__lowerCamelCase							,       cut_all=__lowerCamelCase				):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowerCamelCase				)				)
     return output_tokens



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Union[str, Any]							,       **__lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      [i for i in token_ids if i >= 0]
     _snake_case   	=      [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(__lowerCamelCase							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     return token in self.encoder



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     return "".join(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.encoder.get(__lowerCamelCase							,       self.encoder.get(self.unk_token				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.decoder.get(__lowerCamelCase							,       self.unk_token				)



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if os.path.isdir(__lowerCamelCase				):
       _snake_case   	=      os.path.join(
           __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
     else:
       _snake_case   	=      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
     _snake_case   	=      0
     if " " in self.encoder:
       _snake_case   	=      self.encoder[''' ''']
       del self.encoder[" "]
     if "\n" in self.encoder:
       _snake_case   	=      self.encoder['''\n''']
       del self.encoder["\n"]
     _snake_case   	=      collections.OrderedDict(sorted(self.encoder.items()							,       key=lambda __lowerCamelCase				: x[1]				)				)
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               ''' Please check that the vocabulary is not corrupted!'''				)
           _snake_case   	=      token_index
         writer.write(token + '''\n'''				)
         index += 1
     return (vocab_file,)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	List[int] = None				):



     """simple docstring"""
     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is not None:
       return [1] + ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				))
     return [1] + ([0] * len(__lowerCamelCase				))








import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


lowercase			:				Dict   =     datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
		import pyspark







@dataclass
class 					__lowercase  (						datasets.BuilderConfig				):





							"""simple docstring"""

							UpperCAmelCase_						:		Optional[datasets.Features]								=	None






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_     ,				):
							import pyspark

							def generate_fn():
														A      :					Dict      		=					df.select('''*'''     ,				pyspark.sql.functions.spark_partition_id().alias('''part_id'''		)		)
														for partition_id in partition_order:
																					A      :					Optional[Any]      		=					df_with_partition_id.select('''*'''		).where(F'part_id = {partition_id}'		).drop('''part_id'''		)
																					A      :					Dict      		=					partition_df.collect()
																					A      :					List[str]      		=					0
																					for row in rows:
																												yield F'{partition_id}_{row_id}', row.asDict()
																												row_id += 1

							return generate_fn







class 					__lowercase  (						_BaseExamplesIterable				):





							"""simple docstring"""

							def __init__(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None    ,						)		->       List[str]:
														A      :					List[Any]      		=					df
														A      :					List[str]      		=					partition_order or range(self.df.rdd.getNumPartitions()						)
														A      :					Optional[int]      		=					_generate_iterable_examples(self.df    ,						self.partition_order						)




							def __iter__(					self						)		->       str:
														yield from self.generate_examples_fn()




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       "SparkExamplesIterable":
														A      :					Tuple      		=					list(range(self.df.rdd.getNumPartitions()						)						)
														generator.shuffle(__UpperCAmelCase						)
														return SparkExamplesIterable(self.df    ,						partition_order=__UpperCAmelCase						)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase						)		->       "SparkExamplesIterable":
														A      :					Tuple      		=					self.split_shard_indices_by_worker(__UpperCAmelCase    ,						__UpperCAmelCase						)
														return SparkExamplesIterable(self.df    ,						partition_order=__UpperCAmelCase						)






							@property
							def       snake_case			(					self						)		->       int:
														return len(self.partition_order						)







class 					__lowercase  (						datasets.DatasetBuilder				):





							"""simple docstring"""

							UpperCAmelCase_						:		List[str]								=	SparkConfig




							def __init__(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None    ,						__UpperCAmelCase = None    ,						**__UpperCAmelCase    ,						)		->       Optional[Any]:
														import pyspark

														A      :					Tuple      		=					pyspark.sql.SparkSession.builder.getOrCreate()
														A      :					List[Any]      		=					df
														A      :					List[Any]      		=					working_dir

														super().__init__(
														    cache_dir=__UpperCAmelCase    ,						config_name=str(self.df.semanticHash()						)    ,						**__UpperCAmelCase    ,						)




							def       snake_case			(					self						)		->       Optional[Any]:
														# Returns the path of the created file.
														def create_cache_and_write_probe(__UpperCAmelCase						):
																					# makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
																					# already exist.
																					os.makedirs(self._cache_dir    ,						exist_ok=__UpperCAmelCase						)
																					A      :					Optional[Any]      		=					os.path.join(self._cache_dir    ,						'''fs_test''' + uuid.uuida().hex						)
																					# Opening the file in append mode will create a new file unless it already exists, in which case it will not
																					# change the file contents.
																					open(__UpperCAmelCase    ,						'''a'''						)
																					return [probe_file]

														if self._spark.conf.get('''spark.master'''    ,						''''''						).startswith('''local'''						):
																					return

														# If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
														# accessible to the driver.
														# TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
														if self._cache_dir:
																					A      :					Optional[int]      		=					(
																					    self._spark.sparkContext.parallelize(range(1						)    ,						1						).mapPartitions(__UpperCAmelCase						).collect()
																					)
																					if os.path.isfile(probe[0]						):
																												return

														raise ValueError(
														    '''When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir'''						)




							def       snake_case			(					self						)		->       List[Any]:
														return datasets.DatasetInfo(features=self.config.features						)




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       str:
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN						)]




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       List[Any]:
														import pyspark

														def get_arrow_batch_size(__UpperCAmelCase						):
																					for batch in it:
																												yield pa.RecordBatch.from_pydict({'''batch_bytes''': [batch.nbytes]}						)

														A      :					Any      		=					self.df.count()
														A      :					Tuple      		=					df_num_rows if df_num_rows <= 1_00 else 1_00
														# Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
														A      :					Dict      		=					(
														    self.df.limit(__UpperCAmelCase						)
														    .repartition(1						)
														    .mapInArrow(__UpperCAmelCase    ,						'''batch_bytes: long'''						)
														    .agg(pyspark.sql.functions.sum('''batch_bytes'''						).alias('''sample_bytes'''						)						)
														    .collect()[0]
														    .sample_bytes
														    / sample_num_rows
														)
														A      :					str      		=					approx_bytes_per_row * df_num_rows
														if approx_total_size > max_shard_size:
																					# Make sure there is at least one row per partition.
																					A      :					Union[str, Any]      		=					min(__UpperCAmelCase    ,						int(approx_total_size / max_shard_size						)						)
																					A      :					List[Any]      		=					self.df.repartition(__UpperCAmelCase						)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						)		->       Iterable[Tuple[int, bool, Union[int, tuple]]]:
														import pyspark

														A      :					Dict      		=					ParquetWriter if file_format == '''parquet''' else ArrowWriter
														A      :					Optional[Any]      		=					os.path.join(self._working_dir    ,						os.path.basename(__UpperCAmelCase						)						) if self._working_dir else fpath
														A      :					int      		=					file_format == '''parquet'''

														# Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
														# pickling the SparkContext.
														A      :					Any      		=					self.config.features
														A      :					Any      		=					self._writer_batch_size
														A      :					int      		=					self._fs.storage_options

														def write_arrow(__UpperCAmelCase						):
																					# Within the same SparkContext, no two task attempts will share the same attempt ID.
																					A      :					List[Any]      		=					pyspark.TaskContext().taskAttemptId()
																					A      :					Union[str, Any]      		=					next(__UpperCAmelCase    ,						__UpperCAmelCase						)
																					if first_batch is None:
																												# Some partitions might not receive any data.
																												return pa.RecordBatch.from_arrays(
																												    [[task_id], [0], [0]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)
																					A      :					Optional[int]      		=					0
																					A      :					Optional[int]      		=					writer_class(
																					    features=__UpperCAmelCase    ,						path=working_fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						writer_batch_size=__UpperCAmelCase    ,						storage_options=__UpperCAmelCase    ,						embed_local_files=__UpperCAmelCase    ,						)
																					A      :					Tuple      		=					pa.Table.from_batches([first_batch]						)
																					writer.write_table(__UpperCAmelCase						)
																					for batch in it:
																												if max_shard_size is not None and writer._num_bytes >= max_shard_size:
																																			A		,		A      :					str      		=					writer.finalize()
																																			writer.close()
																																			yield pa.RecordBatch.from_arrays(
																																			    [[task_id], [num_examples], [num_bytes]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)
																																			shard_id += 1
																																			A      :					List[str]      		=					writer_class(
																																			    features=writer._features    ,						path=working_fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						writer_batch_size=__UpperCAmelCase    ,						storage_options=__UpperCAmelCase    ,						embed_local_files=__UpperCAmelCase    ,						)
																												A      :					str      		=					pa.Table.from_batches([batch]						)
																												writer.write_table(__UpperCAmelCase						)

																					if writer._num_bytes > 0:
																												A		,		A      :					Tuple      		=					writer.finalize()
																												writer.close()
																												yield pa.RecordBatch.from_arrays(
																												    [[task_id], [num_examples], [num_bytes]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)

																					if working_fpath != fpath:
																												for file in os.listdir(os.path.dirname(__UpperCAmelCase						)						):
																																			A      :					Union[str, Any]      		=					os.path.join(os.path.dirname(__UpperCAmelCase						)    ,						os.path.basename(__UpperCAmelCase						)						)
																																			shutil.move(__UpperCAmelCase    ,						__UpperCAmelCase						)

														A      :					Union[str, Any]      		=					(
														    self.df.mapInArrow(__UpperCAmelCase    ,						'''task_id: long, num_examples: long, num_bytes: long'''						)
														    .groupBy('''task_id'''						)
														    .agg(
														        pyspark.sql.functions.sum('''num_examples'''						).alias('''total_num_examples'''						)    ,						pyspark.sql.functions.sum('''num_bytes'''						).alias('''total_num_bytes'''						)    ,						pyspark.sql.functions.count('''num_bytes'''						).alias('''num_shards'''						)    ,						pyspark.sql.functions.collect_list('''num_examples'''						).alias('''shard_lengths'''						)    ,						)
														    .collect()
														)
														for row in stats:
																					yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = "arrow"    ,						__UpperCAmelCase = None    ,						__UpperCAmelCase = None    ,						**__UpperCAmelCase    ,						)		->       List[str]:
														self._validate_cache_dir()

														A      :					int      		=					convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE						)
														self._repartition_df_if_needed(__UpperCAmelCase						)
														A      :					Any      		=					not is_remote_filesystem(self._fs						)
														A      :					Union[str, Any]      		=					os.path.join if is_local else posixpath.join

														A      :					Any      		=					'''-TTTTT-SSSSS-of-NNNNN'''
														A      :					Any      		=					f'{self.name}-{split_generator.name}{SUFFIX}.{file_format}'
														A      :					List[Any]      		=					path_join(self._output_dir    ,						__UpperCAmelCase						)

														A      :					Union[str, Any]      		=					0
														A      :					Any      		=					0
														A      :					Tuple      		=					0
														A      :					Union[str, Any]      		=					[]
														A      :					List[str]      		=					[]

														for task_id, content in self._prepare_split_single(__UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase						):
																					(
																					    (
																					    A
																					)		,		(
																					    A
																					)		,		(
																					    A
																					)		,		(
																					    A
																					)		,		
																					)      :					List[Any]      		=					content
																					if num_bytes > 0:
																												total_num_examples += num_examples
																												total_num_bytes += num_bytes
																												total_shards += num_shards
																												task_id_and_num_shards.append((task_id, num_shards)						)
																												all_shard_lengths.extend(__UpperCAmelCase						)

														A      :					List[str]      		=					total_num_examples
														A      :					List[str]      		=					total_num_bytes

														# should rename everything at the end
														logger.debug(f'Renaming {total_shards} shards.'						)
														if total_shards > 1:
																					A      :					Tuple      		=					all_shard_lengths

																					# Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
																					# pickling error due to pickling the SparkContext.
																					A      :					Union[str, Any]      		=					self._fs

																					# use the -SSSSS-of-NNNNN pattern
																					def _rename_shard(
																					    __UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						):
																												rename(
																												    __UpperCAmelCase    ,						fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						fpath.replace('''TTTTT-SSSSS'''    ,						f'{global_shard_id:05d}'						).replace('''NNNNN'''    ,						f'{total_shards:05d}'						)    ,						)

																					A      :					Tuple      		=					[]
																					A      :					Optional[int]      		=					0
																					for i in range(len(__UpperCAmelCase						)						):
																												A		,		A      :					Tuple      		=					task_id_and_num_shards[i]
																												for shard_id in range(__UpperCAmelCase						):
																																			args.append([task_id, shard_id, global_shard_id]						)
																																			global_shard_id += 1
																					self._spark.sparkContext.parallelize(__UpperCAmelCase    ,						len(__UpperCAmelCase						)						).map(lambda __UpperCAmelCase						: _rename_shard(*__UpperCAmelCase						)						).collect()
														else:
																					# don't use any pattern
																					A      :					int      		=					0
																					A      :					Tuple      		=					task_id_and_num_shards[0][0]
																					self._rename(
																					    fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						fpath.replace(__UpperCAmelCase    ,						''''''						)    ,						)






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						)		->       SparkExamplesIterable:
														return SparkExamplesIterable(self.df						)






