





import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


_A   :       Optional[Any]    					=    datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
       import pyspark


@dataclass
class   __SCREAMING_SNAKE_CASE     (					datasets.BuilderConfig   ):
       _UpperCAmelCase :    List[Any]					       =       None



def 			_a		(       UpperCAmelCase     ,    UpperCAmelCase     ,    )							->							str:







   """simple docstring"""
   import pyspark

   def generate_fn():
      lowerCamelCase__  :		Optional[int]     =				df.select('''*'''     ,    pyspark.sql.functions.spark_partition_id().alias('''part_id'''	)	)
      for partition_id in partition_order:
         lowerCamelCase__  :		str     =				df_with_partition_id.select('''*'''	).where(f"part_id = {partition_id}"	).drop('''part_id'''	)
         lowerCamelCase__  :		str     =				partition_df.collect()
         lowerCamelCase__  :		int     =				0
         for row in rows:
            yield f"{partition_id}_{row_id}", row.asDict()
            row_id += 1

   return generate_fn


class   __SCREAMING_SNAKE_CASE     (					_BaseExamplesIterable   ):







       def __init__( self	:      Tuple						,   A	:      "pyspark.sql.DataFrame"						,   A	:      Dict=None						,   )  ->Dict:
          lowerCamelCase__  :		Optional[Any]     =				df
          lowerCamelCase__  :		List[Any]     =				partition_order or range(self.df.rdd.getNumPartitions()     )
          lowerCamelCase__  :		Optional[Any]     =				_generate_iterable_examples(self.df						,   self.partition_order     )







       def __iter__( self	:      Tuple     )  ->Dict:
          yield from self.generate_examples_fn()







       def 		__lowerCamelCase       ( self	:      str						,   A	:      np.random.Generator     )  ->List[Any]:
          lowerCamelCase__  :		int     =				list(range(self.df.rdd.getNumPartitions()     )     )
          generator.shuffle(__lowerCamelCase     )
          return SparkExamplesIterable(self.df						,   partition_order=__lowerCamelCase     )







       def 		__lowerCamelCase       ( self	:      Union[str, Any]						,   A	:      int						,   A	:      int     )  ->List[Any]:
          lowerCamelCase__  :		str     =				self.split_shard_indices_by_worker(__lowerCamelCase						,   __lowerCamelCase     )
          return SparkExamplesIterable(self.df						,   partition_order=__lowerCamelCase     )







       @property
       def 		__lowerCamelCase       ( self	:      Union[str, Any]     )  ->Optional[int]:
          return len(self.partition_order     )


class   __SCREAMING_SNAKE_CASE     (					datasets.DatasetBuilder   ):
       _UpperCAmelCase :    str					       =       SparkConfig







       def __init__( self	:      Tuple						,   A	:      "pyspark.sql.DataFrame"						,   A	:      str = None						,   A	:      str = None						,   **A	:      Any						,   )  ->List[Any]:
          import pyspark

          lowerCamelCase__  :		Dict     =				pyspark.sql.SparkSession.builder.getOrCreate()
          lowerCamelCase__  :		Optional[int]     =				df
          lowerCamelCase__  :		Any     =				working_dir

          super().__init__(
              cache_dir=__lowerCamelCase						,   config_name=str(self.df.semanticHash()     )						,   **__lowerCamelCase						,   )







       def 		__lowerCamelCase       ( self	:      str     )  ->Dict:
          # Returns the path of the created file.
          def create_cache_and_write_probe(A	:      Dict     ):
             # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
             # already exist.
             os.makedirs(self._cache_dir						,   exist_ok=__lowerCamelCase     )
             lowerCamelCase__  :		Tuple     =				os.path.join(self._cache_dir						,   '''fs_test''' + uuid.uuida().hex     )
             # Opening the file in append mode will create a new file unless it already exists, in which case it will not
             # change the file contents.
             open(__lowerCamelCase						,   '''a'''     )
             return [probe_file]

          if self._spark.conf.get('''spark.master'''						,   ''''''     ).startswith('''local'''     ):
             return

          # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
          # accessible to the driver.
          # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
          if self._cache_dir:
             lowerCamelCase__  :		Optional[Any]     =				(
                 self._spark.sparkContext.parallelize(range(1     )						,   1     ).mapPartitions(__lowerCamelCase     ).collect()
             )
             if os.path.isfile(probe[0]     ):
                return

          raise ValueError(
              '''When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir'''     )







       def 		__lowerCamelCase       ( self	:      str     )  ->Tuple:
          return datasets.DatasetInfo(features=self.config.features     )







       def 		__lowerCamelCase       ( self	:      str						,   A	:      datasets.download.download_manager.DownloadManager     )  ->Tuple:
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN     )]







       def 		__lowerCamelCase       ( self	:      Dict						,   A	:      Optional[int]     )  ->Optional[Any]:
          import pyspark

          def get_arrow_batch_size(A	:      Tuple     ):
             for batch in it:
                yield pa.RecordBatch.from_pydict({'''batch_bytes''': [batch.nbytes]}     )

          lowerCamelCase__  :		List[Any]     =				self.df.count()
          lowerCamelCase__  :		str     =				df_num_rows if df_num_rows <= 1_0_0 else 1_0_0
          # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
          lowerCamelCase__  :		Optional[Any]     =				(
              self.df.limit(__lowerCamelCase     )
              .repartition(1     )
              .mapInArrow(__lowerCamelCase						,   '''batch_bytes: long'''     )
              .agg(pyspark.sql.functions.sum('''batch_bytes'''     ).alias('''sample_bytes'''     )     )
              .collect()[0]
              .sample_bytes
              / sample_num_rows
          )
          lowerCamelCase__  :		Union[str, Any]     =				approx_bytes_per_row * df_num_rows
          if approx_total_size > max_shard_size:
             # Make sure there is at least one row per partition.
             lowerCamelCase__  :		int     =				min(__lowerCamelCase						,   int(approx_total_size / max_shard_size     )     )
             lowerCamelCase__  :		str     =				self.df.repartition(__lowerCamelCase     )







       def 		__lowerCamelCase       ( self	:      List[str]						,   A	:      str						,   A	:      str						,   A	:      int						,   )  ->List[str]:
          import pyspark

          lowerCamelCase__  :		Optional[Any]     =				ParquetWriter if file_format == '''parquet''' else ArrowWriter
          lowerCamelCase__  :		Union[str, Any]     =				os.path.join(self._working_dir						,   os.path.basename(__lowerCamelCase     )     ) if self._working_dir else fpath
          lowerCamelCase__  :		Tuple     =				file_format == '''parquet'''

          # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
          # pickling the SparkContext.
          lowerCamelCase__  :		List[Any]     =				self.config.features
          lowerCamelCase__  :		str     =				self._writer_batch_size
          lowerCamelCase__  :		Dict     =				self._fs.storage_options

          def write_arrow(A	:      Any     ):
             # Within the same SparkContext, no two task attempts will share the same attempt ID.
             lowerCamelCase__  :		Union[str, Any]     =				pyspark.TaskContext().taskAttemptId()
             lowerCamelCase__  :		List[Any]     =				next(__lowerCamelCase						,   __lowerCamelCase     )
             if first_batch is None:
                # Some partitions might not receive any data.
                return pa.RecordBatch.from_arrays(
                    [[task_id], [0], [0]]						,   names=['''task_id''', '''num_examples''', '''num_bytes''']						,   )
             lowerCamelCase__  :		List[Any]     =				0
             lowerCamelCase__  :		Optional[Any]     =				writer_class(
                 features=__lowerCamelCase						,   path=working_fpath.replace('''SSSSS'''						,   F"{shard_id:05d}"     ).replace('''TTTTT'''						,   F"{task_id:05d}"     )						,   writer_batch_size=__lowerCamelCase						,   storage_options=__lowerCamelCase						,   embed_local_files=__lowerCamelCase						,   )
             lowerCamelCase__  :		Optional[int]     =				pa.Table.from_batches([first_batch]     )
             writer.write_table(__lowerCamelCase     )
             for batch in it:
                if max_shard_size is not None and writer._num_bytes >= max_shard_size:
                   lowerCamelCase__  :		int     =				writer.finalize()
                   writer.close()
                   yield pa.RecordBatch.from_arrays(
                       [[task_id], [num_examples], [num_bytes]]						,   names=['''task_id''', '''num_examples''', '''num_bytes''']						,   )
                   shard_id += 1
                   lowerCamelCase__  :		Optional[int]     =				writer_class(
                       features=writer._features						,   path=working_fpath.replace('''SSSSS'''						,   F"{shard_id:05d}"     ).replace('''TTTTT'''						,   F"{task_id:05d}"     )						,   writer_batch_size=__lowerCamelCase						,   storage_options=__lowerCamelCase						,   embed_local_files=__lowerCamelCase						,   )
                lowerCamelCase__  :		Optional[int]     =				pa.Table.from_batches([batch]     )
                writer.write_table(__lowerCamelCase     )

             if writer._num_bytes > 0:
                lowerCamelCase__  :		int     =				writer.finalize()
                writer.close()
                yield pa.RecordBatch.from_arrays(
                    [[task_id], [num_examples], [num_bytes]]						,   names=['''task_id''', '''num_examples''', '''num_bytes''']						,   )

             if working_fpath != fpath:
                for file in os.listdir(os.path.dirname(__lowerCamelCase     )     ):
                   lowerCamelCase__  :		Optional[int]     =				os.path.join(os.path.dirname(__lowerCamelCase     )						,   os.path.basename(__lowerCamelCase     )     )
                   shutil.move(__lowerCamelCase						,   __lowerCamelCase     )

          lowerCamelCase__  :		int     =				(
              self.df.mapInArrow(__lowerCamelCase						,   '''task_id: long, num_examples: long, num_bytes: long'''     )
              .groupBy('''task_id'''     )
              .agg(
                  pyspark.sql.functions.sum('''num_examples'''     ).alias('''total_num_examples'''     )						,   pyspark.sql.functions.sum('''num_bytes'''     ).alias('''total_num_bytes'''     )						,   pyspark.sql.functions.count('''num_bytes'''     ).alias('''num_shards'''     )						,   pyspark.sql.functions.collect_list('''num_examples'''     ).alias('''shard_lengths'''     )						,   )
              .collect()
          )
          for row in stats:
             yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)







       def 		__lowerCamelCase       ( self	:      Tuple						,   A	:      "datasets.SplitGenerator"						,   A	:      str = "arrow"						,   A	:      Optional[Union[str, int]] = None						,   A	:      Optional[int] = None						,   **A	:      Optional[Any]						,   )  ->Tuple:
          self._validate_cache_dir()

          lowerCamelCase__  :		Optional[Any]     =				convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE     )
          self._repartition_df_if_needed(__lowerCamelCase     )
          lowerCamelCase__  :		Tuple     =				not is_remote_filesystem(self._fs     )
          lowerCamelCase__  :		Optional[Any]     =				os.path.join if is_local else posixpath.join

          lowerCamelCase__  :		Dict     =				'''-TTTTT-SSSSS-of-NNNNN'''
          lowerCamelCase__  :		List[Any]     =				F"{self.name}-{split_generator.name}{SUFFIX}.{file_format}"
          lowerCamelCase__  :		Union[str, Any]     =				path_join(self._output_dir						,   __lowerCamelCase     )

          lowerCamelCase__  :		Union[str, Any]     =				0
          lowerCamelCase__  :		List[str]     =				0
          lowerCamelCase__  :		Dict     =				0
          lowerCamelCase__  :		str     =				[]
          lowerCamelCase__  :		Optional[Any]     =				[]

          for task_id, content in self._prepare_split_single(__lowerCamelCase						,   __lowerCamelCase						,   __lowerCamelCase     ):
             (
                 lowerCamelCase__						
             )  :		Dict     =				content
             if num_bytes > 0:
                total_num_examples += num_examples
                total_num_bytes += num_bytes
                total_shards += num_shards
                task_id_and_num_shards.append((task_id, num_shards)     )
                all_shard_lengths.extend(__lowerCamelCase     )

          lowerCamelCase__  :		Any     =				total_num_examples
          lowerCamelCase__  :		List[Any]     =				total_num_bytes

          # should rename everything at the end
          logger.debug(F"Renaming {total_shards} shards."     )
          if total_shards > 1:
             lowerCamelCase__  :		Optional[int]     =				all_shard_lengths

             # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
             # pickling error due to pickling the SparkContext.
             lowerCamelCase__  :		int     =				self._fs

             # use the -SSSSS-of-NNNNN pattern
             def _rename_shard(
                 A	:      int						,   A	:      int						,   A	:      int						,   ):
                rename(
                    __lowerCamelCase						,   fpath.replace('''SSSSS'''						,   F"{shard_id:05d}"     ).replace('''TTTTT'''						,   F"{task_id:05d}"     )						,   fpath.replace('''TTTTT-SSSSS'''						,   F"{global_shard_id:05d}"     ).replace('''NNNNN'''						,   F"{total_shards:05d}"     )						,   )

             lowerCamelCase__  :		Optional[Any]     =				[]
             lowerCamelCase__  :		Dict     =				0
             for i in range(len(__lowerCamelCase     )     ):
                lowerCamelCase__  :		List[Any]     =				task_id_and_num_shards[i]
                for shard_id in range(__lowerCamelCase     ):
                   args.append([task_id, shard_id, global_shard_id]     )
                   global_shard_id += 1
             self._spark.sparkContext.parallelize(__lowerCamelCase						,   len(__lowerCamelCase     )     ).map(lambda A     : _rename_shard(*__lowerCamelCase     )     ).collect()
          else:
             # don't use any pattern
             lowerCamelCase__  :		List[str]     =				0
             lowerCamelCase__  :		int     =				task_id_and_num_shards[0][0]
             self._rename(
                 fpath.replace('''SSSSS'''						,   F"{shard_id:05d}"     ).replace('''TTTTT'''						,   F"{task_id:05d}"     )						,   fpath.replace(__lowerCamelCase						,   ''''''     )						,   )







       def 		__lowerCamelCase       ( self	:      int						,   A	:      "datasets.SplitGenerator"						,   )  ->Optional[Any]:
          return SparkExamplesIterable(self.df     )







'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCamelCase												=						{
    '''configuration_maskformer''': ['''MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MaskFormerConfig'''],
    '''configuration_maskformer_swin''': ['''MaskFormerSwinConfig'''],
}

try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase												=						['''MaskFormerFeatureExtractor''']
    __lowerCamelCase												=						['''MaskFormerImageProcessor''']


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase												=						[
        '''MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''MaskFormerForInstanceSegmentation''',
        '''MaskFormerModel''',
        '''MaskFormerPreTrainedModel''',
    ]
    __lowerCamelCase												=						[
        '''MaskFormerSwinBackbone''',
        '''MaskFormerSwinModel''',
        '''MaskFormerSwinPreTrainedModel''',
    ]

if TYPE_CHECKING:
    from .configuration_maskformer import MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskFormerConfig
    from .configuration_maskformer_swin import MaskFormerSwinConfig

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_maskformer import MaskFormerFeatureExtractor
        from .image_processing_maskformer import MaskFormerImageProcessor
    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_maskformer import (
            MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            MaskFormerForInstanceSegmentation,
            MaskFormerModel,
            MaskFormerPreTrainedModel,
        )
        from .modeling_maskformer_swin import (
            MaskFormerSwinBackbone,
            MaskFormerSwinModel,
            MaskFormerSwinPreTrainedModel,
        )


else:
    import sys

    __lowerCamelCase												=						_LazyModule(__name__, globals()['''__file__'''], _import_structure)



