




import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_lowerCamelCase			 =						datasets.utils.logging.get_logger(__name__)



@dataclass
class 			a     (   datasets.BuilderConfig			):

     '''simple docstring'''





     lowerCAmelCase				:       Optional[datasets.Features]   		=  None
     lowerCAmelCase				:       str   		=  "utf-8"
     lowerCAmelCase				:       Optional[str]   		=  None
     lowerCAmelCase				:       int   		=  10 << 20  # 10MB
     lowerCAmelCase				:       bool   		=  False
     lowerCAmelCase				:       str   		=  "line"



class 			a     (   datasets.ArrowBasedBuilder			):

     '''simple docstring'''





     lowerCAmelCase				:       Optional[Any]   		=  TextConfig
     def   lowerCamelCase_   (	self   :			str				):
         return datasets.DatasetInfo(features=self.config.features				)
     def   lowerCamelCase_   (	self   :			List[Any]					,    __snake_case   :			Tuple				):
         if not self.config.data_files:
             raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'				)
         UpperCAmelCase_							 =     dl_manager.download_and_extract(self.config.data_files				)
         if isinstance(__snake_case					,    (str, list, tuple)				):
             UpperCAmelCase_							 =     data_files
             if isinstance(__snake_case					,    __snake_case				):
                 UpperCAmelCase_							 =     [files]
             UpperCAmelCase_							 =     [dl_manager.iter_files(__snake_case				) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,    gen_kwargs={'''files''': files}				)]
         UpperCAmelCase_							 =     []
         for split_name, files in data_files.items():
             if isinstance(__snake_case					,    __snake_case				):
                 UpperCAmelCase_							 =     [files]
             UpperCAmelCase_							 =     [dl_manager.iter_files(__snake_case				) for file in files]
             splits.append(datasets.SplitGenerator(name=__snake_case					,    gen_kwargs={'''files''': files}				)				)
         return splits
     def   lowerCamelCase_   (	self   :			Optional[int]					,    __snake_case   :			pa.Table				):
         if self.config.features is not None:
             UpperCAmelCase_							 =     self.config.features.arrow_schema
             if all(not require_storage_cast(__snake_case				) for feature in self.config.features.values()				):
                 # cheaper cast
                 UpperCAmelCase_							 =     pa_table.cast(__snake_case				)
             else:
                 # more expensive cast; allows str <-> int/float or str to Audio for example
                 UpperCAmelCase_							 =     table_cast(__snake_case					,    __snake_case				)
             return pa_table
         else:
             return pa_table.cast(pa.schema({'''text''': pa.string()}				)				)




     def   lowerCamelCase_   (	self   :			str					,    __snake_case   :			str				):
         UpperCAmelCase_							 =     list(self.config.features				) if self.config.features is not None else ['''text''']
         for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case				)				):
             # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
             with open(__snake_case					,    encoding=self.config.encoding					,    errors=self.config.errors				) as f:
                 if self.config.sample_by == "line":
                     UpperCAmelCase_							 =     0
                     while True:
                         UpperCAmelCase_							 =     f.read(self.config.chunksize				)
                         if not batch:
                             break
                         batch += f.readline()  # finish current line
                         # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                         UpperCAmelCase_							 =     StringIO(__snake_case				).readlines()
                         if not self.config.keep_linebreaks:
                             UpperCAmelCase_							 =     [line.rstrip('''\n'''				) for line in batch]
                         UpperCAmelCase_							 =     pa.Table.from_arrays([pa.array(__snake_case				)]					,    names=__snake_case				)
                         # Uncomment for debugging (will print the Arrow table size and elements)
                         # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                         # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                         yield (file_idx, batch_idx), self._cast_table(__snake_case				)
                         batch_idx += 1
                 elif self.config.sample_by == "paragraph":
                     UpperCAmelCase_							 =     0
                     UpperCAmelCase_							 =     ''''''
                     while True:
                         UpperCAmelCase_							 =     f.read(self.config.chunksize				)
                         if not new_batch:
                             break
                         batch += new_batch
                         batch += f.readline()  # finish current line
                         UpperCAmelCase_							 =     batch.split('''\n\n'''				)
                         UpperCAmelCase_							 =     pa.Table.from_arrays(
                             [pa.array([example for example in batch[:-1] if example]				)]					,    names=__snake_case				)
                         # Uncomment for debugging (will print the Arrow table size and elements)
                         # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                         # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                         yield (file_idx, batch_idx), self._cast_table(__snake_case				)
                         batch_idx += 1
                         UpperCAmelCase_							 =     batch[-1]
                     if batch:
                         UpperCAmelCase_							 =     pa.Table.from_arrays([pa.array([batch]				)]					,    names=__snake_case				)
                         yield (file_idx, batch_idx), self._cast_table(__snake_case				)
                 elif self.config.sample_by == "document":
                     UpperCAmelCase_							 =     f.read()
                     UpperCAmelCase_							 =     pa.Table.from_arrays([pa.array([text]				)]					,    names=__snake_case				)
                     yield file_idx, self._cast_table(__snake_case				)











from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy



class 			a     (   _A			):

     '''simple docstring'''





     lowerCAmelCase				:       Any   		=  'MarkupLMFeatureExtractor'
     lowerCAmelCase				:       Optional[Any]   		=  ('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
     lowerCAmelCase				:       List[Any]   		=  True
     def __call__(	self   :			Dict					,    __snake_case   :			Tuple=None					,    __snake_case   :			List[Any]=None					,    __snake_case   :			Tuple=None					,    __snake_case   :			Tuple=None					,    __snake_case   :			Any=None					,    __snake_case   :			bool = True					,    __snake_case   :			Union[bool, str, PaddingStrategy] = False					,    __snake_case   :			Union[bool, str, TruncationStrategy] = None					,    __snake_case   :			Optional[int] = None					,    __snake_case   :			int = 0					,    __snake_case   :			Optional[int] = None					,    __snake_case   :			Optional[bool] = None					,    __snake_case   :			Optional[bool] = None					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = True					,    __snake_case   :			Optional[Union[str, TensorType]] = None					,    **__snake_case   :			Any					,    ):
         # first, create nodes and xpaths
         if self.parse_html:
             if html_strings is None:
                 raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''				)

             if nodes is not None or xpaths is not None or node_labels is not None:
                 raise ValueError(
                     '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''				)

             UpperCAmelCase_							 =     self.feature_extractor(__snake_case				)
             UpperCAmelCase_							 =     features['''nodes''']
             UpperCAmelCase_							 =     features['''xpaths''']
         else:
             if html_strings is not None:
                 raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''				)
             if nodes is None or xpaths is None:
                 raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''				)

        # # second, apply the tokenizer
         if questions is not None and self.parse_html:
             if isinstance(__snake_case					,    __snake_case				):
                 UpperCAmelCase_							 =     [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

         UpperCAmelCase_							 =     self.tokenizer(
             text=questions if questions is not None else nodes					,    text_pair=nodes if questions is not None else None					,    xpaths=__snake_case					,    node_labels=__snake_case					,    add_special_tokens=__snake_case					,    padding=__snake_case					,    truncation=__snake_case					,    max_length=__snake_case					,    stride=__snake_case					,    pad_to_multiple_of=__snake_case					,    return_token_type_ids=__snake_case					,    return_attention_mask=__snake_case					,    return_overflowing_tokens=__snake_case					,    return_special_tokens_mask=__snake_case					,    return_offsets_mapping=__snake_case					,    return_length=__snake_case					,    verbose=__snake_case					,    return_tensors=__snake_case					,    **__snake_case					,    )

         return encoded_inputs
     def   lowerCamelCase_   (	self   :			List[Any]					,    *__snake_case   :			Dict					,    **__snake_case   :			Optional[int]				):
         return self.tokenizer.batch_decode(*__snake_case					,    **__snake_case				)
     def   lowerCamelCase_   (	self   :			Optional[Any]					,    *__snake_case   :			Any					,    **__snake_case   :			Dict				):
         return self.tokenizer.decode(*__snake_case					,    **__snake_case				)




     @property
     def   lowerCamelCase_   (	self   :			Dict				):
         UpperCAmelCase_							 =     self.tokenizer.model_input_names
         return tokenizer_input_names





