






"""simple docstring"""
import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast



@dataclass
class        UpperCamelCase__ (				datasets.BuilderConfig):







    """simple docstring"""
    __UpperCAmelCase           = None



class        UpperCamelCase__ (				datasets.ArrowBasedBuilder):







    """simple docstring"""
    __UpperCAmelCase           = PandasConfig







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           return datasets.DatasetInfo(features=self.config.features )







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           if not self.config.data_files:
                  raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}""" )
           __magic_name__      				=    dl_manager.download_and_extract(self.config.data_files )
           if isinstance(UpperCamelCase_				,    (str, list, tuple) ):
                  __magic_name__      				=    data_files
                  if isinstance(UpperCamelCase_				,    UpperCamelCase_ ):
                         __magic_name__      				=    [files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  __magic_name__      				=    [dl_manager.iter_files(UpperCamelCase_ ) for file in files]
                  return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,    gen_kwargs={'files': files} )]
           __magic_name__      				=    []
           for split_name, files in data_files.items():
                  if isinstance(UpperCamelCase_				,    UpperCamelCase_ ):
                         __magic_name__      				=    [files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  __magic_name__      				=    [dl_manager.iter_files(UpperCamelCase_ ) for file in files]
                  splits.append(datasets.SplitGenerator(name=UpperCamelCase_				,    gen_kwargs={'files': files} ) )
           return splits







    def 				a__			(		self :	Tuple				,    UpperCamelCase_ :	pa.Table ):





           '''simple docstring'''

           if self.config.features is not None:
                  # more expensive cast to support nested features with keys in a different order
                  # allows str <-> int/float or str to Audio for example
                  __magic_name__      				=    table_cast(UpperCamelCase_				,    self.config.features.arrow_schema )
           return pa_table







    def 				a__			(		self :	Tuple				,    UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           for i, file in enumerate(itertools.chain.from_iterable(UpperCamelCase_ ) ):
                  with open(UpperCamelCase_				,    'rb' ) as f:
                         __magic_name__      				=    pa.Table.from_pandas(pd.read_pickle(UpperCamelCase_ ) )
                         yield i, self._cast_table(UpperCamelCase_ )







"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case : Optional[Any]     		=		logging.get_logger(__name__)

snake_case : str     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

snake_case : Tuple     		=		{"""mobilebert-uncased""": 5_1_2}


snake_case : List[str]     		=		{}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = MobileBertTokenizer







    def __init__(		self :	Optional[int]				,    UpperCamelCase_ :	Optional[int]=None				,    UpperCamelCase_ :	str=None				,    UpperCamelCase_ :	Tuple=True				,    UpperCamelCase_ :	Any="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Any="[PAD]"				,    UpperCamelCase_ :	Dict="[CLS]"				,    UpperCamelCase_ :	str="[MASK]"				,    UpperCamelCase_ :	Union[str, Any]=True				,    UpperCamelCase_ :	Dict=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	List[Any]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )