






from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


snake_case__   :						List[str]													=   logging.get_logger(__name__)

snake_case__   :						Optional[Any]													=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case__   :						Any													=   {
    """vocab_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"""
    },
}

snake_case__   :						str													=   {"""allegro/herbert-base-cased""": 5_1_4}
snake_case__   :						Union[str, Any]													=   {}


class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  A_						       = VOCAB_FILES_NAMES
  A_						       = PRETRAINED_VOCAB_FILES_MAP
  A_						       = PRETRAINED_INIT_CONFIGURATION
  A_						       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_						       = HerbertTokenizer




  def __init__(  self		,		_UpperCAmelCase=None		,		_UpperCAmelCase=None		,		_UpperCAmelCase=None		,		_UpperCAmelCase="<s>"		,		_UpperCAmelCase="<unk>"		,		_UpperCAmelCase="<pad>"		,		_UpperCAmelCase="<mask>"		,		_UpperCAmelCase="</s>"		,		**_UpperCAmelCase		,		)      ->			Optional[int]:
     super().__init__(
         _UpperCAmelCase		,		_UpperCAmelCase		,		tokenizer_file=_UpperCAmelCase		,		cls_token=_UpperCAmelCase		,		unk_token=_UpperCAmelCase		,		pad_token=_UpperCAmelCase		,		mask_token=_UpperCAmelCase		,		sep_token=_UpperCAmelCase		,		**_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase = None			)      ->			List[int]:
     UpperCamelCase_						=  [self.cls_token_id]
     UpperCamelCase_						=  [self.sep_token_id]
     if token_ids_a is None:
        return cls + token_ids_a + sep

     return cls + token_ids_a + sep + token_ids_a + sep




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase = None		,		_UpperCAmelCase = False			)      ->			List[int]:
     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=_UpperCAmelCase		,		token_ids_a=_UpperCAmelCase		,		already_has_special_tokens=_UpperCAmelCase			)

     if token_ids_a is None:
        return [1] + ([0] * len(_UpperCAmelCase			)) + [1]
     return [1] + ([0] * len(_UpperCAmelCase			)) + [1] + ([0] * len(_UpperCAmelCase			)) + [1]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase = None			)      ->			List[int]:
     UpperCamelCase_						=  [self.sep_token_id]
     UpperCamelCase_						=  [self.cls_token_id]

     if token_ids_a is None:
        return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase = None			)      ->			Tuple[str]:
     UpperCamelCase_						=  self._tokenizer.model.save(_UpperCAmelCase		,		name=_UpperCAmelCase			)
     return tuple(_UpperCAmelCase			)













import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     return datasets.DatasetInfo(
         features=datasets.Features({'content': datasets.Value('string'			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_dummy_examples()}			)]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)




class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Any:
     return datasets.DatasetInfo(
         features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     return [
         datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_nested_examples()}			)
     ]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Union[str, Any]:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)






def      _snake_case     ():
   return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar'])]






def      _snake_case     ():
   return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar'])]


class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Dict:
     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_dummy_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_dummy_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     import apache_beam as beam

     UpperCamelCase_						=  beam.io.parquetio.WriteToParquet

     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        with patch('apache_beam.io.parquetio.WriteToParquet'			) as write_parquet_mock:
           UpperCamelCase_						=  partial(_UpperCAmelCase		,		num_shards=2			)
           builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        # Order is not preserved when sharding, so we just check that all the elements are there
        self.assertListEqual(sorted(dset['train']['content']			)		,		sorted(['foo', 'bar', 'foobar']			)			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Any:
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase			)
        self.assertRaises(datasets.builder.MissingBeamOptions		,		builder.download_and_prepare			)




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  len(get_test_nested_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  NestedBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(
            builder.info.features		,		datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_nested_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_nested_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset





