





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_				= logging.get_logger(__name__)

lowerCAmelCase_				= {"""vocab_file""": """spiece.model"""}

lowerCAmelCase_				= {
    """vocab_file""": {
        """bert_for_seq_generation""": (
            """https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model"""
        ),
    }
}

lowerCAmelCase_				= {"""bert_for_seq_generation""": 5_1_2}

class 					A     (UpperCAmelCase__					):
      _SCREAMING_SNAKE_CASE       =				VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE       =				PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE       =				[]
      _SCREAMING_SNAKE_CASE       =				["""input_ids""", """attention_mask"""]



      def __init__(       self  ,	lowercase_  ,	lowercase_="<s>"  ,	lowercase_="</s>"  ,	lowercase_="<unk>"  ,	lowercase_="<pad>"  ,	lowercase_="<::::>"  ,	lowercase_ = None  ,	**lowercase_  ,	)   -> None:



             '''simple docstring'''

             _snake_case   :				Tuple  					=    {} if sp_model_kwargs is None else sp_model_kwargs

             # Add extra_ids to the special token list
             super().__init__(
                 bos_token=_UpperCAmelCase  ,	eos_token=_UpperCAmelCase  ,	unk_token=_UpperCAmelCase  ,	pad_token=_UpperCAmelCase  ,	sep_token=_UpperCAmelCase  ,	sp_model_kwargs=self.sp_model_kwargs  ,	**_UpperCAmelCase  ,	)

             _snake_case   :				Optional[int]  					=    vocab_file

             _snake_case   :				Optional[Any]  					=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(_UpperCAmelCase       )



      @property
      def  __a			(       self       )   -> Union[str, Any]:



             '''simple docstring'''

             return self.sp_model.get_piece_size()



      def  __a			(       self       )   -> int:



             '''simple docstring'''

             _snake_case   :				Union[str, Any]  					=    {self.convert_ids_to_tokens(_UpperCAmelCase       ): i for i in range(self.vocab_size       )}
             vocab.update(self.added_tokens_encoder       )
             return vocab



      def __getstate__(       self       )   -> str:



             '''simple docstring'''

             _snake_case   :				Dict  					=    self.__dict__.copy()
             _snake_case   :				List[Any]  					=    None
             return state



      def __setstate__(       self  ,	lowercase_       )   -> List[str]:



             '''simple docstring'''

             _snake_case   :				List[Any]  					=    d

             # for backward compatibility
             if not hasattr(self  ,	'''sp_model_kwargs'''       ):
                    _snake_case   :				Any  					=    {}

             _snake_case   :				List[str]  					=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(self.vocab_file       )



      def  __a			(       self  ,	lowercase_       )   -> List[str]:



             '''simple docstring'''

             return self.sp_model.encode(_UpperCAmelCase  ,	out_type=_UpperCAmelCase       )



      def  __a			(       self  ,	lowercase_       )   -> Union[str, Any]:



             '''simple docstring'''

             return self.sp_model.piece_to_id(_UpperCAmelCase       )



      def  __a			(       self  ,	lowercase_       )   -> Optional[int]:



             '''simple docstring'''

             _snake_case   :				str  					=    self.sp_model.IdToPiece(_UpperCAmelCase       )
             return token



      def  __a			(       self  ,	lowercase_       )   -> Union[str, Any]:



             '''simple docstring'''

             _snake_case   :				Optional[int]  					=    []
             _snake_case   :				Optional[int]  					=    ''''''
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           out_string += self.sp_model.decode(_UpperCAmelCase       ) + token
                           _snake_case   :				Union[str, Any]  					=    []
                    else:
                           current_sub_tokens.append(_UpperCAmelCase       )
             out_string += self.sp_model.decode(_UpperCAmelCase       )
             return out_string.strip()



      def  __a			(       self  ,	lowercase_  ,	lowercase_ = None       )   -> Tuple[str]:



             '''simple docstring'''

             if not os.path.isdir(_UpperCAmelCase       ):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
                    return
             _snake_case   :				List[Any]  					=    os.path.join(
                 _UpperCAmelCase  ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

             if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCAmelCase       ) and os.path.isfile(self.vocab_file       ):
                    copyfile(self.vocab_file  ,	_UpperCAmelCase       )
             elif not os.path.isfile(self.vocab_file       ):
                    with open(_UpperCAmelCase  ,	'''wb'''       ) as fi:
                           _snake_case   :				str  					=    self.sp_model.serialized_model_proto()
                           fi.write(_UpperCAmelCase       )

             return (out_vocab_file,)














import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     return datasets.DatasetInfo(
         features=datasets.Features({'content': datasets.Value('string'			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_dummy_examples()}			)]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)




class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Any:
     return datasets.DatasetInfo(
         features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     return [
         datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_nested_examples()}			)
     ]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Union[str, Any]:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)






def      _snake_case     ():
   return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar'])]






def      _snake_case     ():
   return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar'])]


class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Dict:
     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_dummy_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_dummy_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     import apache_beam as beam

     UpperCamelCase_						=  beam.io.parquetio.WriteToParquet

     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        with patch('apache_beam.io.parquetio.WriteToParquet'			) as write_parquet_mock:
           UpperCamelCase_						=  partial(_UpperCAmelCase		,		num_shards=2			)
           builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        # Order is not preserved when sharding, so we just check that all the elements are there
        self.assertListEqual(sorted(dset['train']['content']			)		,		sorted(['foo', 'bar', 'foobar']			)			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Any:
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase			)
        self.assertRaises(datasets.builder.MissingBeamOptions		,		builder.download_and_prepare			)




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  len(get_test_nested_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  NestedBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(
            builder.info.features		,		datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_nested_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_nested_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset





