

"""simple docstring"""






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__lowerCamelCase										=		logging.get_logger(__name__)

__lowerCamelCase										=		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase										=		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

__lowerCamelCase										=		{
    "Salesforce/codegen-350M-mono": 20_48,
}
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       VOCAB_FILES_NAMES
     _lowerCamelCase     =       PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase     =       ['''input_ids''', '''attention_mask''']
     _lowerCamelCase     =       CodeGenTokenizer







     def __init__(      self ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_="<|endoftext|>" ,						UpperCamelCase_="<|endoftext|>" ,						UpperCamelCase_="<|endoftext|>" ,						UpperCamelCase_=False ,						**UpperCamelCase_ ,						):
      super().__init__(
          UpperCamelCase_ ,						UpperCamelCase_ ,						tokenizer_file=UpperCamelCase_ ,						unk_token=UpperCamelCase_ ,						bos_token=UpperCamelCase_ ,						eos_token=UpperCamelCase_ ,						add_prefix_space=UpperCamelCase_ ,						**UpperCamelCase_ ,						)

      if kwargs.pop('''add_bos_token''' ,						UpperCamelCase_       ):
       __magic_name__     =      kwargs.pop('''name_or_path''' ,						''''''       )
       raise ValueError(
           '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
           '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
           f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
           f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
           '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
           ''' so that the fast tokenizer works correctly.'''       )

      __magic_name__     =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
      if pre_tok_state.get('''add_prefix_space''' ,						UpperCamelCase_       ) != add_prefix_space:
       __magic_name__     =      getattr(UpperCamelCase_ ,						pre_tok_state.pop('''type'''       )       )
       __magic_name__     =      add_prefix_space
       __magic_name__     =      pre_tok_class(**UpperCamelCase_       )

      __magic_name__     =      add_prefix_space







     def 	lowerCAmelCase__    (      self ,						*UpperCamelCase_ ,						**UpperCamelCase_       ):
      __magic_name__     =      kwargs.get('''is_split_into_words''' ,						UpperCamelCase_       )
      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*UpperCamelCase_ ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						*UpperCamelCase_ ,						**UpperCamelCase_       ):
      __magic_name__     =      kwargs.get('''is_split_into_words''' ,						UpperCamelCase_       )

      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*UpperCamelCase_ ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      self._tokenizer.model.save(UpperCamelCase_ ,						name=UpperCamelCase_       )
      return tuple(UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = False ,						UpperCamelCase_ = None ,						UpperCamelCase_ = None ,						**UpperCamelCase_ ,						):
      __magic_name__     =      super().decode(
          token_ids=UpperCamelCase_ ,						skip_special_tokens=UpperCamelCase_ ,						clean_up_tokenization_spaces=UpperCamelCase_ ,						**UpperCamelCase_ ,						)

      if truncate_before_pattern is not None and len(UpperCamelCase_       ) > 0:
       __magic_name__     =      self.truncate(UpperCamelCase_ ,						UpperCamelCase_       )

      return decoded_text







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      def find_re(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_       ):
       __magic_name__     =      pattern.search(UpperCamelCase_ ,						UpperCamelCase_       )
       return m.start() if m else -1

      __magic_name__     =      [re.compile(UpperCamelCase_ ,						re.MULTILINE       ) for pattern in truncate_before_pattern]

      __magic_name__     =      list(re.finditer('''^print''' ,						UpperCamelCase_ ,						re.MULTILINE       )       )

      if len(UpperCamelCase_       ) > 1:
       __magic_name__     =      completion[: prints[1].start()]

      __magic_name__     =      list(re.finditer('''^def''' ,						UpperCamelCase_ ,						re.MULTILINE       )       )

      if len(UpperCamelCase_       ) > 1:
       __magic_name__     =      completion[: defs[1].start()]

      __magic_name__     =      0

      __magic_name__     =      [
          pos for pos in [find_re(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_       ) for terminal in terminals] if pos != -1
      ]

      if len(UpperCamelCase_       ) > 0:
       return completion[: min(UpperCamelCase_       )]
      else:
       return completion




"""simple docstring"""






import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


__lowerCamelCase										=		[
    {"dataset": "wikipedia", "config_name": "20220301.de"},
    {"dataset": "wikipedia", "config_name": "20220301.en"},
    {"dataset": "wikipedia", "config_name": "20220301.fr"},
    {"dataset": "wikipedia", "config_name": "20220301.frr"},
    {"dataset": "wikipedia", "config_name": "20220301.it"},
    {"dataset": "wikipedia", "config_name": "20220301.simple"},
    {"dataset": "snli", "config_name": "plain_text"},
    {"dataset": "eli5", "config_name": "LFQA_reddit"},
    {"dataset": "wiki40b", "config_name": "en"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.compressed"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.no_index"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.multiset.no_index"},
    {"dataset": "natural_questions", "config_name": "default"},
]





def 	lowercase      (   __UpperCamelCase=True						) ->      List[Any]:
 if with_config:
  return [
      {
          "testcase_name": d["dataset"] + "/" + d["config_name"],
          "dataset": d["dataset"],
          "config_name": d["config_name"],
      }
      for d in DATASETS_ON_HF_GCP
  ]
 else:
  return [
      {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
  ]
@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__UpperCAmelCase     )     )
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       None
     _lowerCamelCase     =       None







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      with TemporaryDirectory() as tmp_dir:
       __magic_name__     =      dataset_module_factory(UpperCamelCase_ ,						cache_dir=UpperCamelCase_       )

       __magic_name__     =      import_main_class(dataset_module.module_path ,						dataset=UpperCamelCase_       )

       __magic_name__     =      builder_cls(
           cache_dir=UpperCamelCase_ ,						config_name=UpperCamelCase_ ,						hash=dataset_module.hash ,						)

       __magic_name__     =      '''/'''.join(
           [
               HF_GCP_BASE_URL,
               builder_instance._relative_data_dir(with_hash=UpperCamelCase_       ).replace(os.sep ,						'''/'''       ),
               config.DATASET_INFO_FILENAME,
           ]       )
       __magic_name__     =      cached_path(UpperCamelCase_ ,						cache_dir=UpperCamelCase_       )
       self.assertTrue(os.path.exists(UpperCamelCase_       )       )





@pytest.mark.integration
def 	lowercase      (   __UpperCamelCase						) ->      Tuple:
 __magic_name__     =      tmp_path_factory.mktemp('''test_hf_gcp'''						) / '''test_wikipedia_simple'''
 __magic_name__     =      dataset_module_factory('''wikipedia'''      ,   cache_dir=__UpperCamelCase						)
 __magic_name__     =      import_main_class(dataset_module.module_path						)
 __magic_name__     =      builder_cls(
     cache_dir=__UpperCamelCase      ,   config_name='''20220301.frr'''      ,   hash=dataset_module.hash      ,   )
 # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
 __magic_name__     =      None
 builder_instance.download_and_prepare()
 __magic_name__     =      builder_instance.as_dataset()
 assert ds





@pytest.mark.integration
def 	lowercase      (   __UpperCamelCase						) ->      Tuple:
 __magic_name__     =      dataset_module_factory('''wikipedia'''      ,   cache_dir=__UpperCamelCase						)
 __magic_name__     =      import_main_class(dataset_module.module_path      ,   dataset=__UpperCamelCase						)
 __magic_name__     =      builder_cls(
     cache_dir=__UpperCamelCase      ,   config_name='''20220301.frr'''      ,   hash=dataset_module.hash      ,   )
 __magic_name__     =      builder_instance.as_streaming_dataset()
 assert ds
 assert isinstance(__UpperCamelCase      ,   __UpperCamelCase						)
 assert "train" in ds
 assert isinstance(ds['''train''']      ,   __UpperCamelCase						)
 assert next(iter(ds['''train''']						)						)

