





'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A__							:  Any    =pytest.mark.integration




@require_faiss
class 					UpperCAmelCase	(     snake_case_  ):




      def 							lowercase__       (     self     :					str    )							-> List[Any]:
            _lowerCAmelCase					=				Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(__snake_case    ) for x in np.arange(30    ).tolist()]}    )
            return dset




      def 							lowercase__       (     self     :					Optional[Any]    )							-> Tuple:
            import faiss

            _lowerCAmelCase					=				self._create_dummy_dataset()
            _lowerCAmelCase					=				dset.map(
                lambda __snake_case     ,     __snake_case    : {"vecs": i * np.ones(5     ,     dtype=np.floataa    )}     ,     with_indices=__snake_case     ,     keep_in_memory=__snake_case    )
            _lowerCAmelCase					=				dset.add_faiss_index("""vecs"""     ,     batch_size=1_00     ,     metric_type=faiss.METRIC_INNER_PRODUCT    )
            _lowerCAmelCase	,				_lowerCAmelCase					=				dset.get_nearest_examples("""vecs"""     ,     np.ones(5     ,     dtype=np.floataa    )    )
            self.assertEqual(examples["""filename"""][0]     ,     """my_name-train_29"""    )
            dset.drop_index("""vecs"""    )




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> Any:
            import faiss

            _lowerCAmelCase					=				self._create_dummy_dataset()
            dset.add_faiss_index_from_external_arrays(
                external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1     ,     1    )     ,     index_name="""vecs"""     ,     batch_size=1_00     ,     metric_type=faiss.METRIC_INNER_PRODUCT     ,     )
            _lowerCAmelCase	,				_lowerCAmelCase					=				dset.get_nearest_examples("""vecs"""     ,     np.ones(5     ,     dtype=np.floataa    )    )
            self.assertEqual(examples["""filename"""][0]     ,     """my_name-train_29"""    )




      def 							lowercase__       (     self     :					Dict    )							-> str:
            import faiss

            _lowerCAmelCase					=				self._create_dummy_dataset()
            dset.add_faiss_index_from_external_arrays(
                external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1     ,     1    )     ,     index_name="""vecs"""     ,     metric_type=faiss.METRIC_INNER_PRODUCT     ,     )

            # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
            # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
            # see https://bugs.python.org/issue14243 and
            # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
            with tempfile.NamedTemporaryFile(delete=__snake_case    ) as tmp_file:
                  dset.save_faiss_index("""vecs"""     ,     tmp_file.name    )
                  dset.load_faiss_index("""vecs2"""     ,     tmp_file.name    )
            os.unlink(tmp_file.name    )

            _lowerCAmelCase	,				_lowerCAmelCase					=				dset.get_nearest_examples("""vecs2"""     ,     np.ones(5     ,     dtype=np.floataa    )    )
            self.assertEqual(examples["""filename"""][0]     ,     """my_name-train_29"""    )




      def 							lowercase__       (     self     :					Optional[Any]    )							-> Tuple:
            _lowerCAmelCase					=				self._create_dummy_dataset()
            dset.add_faiss_index_from_external_arrays(
                external_arrays=np.ones((30, 5)    ) * np.arange(30    ).reshape(-1     ,     1    )     ,     index_name="""vecs"""    )
            dset.drop_index("""vecs"""    )
            self.assertRaises(__snake_case     ,     partial(dset.get_nearest_examples     ,     """vecs2"""     ,     np.ones(5     ,     dtype=np.floataa    )    )    )




      def 							lowercase__       (     self     :					Tuple    )							-> int:
            from elasticsearch import Elasticsearch

            _lowerCAmelCase					=				self._create_dummy_dataset()
            with patch("""elasticsearch.Elasticsearch.search"""    ) as mocked_search, patch(
                """elasticsearch.client.IndicesClient.create"""    ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""    ) as mocked_bulk:
                  _lowerCAmelCase					=				{"""acknowledged""": True}
                  mocked_bulk.return_value([(True, None)] * 30    )
                  _lowerCAmelCase					=				{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
                  _lowerCAmelCase					=				Elasticsearch()

                  dset.add_elasticsearch_index("""filename"""     ,     es_client=__snake_case    )
                  _lowerCAmelCase	,				_lowerCAmelCase					=				dset.get_nearest_examples("""filename"""     ,     """my_name-train_29"""    )
                  self.assertEqual(examples["""filename"""][0]     ,     """my_name-train_29"""    )




@require_faiss
class 					UpperCAmelCase	(     snake_case_  ):




      def 							lowercase__       (     self     :					Tuple    )							-> List[Any]:
            import faiss

            _lowerCAmelCase					=				FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )

            # add vectors
            index.add_vectors(np.eye(5     ,     dtype=np.floataa    )    )
            self.assertIsNotNone(index.faiss_index    )
            self.assertEqual(index.faiss_index.ntotal     ,     5    )
            index.add_vectors(np.zeros((5, 5)     ,     dtype=np.floataa    )    )
            self.assertEqual(index.faiss_index.ntotal     ,     10    )

            # single query
            _lowerCAmelCase					=				np.zeros(5     ,     dtype=np.floataa    )
            _lowerCAmelCase					=				1
            _lowerCAmelCase	,				_lowerCAmelCase					=				index.search(__snake_case    )
            self.assertRaises(__snake_case     ,     index.search     ,     query.reshape(-1     ,     1    )    )
            self.assertGreater(scores[0]     ,     0    )
            self.assertEqual(indices[0]     ,     1    )

            # batched queries
            _lowerCAmelCase					=				np.eye(5     ,     dtype=np.floataa    )[::-1]
            _lowerCAmelCase	,				_lowerCAmelCase					=				index.search_batch(__snake_case    )
            self.assertRaises(__snake_case     ,     index.search_batch     ,     queries[0]    )
            _lowerCAmelCase					=				[scores[0] for scores in total_scores]
            _lowerCAmelCase					=				[indices[0] for indices in total_indices]
            self.assertGreater(np.min(__snake_case    )     ,     0    )
            self.assertListEqual([4, 3, 2, 1, 0]     ,     __snake_case    )




      def 							lowercase__       (     self     :					str    )							-> str:
            import faiss

            _lowerCAmelCase					=				FaissIndex(string_factory="""Flat"""    )
            index.add_vectors(np.eye(5     ,     dtype=np.floataa    )    )
            self.assertIsInstance(index.faiss_index     ,     faiss.IndexFlat    )
            _lowerCAmelCase					=				FaissIndex(string_factory="""LSH"""    )
            index.add_vectors(np.eye(5     ,     dtype=np.floataa    )    )
            self.assertIsInstance(index.faiss_index     ,     faiss.IndexLSH    )
            with self.assertRaises(__snake_case    ):
                  _lowerCAmelCase					=				FaissIndex(string_factory="""Flat"""     ,     custom_index=faiss.IndexFlat(5    )    )




      def 							lowercase__       (     self     :					Tuple    )							-> Union[str, Any]:
            import faiss

            _lowerCAmelCase					=				faiss.IndexFlat(5    )
            _lowerCAmelCase					=				FaissIndex(custom_index=__snake_case    )
            index.add_vectors(np.eye(5     ,     dtype=np.floataa    )    )
            self.assertIsInstance(index.faiss_index     ,     faiss.IndexFlat    )




      def 							lowercase__       (     self     :					str    )							-> str:
            import faiss

            _lowerCAmelCase					=				FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )
            index.add_vectors(np.eye(5     ,     dtype=np.floataa    )    )

            # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
            # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
            # see https://bugs.python.org/issue14243 and
            # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
            with tempfile.NamedTemporaryFile(delete=__snake_case    ) as tmp_file:
                  index.save(tmp_file.name    )
                  _lowerCAmelCase					=				FaissIndex.load(tmp_file.name    )
            os.unlink(tmp_file.name    )

            _lowerCAmelCase					=				np.zeros(5     ,     dtype=np.floataa    )
            _lowerCAmelCase					=				1
            _lowerCAmelCase	,				_lowerCAmelCase					=				index.search(__snake_case    )
            self.assertGreater(scores[0]     ,     0    )
            self.assertEqual(indices[0]     ,     1    )




@require_faiss
def   UpperCamelCase__ (						lowerCAmelCase    ):





      """simple docstring"""


      import faiss

      _lowerCAmelCase					=				FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT    )
      index.add_vectors(np.eye(5				,      dtype=np.floataa    )    )

      _lowerCAmelCase					=				"""index.faiss"""
      _lowerCAmelCase					=				f"mock://{index_name}"
      index.save(lowerCAmelCase				,      storage_options=mockfs.storage_options    )
      _lowerCAmelCase					=				FaissIndex.load(lowerCAmelCase				,      storage_options=mockfs.storage_options    )

      _lowerCAmelCase					=				np.zeros(5				,      dtype=np.floataa    )
      _lowerCAmelCase					=				1
      _lowerCAmelCase	,				_lowerCAmelCase					=				index.search(lowerCAmelCase    )
      assert scores[0] > 0
      assert indices[0] == 1





@require_elasticsearch
class 					UpperCAmelCase	(     snake_case_  ):




      def 							lowercase__       (     self     :					str    )							-> Tuple:
            from elasticsearch import Elasticsearch

            with patch("""elasticsearch.Elasticsearch.search"""    ) as mocked_search, patch(
                """elasticsearch.client.IndicesClient.create"""    ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""    ) as mocked_bulk:
                  _lowerCAmelCase					=				Elasticsearch()
                  _lowerCAmelCase					=				{"""acknowledged""": True}
                  _lowerCAmelCase					=				ElasticSearchIndex(es_client=__snake_case    )
                  mocked_bulk.return_value([(True, None)] * 3    )
                  index.add_documents(["""foo""", """bar""", """foobar"""]    )

                  # single query
                  _lowerCAmelCase					=				"""foo"""
                  _lowerCAmelCase					=				{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
                  _lowerCAmelCase	,				_lowerCAmelCase					=				index.search(__snake_case    )
                  self.assertEqual(scores[0]     ,     1    )
                  self.assertEqual(indices[0]     ,     0    )

                  # single query with timeout
                  _lowerCAmelCase					=				"""foo"""
                  _lowerCAmelCase					=				{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
                  _lowerCAmelCase	,				_lowerCAmelCase					=				index.search(__snake_case     ,     request_timeout=30    )
                  self.assertEqual(scores[0]     ,     1    )
                  self.assertEqual(indices[0]     ,     0    )

                  # batched queries
                  _lowerCAmelCase					=				["""foo""", """bar""", """foobar"""]
                  _lowerCAmelCase					=				{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
                  _lowerCAmelCase	,				_lowerCAmelCase					=				index.search_batch(__snake_case    )
                  _lowerCAmelCase					=				[scores[0] for scores in total_scores]
                  _lowerCAmelCase					=				[indices[0] for indices in total_indices]
                  self.assertGreater(np.min(__snake_case    )     ,     0    )
                  self.assertListEqual([1, 1, 1]     ,     __snake_case    )

                  # batched queries with timeout
                  _lowerCAmelCase					=				["""foo""", """bar""", """foobar"""]
                  _lowerCAmelCase					=				{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
                  _lowerCAmelCase	,				_lowerCAmelCase					=				index.search_batch(__snake_case     ,     request_timeout=30    )
                  _lowerCAmelCase					=				[scores[0] for scores in total_scores]
                  _lowerCAmelCase					=				[indices[0] for indices in total_indices]
                  self.assertGreater(np.min(__snake_case    )     ,     0    )
                  self.assertListEqual([1, 1, 1]     ,     __snake_case    )













'''simple docstring'''






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
A__							:  List[str]    =logging.getLogger(__name__)
def   UpperCamelCase__ (						):





      """simple docstring"""


      _lowerCAmelCase					=				argparse.ArgumentParser(
          description="""Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."""    )
      parser.add_argument("""--file_path"""				,      type=lowerCAmelCase				,      default="""data/dump.txt"""				,      help="""The path to the data."""    )
      parser.add_argument("""--tokenizer_type"""				,      type=lowerCAmelCase				,      default="""bert"""				,      choices=["""bert""", """roberta""", """gpt2"""]    )
      parser.add_argument("""--tokenizer_name"""				,      type=lowerCAmelCase				,      default="""bert-base-uncased"""				,      help="""The tokenizer to use."""    )
      parser.add_argument("""--dump_file"""				,      type=lowerCAmelCase				,      default="""data/dump"""				,      help="""The dump file prefix."""    )
      _lowerCAmelCase					=				parser.parse_args()

      logger.info(f"Loading Tokenizer ({args.tokenizer_name})"    )
      if args.tokenizer_type == "bert":
            _lowerCAmelCase					=				BertTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""cls_token"""]  # `[CLS]`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""sep_token"""]  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            _lowerCAmelCase					=				RobertaTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""cls_token"""]  # `<s>`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""sep_token"""]  # `</s>`
      elif args.tokenizer_type == "gpt2":
            _lowerCAmelCase					=				GPTaTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""bos_token"""]  # `<|endoftext|>`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""eos_token"""]  # `<|endoftext|>`

      logger.info(f"Loading text from {args.file_path}"    )
      with open(args.file_path				,      """r"""				,      encoding="""utf8"""    ) as fp:
            _lowerCAmelCase					=				fp.readlines()

      logger.info("""Start encoding"""    )
      logger.info(f"{len(lowerCAmelCase    )} examples to process."    )

      _lowerCAmelCase					=				[]
      _lowerCAmelCase					=				0
      _lowerCAmelCase					=				1_00_00
      _lowerCAmelCase					=				time.time()
      for text in data:
            _lowerCAmelCase					=				f"{bos} {text.strip()} {sep}"
            _lowerCAmelCase					=				tokenizer.encode(lowerCAmelCase				,      add_special_tokens=lowerCAmelCase    )
            rslt.append(lowerCAmelCase    )

            iter += 1
            if iter % interval == 0:
                  _lowerCAmelCase					=				time.time()
                  logger.info(f"{iter} examples processed. - {(end-start):.2f}s/{interval}expl"    )
                  _lowerCAmelCase					=				time.time()
      logger.info("""Finished binarization"""    )
      logger.info(f"{len(lowerCAmelCase    )} examples processed."    )

      _lowerCAmelCase					=				f"{args.dump_file}.{args.tokenizer_name}.pickle"
      _lowerCAmelCase					=				tokenizer.vocab_size
      if vocab_size < (1 << 16):
            _lowerCAmelCase					=				[np.uintaa(lowerCAmelCase    ) for d in rslt]
      else:
            _lowerCAmelCase					=				[np.intaa(lowerCAmelCase    ) for d in rslt]
      random.shuffle(rslt_    )
      logger.info(f"Dump to {dp_file}"    )
      with open(lowerCAmelCase				,      """wb"""    ) as handle:
            pickle.dump(rslt_				,      lowerCAmelCase				,      protocol=pickle.HIGHEST_PROTOCOL    )


if __name__ == "__main__":
      main()






