



"""simple docstring"""






import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


snake_case   					=				'''\
    Text data.
    Second line of data.'''

snake_case   					=				'''file'''




@pytest.fixture(scope='''session'''	)
def 			snake_case		(		lowerCAmelCase_	)						->				Any:
  _snake_case   	=      tmp_path_factory.mktemp('''data'''	) / (FILE_PATH + '''.zstd''')
  _snake_case   	=      bytes(lowerCAmelCase_  ,						'''utf-8'''	)
  with zstd.open(lowerCAmelCase_  ,						'''wb'''	) as f:
    f.write(lowerCAmelCase_	)
  return path




@pytest.fixture
def 			snake_case		(		lowerCAmelCase_	)						->				Tuple:
  with open(os.path.join(tmpfs.local_root_dir  ,						lowerCAmelCase_	)  ,						'''w'''	) as f:
    f.write(lowerCAmelCase_	)
  return FILE_PATH




@pytest.mark.parametrize('''compression_format'''  ,						['''gzip''', '''xz''', '''zstd''']	)
def 			snake_case		(		lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_	)						->				Tuple:
  _snake_case   	=      {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
  _snake_case   	=      input_paths[compression_format]
  _snake_case   	=      tmp_path / '''cache'''
  _snake_case   	=      DownloadConfig(cache_dir=lowerCAmelCase_  ,						extract_compressed_file=lowerCAmelCase_	)
  _snake_case   	=      cached_path(lowerCAmelCase_  ,						download_config=lowerCAmelCase_	)
  with open(lowerCAmelCase_	) as f:
    _snake_case   	=      f.read()
  with open(lowerCAmelCase_	) as f:
    _snake_case   	=      f.read()
  assert extracted_file_content == expected_file_content




@pytest.mark.parametrize('''default_extracted'''  ,						[True, False]	)
@pytest.mark.parametrize('''default_cache_dir'''  ,						[True, False]	)
def 			snake_case		(		lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_	)						->				Union[str, Any]:
  _snake_case   	=      '''custom_cache'''
  _snake_case   	=      '''custom_extracted_dir'''
  _snake_case   	=      tmp_path / '''custom_extracted_path'''
  if default_extracted:
    _snake_case   	=      ('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
  else:
    monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''  ,						lowerCAmelCase_	)
    monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''  ,						str(lowerCAmelCase_	)	)
    _snake_case   	=      custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

  _snake_case   	=      xz_file
  _snake_case   	=      (
      DownloadConfig(extract_compressed_file=lowerCAmelCase_	)
      if default_cache_dir
      else DownloadConfig(cache_dir=tmp_path / custom_cache_dir  ,						extract_compressed_file=lowerCAmelCase_	)
  )
  _snake_case   	=      cached_path(lowerCAmelCase_  ,						download_config=lowerCAmelCase_	)
  assert Path(lowerCAmelCase_	).parent.parts[-2:] == expected




def 			snake_case		(		lowerCAmelCase_	)						->				List[str]:
  # absolute path
  _snake_case   	=      str(Path(lowerCAmelCase_	).resolve()	)
  assert cached_path(lowerCAmelCase_	) == text_file
  # relative path
  _snake_case   	=      str(Path(lowerCAmelCase_	).resolve().relative_to(Path(os.getcwd()	)	)	)
  assert cached_path(lowerCAmelCase_	) == text_file




def 			snake_case		(		lowerCAmelCase_	)						->				Dict:
  # absolute path
  _snake_case   	=      str(tmp_path.resolve() / '''__missing_file__.txt'''	)
  with pytest.raises(lowerCAmelCase_	):
    cached_path(lowerCAmelCase_	)
  # relative path
  _snake_case   	=      '''./__missing_file__.txt'''
  with pytest.raises(lowerCAmelCase_	):
    cached_path(lowerCAmelCase_	)




def 			snake_case		(		lowerCAmelCase_	)						->				Any:
  _snake_case   	=      get_from_cache(f"""tmp://{tmpfs_file}"""	)
  with open(lowerCAmelCase_	) as f:
    _snake_case   	=      f.read()
  assert output_file_content == FILE_CONTENT




@patch('''datasets.config.HF_DATASETS_OFFLINE'''  ,						lowerCAmelCase_	)
def 			snake_case		(		)						->				Optional[Any]:
  with pytest.raises(lowerCAmelCase_	):
    cached_path('''https://huggingface.co'''	)




@patch('''datasets.config.HF_DATASETS_OFFLINE'''  ,						lowerCAmelCase_	)
def 			snake_case		(		lowerCAmelCase_	)						->				Optional[int]:
  _snake_case   	=      tmp_path_factory.mktemp('''data'''	) / '''file.html'''
  with pytest.raises(lowerCAmelCase_	):
    http_get('''https://huggingface.co'''  ,						temp_file=lowerCAmelCase_	)
  with pytest.raises(lowerCAmelCase_	):
    http_head('''https://huggingface.co'''	)




@patch('''datasets.config.HF_DATASETS_OFFLINE'''  ,						lowerCAmelCase_	)
def 			snake_case		(		lowerCAmelCase_	)						->				int:
  _snake_case   	=      tmp_path_factory.mktemp('''data'''	) / '''file.html'''
  with pytest.raises(lowerCAmelCase_	):
    ftp_get('''ftp://huggingface.co'''  ,						temp_file=lowerCAmelCase_	)
  with pytest.raises(lowerCAmelCase_	):
    ftp_head('''ftp://huggingface.co'''	)







@patch('''datasets.config.HF_DATASETS_OFFLINE'''  ,						lowerCAmelCase_	)
def 			snake_case		(		lowerCAmelCase_	)						->				str:
  _snake_case   	=      tmp_path_factory.mktemp('''data'''	) / '''file.html'''
  with pytest.raises(lowerCAmelCase_	):
    fsspec_get('''s3://huggingface.co'''  ,						temp_file=lowerCAmelCase_	)
  with pytest.raises(lowerCAmelCase_	):
    fsspec_head('''s3://huggingface.co'''	)






"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__			=    logging.get_logger(__name__)

UpperCAmelCase__			=    {
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}


class 							a					(		lowerCAmelCase_ ):
    _snake_case  :	List[Any]  				=				'realm'


    def __init__(       self						:	List[Any] ,					__lowerCAmelCase						:	List[str]=3_0522 ,					__lowerCAmelCase						:	Tuple=768 ,					__lowerCAmelCase						:	List[Any]=128 ,					__lowerCAmelCase						:	Tuple=12 ,					__lowerCAmelCase						:	List[str]=12 ,					__lowerCAmelCase						:	Union[str, Any]=8 ,					__lowerCAmelCase						:	List[str]=3072 ,					__lowerCAmelCase						:	Union[str, Any]="gelu_new" ,					__lowerCAmelCase						:	str=0.1 ,					__lowerCAmelCase						:	Tuple=0.1 ,					__lowerCAmelCase						:	List[str]=512 ,					__lowerCAmelCase						:	Dict=2 ,					__lowerCAmelCase						:	List[Any]=0.02 ,					__lowerCAmelCase						:	Optional[int]=1e-1_2 ,					__lowerCAmelCase						:	Any=256 ,					__lowerCAmelCase						:	Union[str, Any]=10 ,					__lowerCAmelCase						:	Dict=1e-3 ,					__lowerCAmelCase						:	str=5 ,					__lowerCAmelCase						:	str=320 ,					__lowerCAmelCase						:	Tuple=1335_3718 ,					__lowerCAmelCase						:	Optional[Any]=5000 ,					__lowerCAmelCase						:	int=1 ,					__lowerCAmelCase						:	Dict=0 ,					__lowerCAmelCase						:	Any=2 ,					**__lowerCAmelCase						:	Optional[int] ,					):
      super().__init__(pad_token_id=__lowerCAmelCase ,					bos_token_id=__lowerCAmelCase ,					eos_token_id=__lowerCAmelCase ,					**__lowerCAmelCase       )

      # Common config
      _UpperCAmelCase    			=   vocab_size
      _UpperCAmelCase    			=   max_position_embeddings
      _UpperCAmelCase    			=   hidden_size
      _UpperCAmelCase    			=   retriever_proj_size
      _UpperCAmelCase    			=   num_hidden_layers
      _UpperCAmelCase    			=   num_attention_heads
      _UpperCAmelCase    			=   num_candidates
      _UpperCAmelCase    			=   intermediate_size
      _UpperCAmelCase    			=   hidden_act
      _UpperCAmelCase    			=   hidden_dropout_prob
      _UpperCAmelCase    			=   attention_probs_dropout_prob
      _UpperCAmelCase    			=   initializer_range
      _UpperCAmelCase    			=   type_vocab_size
      _UpperCAmelCase    			=   layer_norm_eps

      # Reader config
      _UpperCAmelCase    			=   span_hidden_size
      _UpperCAmelCase    			=   max_span_width
      _UpperCAmelCase    			=   reader_layer_norm_eps
      _UpperCAmelCase    			=   reader_beam_size
      _UpperCAmelCase    			=   reader_seq_len

      # Retrieval config
      _UpperCAmelCase    			=   num_block_records
      _UpperCAmelCase    			=   searcher_beam_size
