



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_      :      List[str]    =     get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
@require_tokenizers
class    __UpperCamelCase			(       _lowercase	,	unittest.TestCase				):

 """simple docstring"""



 _lowercase   :    str      						=					XGLMTokenizer
 _lowercase   :    Tuple      						=					XGLMTokenizerFast
 _lowercase   :    str      						=					True
 _lowercase   :    List[Any]      						=					True

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  super().setUp()

  # We have a SentencePiece fixture for testing
  a__												= XGLMTokenizer(SCREAMING_SNAKE_CASE		,				keep_accents=SCREAMING_SNAKE_CASE      )
  tokenizer.save_pretrained(self.tmpdirname      )

 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  a__												= '''<pad>'''
  a__												= 1

  self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE      )		,				SCREAMING_SNAKE_CASE      )
  self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE      )		,				SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			str:
  a__												= list(self.get_tokenizer().get_vocab().keys()      )

  self.assertEqual(vocab_keys[0]		,				'''<s>'''      )
  self.assertEqual(vocab_keys[1]		,				'''<pad>'''      )
  self.assertEqual(len(SCREAMING_SNAKE_CASE      )		,				1_0_0_8      )

 def 						_UpperCAmelCase       (	self      ) ->			int:
  self.assertEqual(self.get_tokenizer().vocab_size		,				1_0_0_8      )

 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  a__												= XGLMTokenizer(SCREAMING_SNAKE_CASE		,				keep_accents=SCREAMING_SNAKE_CASE      )

  a__												= tokenizer.tokenize('''This is a test'''      )
  self.assertListEqual(SCREAMING_SNAKE_CASE		,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

  self.assertListEqual(
      tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE      )		,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]		,				)

  a__												= tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
  self.assertListEqual(
      SCREAMING_SNAKE_CASE		,				[
          SPIECE_UNDERLINE + '''I''',
          SPIECE_UNDERLINE + '''was''',
          SPIECE_UNDERLINE + '''b''',
          '''or''',
          '''n''',
          SPIECE_UNDERLINE + '''in''',
          SPIECE_UNDERLINE + '''''',
          '''9''',
          '''2''',
          '''0''',
          '''0''',
          '''0''',
          ''',''',
          SPIECE_UNDERLINE + '''and''',
          SPIECE_UNDERLINE + '''this''',
          SPIECE_UNDERLINE + '''is''',
          SPIECE_UNDERLINE + '''f''',
          '''al''',
          '''s''',
          '''é''',
          '''.''',
      ]		,				)
  a__												= tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE      )
  self.assertListEqual(
      SCREAMING_SNAKE_CASE		,				[
          value + tokenizer.fairseq_offset
          for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
      ]		,				)

  a__												= tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE      )
  self.assertListEqual(
      SCREAMING_SNAKE_CASE		,				[
          SPIECE_UNDERLINE + '''I''',
          SPIECE_UNDERLINE + '''was''',
          SPIECE_UNDERLINE + '''b''',
          '''or''',
          '''n''',
          SPIECE_UNDERLINE + '''in''',
          SPIECE_UNDERLINE + '''''',
          '''<unk>''',
          '''2''',
          '''0''',
          '''0''',
          '''0''',
          ''',''',
          SPIECE_UNDERLINE + '''and''',
          SPIECE_UNDERLINE + '''this''',
          SPIECE_UNDERLINE + '''is''',
          SPIECE_UNDERLINE + '''f''',
          '''al''',
          '''s''',
          '''<unk>''',
          '''.''',
      ]		,				)

 @cached_property
 def 						_UpperCAmelCase       (	self      ) ->			Any:
  return XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''      )

 def 						_UpperCAmelCase       (	self      ) ->			Tuple:
  with tempfile.NamedTemporaryFile() as f:
   shutil.copyfile(SCREAMING_SNAKE_CASE		,				f.name      )
   a__												= XGLMTokenizer(f.name		,				keep_accents=SCREAMING_SNAKE_CASE      )
   a__												= pickle.dumps(SCREAMING_SNAKE_CASE      )
  pickle.loads(SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  if not self.test_rust_tokenizer:
   return

  a__												= self.get_tokenizer()
  a__												= self.get_rust_tokenizer()

  a__												= '''I was born in 92000, and this is falsé.'''

  a__												= tokenizer.tokenize(SCREAMING_SNAKE_CASE      )
  a__												= rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE      )
  self.assertListEqual(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      )

  a__												= tokenizer.encode(SCREAMING_SNAKE_CASE		,				add_special_tokens=SCREAMING_SNAKE_CASE      )
  a__												= rust_tokenizer.encode(SCREAMING_SNAKE_CASE		,				add_special_tokens=SCREAMING_SNAKE_CASE      )
  self.assertListEqual(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      )

  a__												= self.get_rust_tokenizer()
  a__												= tokenizer.encode(SCREAMING_SNAKE_CASE      )
  a__												= rust_tokenizer.encode(SCREAMING_SNAKE_CASE      )
  self.assertListEqual(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      )

 @slow
 def 						_UpperCAmelCase       (	self      ) ->			str:
  a__												= '''Hello World!'''
  a__												= [2, 3_1_2_2_7, 4_4_4_7, 3_5]

  self.assertListEqual(SCREAMING_SNAKE_CASE		,				self.big_tokenizer.encode(SCREAMING_SNAKE_CASE      )      )

 @slow
 def 						_UpperCAmelCase       (	self      ) ->			str:
  a__												= (
      '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
      ''' add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth'''
  )
  # fmt: off
  a__												= [2, 1_0_1_8, 6_7, 1_1, 1_9_8_8, 2_6_1_7, 5_6_3_1, 2_7_8, 1_1, 3_4_0_7, 4_8, 7_1_6_3_0, 2_8_0_8_5, 4, 3_2_3_4, 1_5_7, 1_3, 6, 5, 6, 4, 3_5_2_6, 7_6_8, 1_5, 6_5_9, 5_7, 2_9_8, 3_9_8_3, 8_6_4, 1_2_9, 2_1, 6, 5, 1_3_6_7_5, 3_7_7, 6_5_2, 7_5_8_0, 1_0_3_4_1, 1_5_5, 2_8_1_7, 4_2_2, 1_6_6_6, 7, 1_6_7_4, 5_3, 1_1_3, 2_0_2_2_7_7, 1_7_8_9_2, 3_3, 6_0, 8_7, 4, 3_2_3_4, 1_5_7, 6_1, 2_6_6_7, 5_2_3_7_6, 1_9, 8_8, 2_3, 7_3_5]
  # fmt: on

  self.assertListEqual(SCREAMING_SNAKE_CASE		,				self.big_tokenizer.encode(SCREAMING_SNAKE_CASE      )      )



 @slow
 def 						_UpperCAmelCase       (	self      ) ->			Optional[Any]:
  # fmt: off
  a__												= {
      '''input_ids''': [[2, 1_0_8_8_2_5, 1_1_6_3, 1_5, 8_8_0_1_0, 4_7_3, 1_5_8_9_8, 1_5_7, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 2_3_8_0_2_1, 1_1_6_3, 5_3, 1_3_6_7_2, 1_8_5_7, 3_1_2, 8, 5_3_2_8_3, 1_8_2_3_9_6, 8, 1_8_5_6_6, 1_6, 3_6_7_3_3, 4_1_0_1, 8, 2_3_0, 2_4_4_0_1_7, 1_2_2_5_5_3, 7, 1_5, 1_3_2_5_9_7, 4, 2_9_3, 1_2_5_1_1, 7_6_1_0, 4, 3_4_1_4, 1_3_2_5_9_7, 9, 4, 3_2_3_6_1, 3_6_2, 4, 7_3_4, 2_8_5_1_2, 3_2_5_6_9, 1_8, 4, 3_2_3_6_1, 2_6_0_9_6, 1_4_9_8_2, 7_3, 1_8_7_1_5, 2_1_4_3_3, 2_3_5_2_6_1, 1_5, 4_9_2, 1_2_4_2_7, 1_6, 5_3, 1_8_7_1_5, 2_1_4_3_3, 6_5_4_5_4, 1_5, 2_3_6_5_9, 5_6_3, 1_6, 2_7_8, 5_9_7, 2_8_4_3, 5_9_5, 7_9_3_1, 1_8_2_3_9_6, 6_4_1_8_6, 2_2, 8_8_6, 5_9_5, 1_3_2_9_8_1, 5_3, 2_5_5_4_0, 3_4_4_9, 4_3_9_8_2, 3_9_9_0_1, 5_9_5_1, 8_7_8, 3_3_0, 4, 2_7_6_9_4, 8_0_2_6_9, 3_1_2, 5_3, 6_5_1_7, 1_1_7_8_0, 6_1_1, 2_0_4_0_8, 5], [2, 6, 1_3_2_5_9_7, 6_7, 4_2_8_9_7, 3_3, 5_9_2, 8, 1_6_3_7_2_9, 2_5_5_4_0, 3_6_1, 1_3_6_9_9_7, 1_0_9_5_1_4, 1_7_3_2_3_0, 7, 5_0_1, 6_0, 1_0_2_9_1_3, 1_9_6, 5_6_3_1, 2_3_5, 6_3_2_4_3, 4_7_3, 6, 2_3_1_7_5_7, 7_4, 5_2_7_7, 7_9_0_5, 5_3, 3_0_9_5, 3_7_3_1_7, 2_2, 4_5_4, 1_8_3_8_7_4, 5], [2, 2_6_8, 3_1_2_9_8, 4_6_5_3_0, 6, 1_3_2_9_3_5, 4_3_8_3_1, 7, 5_9_7, 3_2, 2_4, 3_6_8_8, 9_8_6_5, 5]],
      '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
  }  # noqa: E501
  # fmt: on

  self.tokenizer_integration_test_util(
      expected_encoding=SCREAMING_SNAKE_CASE		,				model_name='''facebook/xglm-564M'''		,				padding=SCREAMING_SNAKE_CASE		,				)










import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


a_      :      int    =     '\\n    Text data.\n    Second line of data.'

a_      :      Optional[Any]    =     'file'






@pytest.fixture(scope='''session'''       )
def     __a					(		__UpperCAmelCase       ):
 a__												= tmp_path_factory.mktemp('''data'''       ) / (FILE_PATH + '''.zstd''')
 a__												= bytes(__UpperCAmelCase							,  '''utf-8'''       )
 with zstd.open(__UpperCAmelCase							,  '''wb'''       ) as f:
  f.write(__UpperCAmelCase       )
 return path






@pytest.fixture
def     __a					(		__UpperCAmelCase       ):
 with open(os.path.join(tmpfs.local_root_dir							,  __UpperCAmelCase       )							,  '''w'''       ) as f:
  f.write(__UpperCAmelCase       )
 return FILE_PATH






@pytest.mark.parametrize('''compression_format'''							,  ['''gzip''', '''xz''', '''zstd''']       )
def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
 a__												= {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
 a__												= input_paths[compression_format]
 a__												= tmp_path / '''cache'''
 a__												= DownloadConfig(cache_dir=__UpperCAmelCase							,  extract_compressed_file=__UpperCAmelCase       )
 a__												= cached_path(__UpperCAmelCase							,  download_config=__UpperCAmelCase       )
 with open(__UpperCAmelCase       ) as f:
  a__												= f.read()
 with open(__UpperCAmelCase       ) as f:
  a__												= f.read()
 assert extracted_file_content == expected_file_content






@pytest.mark.parametrize('''default_extracted'''							,  [True, False]       )
@pytest.mark.parametrize('''default_cache_dir'''							,  [True, False]       )
def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
 a__												= '''custom_cache'''
 a__												= '''custom_extracted_dir'''
 a__												= tmp_path / '''custom_extracted_path'''
 if default_extracted:
  a__												= ('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
 else:
  monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''							,  __UpperCAmelCase       )
  monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''							,  str(__UpperCAmelCase       )       )
  a__												= custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

 a__												= xz_file
 a__												= (
     DownloadConfig(extract_compressed_file=__UpperCAmelCase       )
     if default_cache_dir
     else DownloadConfig(cache_dir=tmp_path / custom_cache_dir							,  extract_compressed_file=__UpperCAmelCase       )
 )
 a__												= cached_path(__UpperCAmelCase							,  download_config=__UpperCAmelCase       )
 assert Path(__UpperCAmelCase       ).parent.parts[-2:] == expected






def     __a					(		__UpperCAmelCase       ):
 # absolute path
 a__												= str(Path(__UpperCAmelCase       ).resolve()       )
 assert cached_path(__UpperCAmelCase       ) == text_file
 # relative path
 a__												= str(Path(__UpperCAmelCase       ).resolve().relative_to(Path(os.getcwd()       )       )       )
 assert cached_path(__UpperCAmelCase       ) == text_file






def     __a					(		__UpperCAmelCase       ):
 # absolute path
 a__												= str(tmp_path.resolve() / '''__missing_file__.txt'''       )
 with pytest.raises(__UpperCAmelCase       ):
  cached_path(__UpperCAmelCase       )
 # relative path
 a__												= '''./__missing_file__.txt'''
 with pytest.raises(__UpperCAmelCase       ):
  cached_path(__UpperCAmelCase       )






def     __a					(		__UpperCAmelCase       ):
 a__												= get_from_cache(f"tmp://{tmpfs_file}"       )
 with open(__UpperCAmelCase       ) as f:
  a__												= f.read()
 assert output_file_content == FILE_CONTENT






@patch('''datasets.config.HF_DATASETS_OFFLINE'''							,  __UpperCAmelCase       )
def     __a					(		):
 with pytest.raises(__UpperCAmelCase       ):
  cached_path('''https://huggingface.co'''       )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''							,  __UpperCAmelCase       )
def     __a					(		__UpperCAmelCase       ):
 a__												= tmp_path_factory.mktemp('''data'''       ) / '''file.html'''
 with pytest.raises(__UpperCAmelCase       ):
  http_get('''https://huggingface.co'''							,  temp_file=__UpperCAmelCase       )
 with pytest.raises(__UpperCAmelCase       ):
  http_head('''https://huggingface.co'''       )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''							,  __UpperCAmelCase       )
def     __a					(		__UpperCAmelCase       ):
 a__												= tmp_path_factory.mktemp('''data'''       ) / '''file.html'''
 with pytest.raises(__UpperCAmelCase       ):
  ftp_get('''ftp://huggingface.co'''							,  temp_file=__UpperCAmelCase       )
 with pytest.raises(__UpperCAmelCase       ):
  ftp_head('''ftp://huggingface.co'''       )






@patch('''datasets.config.HF_DATASETS_OFFLINE'''							,  __UpperCAmelCase       )
def     __a					(		__UpperCAmelCase       ):
 a__												= tmp_path_factory.mktemp('''data'''       ) / '''file.html'''
 with pytest.raises(__UpperCAmelCase       ):
  fsspec_get('''s3://huggingface.co'''							,  temp_file=__UpperCAmelCase       )
 with pytest.raises(__UpperCAmelCase       ):
  fsspec_head('''s3://huggingface.co'''       )





