




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
							from test_module.custom_tokenization_fast import CustomTokenizerFast




class 		__A      (			unittest.TestCase			):

						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	Any								=mock.Mock()
										__magic_name__	:	Optional[Any]								=5_00
										__magic_name__	:	Optional[int]								={}
										__magic_name__	:	Tuple								=HTTPError
										__magic_name__	:	Any								={}

										# Download this model to make sure it's in the cache.
										__magic_name__	:	Optional[int]								=BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""" )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("""requests.Session.request"""    , return_value=__snake_case ) as mock_head:
														__magic_name__	:	Tuple								=BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""" )
														# This check we did call the fake head request
														mock_head.assert_called()

						@require_tokenizers
						def     A__						(		self   :Optional[int] ):
										'''simple docstring'''

										__magic_name__	:	List[str]								=mock.Mock()
										__magic_name__	:	Union[str, Any]								=5_00
										__magic_name__	:	Dict								={}
										__magic_name__	:	List[str]								=HTTPError
										__magic_name__	:	Tuple								={}

										# Download this model to make sure it's in the cache.
										__magic_name__	:	str								=GPTaTokenizerFast.from_pretrained("""gpt2""" )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("""requests.Session.request"""    , return_value=__snake_case ) as mock_head:
														__magic_name__	:	int								=GPTaTokenizerFast.from_pretrained("""gpt2""" )
														# This check we did call the fake head request
														mock_head.assert_called()

						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										try:
														__magic_name__	:	List[str]								=tempfile.mktemp()
														with open(__snake_case    , """wb""" ) as f:
																		http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""    , __snake_case )

														__magic_name__	:	Any								=AlbertTokenizer.from_pretrained(__snake_case )
										finally:
														os.remove(__snake_case )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("""tokenizer.json""" ):
														# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
														return
										try:
														with open("""tokenizer.json"""    , """wb""" ) as f:
																		http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""    , __snake_case )
														__magic_name__	:	List[str]								=AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2""" )
														# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
														self.assertEqual(tokenizer.vocab_size    , 10_00 )
														# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
														os.remove("""tokenizer.json""" )


						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	List[str]								=AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""" )




@is_staging_test
class 		__A      (			unittest.TestCase			):
						UpperCamelCase     						=					["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

						@classmethod
						def     A__						(		cls   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	str								=TOKEN
										HfFolder.save_token(__snake_case )

						@classmethod
						def     A__						(		cls   :List[Any] ):
										'''simple docstring'''

										try:
														delete_repo(token=cls._token    , repo_id="""test-tokenizer""" )
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token    , repo_id="""valid_org/test-tokenizer-org""" )
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token    , repo_id="""test-dynamic-tokenizer""" )
										except HTTPError:
														pass

						def     A__						(		self   :Any ):
										'''simple docstring'''

										with tempfile.TemporaryDirectory() as tmp_dir:
														__magic_name__	:	List[Any]								=os.path.join(__snake_case    , """vocab.txt""" )
														with open(__snake_case    , """w"""    , encoding="""utf-8""" ) as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														__magic_name__	:	Tuple								=BertTokenizer(__snake_case )

										tokenizer.push_to_hub("""test-tokenizer"""    , use_auth_token=self._token )
										__magic_name__	:	Optional[int]								=BertTokenizer.from_pretrained(f"{USER}/test-tokenizer" )
										self.assertDictEqual(new_tokenizer.vocab    , tokenizer.vocab )

										# Reset repo
										delete_repo(token=self._token    , repo_id="""test-tokenizer""" )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__snake_case    , repo_id="""test-tokenizer"""    , push_to_hub=__snake_case    , use_auth_token=self._token )

										__magic_name__	:	Optional[Any]								=BertTokenizer.from_pretrained(f"{USER}/test-tokenizer" )
										self.assertDictEqual(new_tokenizer.vocab    , tokenizer.vocab )

						def     A__						(		self   :Optional[int] ):
										'''simple docstring'''

										with tempfile.TemporaryDirectory() as tmp_dir:
														__magic_name__	:	int								=os.path.join(__snake_case    , """vocab.txt""" )
														with open(__snake_case    , """w"""    , encoding="""utf-8""" ) as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														__magic_name__	:	Dict								=BertTokenizer(__snake_case )

										tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""    , use_auth_token=self._token )
										__magic_name__	:	int								=BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""" )
										self.assertDictEqual(new_tokenizer.vocab    , tokenizer.vocab )

										# Reset repo
										delete_repo(token=self._token    , repo_id="""valid_org/test-tokenizer-org""" )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(
														    __snake_case    , repo_id="""valid_org/test-tokenizer-org"""    , push_to_hub=__snake_case    , use_auth_token=self._token )

										__magic_name__	:	Optional[int]								=BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""" )
										self.assertDictEqual(new_tokenizer.vocab    , tokenizer.vocab )


						@require_tokenizers
						def     A__						(		self   :List[Any] ):
										'''simple docstring'''

										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
														__magic_name__	:	Optional[Any]								=os.path.join(__snake_case    , """vocab.txt""" )
														with open(__snake_case    , """w"""    , encoding="""utf-8""" ) as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														__magic_name__	:	List[str]								=CustomTokenizer(__snake_case )

										# No fast custom tokenizer
										tokenizer.push_to_hub("""test-dynamic-tokenizer"""    , use_auth_token=self._token )

										__magic_name__	:	int								=AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"    , trust_remote_code=__snake_case )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__    , """CustomTokenizer""" )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
														__magic_name__	:	Optional[Any]								=os.path.join(__snake_case    , """vocab.txt""" )
														with open(__snake_case    , """w"""    , encoding="""utf-8""" ) as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )

														__magic_name__	:	Tuple								=BertTokenizerFast.from_pretrained(__snake_case )
														bert_tokenizer.save_pretrained(__snake_case )
														__magic_name__	:	Optional[Any]								=CustomTokenizerFast.from_pretrained(__snake_case )

										tokenizer.push_to_hub("""test-dynamic-tokenizer"""    , use_auth_token=self._token )

										__magic_name__	:	str								=AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"    , trust_remote_code=__snake_case )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__    , """CustomTokenizerFast""" )
										__magic_name__	:	List[Any]								=AutoTokenizer.from_pretrained(
										    f"{USER}/test-dynamic-tokenizer"    , use_fast=__snake_case    , trust_remote_code=__snake_case )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__    , """CustomTokenizer""" )




class 		__A      (			unittest.TestCase			):

						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	List[str]								=Trie()
										trie.add("""Hello 友達""" )
										self.assertEqual(trie.data    , {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}} )
										trie.add("""Hello""" )
										trie.data
										self.assertEqual(trie.data    , {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}} )

						def     A__						(		self   :str ):
										'''simple docstring'''

										__magic_name__	:	Any								=Trie()
										self.assertEqual(trie.split("""[CLS] This is a extra_id_100""" )    , ["""[CLS] This is a extra_id_100"""] )
										trie.add("""[CLS]""" )
										trie.add("""extra_id_1""" )
										trie.add("""extra_id_100""" )
										self.assertEqual(trie.split("""[CLS] This is a extra_id_100""" )    , ["""[CLS]""", """ This is a """, """extra_id_100"""] )

						def     A__						(		self   :List[Any] ):
										'''simple docstring'''

										__magic_name__	:	Optional[Any]								=Trie()
										trie.add("""A""" )
										self.assertEqual(trie.split("""ABC""" )    , ["""A""", """BC"""] )
										self.assertEqual(trie.split("""BCA""" )    , ["""BC""", """A"""] )

						def     A__						(		self   :Optional[Any] ):
										'''simple docstring'''

										__magic_name__	:	Union[str, Any]								=Trie()
										trie.add("""TOKEN]""" )
										trie.add("""[SPECIAL_TOKEN]""" )
										self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""" )    , ["""This is something """, """[SPECIAL_TOKEN]"""] )

						def     A__						(		self   :Union[str, Any] ):
										'''simple docstring'''

										__magic_name__	:	Tuple								=Trie()
										trie.add("""A""" )
										trie.add("""P""" )
										trie.add("""[SPECIAL_TOKEN]""" )
										self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""" )    , ["""This is something """, """[SPECIAL_TOKEN]"""] )

						def     A__						(		self   :Union[str, Any] ):
										'''simple docstring'''

										__magic_name__	:	Dict								=Trie()
										trie.add("""AB""" )
										trie.add("""B""" )
										trie.add("""C""" )
										self.assertEqual(trie.split("""ABC""" )    , ["""AB""", """C"""] )

						def     A__						(		self   :Union[str, Any] ):
										'''simple docstring'''

										__magic_name__	:	Optional[Any]								=Trie()
										trie.add("""ABC""" )
										trie.add("""B""" )
										trie.add("""CD""" )
										self.assertEqual(trie.split("""ABCD""" )    , ["""ABC""", """D"""] )


						def     A__						(		self   :Optional[Any] ):
										'''simple docstring'''

										__magic_name__	:	Union[str, Any]								=Trie()
										__magic_name__	:	Union[str, Any]								=trie.cut_text("""ABC"""    , [0, 0, 2, 1, 2, 3] )
										self.assertEqual(__snake_case    , ["""AB""", """C"""] )





# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCAmelCase_		:				str									=     "3"

print("Python version:", sys.version)
print("transformers version:", transformers.__version__)

try:
							import torch

							print("Torch version:", torch.__version__)
							print("Cuda available:", torch.cuda.is_available())
							print("Cuda version:", torch.version.cuda)
							print("CuDNN version:", torch.backends.cudnn.version())
							print("Number of GPUs available:", torch.cuda.device_count())
							print("NCCL version:", torch.cuda.nccl.version())
except ImportError:
							print("Torch version:", None)

try:
							import deepspeed

							print("DeepSpeed version:", deepspeed.__version__)
except ImportError:
							print("DeepSpeed version:", None)

try:
							import tensorflow as tf

							print("TensorFlow version:", tf.__version__)
							print("TF GPUs available:", bool(tf.config.list_physical_devices("GPU")))
							print("Number of TF GPUs available:", len(tf.config.list_physical_devices("GPU")))
except ImportError:
							print("TensorFlow version:", None)




