
"""simple docstring"""
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


lowercase_       :						Tuple     					=	[
    '''EAGER''',
    '''AOT_EAGER''',
    '''INDUCTOR''',
    '''NVFUSER''',
    '''AOT_NVFUSER''',
    '''AOT_CUDAGRAPHS''',
    '''OFI''',
    '''FX2TRT''',
    '''ONNXRT''',
    '''IPEX''',
]





def    _lowerCAmelCase    (     lowerCamelCase__							:							Dict,						lowerCamelCase__							:							int=None,						lowerCamelCase__							:							Any=None,						lowerCamelCase__							:							Any=None				)  ->    Tuple:
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = True
  while ask_again:
    _SCREAMING_SNAKE_CASE   :				List[str]							    = input(lowerCamelCase__				)
    try:
      if default is not None and len(lowerCamelCase__				) == 0:
        return default
      return convert_value(lowerCamelCase__				) if convert_value is not None else result
    except Exception:
      if error_message is not None:
        print(lowerCamelCase__				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							Any,						lowerCamelCase__							:							Dict=[],						lowerCamelCase__							:							Optional[int]=None,						lowerCamelCase__							:							str=0				)  ->    str:
  _SCREAMING_SNAKE_CASE   :				int							    = BulletMenu(lowerCamelCase__,						lowerCamelCase__				)
  _SCREAMING_SNAKE_CASE   :				str							    = menu.run(default_choice=lowerCamelCase__				)
  return convert_value(lowerCamelCase__				) if convert_value is not None else result





def    _lowerCAmelCase    (     lowerCamelCase__							:							Optional[int]				)  ->    Optional[int]:
  _SCREAMING_SNAKE_CASE   :				str							    = int(lowerCamelCase__				)
  return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							str				)  ->    List[Any]:
  _SCREAMING_SNAKE_CASE   :				str							    = int(lowerCamelCase__				)
  return DistributedType(["NO", "MULTI_CPU", "MULTI_XPU", "MULTI_GPU", "MULTI_NPU", "TPU"][value]				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							Optional[Any]				)  ->    List[Any]:
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = int(lowerCamelCase__				)
  return DynamoBackend(DYNAMO_BACKENDS[value]				).value





def    _lowerCAmelCase    (     lowerCamelCase__							:							int				)  ->    Dict:
  _SCREAMING_SNAKE_CASE   :				int							    = int(lowerCamelCase__				)
  return PrecisionType(["no", "fp16", "bf16", "fp8"][value]				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							List[Any]				)  ->    Any:
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = int(lowerCamelCase__				)
  return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							List[Any]				)  ->    Optional[Any]:
  return {"yes": True, "no": False}[value.lower()]





class 			UpperCamelCase      (							argparse.RawDescriptionHelpFormatter  ):
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Any							    = super()._format_usage(snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Any							    = usage.replace("<command> [<args>] "  ,     ""				)
       return usage





"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE      ,       unittest.TestCase  ):
     A__      =			None
     A__      =			BloomTokenizerFast
     A__      =			BloomTokenizerFast
     A__      =			True
     A__      =			False
     A__      =			"""tokenizer_file"""
     A__      =			{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       super().setUp()
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = BloomTokenizerFast.from_pretrained("bigscience/tokenizer"				)
       tokenizer.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       kwargs.update(self.special_tokens_map				)
       return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,     **snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = self.get_rust_tokenizer()

       _SCREAMING_SNAKE_CASE   :				Tuple							    = ["The quick brown fox</s>", "jumps over the lazy dog</s>"]
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = [[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tokenizer.batch_encode_plus(snake_case__				)["input_ids"]
       self.assertListEqual(snake_case__  ,     snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = tokenizer.batch_decode(snake_case__				)
       self.assertListEqual(snake_case__  ,     snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__=6				):




       """simple docstring"""


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
           _SCREAMING_SNAKE_CASE   :				Tuple							    = self.rust_tokenizer_class.from_pretrained(snake_case__  ,     **snake_case__				)
           # tokenizer_r.pad_token = None # Hotfixing padding = None
           # Simple input
           _SCREAMING_SNAKE_CASE   :				str							    = "This is a simple input"
           _SCREAMING_SNAKE_CASE   :				Tuple							    = ["This is a simple input 1", "This is a simple input 2"]
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = ("This is a simple input", "This is a pair")
           _SCREAMING_SNAKE_CASE   :				List[str]							    = [
               ("This is a simple input 1", "This is a simple input 2"),
               ("This is a simple pair 1", "This is a simple pair 2"),
           ]

           # Simple input tests
           try:
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode_plus(snake_case__  ,     max_length=snake_case__				)

             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
           except ValueError:
             self.fail("Bloom Tokenizer should be able to deal with padding"				)

           _SCREAMING_SNAKE_CASE   :				Tuple							    = None  # Hotfixing padding = None
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.get_rust_tokenizer()
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = load_dataset("xnli"  ,     "all_languages"  ,     split="test"  ,     streaming=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = next(iter(snake_case__				)				)["premise"]  # pick up one data
       _SCREAMING_SNAKE_CASE   :				int							    = list(sample_data.values()				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = list(map(tokenizer.encode  ,     snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				int							    = [tokenizer.decode(snake_case__  ,     clean_up_tokenization_spaces=snake_case__				) for x in output_tokens]
       self.assertListEqual(snake_case__  ,     snake_case__				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map				)  ,     1				)
       self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()				)[0]				)  ,     1				)



