


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE						:Optional[Any]			=  {}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE						:Dict			=  ['''NllbTokenizer''']

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE						:Dict			=  ['''NllbTokenizerFast''']


if TYPE_CHECKING:
   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_nllb import NllbTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_nllb_fast import NllbTokenizerFast

else:
   import sys

   SCREAMING_SNAKE_CASE						:Optional[int]			=  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)








from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  _SCREAMING_SNAKE_CASE		     =							DistilBertTokenizer
  _SCREAMING_SNAKE_CASE		     =							DistilBertTokenizerFast
  _SCREAMING_SNAKE_CASE		     =							True



  @slow
  def        lowerCAmelCase__	(  self							:							Tuple	) ->			Optional[Any]:




     """simple docstring"""
     snake_case_            =       DistilBertTokenizer.from_pretrained("distilbert-base-uncased"	)

     snake_case_            =       tokenizer.encode("sequence builders"     ,					add_special_tokens=_lowerCAmelCase	)
     snake_case_            =       tokenizer.encode("multi-sequence build"     ,					add_special_tokens=_lowerCAmelCase	)

     snake_case_            =       tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase	)
     snake_case_            =       tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     ,					_lowerCAmelCase	)

     assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
     assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
         tokenizer.sep_token_id
     ]




