





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


lowercase_          =		{"""configuration_speech_encoder_decoder""": ["""SpeechEncoderDecoderConfig"""]}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase_          =		["""SpeechEncoderDecoderModel"""]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase_          =		["""FlaxSpeechEncoderDecoderModel"""]

if TYPE_CHECKING:
      from .configuration_speech_encoder_decoder import SpeechEncoderDecoderConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_speech_encoder_decoder import SpeechEncoderDecoderModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_speech_encoder_decoder import FlaxSpeechEncoderDecoderModel

else:
      import sys

      lowercase_          =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase_          =		logging.get_logger(__name__)


lowercase_          =		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

lowercase_          =		{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

lowercase_          =		{
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def       __UpperCamelCase  ()    ->       Union[str, Any]:
  lowercase__           =  (
      list(range(ord('!'		)		,						ord('~'		) + 1		)		) + list(range(ord('¡'		)		,						ord('¬'		) + 1		)		) + list(range(ord('®'		)		,						ord('ÿ'		) + 1		)		)
  )
  lowercase__           =  bs[:]
  lowercase__           =  0
  for b in range(2**8		):
    if b not in bs:
      bs.append(_SCREAMING_SNAKE_CASE		)
      cs.append(2**8 + n		)
      n += 1
  lowercase__           =  [chr(_SCREAMING_SNAKE_CASE		) for n in cs]
  return dict(zip(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		)		)




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Union[str, Any]:
  lowercase__           =  set()
  lowercase__           =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    lowercase__           =  char
  return pairs


class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):
   _UpperCamelCase   :		str			 =	VOCAB_FILES_NAMES
   _UpperCamelCase   :		Optional[int]			 =	PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase   :		Union[str, Any]			 =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase   :		Any			 =	['input_ids', 'attention_mask']


   def __init__(   self      :    Dict      ,					a      :    Union[str, Any]      ,					a      :    Optional[Any]      ,					a      :    List[str]="replace"      ,					a      :    Optional[int]="<s>"      ,					a      :    List[str]="</s>"      ,					a      :    List[Any]="</s>"      ,					a      :    Union[str, Any]="<s>"      ,					a      :    Any="<unk>"      ,					a      :    Optional[int]="<pad>"      ,					a      :    Optional[Any]="<mask>"      ,					a      :    Tuple=False      ,					**a      :    List[Any]      ,					)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else bos_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else eos_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else sep_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else cls_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else unk_token
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else mask_token

     super().__init__(
         errors=a      ,					bos_token=a      ,					eos_token=a      ,					unk_token=a      ,					sep_token=a      ,					cls_token=a      ,					pad_token=a      ,					mask_token=a      ,					add_prefix_space=a      ,					**a      ,					)

     with open(a      ,					encoding='utf-8'	) as vocab_handle:
       lowercase__           =  json.load(a	)
     lowercase__           =  {v: k for k, v in self.encoder.items()}
     lowercase__           =  errors  # how to handle errors in decoding
     lowercase__           =  bytes_to_unicode()
     lowercase__           =  {v: k for k, v in self.byte_encoder.items()}
     with open(a      ,					encoding='utf-8'	) as merges_handle:
       lowercase__           =  merges_handle.read().split('\n'	)[1:-1]
     lowercase__           =  [tuple(merge.split()	) for merge in bpe_merges]
     lowercase__           =  dict(zip(a      ,					range(len(a	)	)	)	)
     lowercase__           =  {}
     lowercase__           =  add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     lowercase__           =  re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Any:





     """simple docstring"""







     return len(self.encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       Optional[int]:





     """simple docstring"""







     return dict(self.encoder      ,					**self.added_tokens_encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    List[Any]	)->       Dict:





     """simple docstring"""







     if token in self.cache:
       return self.cache[token]
     lowercase__           =  tuple(a	)
     lowercase__           =  get_pairs(a	)

     if not pairs:
       return token

     while True:
       lowercase__           =  min(a      ,					key=lambda a	: self.bpe_ranks.get(a      ,					float('inf'	)	)	)
       if bigram not in self.bpe_ranks:
         break
       lowercase__      ,   lowercase__           =  bigram
       lowercase__           =  []
       lowercase__           =  0
       while i < len(a	):
         try:
           lowercase__           =  word.index(a      ,					a	)
         except ValueError:
           new_word.extend(word[i:]	)
           break
         else:
           new_word.extend(word[i:j]	)
           lowercase__           =  j

         if word[i] == first and i < len(a	) - 1 and word[i + 1] == second:
           new_word.append(first + second	)
           i += 2
         else:
           new_word.append(word[i]	)
           i += 1
       lowercase__           =  tuple(a	)
       lowercase__           =  new_word
       if len(a	) == 1:
         break
       else:
         lowercase__           =  get_pairs(a	)
     lowercase__           =  ' '.join(a	)
     lowercase__           =  word
     return word


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    str	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  []
     for token in re.findall(self.pat      ,					a	):
       lowercase__           =  ''.join(
           self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a	).split(' '	)	)
     return bpe_tokens


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    List[Any]	)->       Optional[int]:





     """simple docstring"""







     return self.encoder.get(a      ,					self.encoder.get(self.unk_token	)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    Optional[Any]	)->       Union[str, Any]:





     """simple docstring"""







     return self.decoder.get(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    Optional[int]	)->       Dict:





     """simple docstring"""







     lowercase__           =  ''.join(a	)
     lowercase__           =  bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'      ,					errors=self.errors	)
     return text


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str      ,					a      :    Optional[str] = None	)->       Tuple[str]:





     """simple docstring"""







     if not os.path.isdir(a	):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
       return
     lowercase__           =  os.path.join(
         a      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
     lowercase__           =  os.path.join(
         a      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

     with open(a      ,					'w'      ,					encoding='utf-8'	) as f:
       f.write(json.dumps(self.encoder      ,					indent=2      ,					sort_keys=a      ,					ensure_ascii=a	) + '\n'	)

     lowercase__           =  0
     with open(a      ,					'w'      ,					encoding='utf-8'	) as writer:
       writer.write('#version: 0.2\n'	)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,					key=lambda a	: kv[1]	):
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
               ' Please check that the tokenizer is not corrupted!'	)
           lowercase__           =  token_index
         writer.write(' '.join(a	) + '\n'	)
         index += 1

     return vocab_file, merge_file


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     lowercase__           =  [self.cls_token_id]
     lowercase__           =  [self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None      ,					a      :    bool = False	)->       List[int]:





     """simple docstring"""







     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=a      ,					token_ids_a=a      ,					already_has_special_tokens=a	)

     if token_ids_a is None:
       return [1] + ([0] * len(a	)) + [1]
     return [1] + ([0] * len(a	)) + [1, 1] + ([0] * len(a	)) + [1]


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     lowercase__           =  [self.sep_token_id]
     lowercase__           =  [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    Dict      ,					a      :    Dict=False      ,					**a      :    Union[str, Any]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  kwargs.pop('add_prefix_space'      ,					self.add_prefix_space	)
     if (is_split_into_words or add_prefix_space) and (len(a	) > 0 and not text[0].isspace()):
       lowercase__           =  ' ' + text
     return (text, kwargs)


