

"""simple docstring"""




__UpperCAmelCase =				tuple[float, float, float]
__UpperCAmelCase =				tuple[float, float, float]
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Union[str, Any]					   =					end_pointa[0] - end_pointa[0]
 UpperCAmelCase__					:  Optional[Any]					   =					end_pointa[1] - end_pointa[1]
 UpperCAmelCase__					:  List[str]					   =					end_pointa[2] - end_pointa[2]
 return (x, y, z)
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[str]					   =					ab[1] * ac[2] - ab[2] * ac[1]  # *i
 UpperCAmelCase__					:  List[Any]					   =					(ab[0] * ac[2] - ab[2] * ac[0]) * -1  # *j
 UpperCAmelCase__					:  List[str]					   =					ab[0] * ac[1] - ab[1] * ac[0]  # *k
 return (x, y, z)
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 return tuple(round(__UpperCamelCase							,    __UpperCamelCase      ) for x in vector      ) == (0, 0, 0)




def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase = 10      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Any					   =					create_vector(__UpperCamelCase							,    __UpperCamelCase      )
 UpperCAmelCase__					:  List[Any]					   =					create_vector(__UpperCamelCase							,    __UpperCamelCase      )
 return is_zero_vector(get_ad_vectors_cross(__UpperCamelCase							,    __UpperCamelCase      )							,    __UpperCamelCase      )








"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


__UpperCAmelCase =				logging.get_logger(__name__)


__UpperCAmelCase =				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
__UpperCAmelCase =				{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

__UpperCAmelCase =				{
    'allenai/led-base-16384': 1_6384,
}
@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def   lowerCAmelCase						(					):
 '''simple docstring'''





 UpperCAmelCase__					:  Tuple					   =					(
     list(range(ord("""!"""      )							,    ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )							,    ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )							,    ord("""ÿ"""      ) + 1      )      )
 )
 UpperCAmelCase__					:  int					   =					bs[:]
 UpperCAmelCase__					:  Union[str, Any]					   =					0
 for b in range(2**8      ):
  if b not in bs:
   bs.append(__UpperCamelCase      )
   cs.append(2**8 + n      )
   n += 1
 UpperCAmelCase__					:  Tuple					   =					[chr(__UpperCamelCase      ) for n in cs]
 return dict(zip(__UpperCamelCase							,    __UpperCamelCase      )      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  int					   =					set()
 UpperCAmelCase__					:  Tuple					   =					word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)      )
  UpperCAmelCase__					:  Optional[Any]					   =					char
 return pairs







class      __lowercase			(    __lowerCamelCase					):
  snake_case_				       =       VOCAB_FILES_NAMES
  snake_case_				       =       PRETRAINED_VOCAB_FILES_MAP
  snake_case_				       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_				       =       ["""input_ids""", """attention_mask"""]

  def __init__(  self		:			Union[str, Any]					,A		:			Any					,A		:			Dict					,A		:			Optional[Any]="replace"					,A		:			Dict="<s>"					,A		:			str="</s>"					,A		:			str="</s>"					,A		:			Dict="<s>"					,A		:			List[str]="<unk>"					,A		:			Union[str, Any]="<pad>"					,A		:			Any="<mask>"					,A		:			str=False					,**A		:			Optional[Any]					,):






   '''simple docstring'''
   UpperCAmelCase__					:  List[str]					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else bos_token
   UpperCAmelCase__					:  Any					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else eos_token
   UpperCAmelCase__					:  List[str]					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else sep_token
   UpperCAmelCase__					:  Tuple					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else cls_token
   UpperCAmelCase__					:  Tuple					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else unk_token
   UpperCAmelCase__					:  List[str]					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else pad_token

   # Mask token behave like a normal word, i.e. include the space before it
   UpperCAmelCase__					:  Union[str, Any]					   =					AddedToken(A					,lstrip=A					,rstrip=A    ) if isinstance(A					,A    ) else mask_token

   super().__init__(
       errors=A					,bos_token=A					,eos_token=A					,unk_token=A					,sep_token=A					,cls_token=A					,pad_token=A					,mask_token=A					,add_prefix_space=A					,**A					,)

   with open(A					,encoding="""utf-8"""    ) as vocab_handle:
    UpperCAmelCase__					:  Tuple					   =					json.load(A    )
   UpperCAmelCase__					:  Any					   =					{v: k for k, v in self.encoder.items()}
   UpperCAmelCase__					:  List[Any]					   =					errors  # how to handle errors in decoding
   UpperCAmelCase__					:  List[str]					   =					bytes_to_unicode()
   UpperCAmelCase__					:  int					   =					{v: k for k, v in self.byte_encoder.items()}
   with open(A					,encoding="""utf-8"""    ) as merges_handle:
    UpperCAmelCase__					:  List[Any]					   =					merges_handle.read().split("""\n"""    )[1:-1]
   UpperCAmelCase__					:  Tuple					   =					[tuple(merge.split()    ) for merge in bpe_merges]
   UpperCAmelCase__					:  Any					   =					dict(zip(A					,range(len(A    )    )    )    )
   UpperCAmelCase__					:  Optional[Any]					   =					{}
   UpperCAmelCase__					:  Optional[int]					   =					add_prefix_space

   # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
   UpperCAmelCase__					:  Union[str, Any]					   =					re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""    )

  @property
  # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   return len(self.encoder    )

  def    __lowercase       (  self		:			Optional[Any]    ):






   '''simple docstring'''
   return dict(self.encoder					,**self.added_tokens_encoder    )

  def    __lowercase       (  self		:			Optional[int]					,A		:			Union[str, Any]    ):






   '''simple docstring'''
   if token in self.cache:
    return self.cache[token]
   UpperCAmelCase__					:  Optional[Any]					   =					tuple(A    )
   UpperCAmelCase__					:  int					   =					get_pairs(A    )

   if not pairs:
    return token

   while True:
    UpperCAmelCase__					:  str					   =					min(A					,key=lambda A    : self.bpe_ranks.get(A					,float("""inf"""    )    )    )
    if bigram not in self.bpe_ranks:
     break
    UpperCAmelCase__   ,      UpperCAmelCase__					:  Tuple					   =					bigram
    UpperCAmelCase__					:  Optional[Any]					   =					[]
    UpperCAmelCase__					:  Any					   =					0
    while i < len(A    ):
     try:
      UpperCAmelCase__					:  Optional[Any]					   =					word.index(A					,A    )
     except ValueError:
      new_word.extend(word[i:]    )
      break
     else:
      new_word.extend(word[i:j]    )
      UpperCAmelCase__					:  int					   =					j

     if word[i] == first and i < len(A    ) - 1 and word[i + 1] == second:
      new_word.append(first + second    )
      i += 2
     else:
      new_word.append(word[i]    )
      i += 1
    UpperCAmelCase__					:  List[str]					   =					tuple(A    )
    UpperCAmelCase__					:  str					   =					new_word
    if len(A    ) == 1:
     break
    else:
     UpperCAmelCase__					:  str					   =					get_pairs(A    )
   UpperCAmelCase__					:  int					   =					""" """.join(A    )
   UpperCAmelCase__					:  List[str]					   =					word
   return word

  def    __lowercase       (  self		:			Optional[Any]					,A		:			Any    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					[]
   for token in re.findall(self.pat					,A    ):
    UpperCAmelCase__					:  Any					   =					"""""".join(
        self.byte_encoder[b] for b in token.encode("""utf-8"""    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
    bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A    ).split(""" """    )    )
   return bpe_tokens

  def    __lowercase       (  self		:			Dict					,A		:			int    ):






   '''simple docstring'''
   return self.encoder.get(A					,self.encoder.get(self.unk_token    )    )

  def    __lowercase       (  self		:			Dict					,A		:			Optional[Any]    ):






   '''simple docstring'''
   return self.decoder.get(A    )

  def    __lowercase       (  self		:			Optional[Any]					,A		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					"""""".join(A    )
   UpperCAmelCase__					:  Dict					   =					bytearray([self.byte_decoder[c] for c in text]    ).decode("""utf-8"""					,errors=self.errors    )
   return text

  def    __lowercase       (  self		:			Optional[int]					,A		:			str					,A		:			Optional[str] = None    ):






   '''simple docstring'''
   if not os.path.isdir(A    ):
    logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
    return
   UpperCAmelCase__					:  Any					   =					os.path.join(
       A					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
   UpperCAmelCase__					:  List[str]					   =					os.path.join(
       A					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

   with open(A					,"""w"""					,encoding="""utf-8"""    ) as f:
    f.write(json.dumps(self.encoder					,indent=2					,sort_keys=A					,ensure_ascii=A    ) + """\n"""    )

   UpperCAmelCase__					:  Any					   =					0
   with open(A					,"""w"""					,encoding="""utf-8"""    ) as writer:
    writer.write("""#version: 0.2\n"""    )
    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,key=lambda A    : kv[1]    ):
     if index != token_index:
      logger.warning(
          f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
          """ Please check that the tokenizer is not corrupted!"""    )
      UpperCAmelCase__					:  Optional[int]					   =					token_index
     writer.write(""" """.join(A    ) + """\n"""    )
     index += 1

   return vocab_file, merge_file

  def    __lowercase       (  self		:			Union[str, Any]					,A		:			List[int]					,A		:			Optional[List[int]] = None    ):






   '''simple docstring'''
   if token_ids_a is None:
    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   UpperCAmelCase__					:  Dict					   =					[self.cls_token_id]
   UpperCAmelCase__					:  Optional[Any]					   =					[self.sep_token_id]
   return cls + token_ids_a + sep + sep + token_ids_a + sep

  def    __lowercase       (  self		:			int					,A		:			List[int]					,A		:			Optional[List[int]] = None					,A		:			bool = False    ):






   '''simple docstring'''
   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=A					,token_ids_a=A					,already_has_special_tokens=A    )

   if token_ids_a is None:
    return [1] + ([0] * len(A    )) + [1]
   return [1] + ([0] * len(A    )) + [1, 1] + ([0] * len(A    )) + [1]

  def    __lowercase       (  self		:			Tuple					,A		:			List[int]					,A		:			Optional[List[int]] = None    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					[self.sep_token_id]
   UpperCAmelCase__					:  List[str]					   =					[self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep    ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]

  def    __lowercase       (  self		:			Any					,A		:			str					,A		:			List[Any]=False					,**A		:			Tuple    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					kwargs.pop("""add_prefix_space"""					,self.add_prefix_space    )
   if (is_split_into_words or add_prefix_space) and (len(A    ) > 0 and not text[0].isspace()):
    UpperCAmelCase__					:  Dict					   =					""" """ + text
   return (text, kwargs)




  def    __lowercase       (  self		:			Dict					,A		:			Union[Dict[str, EncodedInput], BatchEncoding]					,A		:			Optional[int] = None					,A		:			PaddingStrategy = PaddingStrategy.DO_NOT_PAD					,A		:			Optional[int] = None					,A		:			Optional[bool] = None					,):






   '''simple docstring'''
   UpperCAmelCase__					:  Union[str, Any]					   =					super()._pad(
       encoded_inputs=A					,max_length=A					,padding_strategy=A					,pad_to_multiple_of=A					,return_attention_mask=A					,)

   # Load from model defaults
   if return_attention_mask is None:
    UpperCAmelCase__					:  str					   =					"""attention_mask""" in self.model_input_names

   if return_attention_mask and "global_attention_mask" in encoded_inputs:
    UpperCAmelCase__					:  Optional[int]					   =					encoded_inputs[self.model_input_names[0]]
    # `global_attention_mask` need to have the same length as other (sequential) inputs.
    UpperCAmelCase__					:  Tuple					   =					len(encoded_inputs["""global_attention_mask"""]    ) != len(A    )

    if needs_to_be_padded:
     UpperCAmelCase__					:  List[Any]					   =					len(A    ) - len(encoded_inputs["""global_attention_mask"""]    )

     if self.padding_side == "right":
      # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
      UpperCAmelCase__					:  Tuple					   =					(
          encoded_inputs["""global_attention_mask"""] + [-1] * difference
      )
     elif self.padding_side == "left":
      UpperCAmelCase__					:  Dict					   =					[-1] * difference + encoded_inputs[
          """global_attention_mask"""
      ]
     else:
      raise ValueError("""Invalid padding strategy:""" + str(self.padding_side    )    )

   return encoded_inputs





