



import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			UpperCAmelCase__    (		lowercase_		,	unittest.TestCase					):



  """simple docstring"""






  lowerCAmelCase__		:	int    		= LongformerTokenizer
  lowerCAmelCase__		:	Dict    		= True
  lowerCAmelCase__		:	Union[str, Any]    		= LongformerTokenizerFast
  lowerCAmelCase__		:	Optional[Any]    		= True






  def 	_UpperCAmelCase				(				self:			Union[str, Any]   ) -> int:




    '''simple docstring'''




    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    __UpperCAmelCase						    =       [
        "l",
        "o",
        "w",
        "e",
        "r",
        "s",
        "t",
        "i",
        "d",
        "n",
        "\u0120",
        "\u0120l",
        "\u0120n",
        "\u0120lo",
        "\u0120low",
        "er",
        "\u0120lowest",
        "\u0120newer",
        "\u0120wider",
        "<unk>",
    ]
    __UpperCAmelCase						    =       dict(zip(__lowerCAmelCase				,				range(len(__lowerCAmelCase   )   )   )   )
    __UpperCAmelCase						    =       ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
    __UpperCAmelCase						    =       {"unk_token": "<unk>"}

    __UpperCAmelCase						    =       os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["vocab_file"]   )
    __UpperCAmelCase						    =       os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["merges_file"]   )
    with open(self.vocab_file				,				"w"				,				encoding="utf-8"   ) as fp:
      fp.write(json.dumps(__lowerCAmelCase   ) + "\n"   )
    with open(self.merges_file				,				"w"				,				encoding="utf-8"   ) as fp:
      fp.write("\n".join(__lowerCAmelCase   )   )






  def 	_UpperCAmelCase				(				self:			Any				,				**__lowerCAmelCase:			Union[str, Any]   ) -> str:




    '''simple docstring'''




    kwargs.update(self.special_tokens_map   )
    return self.tokenizer_class.from_pretrained(self.tmpdirname				,				**__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Any				,				**__lowerCAmelCase:			Union[str, Any]   ) -> int:




    '''simple docstring'''




    kwargs.update(self.special_tokens_map   )
    return self.rust_tokenizer_class.from_pretrained(self.tmpdirname				,				**__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]				,				__lowerCAmelCase:			Any   ) -> Any:




    '''simple docstring'''




    __UpperCAmelCase						    =       "lower newer"
    __UpperCAmelCase						    =       "lower newer"
    return input_text, output_text






  def 	_UpperCAmelCase				(				self:			int   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.tokenizer_class(self.vocab_file				,				self.merges_file				,				**self.special_tokens_map   )
    __UpperCAmelCase						    =       "lower newer"
    __UpperCAmelCase						    =       ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
    __UpperCAmelCase						    =       tokenizer.tokenize(__lowerCAmelCase   )  # , add_prefix_space=True)
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokens + [tokenizer.unk_token]
    __UpperCAmelCase						    =       [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase   )				,				__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Union[str, Any]   ) -> List[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.get_tokenizer()

    self.assertListEqual(tokenizer.encode("Hello world!"				,				add_special_tokens=__lowerCAmelCase   )				,				[0, 31_414, 232, 328, 2]   )
    self.assertListEqual(
        tokenizer.encode("Hello world! cécé herlolip 418"				,				add_special_tokens=__lowerCAmelCase   )				,				[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]				,				)






  @slow
  def 	_UpperCAmelCase				(				self:			Any   ) -> Union[str, Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.tokenizer_class.from_pretrained("allenai/longformer-base-4096"   )

    __UpperCAmelCase						    =       tokenizer.encode("sequence builders"				,				add_special_tokens=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.encode("multi-sequence build"				,				add_special_tokens=__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.encode(
        "sequence builders"				,				add_special_tokens=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.encode(
        "sequence builders"				,				"multi-sequence build"				,				add_special_tokens=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase				,				__lowerCAmelCase   )

    assert encoded_sentence == encoded_text_from_decode
    assert encoded_pair == encoded_pair_from_decode






  def 	_UpperCAmelCase				(				self:			int   ) -> List[str]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.get_tokenizer()

    __UpperCAmelCase						    =       "Encode this sequence."
    __UpperCAmelCase						    =       tokenizer.byte_encoder[" ".encode("utf-8"   )[0]]

    # Testing encoder arguments
    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
    self.assertNotEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
    self.assertEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    tokenizer.add_special_tokens({"bos_token": "<s>"}   )
    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.convert_ids_to_tokens(encoded[1]   )[0]
    self.assertNotEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    # Testing spaces after special tokens
    __UpperCAmelCase						    =       "<mask>"
    tokenizer.add_special_tokens(
        {"mask_token": AddedToken(__lowerCAmelCase				,				lstrip=__lowerCAmelCase				,				rstrip=__lowerCAmelCase   )}   )  # mask token has a left space
    __UpperCAmelCase						    =       tokenizer.convert_tokens_to_ids(__lowerCAmelCase   )

    __UpperCAmelCase						    =       "Encode <mask> sequence"
    __UpperCAmelCase						    =       "Encode <mask>sequence"

    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase   )
    __UpperCAmelCase						    =       encoded.index(__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
    self.assertEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase   )
    __UpperCAmelCase						    =       encoded.index(__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
    self.assertNotEqual(__lowerCAmelCase				,				__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Any   ) -> Any:




    '''simple docstring'''




    pass






  def 	_UpperCAmelCase				(				self:			List[Any]   ) -> int:




    '''simple docstring'''




    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )
        __UpperCAmelCase						    =       self.tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )
        __UpperCAmelCase						    =       "A, <mask> AllenNLP sentence."
        __UpperCAmelCase						    =       tokenizer_r.encode_plus(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase				,				return_token_type_ids=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_p.encode_plus(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase				,				return_token_type_ids=__lowerCAmelCase   )

        # token_type_ids should put 0 everywhere
        self.assertEqual(sum(tokens_r["token_type_ids"]   )				,				sum(tokens_p["token_type_ids"]   )   )

        # attention_mask should put 1 everywhere, so sum over length should be 1
        self.assertEqual(
            sum(tokens_r["attention_mask"]   ) / len(tokens_r["attention_mask"]   )				,				sum(tokens_p["attention_mask"]   ) / len(tokens_p["attention_mask"]   )				,				)

        __UpperCAmelCase						    =       tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]   )
        __UpperCAmelCase						    =       tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]   )

        # Rust correctly handles the space before the mask while python doesnt
        self.assertSequenceEqual(tokens_p["input_ids"]				,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]   )
        self.assertSequenceEqual(tokens_r["input_ids"]				,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]   )

        self.assertSequenceEqual(
            __lowerCAmelCase				,				["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]   )
        self.assertSequenceEqual(
            __lowerCAmelCase				,				["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]   ) -> Dict:




    '''simple docstring'''




    for trim_offsets, add_prefix_space in itertools.product([True, False]				,				repeat=2   ):
      __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
          self.tmpdirname				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )

      __UpperCAmelCase						    =       json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()   )
      __UpperCAmelCase						    =       json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()   )

      self.assertEqual(pre_tokenizer_state["add_prefix_space"]				,				__lowerCAmelCase   )

      self.assertEqual(post_processor_state["add_prefix_space"]				,				__lowerCAmelCase   )
      self.assertEqual(post_processor_state["trim_offsets"]				,				__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]   ) -> str:




    '''simple docstring'''




    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
        __UpperCAmelCase						    =       "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
        __UpperCAmelCase						    =       F'''{text_of_1_token} {text_of_1_token}'''

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(len(__lowerCAmelCase   ) + 1, len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(len(__lowerCAmelCase   ) + 1, len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(len(__lowerCAmelCase   ), len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(len(__lowerCAmelCase   ), len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       F''' {text}'''

        # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
        #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
        # )
        # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
        # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
        # self.assertEqual(
        #     encoding.offset_mapping[1],
        #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
        # )

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(1, 1 + len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(1 + len(__lowerCAmelCase   ) + 1, 1 + len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, 1 + len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(1 + len(__lowerCAmelCase   ), 1 + len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)

        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(
            __lowerCAmelCase				,				use_fast=__lowerCAmelCase				,				add_prefix_space=__lowerCAmelCase				,				trim_offsets=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r(__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        self.assertEqual(encoding.offset_mapping[0]				,				(0, 1 + len(__lowerCAmelCase   ))   )
        self.assertEqual(
            encoding.offset_mapping[1]				,				(1 + len(__lowerCAmelCase   ), 1 + len(__lowerCAmelCase   ) + 1 + len(__lowerCAmelCase   ))				,				)




from datetime import datetime

import requests




def       __lowerCAmelCase      (						A_   : str					)    ->  bytes:
  __UpperCAmelCase						    =       "https://downloadgram.net/wp-json/wppress/video-downloader/video?url="
  __UpperCAmelCase						    =       requests.get(base_url + url					).json()[0]["urls"][0]["src"]
  return requests.get(A_					).content


if __name__ == "__main__":
       a_							=      input("""Enter Video/IGTV url: """).strip()
       a_							=      F"{datetime.now():%Y-%m-%d_%H:%M:%S}.mp4"
       with open(file_name, """wb""") as fp:
              fp.write(download_video(url))
       print(F"Done. Video saved to disk as {file_name}.")



