





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_snake_case				= logging.get_logger(__name__)

_snake_case				= {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_snake_case				= {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

_snake_case				= {
    """Salesforce/codegen-350M-mono""": 2048,
}


@lru_cache()
def _A   (    ):
   lowercase__    =  (
       list(range(ord("!"       )  ,					ord("~"       ) + 1       )       ) + list(range(ord("¡"       )  ,					ord("¬"       ) + 1       )       ) + list(range(ord("®"       )  ,					ord("ÿ"       ) + 1       )       )
   )
   lowercase__    =  bs[:]
   lowercase__    =  0
   for b in range(2**8       ):
      if b not in bs:
         bs.append(__magic_name__       )
         cs.append(2**8 + n       )
         n += 1
   lowercase__    =  [chr(__magic_name__       ) for n in cs]
   return dict(zip(__magic_name__  ,					__magic_name__       )       )


def _A   (    __magic_name__       ):
   lowercase__    =  set()
   lowercase__    =  word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)       )
      lowercase__    =  char
   return pairs







class 					lowerCAmelCase							(   lowercase_					):
  __lowerCamelCase		=			VOCAB_FILES_NAMES
  __lowerCamelCase		=			PRETRAINED_VOCAB_FILES_MAP
  __lowerCamelCase		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCamelCase		=			['input_ids', 'attention_mask']







  def __init__(			self  :Optional[Any]    ,							_lowercase  :Tuple    ,							_lowercase  :List[Any]    ,							_lowercase  :Optional[int]="replace"    ,							_lowercase  :str="<|endoftext|>"    ,							_lowercase  :List[str]="<|endoftext|>"    ,							_lowercase  :Any="<|endoftext|>"    ,							_lowercase  :Union[str, Any]=None    ,							_lowercase  :Optional[int]=False    ,							_lowercase  :List[Any]=False    ,							**_lowercase  :List[str]    ,							):






     '''simple docstring'''





     lowercase__    =  AddedToken(_lowercase    ,							lstrip=_lowercase    ,							rstrip=_lowercase					) if isinstance(_lowercase    ,							_lowercase					) else bos_token
     lowercase__    =  AddedToken(_lowercase    ,							lstrip=_lowercase    ,							rstrip=_lowercase					) if isinstance(_lowercase    ,							_lowercase					) else eos_token
     lowercase__    =  AddedToken(_lowercase    ,							lstrip=_lowercase    ,							rstrip=_lowercase					) if isinstance(_lowercase    ,							_lowercase					) else unk_token
     lowercase__    =  AddedToken(_lowercase    ,							lstrip=_lowercase    ,							rstrip=_lowercase					) if isinstance(_lowercase    ,							_lowercase					) else pad_token
     super().__init__(
         errors=_lowercase    ,							unk_token=_lowercase    ,							bos_token=_lowercase    ,							eos_token=_lowercase    ,							pad_token=_lowercase    ,							add_prefix_space=_lowercase    ,							add_bos_token=_lowercase    ,							**_lowercase    ,							)
     lowercase__    =  add_bos_token

     with open(_lowercase    ,							encoding="utf-8"					) as vocab_handle:
        lowercase__    =  json.load(_lowercase					)
     lowercase__    =  {v: k for k, v in self.encoder.items()}
     lowercase__    =  errors  # how to handle errors in decoding
     lowercase__    =  bytes_to_unicode()
     lowercase__    =  {v: k for k, v in self.byte_encoder.items()}
     with open(_lowercase    ,							encoding="utf-8"					) as merges_handle:
        lowercase__    =  merges_handle.read().split("\n"					)[1:-1]
     lowercase__    =  [tuple(merge.split()					) for merge in bpe_merges]
     lowercase__    =  dict(zip(_lowercase    ,							range(len(_lowercase					)					)					)					)
     lowercase__    =  {}
     lowercase__    =  add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     lowercase__    =  re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"					)







  @property
  def   UpperCAmelCase     (			self  :Dict					):






     '''simple docstring'''





     return len(self.encoder					)







  def   UpperCAmelCase     (			self  :List[str]					):






     '''simple docstring'''





     return dict(self.encoder    ,							**self.added_tokens_encoder					)







  def   UpperCAmelCase     (			self  :List[Any]    ,							_lowercase  :int					):






     '''simple docstring'''





     if token in self.cache:
        return self.cache[token]
     lowercase__    =  tuple(_lowercase					)
     lowercase__    =  get_pairs(_lowercase					)

     if not pairs:
        return token

     while True:
        lowercase__    =  min(_lowercase    ,							key=lambda _lowercase					: self.bpe_ranks.get(_lowercase    ,							float("inf"					)					)					)
        if bigram not in self.bpe_ranks:
           break
        lowercase__   ,							lowercase__    =  bigram
        lowercase__    =  []
        lowercase__    =  0
        while i < len(_lowercase					):
           try:
              lowercase__    =  word.index(_lowercase    ,							_lowercase					)
           except ValueError:
              new_word.extend(word[i:]					)
              break
           else:
              new_word.extend(word[i:j]					)
              lowercase__    =  j

           if word[i] == first and i < len(_lowercase					) - 1 and word[i + 1] == second:
              new_word.append(first + second					)
              i += 2
           else:
              new_word.append(word[i]					)
              i += 1
        lowercase__    =  tuple(_lowercase					)
        lowercase__    =  new_word
        if len(_lowercase					) == 1:
           break
        else:
           lowercase__    =  get_pairs(_lowercase					)
     lowercase__    =  " ".join(_lowercase					)
     lowercase__    =  word
     return word







  def   UpperCAmelCase     (			self  :Union[str, Any]    ,							_lowercase  :int    ,							_lowercase  :Optional[int]=None					):






     '''simple docstring'''





     if self.add_bos_token:
        lowercase__    =  [self.bos_token_id]
     else:
        lowercase__    =  []

     lowercase__    =  bos_token_ids + token_ids_a

     if token_ids_a is None:
        return output

     return output + bos_token_ids + token_ids_a







  def   UpperCAmelCase     (			self  :List[Any]    ,							_lowercase  :Optional[Any]					):






     '''simple docstring'''





     lowercase__    =  []
     for token in re.findall(self.pat    ,							_lowercase					):
        lowercase__    =  "".join(
            self.byte_encoder[b] for b in token.encode("utf-8"					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowercase					).split(" "					)					)
     return bpe_tokens







  def   UpperCAmelCase     (			self  :Dict    ,							_lowercase  :int					):






     '''simple docstring'''





     return self.encoder.get(_lowercase    ,							self.encoder.get(self.unk_token					)					)







  def   UpperCAmelCase     (			self  :Tuple    ,							_lowercase  :Any					):






     '''simple docstring'''





     return self.decoder.get(_lowercase					)







  def   UpperCAmelCase     (			self  :Any    ,							_lowercase  :Optional[int]					):






     '''simple docstring'''





     lowercase__    =  "".join(_lowercase					)
     lowercase__    =  bytearray([self.byte_decoder[c] for c in text]					).decode("utf-8"    ,							errors=self.errors					)
     return text







  def   UpperCAmelCase     (			self  :Dict    ,							_lowercase  :str    ,							_lowercase  :Optional[str] = None					):






     '''simple docstring'''





     if not os.path.isdir(_lowercase					):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
        return
     lowercase__    =  os.path.join(
         _lowercase    ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
     lowercase__    =  os.path.join(
         _lowercase    ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

     with open(_lowercase    ,							"w"    ,							encoding="utf-8"					) as f:
        f.write(json.dumps(self.encoder    ,							indent=2    ,							sort_keys=_lowercase    ,							ensure_ascii=_lowercase					) + "\n"					)

     lowercase__    =  0
     with open(_lowercase    ,							"w"    ,							encoding="utf-8"					) as writer:
        writer.write("#version: 0.2\n"					)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,							key=lambda _lowercase					: kv[1]					):
           if index != token_index:
              logger.warning(
                  f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                  " Please check that the tokenizer is not corrupted!"					)
              lowercase__    =  token_index
           writer.write(" ".join(_lowercase					) + "\n"					)
           index += 1

     return vocab_file, merge_file







  def   UpperCAmelCase     (			self  :Union[str, Any]    ,							_lowercase  :Tuple    ,							_lowercase  :Optional[int]=False    ,							**_lowercase  :int					):






     '''simple docstring'''





     lowercase__    =  kwargs.pop("add_prefix_space"    ,							self.add_prefix_space					)
     if is_split_into_words or add_prefix_space:
        lowercase__    =  " " + text
     return (text, kwargs)







  def   UpperCAmelCase     (			self  :int    ,							_lowercase  :Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]    ,							_lowercase  :bool = False    ,							_lowercase  :bool = None    ,							_lowercase  :Optional[List[str]] = None    ,							**_lowercase  :int    ,							):






     '''simple docstring'''





     lowercase__    =  super()._decode(
         token_ids=_lowercase    ,							skip_special_tokens=_lowercase    ,							clean_up_tokenization_spaces=_lowercase    ,							**_lowercase    ,							)

     if truncate_before_pattern is not None and len(_lowercase					) > 0:
        lowercase__    =  self.truncate(_lowercase    ,							_lowercase					)

     return decoded_text







  def   UpperCAmelCase     (			self  :int    ,							_lowercase  :List[Any]    ,							_lowercase  :Any					):






     '''simple docstring'''





     def find_re(_lowercase  :str    ,							_lowercase  :str    ,							_lowercase  :int					):
        lowercase__    =  pattern.search(_lowercase    ,							_lowercase					)
        return m.start() if m else -1

     lowercase__    =  [re.compile(_lowercase    ,							re.MULTILINE					) for pattern in truncate_before_pattern]

     lowercase__    =  list(re.finditer("^print"    ,							_lowercase    ,							re.MULTILINE					)					)

     if len(_lowercase					) > 1:
        lowercase__    =  completion[: prints[1].start()]

     lowercase__    =  list(re.finditer("^def"    ,							_lowercase    ,							re.MULTILINE					)					)

     if len(_lowercase					) > 1:
        lowercase__    =  completion[: defs[1].start()]

     lowercase__    =  0

     lowercase__    =  [
         pos for pos in [find_re(_lowercase    ,							_lowercase    ,							_lowercase					) for terminal in terminals] if pos != -1
     ]

     if len(_lowercase					) > 0:
        return completion[: min(_lowercase					)]
     else:
        return completion













import argparse
import os
import sys
import urllib.request
import zipfile


_snake_case				= ["""CoLA""", """SST""", """MRPC""", """QQP""", """STS""", """MNLI""", """SNLI""", """QNLI""", """RTE""", """WNLI""", """diagnostic"""]
_snake_case				= {
    """CoLA""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4""",
    """SST""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8""",
    """MRPC""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc""",
    """QQP""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5""",
    """STS""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5""",
    """MNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce""",
    """SNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df""",
    """QNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601""",
    """RTE""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb""",
    """WNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf""",
    """diagnostic""": """https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D""",
}

_snake_case				= """https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt"""
_snake_case				= """https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt"""


def _A   (    __magic_name__  ,					__magic_name__       ):
   print(f'''Downloading and extracting {task}...'''       )
   lowercase__    =  f'''{task}.zip'''
   urllib.request.urlretrieve(TASKaPATH[task]  ,					__magic_name__       )
   with zipfile.ZipFile(__magic_name__       ) as zip_ref:
      zip_ref.extractall(__magic_name__       )
   os.remove(__magic_name__       )
   print("\tCompleted!"       )


def _A   (    __magic_name__  ,					__magic_name__       ):
   print("Processing MRPC..."       )
   lowercase__    =  os.path.join(__magic_name__  ,					"MRPC"       )
   if not os.path.isdir(__magic_name__       ):
      os.mkdir(__magic_name__       )
   if path_to_data:
      lowercase__    =  os.path.join(__magic_name__  ,					"msr_paraphrase_train.txt"       )
      lowercase__    =  os.path.join(__magic_name__  ,					"msr_paraphrase_test.txt"       )
   else:
      print("Local MRPC data not specified, downloading data from %s" % MRPC_TRAIN       )
      lowercase__    =  os.path.join(__magic_name__  ,					"msr_paraphrase_train.txt"       )
      lowercase__    =  os.path.join(__magic_name__  ,					"msr_paraphrase_test.txt"       )
      urllib.request.urlretrieve(__magic_name__  ,					__magic_name__       )
      urllib.request.urlretrieve(__magic_name__  ,					__magic_name__       )
   if not os.path.isfile(__magic_name__       ):
      raise ValueError(f'''Train data not found at {mrpc_train_file}'''       )
   if not os.path.isfile(__magic_name__       ):
      raise ValueError(f'''Test data not found at {mrpc_test_file}'''       )
   urllib.request.urlretrieve(TASKaPATH["MRPC"]  ,					os.path.join(__magic_name__  ,					"dev_ids.tsv"       )       )

   lowercase__    =  []
   with open(os.path.join(__magic_name__  ,					"dev_ids.tsv"       )  ,					encoding="utf8"       ) as ids_fh:
      for row in ids_fh:
         dev_ids.append(row.strip().split("\t"       )       )

   with open(__magic_name__  ,					encoding="utf8"       ) as data_fh, open(
       os.path.join(__magic_name__  ,					"train.tsv"       )  ,					"w"  ,					encoding="utf8"       ) as train_fh, open(os.path.join(__magic_name__  ,					"dev.tsv"       )  ,					"w"  ,					encoding="utf8"       ) as dev_fh:
      lowercase__    =  data_fh.readline()
      train_fh.write(__magic_name__       )
      dev_fh.write(__magic_name__       )
      for row in data_fh:
         lowercase__   ,							lowercase__   ,							lowercase__   ,							lowercase__   ,							lowercase__    =  row.strip().split("\t"       )
         if [ida, ida] in dev_ids:
            dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, ida, ida, sa, sa)       )
         else:
            train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, ida, ida, sa, sa)       )

   with open(__magic_name__  ,					encoding="utf8"       ) as data_fh, open(
       os.path.join(__magic_name__  ,					"test.tsv"       )  ,					"w"  ,					encoding="utf8"       ) as test_fh:
      lowercase__    =  data_fh.readline()
      test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n"       )
      for idx, row in enumerate(__magic_name__       ):
         lowercase__   ,							lowercase__   ,							lowercase__   ,							lowercase__   ,							lowercase__    =  row.strip().split("\t"       )
         test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, ida, ida, sa, sa)       )
   print("\tCompleted!"       )


def _A   (    __magic_name__       ):
   print("Downloading and extracting diagnostic..."       )
   if not os.path.isdir(os.path.join(__magic_name__  ,					"diagnostic"       )       ):
      os.mkdir(os.path.join(__magic_name__  ,					"diagnostic"       )       )
   lowercase__    =  os.path.join(__magic_name__  ,					"diagnostic"  ,					"diagnostic.tsv"       )
   urllib.request.urlretrieve(TASKaPATH["diagnostic"]  ,					__magic_name__       )
   print("\tCompleted!"       )
   return


def _A   (    __magic_name__       ):
   lowercase__    =  task_names.split(","       )
   if "all" in task_names:
      lowercase__    =  TASKS
   else:
      lowercase__    =  []
      for task_name in task_names:
         if task_name not in TASKS:
            raise ValueError(f'''Task {task_name} not found!'''       )
         tasks.append(__magic_name__       )
   return tasks


def _A   (    __magic_name__       ):
   lowercase__    =  argparse.ArgumentParser()
   parser.add_argument("--data_dir"  ,					help="directory to save data to"  ,					type=__magic_name__  ,					default="glue_data"       )
   parser.add_argument(
       "--tasks"  ,					help="tasks to download data for as a comma separated string"  ,					type=__magic_name__  ,					default="all"       )
   parser.add_argument(
       "--path_to_mrpc"  ,					help="path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt"  ,					type=__magic_name__  ,					default=""  ,					)
   lowercase__    =  parser.parse_args(__magic_name__       )

   if not os.path.isdir(args.data_dir       ):
      os.mkdir(args.data_dir       )
   lowercase__    =  get_tasks(args.tasks       )

   for task in tasks:
      if task == "MRPC":
         format_mrpc(args.data_dir  ,					args.path_to_mrpc       )
      elif task == "diagnostic":
         download_diagnostic(args.data_dir       )
      else:
         download_and_extract(__magic_name__  ,					args.data_dir       )


if __name__ == "__main__":
  sys.exit(main(sys.argv[1:]))






