

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
       if is_torch_available():
              import torch
       if is_tf_available():
              import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


UpperCamelCase__     :					Any								=			logging.get_logger(__name__)

UpperCamelCase__     :					List[Any]								=			{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

UpperCamelCase__     :					int								=			{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

UpperCamelCase__     :					int								=			{
    "Salesforce/codegen-350M-mono": 2_048,
}


@lru_cache()
def     _UpperCAmelCase				(     ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_													=  (
        list(range(ord('!'				)       ,      ord('~'				) + 1				)				) + list(range(ord('¡'				)       ,      ord('¬'				) + 1				)				) + list(range(ord('®'				)       ,      ord('ÿ'				) + 1				)				)
    )
    SCREAMING_SNAKE_CASE_													=  bs[:]
    SCREAMING_SNAKE_CASE_													=  0
    for b in range(2**8				):
        if b not in bs:
            bs.append(_SCREAMING_SNAKE_CASE				)
            cs.append(2**8 + n				)
            n += 1
    SCREAMING_SNAKE_CASE_													=  [chr(_SCREAMING_SNAKE_CASE				) for n in cs]
    return dict(zip(_SCREAMING_SNAKE_CASE       ,      _SCREAMING_SNAKE_CASE				)				)


def     _UpperCAmelCase				(     _SCREAMING_SNAKE_CASE    :						Optional[int]				):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_													=  set()
    SCREAMING_SNAKE_CASE_													=  word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)				)
        SCREAMING_SNAKE_CASE_													=  char
    return pairs





class 					__snake_case     (			lowercase_				):
  __lowerCAmelCase   :		Dict         =     VOCAB_FILES_NAMES
  __lowerCAmelCase   :		Optional[int]         =     PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase   :		List[str]         =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase   :		Optional[Any]         =     ['input_ids', 'attention_mask']

  def __init__(    self  ,						_A  ,						_A  ,						_A="replace"  ,						_A="<|endoftext|>"  ,						_A="<|endoftext|>"  ,						_A="<|endoftext|>"  ,						_A=None  ,						_A=False  ,						_A=False  ,						**_A  ,						):
      SCREAMING_SNAKE_CASE_													=  AddedToken(_lowercase  ,						lstrip=_lowercase  ,						rstrip=_lowercase) if isinstance(_lowercase  ,						_lowercase) else bos_token
      SCREAMING_SNAKE_CASE_													=  AddedToken(_lowercase  ,						lstrip=_lowercase  ,						rstrip=_lowercase) if isinstance(_lowercase  ,						_lowercase) else eos_token
      SCREAMING_SNAKE_CASE_													=  AddedToken(_lowercase  ,						lstrip=_lowercase  ,						rstrip=_lowercase) if isinstance(_lowercase  ,						_lowercase) else unk_token
      SCREAMING_SNAKE_CASE_													=  AddedToken(_lowercase  ,						lstrip=_lowercase  ,						rstrip=_lowercase) if isinstance(_lowercase  ,						_lowercase) else pad_token
      super().__init__(
          errors=_lowercase  ,						unk_token=_lowercase  ,						bos_token=_lowercase  ,						eos_token=_lowercase  ,						pad_token=_lowercase  ,						add_prefix_space=_lowercase  ,						add_bos_token=_lowercase  ,						**_lowercase  ,						)
      SCREAMING_SNAKE_CASE_													=  add_bos_token

      with open(_lowercase  ,						encoding='utf-8') as vocab_handle:
          SCREAMING_SNAKE_CASE_													=  json.load(_lowercase)
      SCREAMING_SNAKE_CASE_													=  {v: k for k, v in self.encoder.items()}
      SCREAMING_SNAKE_CASE_													=  errors  # how to handle errors in decoding
      SCREAMING_SNAKE_CASE_													=  bytes_to_unicode()
      SCREAMING_SNAKE_CASE_													=  {v: k for k, v in self.byte_encoder.items()}
      with open(_lowercase  ,						encoding='utf-8') as merges_handle:
          SCREAMING_SNAKE_CASE_													=  merges_handle.read().split('\n')[1:-1]
      SCREAMING_SNAKE_CASE_													=  [tuple(merge.split()) for merge in bpe_merges]
      SCREAMING_SNAKE_CASE_													=  dict(zip(_lowercase  ,						range(len(_lowercase))))
      SCREAMING_SNAKE_CASE_													=  {}
      SCREAMING_SNAKE_CASE_													=  add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      SCREAMING_SNAKE_CASE_													=  re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')

  @property
  def   lowerCAmelCase__   (    self):
      return len(self.encoder)

  def   lowerCAmelCase__   (    self):
      return dict(self.encoder  ,						**self.added_tokens_encoder)

  def   lowerCAmelCase__   (    self  ,						_A):





      if token in self.cache:
          return self.cache[token]
      SCREAMING_SNAKE_CASE_													=  tuple(_lowercase)
      SCREAMING_SNAKE_CASE_													=  get_pairs(_lowercase)

      if not pairs:
          return token

      while True:
          SCREAMING_SNAKE_CASE_													=  min(_lowercase  ,						key=lambda _A: self.bpe_ranks.get(_lowercase  ,						float('inf')))
          if bigram not in self.bpe_ranks:
              break
          SCREAMING_SNAKE_CASE_							,	SCREAMING_SNAKE_CASE_													=  bigram
          SCREAMING_SNAKE_CASE_													=  []
          SCREAMING_SNAKE_CASE_													=  0
          while i < len(_lowercase):
              try:
                  SCREAMING_SNAKE_CASE_													=  word.index(_lowercase  ,						_lowercase)
              except ValueError:
                  new_word.extend(word[i:])
                  break
              else:
                  new_word.extend(word[i:j])
                  SCREAMING_SNAKE_CASE_													=  j

              if word[i] == first and i < len(_lowercase) - 1 and word[i + 1] == second:
                  new_word.append(first + second)
                  i += 2
              else:
                  new_word.append(word[i])
                  i += 1
          SCREAMING_SNAKE_CASE_													=  tuple(_lowercase)
          SCREAMING_SNAKE_CASE_													=  new_word
          if len(_lowercase) == 1:
              break
          else:
              SCREAMING_SNAKE_CASE_													=  get_pairs(_lowercase)
      SCREAMING_SNAKE_CASE_													=  ' '.join(_lowercase)
      SCREAMING_SNAKE_CASE_													=  word
      return word

  def   lowerCAmelCase__   (    self  ,						_A  ,						_A=None):





      if self.add_bos_token:
          SCREAMING_SNAKE_CASE_													=  [self.bos_token_id]
      else:
          SCREAMING_SNAKE_CASE_													=  []

      SCREAMING_SNAKE_CASE_													=  bos_token_ids + token_ids_a

      if token_ids_a is None:
          return output

      return output + bos_token_ids + token_ids_a

  def   lowerCAmelCase__   (    self  ,						_A):
      SCREAMING_SNAKE_CASE_													=  []
      for token in re.findall(self.pat  ,						_lowercase):
          SCREAMING_SNAKE_CASE_													=  ''.join(
              self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowercase).split(' '))
      return bpe_tokens

  def   lowerCAmelCase__   (    self  ,						_A):
      return self.encoder.get(_lowercase  ,						self.encoder.get(self.unk_token))

  def   lowerCAmelCase__   (    self  ,						_A):
      return self.decoder.get(_lowercase)

  def   lowerCAmelCase__   (    self  ,						_A):
      SCREAMING_SNAKE_CASE_													=  ''.join(_lowercase)
      SCREAMING_SNAKE_CASE_													=  bytearray([self.byte_decoder[c] for c in text]).decode('utf-8'  ,						errors=self.errors)
      return text

  def   lowerCAmelCase__   (    self  ,						_A  ,						_A = None):





      if not os.path.isdir(_lowercase):
          logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
          return
      SCREAMING_SNAKE_CASE_													=  os.path.join(
          _lowercase  ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
      SCREAMING_SNAKE_CASE_													=  os.path.join(
          _lowercase  ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

      with open(_lowercase  ,						'w'  ,						encoding='utf-8') as f:
          f.write(json.dumps(self.encoder  ,						indent=2  ,						sort_keys=_lowercase  ,						ensure_ascii=_lowercase) + '\n')

      SCREAMING_SNAKE_CASE_													=  0
      with open(_lowercase  ,						'w'  ,						encoding='utf-8') as writer:
          writer.write('#version: 0.2\n')
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,						key=lambda _A: kv[1]):
              if index != token_index:
                  logger.warning(
                      f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      ' Please check that the tokenizer is not corrupted!')
                  SCREAMING_SNAKE_CASE_													=  token_index
              writer.write(' '.join(_lowercase) + '\n')
              index += 1

      return vocab_file, merge_file

  def   lowerCAmelCase__   (    self  ,						_A  ,						_A=False  ,						**_A):
      SCREAMING_SNAKE_CASE_													=  kwargs.pop('add_prefix_space'  ,						self.add_prefix_space)
      if is_split_into_words or add_prefix_space:
          SCREAMING_SNAKE_CASE_													=  ' ' + text
      return (text, kwargs)

  def   lowerCAmelCase__   (    self  ,						_A  ,						_A = False  ,						_A = None  ,						_A = None  ,						**_A  ,						):
      SCREAMING_SNAKE_CASE_													=  super()._decode(
          token_ids=_lowercase  ,						skip_special_tokens=_lowercase  ,						clean_up_tokenization_spaces=_lowercase  ,						**_lowercase  ,						)

      if truncate_before_pattern is not None and len(_lowercase) > 0:
          SCREAMING_SNAKE_CASE_													=  self.truncate(_lowercase  ,						_lowercase)

      return decoded_text


  def   lowerCAmelCase__   (    self  ,						_A  ,						_A):





      def find_re(_A  ,						_A  ,						_A):
          SCREAMING_SNAKE_CASE_													=  pattern.search(_lowercase  ,						_lowercase)
          return m.start() if m else -1

      SCREAMING_SNAKE_CASE_													=  [re.compile(_lowercase  ,						re.MULTILINE) for pattern in truncate_before_pattern]

      SCREAMING_SNAKE_CASE_													=  list(re.finditer('^print'  ,						_lowercase  ,						re.MULTILINE))

      if len(_lowercase) > 1:
          SCREAMING_SNAKE_CASE_													=  completion[: prints[1].start()]

      SCREAMING_SNAKE_CASE_													=  list(re.finditer('^def'  ,						_lowercase  ,						re.MULTILINE))

      if len(_lowercase) > 1:
          SCREAMING_SNAKE_CASE_													=  completion[: defs[1].start()]

      SCREAMING_SNAKE_CASE_													=  0

      SCREAMING_SNAKE_CASE_													=  [
          pos for pos in [find_re(_lowercase  ,						_lowercase  ,						_lowercase) for terminal in terminals] if pos != -1
      ]

      if len(_lowercase) > 0:
          return completion[: min(_lowercase)]
      else:
          return completion









import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


_snake_case				= logging.getLogger(__name__)





@dataclass
class 					lowerCAmelCase							:
  __lowerCamelCase		=			field(
      metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}					)
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Pretrained config name or path if not the same as model_name'}					)
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}					)
  __lowerCamelCase		=			field(default=lowercase_							,							metadata={'help': 'Set this flag to use fast tokenization.'}					)
  # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
  # or just modify its tokenizer_config.json.
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}							,							)


def _A   (    ):
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   lowercase__    =  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

   if len(sys.argv       ) == 2 and sys.argv[1].endswith(".json"       ):
      # If we pass only one argument to the script and it's the path to a json file,
      # let's parse it to get our arguments.
      lowercase__   ,							lowercase__   ,							lowercase__    =  parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]       )       )
   else:
      lowercase__   ,							lowercase__   ,							lowercase__    =  parser.parse_args_into_dataclasses()

   if (
       os.path.exists(training_args.output_dir       )
       and os.listdir(training_args.output_dir       )
       and training_args.do_train
       and not training_args.overwrite_output_dir
   ):
      raise ValueError(
          f'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
          " --overwrite_output_dir to overcome."       )

   # Setup logging
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"  ,					datefmt="%m/%d/%Y %H:%M:%S"  ,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN  ,					)
   logger.warning(
       "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"  ,					training_args.local_rank  ,					training_args.device  ,					training_args.n_gpu  ,					bool(training_args.local_rank != -1       )  ,					training_args.fpaa  ,					)
   # Set the verbosity to info of the Transformers logger (on main process only):
   if is_main_process(training_args.local_rank       ):
      transformers.utils.logging.set_verbosity_info()
      transformers.utils.logging.enable_default_handler()
      transformers.utils.logging.enable_explicit_format()
   logger.info("Training/evaluation parameters %s"  ,					__magic_name__       )

   # Prepare Question-Answering task
   # Load pretrained model and tokenizer
   #
   # Distributed training:
   # The .from_pretrained methods guarantee that only one local process can concurrently
   # download model & vocab.

   lowercase__    =  AutoConfig.from_pretrained(
       model_args.config_name if model_args.config_name else model_args.model_name_or_path  ,					cache_dir=model_args.cache_dir  ,					)
   lowercase__    =  AutoTokenizer.from_pretrained(
       model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path  ,					cache_dir=model_args.cache_dir  ,					use_fast=__magic_name__  ,					)
   lowercase__    =  AutoModelForQuestionAnswering.from_pretrained(
       model_args.model_name_or_path  ,					from_tf=bool(".ckpt" in model_args.model_name_or_path       )  ,					config=__magic_name__  ,					cache_dir=model_args.cache_dir  ,					)

   # Get datasets
   lowercase__    =  hasattr(model.config  ,					"lang2id"       )
   lowercase__    =  (
       SquadDataset(
           __magic_name__  ,					tokenizer=__magic_name__  ,					is_language_sensitive=__magic_name__  ,					cache_dir=model_args.cache_dir       )
       if training_args.do_train
       else None
   )
   lowercase__    =  (
       SquadDataset(
           __magic_name__  ,					tokenizer=__magic_name__  ,					mode="dev"  ,					is_language_sensitive=__magic_name__  ,					cache_dir=model_args.cache_dir  ,					)
       if training_args.do_eval
       else None
   )

   # Data collator
   lowercase__    =  DataCollatorWithPadding(__magic_name__  ,					pad_to_multiple_of=8       ) if training_args.fpaa else None

   # Initialize our Trainer
   lowercase__    =  Trainer(
       model=__magic_name__  ,					args=__magic_name__  ,					train_dataset=__magic_name__  ,					eval_dataset=__magic_name__  ,					data_collator=__magic_name__  ,					)

   # Training
   if training_args.do_train:
      trainer.train(
          model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path       ) else None       )
      trainer.save_model()
      # For convenience, we also re-save the tokenizer to the same directory,
      # so that you can share your model easily on huggingface.co/models =)
      if trainer.is_world_master():
         tokenizer.save_pretrained(training_args.output_dir       )


def _A   (    __magic_name__       ):
   # For xla_spawn (TPUs)
   main()


if __name__ == "__main__":
  main()






