






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_						:						Optional[int]						      =					logging.get_logger(__name__)


A_						:						List[str]						      =					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
A_						:						List[str]						      =					{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

A_						:						List[str]						      =					{
    'facebook/bart-base': 1024,
    'facebook/bart-large': 1024,
    'facebook/bart-large-mnli': 1024,
    'facebook/bart-large-cnn': 1024,
    'facebook/bart-large-xsum': 1024,
    'yjernite/bart_eli5': 1024,
}

@lru_cache()
def     snake_case   ()			->							List[Any]:
  UpperCamelCase_:   Tuple        =							(
      list(range(ord('!'    )      ,       ord('~'    ) + 1    )    ) + list(range(ord('¡'    )      ,       ord('¬'    ) + 1    )    ) + list(range(ord('®'    )      ,       ord('ÿ'    ) + 1    )    )
  )
  UpperCamelCase_:   int        =							bs[:]
  UpperCamelCase_:   Any        =							0
  for b in range(2**8    ):
    if b not in bs:
      bs.append(UpperCAmelCase__    )
      cs.append(2**8 + n    )
      n += 1
  UpperCamelCase_:   Tuple        =							[chr(UpperCAmelCase__    ) for n in cs]
  return dict(zip(UpperCAmelCase__      ,       UpperCAmelCase__    )    )

def     snake_case   (UpperCAmelCase__    )			->							Tuple:
  UpperCamelCase_:   List[Any]        =							set()
  UpperCamelCase_:   Optional[Any]        =							word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)    )
    UpperCamelCase_:   Tuple        =							char
  return pairs




class 							_lowerCAmelCase(			UpperCAmelCase_   ):





       """simple docstring"""







       a							:						int										=VOCAB_FILES_NAMES
       a							:						Tuple										=PRETRAINED_VOCAB_FILES_MAP
       a							:						Dict										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a							:						Optional[int]										=['''input_ids''', '''attention_mask''']





       def __init__(   self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase="replace"    ,     _lowerCamelCase="<s>"    ,     _lowerCamelCase="</s>"    ,     _lowerCamelCase="</s>"    ,     _lowerCamelCase="<s>"    ,     _lowerCamelCase="<unk>"    ,     _lowerCamelCase="<pad>"    ,     _lowerCamelCase="<mask>"    ,     _lowerCamelCase=False    ,     **_lowerCamelCase    ,     ):
         UpperCamelCase_:   List[str]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else bos_token
         UpperCamelCase_:   List[Any]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else eos_token
         UpperCamelCase_:   List[str]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else sep_token
         UpperCamelCase_:   List[str]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else cls_token
         UpperCamelCase_:   List[Any]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else unk_token
         UpperCamelCase_:   Optional[Any]        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         UpperCamelCase_:   Any        =							AddedToken(_lowerCamelCase    ,     lstrip=_lowerCamelCase    ,     rstrip=_lowerCamelCase				) if isinstance(_lowerCamelCase    ,     _lowerCamelCase				) else mask_token

         super().__init__(
             errors=_lowerCamelCase    ,     bos_token=_lowerCamelCase    ,     eos_token=_lowerCamelCase    ,     unk_token=_lowerCamelCase    ,     sep_token=_lowerCamelCase    ,     cls_token=_lowerCamelCase    ,     pad_token=_lowerCamelCase    ,     mask_token=_lowerCamelCase    ,     add_prefix_space=_lowerCamelCase    ,     **_lowerCamelCase    ,     )

         with open(_lowerCamelCase    ,     encoding='utf-8'				) as vocab_handle:
           UpperCamelCase_:   Optional[int]        =							json.load(_lowerCamelCase				)
         UpperCamelCase_:   Tuple        =							{v: k for k, v in self.encoder.items()}
         UpperCamelCase_:   Dict        =							errors  # how to handle errors in decoding
         UpperCamelCase_:   Optional[int]        =							bytes_to_unicode()
         UpperCamelCase_:   Dict        =							{v: k for k, v in self.byte_encoder.items()}
         with open(_lowerCamelCase    ,     encoding='utf-8'				) as merges_handle:
           UpperCamelCase_:   Any        =							merges_handle.read().split('\n'				)[1:-1]
         UpperCamelCase_:   Union[str, Any]        =							[tuple(merge.split()				) for merge in bpe_merges]
         UpperCamelCase_:   int        =							dict(zip(_lowerCamelCase    ,     range(len(_lowerCamelCase				)				)				)				)
         UpperCamelCase_:   str        =							{}
         UpperCamelCase_:   Optional[Any]        =							add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         UpperCamelCase_:   Optional[Any]        =							re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'				)





       @property
       def      _a       (   self				):
         return len(self.encoder				)





       def      _a       (   self				):
         return dict(self.encoder    ,     **self.added_tokens_encoder				)





       def      _a       (   self    ,     _lowerCamelCase				):
         if token in self.cache:
           return self.cache[token]
         UpperCamelCase_:   Union[str, Any]        =							tuple(_lowerCamelCase				)
         UpperCamelCase_:   Union[str, Any]        =							get_pairs(_lowerCamelCase				)

         if not pairs:
           return token

         while True:
           UpperCamelCase_:   str        =							min(_lowerCamelCase    ,     key=lambda _lowerCamelCase				: self.bpe_ranks.get(_lowerCamelCase    ,     float('inf'				)				)				)
           if bigram not in self.bpe_ranks:
             break
           UpperCamelCase_ ,UpperCamelCase_:   int        =							bigram
           UpperCamelCase_:   List[str]        =							[]
           UpperCamelCase_:   Tuple        =							0
           while i < len(_lowerCamelCase				):
             try:
               UpperCamelCase_:   List[str]        =							word.index(_lowerCamelCase    ,     _lowerCamelCase				)
             except ValueError:
               new_word.extend(word[i:]				)
               break
             else:
               new_word.extend(word[i:j]				)
               UpperCamelCase_:   Tuple        =							j

             if word[i] == first and i < len(_lowerCamelCase				) - 1 and word[i + 1] == second:
               new_word.append(first + second				)
               i += 2
             else:
               new_word.append(word[i]				)
               i += 1
           UpperCamelCase_:   Any        =							tuple(_lowerCamelCase				)
           UpperCamelCase_:   Dict        =							new_word
           if len(_lowerCamelCase				) == 1:
             break
           else:
             UpperCamelCase_:   Optional[int]        =							get_pairs(_lowerCamelCase				)
         UpperCamelCase_:   List[str]        =							' '.join(_lowerCamelCase				)
         UpperCamelCase_:   Dict        =							word
         return word





       def      _a       (   self    ,     _lowerCamelCase				):
         UpperCamelCase_:   Dict        =							[]
         for token in re.findall(self.pat    ,     _lowerCamelCase				):
           UpperCamelCase_:   Optional[Any]        =							''.join(
               self.byte_encoder[b] for b in token.encode('utf-8'				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase				).split(' '				)				)
         return bpe_tokens





       def      _a       (   self    ,     _lowerCamelCase				):
         return self.encoder.get(_lowerCamelCase    ,     self.encoder.get(self.unk_token				)				)





       def      _a       (   self    ,     _lowerCamelCase				):
         return self.decoder.get(_lowerCamelCase				)





       def      _a       (   self    ,     _lowerCamelCase				):
         UpperCamelCase_:   Optional[int]        =							''.join(_lowerCamelCase				)
         UpperCamelCase_:   Optional[int]        =							bytearray([self.byte_decoder[c] for c in text]				).decode('utf-8'    ,     errors=self.errors				)
         return text





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase = None				):
         if not os.path.isdir(_lowerCamelCase				):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
           return
         UpperCamelCase_:   Union[str, Any]        =							os.path.join(
             _lowerCamelCase    ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
         UpperCamelCase_:   Optional[int]        =							os.path.join(
             _lowerCamelCase    ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']				)

         with open(_lowerCamelCase    ,     'w'    ,     encoding='utf-8'				) as f:
           f.write(json.dumps(self.encoder    ,     indent=2    ,     sort_keys=_lowerCamelCase    ,     ensure_ascii=_lowerCamelCase				) + '\n'				)

         UpperCamelCase_:   str        =							0
         with open(_lowerCamelCase    ,     'w'    ,     encoding='utf-8'				) as writer:
           writer.write('#version: 0.2\n'				)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,     key=lambda _lowerCamelCase				: kv[1]				):
             if index != token_index:
               logger.warning(
                   f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                   ' Please check that the tokenizer is not corrupted!'				)
               UpperCamelCase_:   str        =							token_index
             writer.write(' '.join(_lowerCamelCase				) + '\n'				)
             index += 1

         return vocab_file, merge_file





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase = None				):
         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         UpperCamelCase_:   Optional[int]        =							[self.cls_token_id]
         UpperCamelCase_:   Union[str, Any]        =							[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase = None    ,     _lowerCamelCase = False				):
         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCamelCase    ,     token_ids_a=_lowerCamelCase    ,     already_has_special_tokens=_lowerCamelCase				)

         if token_ids_a is None:
           return [1] + ([0] * len(_lowerCamelCase				)) + [1]
         return [1] + ([0] * len(_lowerCamelCase				)) + [1, 1] + ([0] * len(_lowerCamelCase				)) + [1]





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase = None				):
         UpperCamelCase_:   List[str]        =							[self.sep_token_id]
         UpperCamelCase_:   List[Any]        =							[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase=False    ,     **_lowerCamelCase				):
         UpperCamelCase_:   str        =							kwargs.pop('add_prefix_space'    ,     self.add_prefix_space				)
         if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase				) > 0 and not text[0].isspace()):
           UpperCamelCase_:   Optional[Any]        =							' ' + text
         return (text, kwargs)







'''simple docstring'''





import argparse
import os
import re

import packaging.version


_UpperCamelCase						:						Union[str, Any]  =					'examples/'
_UpperCamelCase						:						List[str]  =					{
    'examples': (re.compile(R'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(R'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(R'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), R'\1version="VERSION",'),
    'doc': (re.compile(R'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
_UpperCamelCase						:						Optional[Any]  =					{
    'init': 'src/diffusers/__init__.py',
    'setup': 'setup.py',
}
_UpperCamelCase						:						Any  =					'README.md'
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			List[Any]			,       A					:			Tuple		)    ->   Dict:
			with open(A			,       '''r'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						UpperCAmelCase_      :       int          =       f.read()
			UpperCAmelCase_    ,      UpperCAmelCase_      :       Union[str, Any]          =       REPLACE_PATTERNS[pattern]
			UpperCAmelCase_      :       Tuple          =       replace.replace('''VERSION'''			,       A		)
			UpperCAmelCase_      :       Tuple          =       re_pattern.sub(A			,       A		)
			with open(A			,       '''w'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						f.write(A		)
def __UpperCAmelCase   (					A					:			List[str]		)    ->   Any:
			for folder, directories, fnames in os.walk(A		):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove('''research_projects'''		)
						if "legacy" in directories:
									directories.remove('''legacy'''		)
						for fname in fnames:
									if fname.endswith('''.py'''		):
												update_version_in_file(os.path.join(A			,       A		)			,       A			,       pattern='''examples'''		)
def __UpperCAmelCase   (					A					:			Tuple			,       A					:			Optional[int]=False		)    ->   Tuple:
			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(A			,       A			,       A		)
			if not patch:
						update_version_in_examples(A		)
def __UpperCAmelCase   (					)    ->   int:
			UpperCAmelCase_      :       List[str]          =       '''🤗 Transformers currently provides the following architectures'''
			UpperCAmelCase_      :       int          =       '''1. Want to contribute a new model?'''
			with open(A			,       '''r'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						UpperCAmelCase_      :       Optional[int]          =       f.readlines()

			# Find the start of the list.
			UpperCAmelCase_      :       List[str]          =       0
			while not lines[start_index].startswith(_start_prompt		):
						start_index += 1
			start_index += 1

			UpperCAmelCase_      :       Any          =       start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt		):
						if lines[index].startswith('''1.'''		):
									UpperCAmelCase_      :       Optional[Any]          =       lines[index].replace(
									    '''https://huggingface.co/docs/diffusers/main/model_doc'''			,       '''https://huggingface.co/docs/diffusers/model_doc'''			,       )
						index += 1

			with open(A			,       '''w'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						f.writelines(A		)
def __UpperCAmelCase   (					)    ->   Dict:
			with open(REPLACE_FILES['''init''']			,       '''r'''		) as f:
						UpperCAmelCase_      :       Any          =       f.read()
			UpperCAmelCase_      :       Union[str, Any]          =       REPLACE_PATTERNS['''init'''][0].search(A		).groups()[0]
			return packaging.version.parse(A		)
def __UpperCAmelCase   (					A					:			str=False		)    ->   List[Any]:
			UpperCAmelCase_      :       Union[str, Any]          =       get_version()
			if patch and default_version.is_devrelease:
						raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''		)
			if default_version.is_devrelease:
						UpperCAmelCase_      :       Optional[int]          =       default_version.base_version
			elif patch:
						UpperCAmelCase_      :       Any          =       F"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
			else:
						UpperCAmelCase_      :       Optional[Any]          =       F"{default_version.major}.{default_version.minor + 1}.0"

			# Now let's ask nicely if that's the right one.
			UpperCAmelCase_      :       List[str]          =       input(F"Which version are you releasing? [{default_version}]"		)
			if len(A		) == 0:
						UpperCAmelCase_      :       Tuple          =       default_version

			print(F"Updating version to {version}."		)
			global_version_update(A			,       patch=A		)
def __UpperCAmelCase   (					)    ->   Optional[int]:
			UpperCAmelCase_      :       str          =       get_version()
			UpperCAmelCase_      :       Tuple          =       F"{current_version.major}.{current_version.minor + 1}.0.dev0"
			UpperCAmelCase_      :       Any          =       current_version.base_version

			# Check with the user we got that right.
			UpperCAmelCase_      :       List[Any]          =       input(F"Which version are we developing now? [{dev_version}]"		)
			if len(A		) == 0:
						UpperCAmelCase_      :       Optional[int]          =       dev_version

			print(F"Updating version to {version}."		)
			global_version_update(A		)


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
						_UpperCamelCase						:						Tuple  =					argparse.ArgumentParser()
						parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
						parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
						_UpperCamelCase						:						Optional[Any]  =					parser.parse_args()
						if not args.post_release:
												pre_release_work(patch=args.patch)
						elif args.patch:
												print('Nothing to do after a patch :-)')
						else:
												post_release_work()


