





'''simple docstring'''






import argparse
import os
import re

import packaging.version


A__							:  Dict    ='''examples/'''
A__							:  Dict    ={
    '''examples''': (re.compile(r'''^check_min_version\("[^"]+"\)\s*$''', re.MULTILINE), '''check_min_version("VERSION")\n'''),
    '''init''': (re.compile(r'''^__version__\s+=\s+"([^"]+)"\s*$''', re.MULTILINE), '''__version__ = "VERSION"\n'''),
    '''setup''': (re.compile(r'''^(\s*)version\s*=\s*"[^"]+",''', re.MULTILINE), r'''\1version="VERSION",'''),
    '''doc''': (re.compile(r'''^(\s*)release\s*=\s*"[^"]+"$''', re.MULTILINE), '''release = "VERSION"\n'''),
}
A__							:  List[str]    ={
    '''init''': '''src/diffusers/__init__.py''',
    '''setup''': '''setup.py''',
}
A__							:  List[str]    ='''README.md'''
def   UpperCamelCase__ (						lowerCAmelCase				,      lowerCAmelCase				,      lowerCAmelCase    ):





      """simple docstring"""


      with open(lowerCAmelCase				,      """r"""				,      encoding="""utf-8"""				,      newline="""\n"""    ) as f:
            _lowerCAmelCase					=				f.read()
      _lowerCAmelCase	,				_lowerCAmelCase					=				REPLACE_PATTERNS[pattern]
      _lowerCAmelCase					=				replace.replace("""VERSION"""				,      lowerCAmelCase    )
      _lowerCAmelCase					=				re_pattern.sub(lowerCAmelCase				,      lowerCAmelCase    )
      with open(lowerCAmelCase				,      """w"""				,      encoding="""utf-8"""				,      newline="""\n"""    ) as f:
            f.write(lowerCAmelCase    )
def   UpperCamelCase__ (						lowerCAmelCase    ):





      """simple docstring"""


      for folder, directories, fnames in os.walk(lowerCAmelCase    ):
            # Removing some of the folders with non-actively maintained examples from the walk
            if "research_projects" in directories:
                  directories.remove("""research_projects"""    )
            if "legacy" in directories:
                  directories.remove("""legacy"""    )
            for fname in fnames:
                  if fname.endswith(""".py"""    ):
                        update_version_in_file(os.path.join(lowerCAmelCase				,      lowerCAmelCase    )				,      lowerCAmelCase				,      pattern="""examples"""    )
def   UpperCamelCase__ (						lowerCAmelCase				,      lowerCAmelCase=False    ):





      """simple docstring"""


      for pattern, fname in REPLACE_FILES.items():
            update_version_in_file(lowerCAmelCase				,      lowerCAmelCase				,      lowerCAmelCase    )
      if not patch:
            update_version_in_examples(lowerCAmelCase    )
def   UpperCamelCase__ (						):





      """simple docstring"""


      _lowerCAmelCase					=				"""🤗 Transformers currently provides the following architectures"""
      _lowerCAmelCase					=				"""1. Want to contribute a new model?"""
      with open(lowerCAmelCase				,      """r"""				,      encoding="""utf-8"""				,      newline="""\n"""    ) as f:
            _lowerCAmelCase					=				f.readlines()

      # Find the start of the list.
      _lowerCAmelCase					=				0
      while not lines[start_index].startswith(_start_prompt    ):
            start_index += 1
      start_index += 1

      _lowerCAmelCase					=				start_index
      # Update the lines in the model list.
      while not lines[index].startswith(_end_prompt    ):
            if lines[index].startswith("""1."""    ):
                  _lowerCAmelCase					=				lines[index].replace(
                      """https://huggingface.co/docs/diffusers/main/model_doc"""				,      """https://huggingface.co/docs/diffusers/model_doc"""				,      )
            index += 1

      with open(lowerCAmelCase				,      """w"""				,      encoding="""utf-8"""				,      newline="""\n"""    ) as f:
            f.writelines(lowerCAmelCase    )
def   UpperCamelCase__ (						):





      """simple docstring"""


      with open(REPLACE_FILES["""init"""]				,      """r"""    ) as f:
            _lowerCAmelCase					=				f.read()
      _lowerCAmelCase					=				REPLACE_PATTERNS["""init"""][0].search(lowerCAmelCase    ).groups()[0]
      return packaging.version.parse(lowerCAmelCase    )
def   UpperCamelCase__ (						lowerCAmelCase=False    ):





      """simple docstring"""


      _lowerCAmelCase					=				get_version()
      if patch and default_version.is_devrelease:
            raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!"""    )
      if default_version.is_devrelease:
            _lowerCAmelCase					=				default_version.base_version
      elif patch:
            _lowerCAmelCase					=				f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
      else:
            _lowerCAmelCase					=				f"{default_version.major}.{default_version.minor + 1}.0"

      # Now let's ask nicely if that's the right one.
      _lowerCAmelCase					=				input(f"Which version are you releasing? [{default_version}]"    )
      if len(lowerCAmelCase    ) == 0:
            _lowerCAmelCase					=				default_version

      print(f"Updating version to {version}."    )
      global_version_update(lowerCAmelCase				,      patch=lowerCAmelCase    )
def   UpperCamelCase__ (						):





      """simple docstring"""


      _lowerCAmelCase					=				get_version()
      _lowerCAmelCase					=				f"{current_version.major}.{current_version.minor + 1}.0.dev0"
      _lowerCAmelCase					=				current_version.base_version

      # Check with the user we got that right.
      _lowerCAmelCase					=				input(f"Which version are we developing now? [{dev_version}]"    )
      if len(lowerCAmelCase    ) == 0:
            _lowerCAmelCase					=				dev_version

      print(f"Updating version to {version}."    )
      global_version_update(lowerCAmelCase    )


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
      A__							:  str    =argparse.ArgumentParser()
      parser.add_argument('''--post_release''', action='''store_true''', help='''Whether this is pre or post release.''')
      parser.add_argument('''--patch''', action='''store_true''', help='''Whether or not this is a patch release.''')
      A__							:  Tuple    =parser.parse_args()
      if not args.post_release:
            pre_release_work(patch=args.patch)
      elif args.patch:
            print('''Nothing to do after a patch :-)''')
      else:
            post_release_work()













'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__							:  Any    =None


A__							:  Optional[int]    =logging.get_logger(__name__)

A__							:  Union[str, Any]    ={'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A__							:  List[str]    ={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

A__							:  List[str]    ={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

A__							:  List[str]    ='''▁'''

# Segments (not really needed)
A__							:  str    =0
A__							:  str    =1
A__							:  List[Any]    =2
A__							:  str    =3
A__							:  Optional[Any]    =4




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Optional[int]			=  VOCAB_FILES_NAMES
      _lowercase:     Optional[int]			=  PRETRAINED_VOCAB_FILES_MAP
      _lowercase:     Dict			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:     Optional[Any]			=  '''left'''
      _lowercase:     Dict			=  XLNetTokenizer




      def __init__(     self     :					List[str]     ,     __snake_case     :					Optional[Any]=None     ,     __snake_case     :					str=None     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					str=True     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					List[Any]="<s>"     ,     __snake_case     :					List[Any]="</s>"     ,     __snake_case     :					str="<unk>"     ,     __snake_case     :					int="<sep>"     ,     __snake_case     :					int="<pad>"     ,     __snake_case     :					Dict="<cls>"     ,     __snake_case     :					int="<mask>"     ,     __snake_case     :					Optional[int]=["<eop>", "<eod>"]     ,     **__snake_case     :					List[str]     ,     )							-> List[str]:
            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase					=				AddedToken(__snake_case     ,     lstrip=__snake_case     ,     rstrip=__snake_case    ) if isinstance(__snake_case     ,     __snake_case    ) else mask_token

            super().__init__(
                vocab_file=__snake_case     ,     tokenizer_file=__snake_case     ,     do_lower_case=__snake_case     ,     remove_space=__snake_case     ,     keep_accents=__snake_case     ,     bos_token=__snake_case     ,     eos_token=__snake_case     ,     unk_token=__snake_case     ,     sep_token=__snake_case     ,     pad_token=__snake_case     ,     cls_token=__snake_case     ,     mask_token=__snake_case     ,     additional_special_tokens=__snake_case     ,     **__snake_case     ,     )

            _lowerCAmelCase					=				3
            _lowerCAmelCase					=				do_lower_case
            _lowerCAmelCase					=				remove_space
            _lowerCAmelCase					=				keep_accents
            _lowerCAmelCase					=				vocab_file
            _lowerCAmelCase					=				False if not self.vocab_file else True




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[self.cls_token_id]
            if token_ids_a is None:
                  return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[2]

            if token_ids_a is None:
                  return len(token_ids_a + sep    ) * [0] + cls_segment_id
            return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str] = None    )							-> Tuple[str]:
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""    )

            if not os.path.isdir(__snake_case    ):
                  logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
                  return
            _lowerCAmelCase					=				os.path.join(
                __snake_case     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
                  copyfile(self.vocab_file     ,     __snake_case    )

            return (out_vocab_file,)






