




"""simple docstring"""

import argparse
import glob
import importlib.util
import os
import re

import black
from doc_builder.style_doc import style_docstrings_in_code


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
a							      =     '''src/diffusers'''
a							      =     '''.'''


# This is to make sure the diffusers module imported is the one in the repo.
a							      =     importlib.util.spec_from_file_location(
    '''diffusers''',
    os.path.join(DIFFUSERS_PATH, '''__init__.py'''),
    submodule_search_locations=[DIFFUSERS_PATH],
)
a							      =     spec.loader.load_module()






def 		_snake_case	(   _snake_case				:   Union[str, Any]							,    _snake_case				:   int				)       ->   Union[str, Any]:






      '''simple docstring'''


      return line.startswith(_snake_case				) or len(_snake_case				) <= 1 or re.search(R'^\s*\)(\s*->.*:|:)\s*$'							,    _snake_case				) is not None






def 		_snake_case	(   _snake_case				:   Optional[int]				)       ->   int:






      '''simple docstring'''


      _A     =  object_name.split('.'				)
      _A     =  0

      # First let's find the module where our object lives.
      _A     =  parts[i]
      while i < len(_snake_case				) and not os.path.isfile(os.path.join(_snake_case							,    F'''{module}.py'''				)				):
            i += 1
            if i < len(_snake_case				):
                  _A     =  os.path.join(_snake_case							,    parts[i]				)
      if i >= len(_snake_case				):
            raise ValueError(F'''`object_name` should begin with the name of a module of diffusers but got {object_name}.'''				)

      with open(os.path.join(_snake_case							,    F'''{module}.py'''				)							,    'r'							,    encoding='utf-8'							,    newline='\n'				) as f:
            _A     =  f.readlines()

      # Now let's find the class / func in the code!
      _A     =  ''
      _A     =  0
      for name in parts[i + 1 :]:
            while (
                line_index < len(_snake_case				) and re.search(RF'''^{indent}(class|def)\s+{name}(\(|\:)'''							,    lines[line_index]				) is None
            ):
                  line_index += 1
            indent += "    "
            line_index += 1

      if line_index >= len(_snake_case				):
            raise ValueError(F''' {object_name} does not match any function or class in {module}.'''				)

      # We found the beginning of the class / func, now let's find the end (when the indent diminishes).
      _A     =  line_index
      while line_index < len(_snake_case				) and _should_continue(lines[line_index]							,    _snake_case				):
            line_index += 1
      # Clean up empty lines at the end (if any).
      while len(lines[line_index - 1]				) <= 1:
            line_index -= 1

      _A     =  lines[start_index:line_index]
      return "".join(_snake_case				)


a							      =     re.compile(r'''^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)''')
a							      =     re.compile(r'''^\s*(\S+)->(\S+)(\s+.*|$)''')
a							      =     re.compile(r'''<FILL\s+[^>]*>''')






def 		_snake_case	(   _snake_case				:   Optional[int]				)       ->   Any:






      '''simple docstring'''


      _A     =  code.split('\n'				)
      _A     =  0
      while idx < len(_snake_case				) and len(lines[idx]				) == 0:
            idx += 1
      if idx < len(_snake_case				):
            return re.search(R'^(\s*)\S'							,    lines[idx]				).groups()[0]
      return ""






def 		_snake_case	(   _snake_case				:   Union[str, Any]				)       ->   List[Any]:






      '''simple docstring'''


      _A     =  len(get_indent(_snake_case				)				) > 0
      if has_indent:
            _A     =  F'''class Bla:\n{code}'''
      _A     =  black.Mode(target_versions={black.TargetVersion.PYaa}							,    line_length=1_19							,    preview=_snake_case				)
      _A     =  black.format_str(_snake_case							,    mode=_snake_case				)
      _A  ,       _A     =  style_docstrings_in_code(_snake_case				)
      return result[len('class Bla:\n'				) :] if has_indent else result






def 		_snake_case	(   _snake_case				:   Tuple							,    _snake_case				:   List[Any]=False				)       ->   Tuple:






      '''simple docstring'''


      with open(_snake_case							,    'r'							,    encoding='utf-8'							,    newline='\n'				) as f:
            _A     =  f.readlines()
      _A     =  []
      _A     =  0
      # Not a for loop cause `lines` is going to change (if `overwrite=True`).
      while line_index < len(_snake_case				):
            _A     =  _re_copy_warning.search(lines[line_index]				)
            if search is None:
                  line_index += 1
                  continue

            # There is some copied code here, let's retrieve the original.
            _A  ,       _A  ,       _A     =  search.groups()
            _A     =  find_code_in_diffusers(_snake_case				)
            _A     =  get_indent(_snake_case				)

            _A     =  line_index + 1 if indent == theoretical_indent else line_index + 2
            _A     =  theoretical_indent
            _A     =  start_index

            # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
            _A     =  True
            while line_index < len(_snake_case				) and should_continue:
                  line_index += 1
                  if line_index >= len(_snake_case				):
                        break
                  _A     =  lines[line_index]
                  _A     =  _should_continue(_snake_case							,    _snake_case				) and re.search(F'''^{indent}# End copy'''							,    _snake_case				) is None
            # Clean up empty lines at the end (if any).
            while len(lines[line_index - 1]				) <= 1:
                  line_index -= 1

            _A     =  lines[start_index:line_index]
            _A     =  ''.join(_snake_case				)

            # Remove any nested `Copied from` comments to avoid circular copies
            _A     =  [line for line in theoretical_code.split('\n'				) if _re_copy_warning.search(_snake_case				) is None]
            _A     =  '\n'.join(_snake_case				)

            # Before comparing, use the `replace_pattern` on the original code.
            if len(_snake_case				) > 0:
                  _A     =  replace_pattern.replace('with'							,    ''				).split(','				)
                  _A     =  [_re_replace_pattern.search(_snake_case				) for p in patterns]
                  for pattern in patterns:
                        if pattern is None:
                              continue
                        _A  ,       _A  ,       _A     =  pattern.groups()
                        _A     =  re.sub(_snake_case							,    _snake_case							,    _snake_case				)
                        if option.strip() == "all-casing":
                              _A     =  re.sub(obja.lower()							,    obja.lower()							,    _snake_case				)
                              _A     =  re.sub(obja.upper()							,    obja.upper()							,    _snake_case				)

            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
                  _A     =  blackify(lines[start_index - 1] + theoretical_code				)
                  _A     =  theoretical_code[len(lines[start_index - 1]				) :]

            # Test for a diff and act accordingly.
            if observed_code != theoretical_code:
                  diffs.append([object_name, start_index]				)
                  if overwrite:
                        _A     =  lines[:start_index] + [theoretical_code] + lines[line_index:]
                        _A     =  start_index + 1

      if overwrite and len(_snake_case				) > 0:
            # Warn the user a file has been modified.
            print(F'''Detected changes, rewriting {filename}.'''				)
            with open(_snake_case							,    'w'							,    encoding='utf-8'							,    newline='\n'				) as f:
                  f.writelines(_snake_case				)
      return diffs






def 		_snake_case	(   _snake_case				:   bool = False				)       ->   Optional[Any]:






      '''simple docstring'''


      _A     =  glob.glob(os.path.join(_snake_case							,    '**/*.py'				)							,    recursive=_snake_case				)
      _A     =  []
      for filename in all_files:
            _A     =  is_copy_consistent(_snake_case							,    _snake_case				)
            diffs += [F'''- {filename}: copy does not match {d[0]} at line {d[1]}''' for d in new_diffs]
      if not overwrite and len(_snake_case				) > 0:
            _A     =  '\n'.join(_snake_case				)
            raise Exception(
                'Found the following copy inconsistencies:\n'
                + diff
                + '\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them.'				)


if __name__ == "__main__":
       a							      =     argparse.ArgumentParser()
       parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
       a							      =     parser.parse_args()

       check_copies(args.fix_and_overwrite)



"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)
__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'spiece.model'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

__SCREAMING_SNAKE_CASE							=     '▁'





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



     def __init__(							self :Optional[Any]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :int=True      ,  _lowerCamelCase :Tuple=False      ,  _lowerCamelCase :Optional[Any]="[CLS]"      ,  _lowerCamelCase :List[Any]="[SEP]"      ,  _lowerCamelCase :Dict="<unk>"      ,  _lowerCamelCase :int="[SEP]"      ,  _lowerCamelCase :int="<pad>"      ,  _lowerCamelCase :Optional[Any]="[CLS]"      ,  _lowerCamelCase :str="[MASK]"      ,  _lowerCamelCase :Optional[Dict[str, Any]] = None      ,  **_lowerCamelCase :Union[str, Any]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					Union[str, Any]   	=(
          AddedToken(_lowerCamelCase      ,  lstrip=_lowerCamelCase      ,  rstrip=_lowerCamelCase      ,  normalized=_lowerCamelCase   )
          if isinstance(_lowerCamelCase      ,  _lowerCamelCase   )
          else mask_token
      )

      UpperCamelCase_     :					Any   	={} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=_lowerCamelCase      ,  remove_space=_lowerCamelCase      ,  keep_accents=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  sp_model_kwargs=self.sp_model_kwargs      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=do_lower_case
      UpperCamelCase_     :					Any   	=remove_space
      UpperCamelCase_     :					List[Any]   	=keep_accents
      UpperCamelCase_     :					Optional[Any]   	=vocab_file

      UpperCamelCase_     :					List[str]   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(_lowerCamelCase   )



     @property
     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      return len(self.sp_model   )



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	={self.convert_ids_to_tokens(_lowerCamelCase   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab



     def __getstate__(							self :Optional[int]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=self.__dict__.copy()
      UpperCamelCase_     :					Optional[Any]   	=None
      return state



     def __setstate__(							self :List[Any]      ,  _lowerCamelCase :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					Union[str, Any]   	=d

      # for backward compatibility
      if not hasattr(self      ,  'sp_model_kwargs'   ):
       UpperCamelCase_     :					Tuple   	={}

      UpperCamelCase_     :					List[Any]   	=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(self.vocab_file   )



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :Tuple   ):







      '''simple docstring'''



      if self.remove_space:
       UpperCamelCase_     :					Union[str, Any]   	=' '.join(inputs.strip().split()   )
      else:
       UpperCamelCase_     :					List[Any]   	=inputs
      UpperCamelCase_     :					Optional[int]   	=outputs.replace('``'      ,  '"'   ).replace('\'\''      ,  '"'   )

      if not self.keep_accents:
       UpperCamelCase_     :					Optional[Any]   	=unicodedata.normalize('NFKD'      ,  _lowerCamelCase   )
       UpperCamelCase_     :					Tuple   	=''.join([c for c in outputs if not unicodedata.combining(_lowerCamelCase   )]   )
      if self.do_lower_case:
       UpperCamelCase_     :					Optional[Any]   	=outputs.lower()

      return outputs



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :str   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.preprocess_text(_lowerCamelCase   )
      UpperCamelCase_     :					Any   	=self.sp_model.encode(_lowerCamelCase      ,  out_type=_lowerCamelCase   )
      UpperCamelCase_     :					int   	=[]
      for piece in pieces:
       if len(_lowerCamelCase   ) > 1 and piece[-1] == str(','   ) and piece[-2].isdigit():
        UpperCamelCase_     :					Union[str, Any]   	=self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCamelCase      ,  ''   )   )
        if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]   ) == 1:
          UpperCamelCase_     :					Any   	=cur_pieces[1:]
         else:
          UpperCamelCase_     :					List[str]   	=cur_pieces[0][1:]
        cur_pieces.append(piece[-1]   )
        new_pieces.extend(_lowerCamelCase   )
       else:
        new_pieces.append(_lowerCamelCase   )

      return new_pieces



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      return self.sp_model.PieceToId(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :Union[str, Any]   ):







      '''simple docstring'''



      return self.sp_model.IdToPiece(_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=[]
      UpperCamelCase_     :					int   	=''
      UpperCamelCase_     :					int   	=False
      for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
        if not prev_is_special:
         out_string += " "
        out_string += self.sp_model.decode(_lowerCamelCase   ) + token
        UpperCamelCase_     :					List[Any]   	=True
        UpperCamelCase_     :					str   	=[]
       else:
        current_sub_tokens.append(_lowerCamelCase   )
        UpperCamelCase_     :					List[Any]   	=False
      out_string += self.sp_model.decode(_lowerCamelCase   )
      return out_string.strip()



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=[self.sep_token_id]
      UpperCamelCase_     :					Tuple   	=[self.cls_token_id]
      if token_ids_a is None:
       return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None      ,  _lowerCamelCase :bool = False   ):







      '''simple docstring'''



      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowerCamelCase      ,  token_ids_a=_lowerCamelCase      ,  already_has_special_tokens=_lowerCamelCase   )

      if token_ids_a is not None:
       return [1] + ([0] * len(_lowerCamelCase   )) + [1] + ([0] * len(_lowerCamelCase   )) + [1]
      return [1] + ([0] * len(_lowerCamelCase   )) + [1]



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=[self.sep_token_id]
      UpperCamelCase_     :					List[str]   	=[self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      if not os.path.isdir(_lowerCamelCase   ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
       return
      UpperCamelCase_     :					int   	=os.path.join(
          _lowerCamelCase      ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
       copyfile(self.vocab_file      ,  _lowerCamelCase   )
      elif not os.path.isfile(self.vocab_file   ):
       with open(_lowerCamelCase      ,  'wb'   ) as fi:
        UpperCamelCase_     :					Dict   	=self.sp_model.serialized_model_proto()
        fi.write(_lowerCamelCase   )

      return (out_vocab_file,)
