



import argparse
import os
import re

import packaging.version


UpperCAmelCase_		 =					"""examples/"""
UpperCAmelCase_		 =					{
    """examples""": (re.compile(r"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(r"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(r"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), r"""\1version=\"VERSION\","""),
    """doc""": (re.compile(r"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
UpperCAmelCase_		 =					{
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
UpperCAmelCase_		 =					"""README.md"""

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:Tuple	,   _snake_case	:int	,   _snake_case	:List[str]				)  ->     Dict:
   with open(_snake_case	,   '''r'''	,   encoding='''utf-8'''	,   newline='''\n'''				) as f:
      _A      =       f.read()
   _A  ,    _A      =       REPLACE_PATTERNS[pattern]
   _A      =       replace.replace('''VERSION'''	,   _snake_case				)
   _A      =       re_pattern.sub(_snake_case	,   _snake_case				)
   with open(_snake_case	,   '''w'''	,   encoding='''utf-8'''	,   newline='''\n'''				) as f:
      f.write(_snake_case				)

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:str				)  ->     List[Any]:
   for folder, directories, fnames in os.walk(_snake_case				):
      # Removing some of the folders with non-actively maintained examples from the walk
      if "research_projects" in directories:
         directories.remove('''research_projects'''				)
      if "legacy" in directories:
         directories.remove('''legacy'''				)
      for fname in fnames:
         if fname.endswith('''.py'''				):
            update_version_in_file(os.path.join(_snake_case	,   _snake_case				)	,   _snake_case	,   pattern='''examples'''				)

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:Union[str, Any]	,   _snake_case	:Optional[int]=False				)  ->     Optional[Any]:
   for pattern, fname in REPLACE_FILES.items():
      update_version_in_file(_snake_case	,   _snake_case	,   _snake_case				)
   if not patch:
      update_version_in_examples(_snake_case				)

def 	SCREAMING_SNAKE_CASE_							(	)  ->     Any:
   _A      =       '''🤗 Transformers currently provides the following architectures'''
   _A      =       '''1. Want to contribute a new model?'''
   with open(_snake_case	,   '''r'''	,   encoding='''utf-8'''	,   newline='''\n'''				) as f:
      _A      =       f.readlines()

   # Find the start of the list.
   _A      =       0
   while not lines[start_index].startswith(_start_prompt				):
      start_index += 1
   start_index += 1

   _A      =       start_index
   # Update the lines in the model list.
   while not lines[index].startswith(_end_prompt				):
      if lines[index].startswith('''1.'''				):
         _A      =       lines[index].replace(
             '''https://huggingface.co/docs/transformers/main/model_doc'''	,   '''https://huggingface.co/docs/transformers/model_doc'''	,   )
      index += 1

   with open(_snake_case	,   '''w'''	,   encoding='''utf-8'''	,   newline='''\n'''				) as f:
      f.writelines(_snake_case				)

def 	SCREAMING_SNAKE_CASE_							(	)  ->     str:
   with open(REPLACE_FILES['''init''']	,   '''r'''				) as f:
      _A      =       f.read()
   _A      =       REPLACE_PATTERNS['''init'''][0].search(_snake_case				).groups()[0]
   return packaging.version.parse(_snake_case				)

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:List[Any]=False				)  ->     Dict:
   _A      =       get_version()
   if patch and default_version.is_devrelease:
      raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''				)
   if default_version.is_devrelease:
      _A      =       default_version.base_version
   elif patch:
      _A      =       F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
   else:
      _A      =       F'''{default_version.major}.{default_version.minor + 1}.0'''

   # Now let's ask nicely if that's the right one.
   _A      =       input(F'''Which version are you releasing? [{default_version}]'''				)
   if len(_snake_case				) == 0:
      _A      =       default_version

   print(F'''Updating version to {version}.'''				)
   global_version_update(_snake_case	,   patch=_snake_case				)
   if not patch:
      print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''				)
      clean_main_ref_in_model_list()

def 	SCREAMING_SNAKE_CASE_							(	)  ->     str:
   _A      =       get_version()
   _A      =       F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
   _A      =       current_version.base_version

   # Check with the user we got that right.
   _A      =       input(F'''Which version are we developing now? [{dev_version}]'''				)
   if len(_snake_case				) == 0:
      _A      =       dev_version

   print(F'''Updating version to {version}.'''				)
   global_version_update(_snake_case				)
   print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''				)
   clean_main_ref_in_model_list()


if __name__ == "__main__":
   UpperCAmelCase_		 =					argparse.ArgumentParser()
   parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
   parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
   UpperCAmelCase_		 =					parser.parse_args()
   if not args.post_release:
      pre_release_work(patch=args.patch)
   elif args.patch:
      print("""Nothing to do after a patch :-)""")
   else:
      post_release_work()








import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_		 =					logging.get_logger(__name__)


UpperCAmelCase_		 =					{
    """vocab_file""": """vocab.json""",
    """tokenizer_config_file""": """tokenizer_config.json""",
    """merges_file""": """merges.txt""",
}

UpperCAmelCase_		 =					{
    """vocab_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json"""
        ),
    },
    """tokenizer_config_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json"""
        ),
    },
    """merges_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt"""
        ),
    },
}

UpperCAmelCase_		 =					"""</w>"""
UpperCAmelCase_		 =					"""@@ """

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:Optional[Any]				)  ->     List[str]:
   _A      =       set()
   _A      =       word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)				)
      _A      =       char
   return pairs


# Speech2Text2 has no max input length
UpperCAmelCase_		 =					{"""facebook/s2t-wav2vec2-large-en-de""": 1_0_2_4}


class 	lowerCamelCase__	(       _A):



 """simple docstring"""

 a__					: Dict =  VOCAB_FILES_NAMES
 a__					: str =  PRETRAINED_VOCAB_FILES_MAP
 a__					: Optional[Any] =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 a__					: List[Any] =  ["input_ids", "attention_mask"]

 def __init__(    self					:					Optional[Any]			,						__lowerCAmelCase					:					Optional[int]			,						__lowerCAmelCase					:					List[str]="<s>"			,						__lowerCAmelCase					:					Tuple="<pad>"			,						__lowerCAmelCase					:					Optional[Any]="</s>"			,						__lowerCAmelCase					:					Dict="<unk>"			,						__lowerCAmelCase					:					Union[str, Any]=False			,						__lowerCAmelCase					:					Optional[int]=None			,						**__lowerCAmelCase					:					str			,						)   ->	Dict:
    super().__init__(
        unk_token=__lowerCAmelCase			,						bos_token=__lowerCAmelCase			,						eos_token=__lowerCAmelCase			,						pad_token=__lowerCAmelCase			,						do_lower_case=__lowerCAmelCase			,						**__lowerCAmelCase			,						)

    _A      =       do_lower_case

    with open(__lowerCAmelCase			,						encoding='''utf-8'''      ) as vocab_handle:
       _A      =       json.load(__lowerCAmelCase      )
    _A      =       {v: k for k, v in self.encoder.items()}

    if merges_file is None:
       logger.info(f'''No merges files provided. {self.__class__.__name__} can only be used for decoding.'''      )

       _A      =       None
       _A      =       None
    else:
       with open(__lowerCAmelCase			,						encoding='''utf-8'''      ) as merges_handle:
          _A      =       merges_handle.read().split('''\n'''      )[:-1]

       _A      =       [tuple(merge.split()[:2]      ) for merge in merges]
       _A      =       dict(zip(__lowerCAmelCase			,						range(len(__lowerCAmelCase      )      )      )      )
       _A      =       {}

 @property
 def     snake_case_      (    self					:					List[str]      )   ->	int:
    return len(self.decoder      )

 def     snake_case_      (    self					:					Dict      )   ->	Dict:
    return dict(self.encoder			,						**self.added_tokens_encoder      )

 def     snake_case_      (    self					:					Union[str, Any]			,						__lowerCAmelCase					:					Any      )   ->	Union[str, Any]:
    _A      =       tuple(token[:-1]      ) + (token[-1] + BPE_TOKEN_MERGES,)
    if token in self.cache:
       return self.cache[token]
    _A      =       get_pairs(__lowerCAmelCase      )

    if not pairs:
       return token

    while True:
       _A      =       min(__lowerCAmelCase			,						key=lambda __lowerCAmelCase      : self.bpe_ranks.get(__lowerCAmelCase			,						float('''inf'''      )      )      )
       if bigram not in self.bpe_ranks:
          break
       _A  ,    _A      =       bigram
       _A      =       []
       _A      =       0
       while i < len(__lowerCAmelCase      ):
          try:
             _A      =       word.index(__lowerCAmelCase			,						__lowerCAmelCase      )
          except ValueError:
             new_word.extend(word[i:]      )
             break
          else:
             new_word.extend(word[i:j]      )
             _A      =       j

          if word[i] == first and i < len(__lowerCAmelCase      ) - 1 and word[i + 1] == second:
             new_word.append(first + second      )
             i += 2
          else:
             new_word.append(word[i]      )
             i += 1
       _A      =       tuple(__lowerCAmelCase      )
       _A      =       new_word
       if len(__lowerCAmelCase      ) == 1:
          break
       else:
          _A      =       get_pairs(__lowerCAmelCase      )
    _A      =       ''' '''.join(__lowerCAmelCase      )
    if word == "\n  " + BPE_TOKEN_MERGES:
       _A      =       '''\n''' + BPE_TOKEN_MERGES

    if word.endswith(__lowerCAmelCase      ):
       _A      =       word.replace(__lowerCAmelCase			,						''''''      )

    _A      =       word.replace(''' '''			,						__lowerCAmelCase      )
    _A      =       word
    return word

 def     snake_case_      (    self					:					Optional[Any]			,						__lowerCAmelCase					:					Tuple      )   ->	Optional[int]:

    if self.bpe_ranks is None:
       raise ValueError(
           '''This tokenizer was instantiated without a `merges.txt` file, so'''
           ''' that it can only be used for decoding, not for encoding.'''
           '''Make sure to provide `merges.txt` file at instantiation to enable '''
           '''encoding.'''      )

    if self.do_lower_case:
       _A      =       text.lower()

    _A      =       text.split()

    _A      =       []
    for token in text:
       if token:
          split_tokens.extend(list(self.bpe(__lowerCAmelCase      ).split(''' '''      )      )      )

    return split_tokens

 def     snake_case_      (    self					:					List[Any]			,						__lowerCAmelCase					:					str      )   ->	int:
    return self.encoder.get(__lowerCAmelCase			,						self.encoder.get(self.unk_token      )      )

 def     snake_case_      (    self					:					str			,						__lowerCAmelCase					:					int      )   ->	str:
    _A      =       self.decoder.get(__lowerCAmelCase			,						self.unk_token      )
    return result

 def     snake_case_      (    self					:					List[str]			,						__lowerCAmelCase					:					List[str]      )   ->	str:
    _A      =       ''' '''.join(__lowerCAmelCase      )

    # make sure @@ tokens are concatenated
    _A      =       ''''''.join(string.split(__lowerCAmelCase      )      )

    return string

 def     snake_case_      (    self					:					Tuple			,						__lowerCAmelCase					:					str			,						__lowerCAmelCase					:					Optional[str] = None      )   ->	Tuple[str]:
    if not os.path.isdir(__lowerCAmelCase      ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
       return
    _A      =       os.path.join(
        __lowerCAmelCase			,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
    _A      =       os.path.join(
        __lowerCAmelCase			,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

    with open(__lowerCAmelCase			,						'''w'''			,						encoding='''utf-8'''      ) as f:
       f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=__lowerCAmelCase			,						ensure_ascii=__lowerCAmelCase      ) + '''\n'''      )

    _A      =       0
    if self.bpe_ranks is None:
       return (vocab_file,)

    with open(__lowerCAmelCase			,						'''w'''			,						encoding='''utf-8'''      ) as writer:
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda __lowerCAmelCase      : kv[1]      ):
          if index != token_index:
             logger.warning(
                 f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
                 ''' Please check that the tokenizer is not corrupted!'''      )
             _A      =       token_index
          writer.write(''' '''.join(__lowerCAmelCase      ) + '''\n'''      )
          index += 1

    return (vocab_file, merges_file)



