





"""simple docstring"""





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCamelCase       =	logging.get_logger(__name__)

_lowerCamelCase       =	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCamelCase       =	{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_lowerCamelCase       =	{
    '''openai-gpt''': 5_12,
}




def 		lowerCAmelCase_	(  lowercase_   :							Dict		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[int]												=       text.strip()
 if not text:
  return []
 __SCREAMING_SNAKE_CASE :			str												=       text.split()
 return tokens





class 			snake_case		(   __UpperCAmelCase   ):

      def __init__(			self   :Union[str, Any]						,							_lowerCamelCase   :int=True						,							_lowerCamelCase   :Union[str, Any]=None						,							_lowerCamelCase   :Union[str, Any]=True						,							_lowerCamelCase   :Any=None						,							_lowerCamelCase   :Optional[int]=True						,							):
       if never_split is None:
        __SCREAMING_SNAKE_CASE :			Any												=       []
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       do_lower_case
       __SCREAMING_SNAKE_CASE :			Optional[int]												=       set(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       tokenize_chinese_chars
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       strip_accents
       __SCREAMING_SNAKE_CASE :			List[Any]												=       do_split_on_punc

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[Any]						,							_lowerCamelCase   :int						,							_lowerCamelCase   :Tuple=None						):
       __SCREAMING_SNAKE_CASE :			Optional[int]												=       self.never_split.union(set(_lowerCamelCase						)						) if never_split else self.never_split
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       self._clean_text(_lowerCamelCase						)

       # This was added on November 1st, 2018 for the multilingual and Chinese
       # models. This is also applied to the English models now, but it doesn't
       # matter since the English models were not trained on any Chinese data
       # and generally don't have any Chinese data in them (there are Chinese
       # characters in the vocabulary because Wikipedia does have some Chinese
       # words in the English Wikipedia.).
       if self.tokenize_chinese_chars:
        __SCREAMING_SNAKE_CASE :			Optional[int]												=       self._tokenize_chinese_chars(_lowerCamelCase						)
       # prevents treating the same character with different unicode codepoints as different characters
       __SCREAMING_SNAKE_CASE :			str												=       unicodedata.normalize('''NFC'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			Dict												=       whitespace_tokenize(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			List[str]												=       []
       for token in orig_tokens:
        if token not in never_split:
         if self.do_lower_case:
          __SCREAMING_SNAKE_CASE :			List[str]												=       token.lower()
          if self.strip_accents is not False:
           __SCREAMING_SNAKE_CASE :			Dict												=       self._run_strip_accents(_lowerCamelCase						)
         elif self.strip_accents:
          __SCREAMING_SNAKE_CASE :			Tuple												=       self._run_strip_accents(_lowerCamelCase						)
        split_tokens.extend(self._run_split_on_punc(_lowerCamelCase						,							_lowerCamelCase						)						)

       __SCREAMING_SNAKE_CASE :			int												=       whitespace_tokenize(''' '''.join(_lowerCamelCase						)						)
       return output_tokens

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[str]						,							_lowerCamelCase   :int						):
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       unicodedata.normalize('''NFD'''						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			int												=       []
       for char in text:
        __SCREAMING_SNAKE_CASE :			Optional[Any]												=       unicodedata.category(_lowerCamelCase						)
        if cat == "Mn":
         continue
        output.append(_lowerCamelCase						)
       return "".join(_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[Any]						,							_lowerCamelCase   :Optional[int]						,							_lowerCamelCase   :Optional[int]=None						):
       if not self.do_split_on_punc or (never_split is not None and text in never_split):
        return [text]
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       list(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			List[Any]												=       0
       __SCREAMING_SNAKE_CASE :			Any												=       True
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       []
       while i < len(_lowerCamelCase						):
        __SCREAMING_SNAKE_CASE :			Dict												=       chars[i]
        if _is_punctuation(_lowerCamelCase						):
         output.append([char]						)
         __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       True
        else:
         if start_new_word:
          output.append([]						)
         __SCREAMING_SNAKE_CASE :			str												=       False
         output[-1].append(_lowerCamelCase						)
        i += 1

       return ["".join(_lowerCamelCase						) for x in output]

      def 						SCREAMING_SNAKE_CASE_			(			self   :Dict						,							_lowerCamelCase   :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE :			Dict												=       []
       for char in text:
        __SCREAMING_SNAKE_CASE :			int												=       ord(_lowerCamelCase						)
        if self._is_chinese_char(_lowerCamelCase						):
         output.append(''' '''						)
         output.append(_lowerCamelCase						)
         output.append(''' '''						)
        else:
         output.append(_lowerCamelCase						)
       return "".join(_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[Any]						,							_lowerCamelCase   :Union[str, Any]						):
       # This defines a "chinese character" as anything in the CJK Unicode block:
       #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
       #
       # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
       # despite its name. The modern Korean Hangul alphabet is a different block,
       # as is Japanese Hiragana and Katakana. Those alphabets are used to write
       # space-separated words, so they are not treated specially and handled
       # like the all of the other languages.
       if (
           (cp >= 0X4e00 and cp <= 0X9fff)
           or (cp >= 0X3400 and cp <= 0X4dbf)  #
           or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
           or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
           or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
           or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
           or (cp >= 0Xf900 and cp <= 0Xfaff)
           or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
       ):  #
        return True

       return False



      def 						SCREAMING_SNAKE_CASE_			(			self   :List[str]						,							_lowerCamelCase   :Any						):
       __SCREAMING_SNAKE_CASE :			Dict												=       []
       for char in text:
        __SCREAMING_SNAKE_CASE :			str												=       ord(_lowerCamelCase						)
        if cp == 0 or cp == 0Xfffd or _is_control(_lowerCamelCase						):
         continue
        if _is_whitespace(_lowerCamelCase						):
         output.append(''' '''						)
        else:
         output.append(_lowerCamelCase						)
       return "".join(_lowerCamelCase						)







def 		lowerCAmelCase_	(  lowercase_   :							Tuple		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[int]												=       set()
 __SCREAMING_SNAKE_CASE :			List[str]												=       word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)		)
  __SCREAMING_SNAKE_CASE :			List[str]												=       char
 return pairs




def 		lowerCAmelCase_	(  lowercase_   :							str		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[Any]												=       text.replace('''—''' ,      '''-'''		)
 __SCREAMING_SNAKE_CASE :			str												=       text.replace('''–''' ,      '''-'''		)
 __SCREAMING_SNAKE_CASE :			int												=       text.replace('''―''' ,      '''-'''		)
 __SCREAMING_SNAKE_CASE :			List[str]												=       text.replace('''…''' ,      '''...'''		)
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       text.replace('''´''' ,      '''\''''		)
 __SCREAMING_SNAKE_CASE :			List[str]												=       re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)''' ,      r''' \1 ''' ,      lowercase_		)
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       re.sub(r'''\s*\n\s*''' ,      ''' \n ''' ,      lowercase_		)
 __SCREAMING_SNAKE_CASE :			Any												=       re.sub(r'''[^\S\n]+''' ,      ''' ''' ,      lowercase_		)
 return text.strip()






class 			snake_case		(   __UpperCAmelCase   ):
      lowerCamelCase__						= VOCAB_FILES_NAMES
      lowerCamelCase__						= PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__						= ['''input_ids''', '''attention_mask''']

      def __init__(			self   :Union[str, Any]						,							_lowerCamelCase   :List[str]						,							_lowerCamelCase   :Optional[int]						,							_lowerCamelCase   :Union[str, Any]="<unk>"						,							**_lowerCamelCase   :List[str]						):
       super().__init__(unk_token=_lowerCamelCase						,							**_lowerCamelCase						)

       try:
        import ftfy
        from spacy.lang.en import English

        __SCREAMING_SNAKE_CASE :			List[str]												=       English()
        __SCREAMING_SNAKE_CASE :			Any												=       _nlp.tokenizer
        __SCREAMING_SNAKE_CASE :			Tuple												=       ftfy.fix_text
       except ImportError:
        logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''						)
        __SCREAMING_SNAKE_CASE :			str												=       BasicTokenizer(do_lower_case=_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE :			List[Any]												=       None

       with open(_lowerCamelCase						,							encoding='''utf-8'''						) as vocab_handle:
        __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       json.load(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			int												=       {v: k for k, v in self.encoder.items()}
       with open(_lowerCamelCase						,							encoding='''utf-8'''						) as merges_handle:
        __SCREAMING_SNAKE_CASE :			int												=       merges_handle.read().split('''\n'''						)[1:-1]
       __SCREAMING_SNAKE_CASE :			int												=       [tuple(merge.split()						) for merge in merges]
       __SCREAMING_SNAKE_CASE :			Tuple												=       dict(zip(_lowerCamelCase						,							range(len(_lowerCamelCase						)						)						)						)
       __SCREAMING_SNAKE_CASE :			List[Any]												=       {}

      @property
      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[Any]						):
       return True

      @property
      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[int]						):
       return len(self.encoder						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[Any]						):
       return dict(self.encoder						,							**self.added_tokens_encoder						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :str						,							_lowerCamelCase   :Optional[Any]						):
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       tuple(token[:-1]						) + (token[-1] + '''</w>''',)
       if token in self.cache:
        return self.cache[token]
       __SCREAMING_SNAKE_CASE :			Tuple												=       get_pairs(_lowerCamelCase						)

       if not pairs:
        return token + "</w>"

       while True:
        __SCREAMING_SNAKE_CASE :			List[str]												=       min(_lowerCamelCase						,							key=lambda _lowerCamelCase						: self.bpe_ranks.get(_lowerCamelCase						,							float('''inf'''						)						)						)
        if bigram not in self.bpe_ranks:
         break
        __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Optional[Any]												=       bigram
        __SCREAMING_SNAKE_CASE :			Dict												=       []
        __SCREAMING_SNAKE_CASE :			str												=       0
        while i < len(_lowerCamelCase						):
         try:
          __SCREAMING_SNAKE_CASE :			List[str]												=       word.index(_lowerCamelCase						,							_lowerCamelCase						)
         except ValueError:
          new_word.extend(word[i:]						)
          break
         else:
          new_word.extend(word[i:j]						)
          __SCREAMING_SNAKE_CASE :			int												=       j

         if word[i] == first and i < len(_lowerCamelCase						) - 1 and word[i + 1] == second:
          new_word.append(first + second						)
          i += 2
         else:
          new_word.append(word[i]						)
          i += 1
        __SCREAMING_SNAKE_CASE :			int												=       tuple(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE :			Tuple												=       new_word
        if len(_lowerCamelCase						) == 1:
         break
        else:
         __SCREAMING_SNAKE_CASE :			List[Any]												=       get_pairs(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			Any												=       ''' '''.join(_lowerCamelCase						)
       if word == "\n  </w>":
        __SCREAMING_SNAKE_CASE :			int												=       '''\n</w>'''
       __SCREAMING_SNAKE_CASE :			List[str]												=       word
       return word

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[str]						,							_lowerCamelCase   :List[Any]						):
       __SCREAMING_SNAKE_CASE :			Optional[int]												=       []
       if self.fix_text is None:
        # Using BERT's BasicTokenizer
        __SCREAMING_SNAKE_CASE :			str												=       self.nlp.tokenize(_lowerCamelCase						)
        for token in text:
         split_tokens.extend(list(self.bpe(_lowerCamelCase						).split(''' '''						)						)						)
       else:
        # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
        __SCREAMING_SNAKE_CASE :			int												=       self.nlp(text_standardize(self.fix_text(_lowerCamelCase						)						)						)
        for token in text:
         split_tokens.extend(list(self.bpe(token.text.lower()						).split(''' '''						)						)						)
       return split_tokens

      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[Any]						,							_lowerCamelCase   :Any						):
       return self.encoder.get(_lowerCamelCase						,							self.encoder.get(self.unk_token						)						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :int						,							_lowerCamelCase   :Optional[Any]						):
       return self.decoder.get(_lowerCamelCase						,							self.unk_token						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Union[str, Any]						,							_lowerCamelCase   :Dict						):
       __SCREAMING_SNAKE_CASE :			Tuple												=       ''''''.join(_lowerCamelCase						).replace('''</w>'''						,							''' '''						).strip()
       return out_string



      def 						SCREAMING_SNAKE_CASE_			(			self   :Union[str, Any]						,							_lowerCamelCase   :str						,							_lowerCamelCase   :Optional[str] = None						):
       if not os.path.isdir(_lowerCamelCase						):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
        return
       __SCREAMING_SNAKE_CASE :			Dict												=       os.path.join(
           _lowerCamelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
       __SCREAMING_SNAKE_CASE :			List[str]												=       os.path.join(
           _lowerCamelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

       with open(_lowerCamelCase						,							'''w'''						,							encoding='''utf-8'''						) as f:
        f.write(json.dumps(self.encoder						,							indent=2						,							sort_keys=_lowerCamelCase						,							ensure_ascii=_lowerCamelCase						) + '''\n'''						)

       __SCREAMING_SNAKE_CASE :			Any												=       0
       with open(_lowerCamelCase						,							'''w'''						,							encoding='''utf-8'''						) as writer:
        writer.write('''#version: 0.2\n'''						)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,							key=lambda _lowerCamelCase						: kv[1]						):
         if index != token_index:
          logger.warning(
              f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              ''' Please check that the tokenizer is not corrupted!'''						)
          __SCREAMING_SNAKE_CASE :			Optional[int]												=       token_index
         writer.write(''' '''.join(_lowerCamelCase						) + '''\n'''						)
         index += 1

       return vocab_file, merge_file








"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_lowerCamelCase       =	{
    '''configuration_mobilenet_v1''': [
        '''MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''MobileNetV1Config''',
        '''MobileNetV1OnnxConfig''',
    ],
}

try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase       =	['''MobileNetV1FeatureExtractor''']
    _lowerCamelCase       =	['''MobileNetV1ImageProcessor''']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase       =	[
        '''MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''MobileNetV1ForImageClassification''',
        '''MobileNetV1Model''',
        '''MobileNetV1PreTrainedModel''',
        '''load_tf_weights_in_mobilenet_v1''',
    ]


if TYPE_CHECKING:
    from .configuration_mobilenet_va import (
        MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP,
        MobileNetVaConfig,
        MobileNetVaOnnxConfig,
    )

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_mobilenet_va import MobileNetVaFeatureExtractor
        from .image_processing_mobilenet_va import MobileNetVaImageProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mobilenet_va import (
            MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST,
            MobileNetVaForImageClassification,
            MobileNetVaModel,
            MobileNetVaPreTrainedModel,
            load_tf_weights_in_mobilenet_va,
        )


else:
    import sys

    _lowerCamelCase       =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

