





import os
import sys
import unittest


__A     =    os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


__A     =    os.path.join("tests", "models", "bert", "test_modeling_bert.py")
__A     =    os.path.join("tests", "models", "blip", "test_modeling_blip.py")



class        _SCREAMING_SNAKE_CASE						(     unittest.TestCase   ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any)	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:				int        =get_test_to_tester_mapping(UpperCAmelCase_)
   lowerCamelCase__:				Any        =get_test_to_tester_mapping(UpperCAmelCase_)

   lowerCamelCase__:				str        ={"BertModelTest": "BertModelTester"}

   lowerCamelCase__:				Optional[int]        ={
       "BlipModelTest": "BlipModelTester",
       "BlipTextImageModelTest": "BlipTextImageModelsModelTester",
       "BlipTextModelTest": "BlipTextModelTester",
       "BlipTextRetrievalModelTest": "BlipTextRetrievalModelTester",
       "BlipVQAModelTest": "BlipVQAModelTester",
       "BlipVisionModelTest": "BlipVisionModelTester",
   }

   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->Dict:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =get_model_to_test_mapping(UpperCAmelCase_)
   lowerCamelCase__:				Dict        =get_model_to_test_mapping(UpperCAmelCase_)

   lowerCamelCase__:				Optional[Any]        ={
       "BertForMaskedLM": ["BertModelTest"],
       "BertForMultipleChoice": ["BertModelTest"],
       "BertForNextSentencePrediction": ["BertModelTest"],
       "BertForPreTraining": ["BertModelTest"],
       "BertForQuestionAnswering": ["BertModelTest"],
       "BertForSequenceClassification": ["BertModelTest"],
       "BertForTokenClassification": ["BertModelTest"],
       "BertLMHeadModel": ["BertModelTest"],
       "BertModel": ["BertModelTest"],
   }

   lowerCamelCase__:				Any        ={
       "BlipForConditionalGeneration": ["BlipTextImageModelTest"],
       "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTest"],
       "BlipForQuestionAnswering": ["BlipVQAModelTest"],
       "BlipModel": ["BlipModelTest"],
       "BlipTextModel": ["BlipTextModelTest"],
       "BlipVisionModel": ["BlipVisionModelTest"],
   }

   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)







 def 							SCREAMING_SNAKE_CASE_ (self    :       Dict)	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:				int        =get_model_to_tester_mapping(UpperCAmelCase_)
   lowerCamelCase__:				Any        =get_model_to_tester_mapping(UpperCAmelCase_)

   lowerCamelCase__:				Dict        ={
       "BertForMaskedLM": ["BertModelTester"],
       "BertForMultipleChoice": ["BertModelTester"],
       "BertForNextSentencePrediction": ["BertModelTester"],
       "BertForPreTraining": ["BertModelTester"],
       "BertForQuestionAnswering": ["BertModelTester"],
       "BertForSequenceClassification": ["BertModelTester"],
       "BertForTokenClassification": ["BertModelTester"],
       "BertLMHeadModel": ["BertModelTester"],
       "BertModel": ["BertModelTester"],
   }

   lowerCamelCase__:				str        ={
       "BlipForConditionalGeneration": ["BlipTextImageModelsModelTester"],
       "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTester"],
       "BlipForQuestionAnswering": ["BlipVQAModelTester"],
       "BlipModel": ["BlipModelTester"],
       "BlipTextModel": ["BlipTextModelTester"],
       "BlipVisionModel": ["BlipVisionModelTester"],
   }

   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(get_test_info.to_json(UpperCAmelCase_)	,			UpperCAmelCase_)








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

