


import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
_snake_case	:						Dict         =							logging.getLogger(__name__)

if __name__ == "__main__":
			_snake_case	:						int         =							argparse.ArgumentParser(
			    description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
			)
			parser.add_argument(
			    "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
			)
			parser.add_argument(
			    "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
			)
			parser.add_argument("--vocab_size", default=30_522, type=int)
			_snake_case	:						str         =							parser.parse_args()

			logger.info(f'''Loading data from {args.data_file}''')
			with open(args.data_file, "rb") as fp:
						_snake_case	:						str         =							pickle.load(fp)

			logger.info("Counting occurrences for MLM.")
			_snake_case	:						Optional[int]         =							Counter()
			for tk_ids in data:
						counter.update(tk_ids)
			_snake_case	:						int         =							[0] * args.vocab_size
			for k, v in counter.items():
						_snake_case	:						List[Any]         =							v

			logger.info(f'''Dump to {args.token_counts_dump}''')
			with open(args.token_counts_dump, "wb") as handle:
						pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)







import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
     import torch

     from transformers import AutoModelForImageClassification

if is_vision_available():
     from transformers import AutoImageProcessor

@require_torch
@require_vision
class     __a(	unittest.TestCase  ):






    """simple docstring"""





    @slow
    def 				a__ ( self    )					->						List[str]:
         UpperCAmelCase_     :				Tuple  =     AutoImageProcessor.from_pretrained('''microsoft/dit-base-finetuned-rvlcdip'''    )
         UpperCAmelCase_     :				Union[str, Any]  =     AutoModelForImageClassification.from_pretrained('''microsoft/dit-base-finetuned-rvlcdip'''    )
         model.to(_SCREAMING_SNAKE_CASE    )

         from datasets import load_dataset

         UpperCAmelCase_     :				Optional[int]  =     load_dataset('''nielsr/rvlcdip-demo'''    )

         UpperCAmelCase_     :				Optional[Any]  =     dataset['''train'''][0]['''image'''].convert('''RGB'''    )

         UpperCAmelCase_     :				str  =     image_processor(_SCREAMING_SNAKE_CASE  ,return_tensors='''pt'''    ).to(_SCREAMING_SNAKE_CASE    )

         # forward pass
         with torch.no_grad():
              UpperCAmelCase_     :				Optional[int]  =     model(**_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				List[Any]  =     outputs.logits

         UpperCAmelCase_     :				Tuple  =     torch.Size((1, 16)    )
         self.assertEqual(logits.shape  ,_SCREAMING_SNAKE_CASE    )

         UpperCAmelCase_     :				int  =     torch.tensor(
             [-0.41_58, -0.40_92, -0.43_47]  ,device=_SCREAMING_SNAKE_CASE  ,dtype=torch.float  ,)
         self.assertTrue(torch.allclose(logits[0, :3]  ,_SCREAMING_SNAKE_CASE  ,atol=1e-4    )    )