


"""simple docstring"""


import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
A  :     int    					=    logging.getLogger(__name__)

if __name__ == "__main__":
	A  :     str    					=    argparse.ArgumentParser(
	    description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
	)
	parser.add_argument(
	    "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
	)
	parser.add_argument(
	    "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
	)
	parser.add_argument("--vocab_size", default=3_0_5_2_2, type=int)
	A  :     str    					=    parser.parse_args()

	logger.info(f'''Loading data from {args.data_file}''')
	with open(args.data_file, "rb") as fp:
		A  :     int    					=    pickle.load(fp)

	logger.info("Counting occurrences for MLM.")
	A  :     Any    					=    Counter()
	for tk_ids in data:
		counter.update(tk_ids)
	A  :     Tuple    					=    [0] * args.vocab_size
	for k, v in counter.items():
		A  :     List[Any]    					=    v

	logger.info(f'''Dump to {args.token_counts_dump}''')
	with open(args.token_counts_dump, "wb") as handle:
		pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)








"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A  :     Union[str, Any]    					=    {
    "configuration_x_clip": [
        "XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "XCLIPConfig",
        "XCLIPTextConfig",
        "XCLIPVisionConfig",
    ],
    "processing_x_clip": ["XCLIPProcessor"],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A  :     Optional[int]    					=    [
	    "XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "XCLIPModel",
	    "XCLIPPreTrainedModel",
	    "XCLIPTextModel",
	    "XCLIPVisionModel",
	]

if TYPE_CHECKING:
	from .configuration_x_clip import (
	    XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    XCLIPConfig,
	    XCLIPTextConfig,
	    XCLIPVisionConfig,
	)
	from .processing_x_clip import XCLIPProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_x_clip import (
		    XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XCLIPModel,
		    XCLIPPreTrainedModel,
		    XCLIPTextModel,
		    XCLIPVisionModel,
		)

else:
	import sys

	A  :     Union[str, Any]    					=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




