






import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
a_		:    int   							=     logging.getLogger(__name__)

if __name__ == "__main__":
				a_		:    List[str]   							=     argparse.ArgumentParser(
				    description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
				)
				parser.add_argument(
				    '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
				)
				parser.add_argument(
				    '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
				)
				parser.add_argument('--vocab_size', default=3_05_22, type=int)
				a_		:    Union[str, Any]   							=     parser.parse_args()

				logger.info(f"""Loading data from {args.data_file}""")
				with open(args.data_file, 'rb') as fp:
								a_		:    Any   							=     pickle.load(fp)

				logger.info('Counting occurrences for MLM.')
				a_		:    int   							=     Counter()
				for tk_ids in data:
								counter.update(tk_ids)
				a_		:    Optional[Any]   							=     [0] * args.vocab_size
				for k, v in counter.items():
								a_		:    Dict   							=     v

				logger.info(f"""Dump to {args.token_counts_dump}""")
				with open(args.token_counts_dump, 'wb') as handle:
								pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)







__snake_case				:      int   		=			[
    """Audio""",
    """Array2D""",
    """Array3D""",
    """Array4D""",
    """Array5D""",
    """ClassLabel""",
    """Features""",
    """Sequence""",
    """Value""",
    """Image""",
    """Translation""",
    """TranslationVariableLanguages""",
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages


