
"""simple docstring"""


import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
SCREAMING_SNAKE_CASE      =     logging.getLogger(__name__)

if __name__ == "__main__":
      SCREAMING_SNAKE_CASE      =     argparse.ArgumentParser(
          description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
      )
      parser.add_argument(
          """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
      )
      parser.add_argument(
          """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
      )
      parser.add_argument("""--vocab_size""", default=30_522, type=int)
      SCREAMING_SNAKE_CASE      =     parser.parse_args()

      logger.info(F'''Loading data from {args.data_file}''')
      with open(args.data_file, """rb""") as fp:
            SCREAMING_SNAKE_CASE      =     pickle.load(fp)

      logger.info("""Counting occurrences for MLM.""")
      SCREAMING_SNAKE_CASE      =     Counter()
      for tk_ids in data:
            counter.update(tk_ids)
      SCREAMING_SNAKE_CASE      =     [0] * args.vocab_size
      for k, v in counter.items():
            SCREAMING_SNAKE_CASE      =     v

      logger.info(F'''Dump to {args.token_counts_dump}''')
      with open(args.token_counts_dump, """wb""") as handle:
            pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)







"""simple docstring"""


import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)



class   __a  (      unittest.TestCase	):






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[Any]			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		["a", "b", "c"]

              # Defaults to last layer if both are None
              UpperCamelCase ,       UpperCamelCase					 =		get_aligned_output_features_output_indices(UpperCAmelCase_       ,      UpperCAmelCase_       ,      UpperCAmelCase_			)
              self.assertEqual(UpperCAmelCase_       ,      ["c"]			)
              self.assertEqual(UpperCAmelCase_       ,      [2]			)

              # Out indices set to match out features
              UpperCamelCase ,       UpperCamelCase					 =		get_aligned_output_features_output_indices(["a", "c"]       ,      UpperCAmelCase_       ,      UpperCAmelCase_			)
              self.assertEqual(UpperCAmelCase_       ,      ["a", "c"]			)
              self.assertEqual(UpperCAmelCase_       ,      [0, 2]			)

              # Out features set to match out indices
              UpperCamelCase ,       UpperCamelCase					 =		get_aligned_output_features_output_indices(UpperCAmelCase_       ,      [0, 2]       ,      UpperCAmelCase_			)
              self.assertEqual(UpperCAmelCase_       ,      ["a", "c"]			)
              self.assertEqual(UpperCAmelCase_       ,      [0, 2]			)

              # Out features selected from negative indices
              UpperCamelCase ,       UpperCamelCase					 =		get_aligned_output_features_output_indices(UpperCAmelCase_       ,      [-3, -1]       ,      UpperCAmelCase_			)
              self.assertEqual(UpperCAmelCase_       ,      ["a", "c"]			)
              self.assertEqual(UpperCAmelCase_       ,      [-3, -1]			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							int			)->  Union[str, Any]:







              """simple docstring"""
              # Stage names must be set
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(["a", "b"]       ,      (0, 1)       ,      UpperCAmelCase_			)

              # Out features must be a list
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(("a", "b")       ,      (0, 1)       ,      ["a", "b"]			)

              # Out features must be a subset of stage names
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(["a", "b"]       ,      (0, 1)       ,      ["a"]			)

              # Out indices must be a list or tuple
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(UpperCAmelCase_       ,      0       ,      ["a", "b"]			)

              # Out indices must be a subset of stage names
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(UpperCAmelCase_       ,      (0, 1)       ,      ["a"]			)

              # Out features and out indices must be the same length
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(["a", "b"]       ,      (0,)       ,      ["a", "b", "c"]			)

              # Out features should match out indices
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(["a", "b"]       ,      (0, 2)       ,      ["a", "b", "c"]			)

              # Out features and out indices should be in order
              with self.assertRaises(UpperCAmelCase_			):
                     verify_out_features_out_indices(["b", "a"]       ,      (0, 1)       ,      ["a", "b"]			)

              # Check passes with valid inputs
              verify_out_features_out_indices(["a", "b", "d"]       ,      (0, 1, -1)       ,      ["a", "b", "c", "d"]			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict			)->  List[Any]:







              """simple docstring"""
              UpperCamelCase					 =		BackboneMixin()

              UpperCamelCase					 =		["a", "b", "c"]
              UpperCamelCase					 =		["a", "c"]
              UpperCamelCase					 =		[0, 2]

              # Check that the output features and indices are set correctly
              self.assertEqual(backbone.out_features       ,      ["a", "c"]			)
              self.assertEqual(backbone.out_indices       ,      [0, 2]			)

              # Check out features and indices are updated correctly
              UpperCamelCase					 =		["a", "b"]
              self.assertEqual(backbone.out_features       ,      ["a", "b"]			)
              self.assertEqual(backbone.out_indices       ,      [0, 1]			)

              UpperCamelCase					 =		[-3, -1]
              self.assertEqual(backbone.out_features       ,      ["a", "c"]			)
              self.assertEqual(backbone.out_indices       ,      [-3, -1]			)





