class_docstr: The goal here is to compare the embeddings generated by the model trained
  using Megatron-LM with the one from the transformers library, with a small GPT2-like model
  to ensure that the conversion from Megatron-LM to transformers has been done successfully.
  The script compares the logits of the embedding layer and the transformer layers.
  
  WARNING: It is expected that these logits will not have exactly the same statistics when running
  the code on CPU or GPU. For more info, please visit:
    - https://github.com/pytorch/pytorch/issues/76052#issuecomment-1103193548
    - https://discuss.pytorch.org/t/reproducibility-issue-between-intel-and-amd-cpus/144779/9
  
  
  You need to install tokenizers following this readme:
      - https://huggingface.co/bigscience-catalogue-data-dev/byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles
  
  Tokenizer used during training:
      - https://huggingface.co/bigscience-catalogue-data-dev/byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles
  
  # TODO change the script (or just add skip) when building the env with tokenizers 0.12.0
code: def test_embeddings(self):
      # The config in this checkpoint has `bfloat16` as `torch_dtype` -> model in `bfloat16`
      model = BloomForCausalLM.from_pretrained(
          self.path_bigscience_model, torch_dtype="auto"
      )
      model.eval()
  
      EMBEDDINGS_DS_BEFORE_LN_BF_16_MEAN = {
          3478: 0.0002307891845703125,
          368: -0.000568389892578125,
          109586: -0.0003910064697265625,
          35433: -0.000194549560546875,
          2: 0.0004138946533203125,
          77: 0.000659942626953125,
          132619: -0.00031280517578125,
          2175: 0.000457763671875,
          23714: 0.000263214111328125,
          73173: -0.000286102294921875,
          144252: 0.00052642822265625,
      }
      EMBEDDINGS_DS_BEFORE_LN_BF_16_MIN = {
          3478: -0.00921630859375,
          368: -0.010009765625,
          109586: -0.01031494140625,
          35433: -0.01177978515625,
          2: -0.0074462890625,
          77: -0.00848388671875,
          132619: -0.009521484375,
          2175: -0.0074462890625,
          23714: -0.0145263671875,
          73173: -0.007415771484375,
          144252: -0.01007080078125,
      }
      EMBEDDINGS_DS_BEFORE_LN_BF_16_MAX = {
          3478: 0.0128173828125,
          368: 0.01214599609375,
          109586: 0.0111083984375,
          35433: 0.01019287109375,
          2: 0.0157470703125,
          77: 0.0174560546875,
          132619: 0.0078125,
          2175: 0.0113525390625,
          23714: 0.0146484375,
          73173: 0.01116943359375,
          144252: 0.01141357421875,
      }
      EMBEDDINGS_DS_BEFORE_LN_BF_16_SUM = {"value": 0.08203125}
  
      EMBEDDINGS_DS_BEFORE_LN_F_16_MEAN = {
          132619: -0.00031256675720214844,
          3478: 0.00023090839385986328,
          368: -0.0005702972412109375,
          109586: -0.00039124488830566406,
          35433: -0.000194549560546875,
          2: 0.0004146099090576172,
          2175: 0.0004572868347167969,
          23714: 0.00026416778564453125,
          73173: -0.0002865791320800781,
          144252: 0.0005254745483398438,
          77: 0.0006618499755859375,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_16_MIN = {
          3478: -0.00921630859375,
          368: -0.010009765625,
          109586: -0.01031494140625,
          35433: -0.01177978515625,
          2: -0.0074462890625,
          77: -0.00848388671875,
          132619: -0.009521484375,
          2175: -0.0074462890625,
          23714: -0.0145263671875,
          73173: -0.007415771484375,
          144252: -0.01007080078125,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_16_MAX = {
          3478: 0.0128173828125,
          368: 0.01214599609375,
          109586: 0.0111083984375,
          35433: 0.01019287109375,
          2: 0.0157470703125,
          77: 0.0174560546875,
          132619: 0.0078125,
          2175: 0.0113525390625,
          23714: 0.0146484375,
          73173: 0.01116943359375,
          144252: 0.01141357421875,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_16_SUM = {"value": 0.0821533203125}
  
      EMBEDDINGS_DS_BEFORE_LN_F_32_MEAN = {
          132619: -0.00031267106533050537,
          3478: 0.00023087859153747559,
          368: -0.0005701072514057159,
          109586: -0.0003911703824996948,
          35433: -0.0001944899559020996,
          2: 0.0004146844148635864,
          2175: 0.00045740045607089996,
          23714: 0.0002641640603542328,
          73173: -0.0002864748239517212,
          144252: 0.0005256589502096176,
          77: 0.0006617321632802486,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_32_MIN = {
          3478: -0.00921630859375,
          368: -0.010009765625,
          109586: -0.01031494140625,
          35433: -0.01177978515625,
          2: -0.0074462890625,
          77: -0.00848388671875,
          132619: -0.009521484375,
          2175: -0.0074462890625,
          23714: -0.0145263671875,
          73173: -0.007415771484375,
          144252: -0.01007080078125,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_32_MAX = {
          3478: 0.0128173828125,
          368: 0.01214599609375,
          109586: 0.0111083984375,
          35433: 0.01019287109375,
          2: 0.0157470703125,
          77: 0.0174560546875,
          132619: 0.0078125,
          2175: 0.0113525390625,
          23714: 0.0146484375,
          73173: 0.01116943359375,
          144252: 0.01141357421875,
      }
      EMBEDDINGS_DS_BEFORE_LN_F_32_SUM = {"value": 0.08217757940292358}
  
      TEST_EMBEDDINGS = {
          "torch.bfloat16": {
              "mean": EMBEDDINGS_DS_BEFORE_LN_BF_16_MEAN,
              "max": EMBEDDINGS_DS_BEFORE_LN_BF_16_MAX,
              "min": EMBEDDINGS_DS_BEFORE_LN_BF_16_MIN,
              "sum": EMBEDDINGS_DS_BEFORE_LN_BF_16_SUM,
          },
          "torch.float32": {
              "mean": EMBEDDINGS_DS_BEFORE_LN_F_32_MEAN,
              "max": EMBEDDINGS_DS_BEFORE_LN_F_32_MAX,
              "min": EMBEDDINGS_DS_BEFORE_LN_F_32_MIN,
              "sum": EMBEDDINGS_DS_BEFORE_LN_F_32_SUM,
          },
          "torch.float": {
              "mean": EMBEDDINGS_DS_BEFORE_LN_F_32_MEAN,
              "max": EMBEDDINGS_DS_BEFORE_LN_F_32_MAX,
              "min": EMBEDDINGS_DS_BEFORE_LN_F_32_MIN,
              "sum": EMBEDDINGS_DS_BEFORE_LN_F_32_SUM,
          },
          "torch.float16": {
              "mean": EMBEDDINGS_DS_BEFORE_LN_F_16_MEAN,
              "max": EMBEDDINGS_DS_BEFORE_LN_F_16_MAX,
              "min": EMBEDDINGS_DS_BEFORE_LN_F_16_MIN,
              "sum": EMBEDDINGS_DS_BEFORE_LN_F_16_SUM,
          },
      }
  
      EXAMPLE_IDS = [3478, 368, 109586, 35433, 2, 77, 132619, 3478, 368, 109586, 35433, 2, 2175, 23714, 73173, 144252, 2, 77, 132619, 3478]  # fmt: skip
  
      EMBEDDINGS_DS_AFTER_LN_MEAN = {
          3478: -6.580352783203125e-05,
          368: 0.0001316070556640625,
          109586: -0.00030517578125,
          35433: 4.00543212890625e-05,
          2: -7.2479248046875e-05,
          77: -8.96453857421875e-05,
          132619: 0.0001583099365234375,
          2175: 2.1219253540039062e-05,
          23714: -0.000247955322265625,
          73173: -0.00021839141845703125,
          144252: -0.0001430511474609375,
      }
      EMBEDDINGS_DS_AFTER_LN_MIN = {
          3478: -1.6953125,
          368: -1.6875,
          109586: -1.6875,
          35433: -2.125,
          2: -1.390625,
          77: -1.5390625,
          132619: -1.875,
          2175: -1.4609375,
          23714: -2.296875,
          73173: -1.3515625,
          144252: -1.78125,
      }
      EMBEDDINGS_DS_AFTER_LN_MAX = {
          3478: 2.265625,
          368: 2.28125,
          109586: 1.953125,
          35433: 1.90625,
          2: 2.703125,
          77: 2.828125,
          132619: 1.65625,
          2175: 2.015625,
          23714: 2.234375,
          73173: 2.171875,
          144252: 1.828125,
      }
  
      EMBEDDINGS_DS_AFTER_LN = {
          "mean": EMBEDDINGS_DS_AFTER_LN_MEAN,
          "min": EMBEDDINGS_DS_AFTER_LN_MIN,
          "max": EMBEDDINGS_DS_AFTER_LN_MAX,
      }
  
      tensor_ids = torch.LongTensor([EXAMPLE_IDS])
      with torch.no_grad():
          embeddings = model.transformer.word_embeddings(tensor_ids)
          embeddings_ln = model.transformer.word_embeddings_layernorm(
              embeddings
          )  #
      # first check the embeddings before LN
      output_dict = {
          "min": {},
          "max": {},
          "mean": {},
          "sum": {"value": embeddings.sum().item()},
      }
      for i, idx in enumerate(EXAMPLE_IDS):
          output_dict["min"][idx] = (
              embeddings.min(dim=-1).values[0][i].item()
          )
          output_dict["max"][idx] = (
              embeddings.max(dim=-1).values[0][i].item()
          )
          output_dict["mean"][idx] = embeddings.mean(dim=-1)[0][
              i
          ].item()
  
      for key in TEST_EMBEDDINGS[str(model.dtype)].keys():
          self.assertDictEqual(
              TEST_EMBEDDINGS[str(model.dtype)][key], output_dict[key]
          )
  
      output_dict_norm = {"min": {}, "max": {}, "mean": {}}
      for i, idx in enumerate(EXAMPLE_IDS):
          output_dict_norm["min"][idx] = (
              embeddings_ln.min(dim=-1).values[0][i].item()
          )
          output_dict_norm["max"][idx] = (
              embeddings_ln.max(dim=-1).values[0][i].item()
          )
          output_dict_norm["mean"][idx] = embeddings_ln.mean(dim=-1)[0][
              i
          ].item()
  
      # This test does not pass when places = 2
      for i, key in enumerate(output_dict_norm.keys()):
          for j, idx in enumerate(output_dict[key].keys()):
              self.assertAlmostEqual(
                  EMBEDDINGS_DS_AFTER_LN[key][idx],
                  output_dict_norm[key][idx],
                  places=1,
              )
  
imports: import math
  import unittest
  from transformers import BloomConfig, is_torch_available
  from transformers.testing_utils import require_torch, require_torch_accelerator, slow, torch_device
  from ...generation.test_utils import GenerationTesterMixin
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
label_desc: Use image node_id 2 for calling the BloomEmbeddingTest obj's underlying member method code with example usage: obj.test_embeddings() without return types