code: def test_pack_dataset(self):
      tokenizer = AutoTokenizer.from_pretrained(
          "facebook/mbart-large-cc25"
      )
  
      tmp_dir = Path(
          make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
      )
      orig_examples = (
          tmp_dir.joinpath("train.source").open().readlines()
      )
      save_dir = Path(
          make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
      )
      pack_data_dir(tokenizer, tmp_dir, 128, save_dir)
      orig_paths = {x.name for x in tmp_dir.iterdir()}
      new_paths = {x.name for x in save_dir.iterdir()}
      packed_examples = (
          save_dir.joinpath("train.source").open().readlines()
      )
      # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
      # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
      assert len(packed_examples) < len(orig_examples)
      assert len(packed_examples) == 1
      assert len(packed_examples[0]) == sum(
          len(x) for x in orig_examples
      )
      assert orig_paths == new_paths
  
imports: import os
  from pathlib import Path
  import numpy
  import pytest
  from torch.utils.data import DataLoader
  from pack_dataset import pack_data_dir
  from parameterized import parameterized
  from save_len_file import save_len_file
  from transformers import AutoTokenizer
  from transformers.models.mbart.modeling_mbart import shift_tokens_right
  from transformers.testing_utils import TestCasePlus, slow
  from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
label_desc: Use image node_id 3 for calling the TestAll obj's underlying member method code with example usage: obj.test_pack_dataset() without return types