code: def map_dataset(
      examples: dict[str, list[str]],
      args: "Args",
      context: TokenizationContext,
  ) -> dict:
      prompts = examples["prompt"]
      completions = examples["completion"]
      print(
          "[Prompt]",
          prompts[0],
          "[Completion]",
          completions[0],
          sep="\n",
      )
  
      assert len(prompts) == len(completions)
      prompt_config = EncodingConfig(add_bos=True, add_eos=False)
      completion_config = EncodingConfig(add_bos=False, add_eos=True)
      prompt_id_batches = context.encode(prompt_config, prompts)
      completion_id_batches = context.encode(
          completion_config, completions
      )
      # prompt_id_batches = context.tokenization_context.encode(prompt_config, prompts)
      # completion_id_batches = context.tokenization_context.encode(
      #     completion_config, completions
      # )
      assert len(prompt_id_batches) == len(completion_id_batches)
      untruncated_input_ids = [
          (instruction_ids + response_ids)
          for instruction_ids, response_ids in zip(
              prompt_id_batches, completion_id_batches
          )
      ]
      exceeding_length = [
          len(input_id) > args.max_training_seq_length
          for input_id in untruncated_input_ids
      ]
      input_ids = [
          input_id[: args.max_training_seq_length]
          for input_id in untruncated_input_ids
      ]
      # NOTE: no need to set EOF to IGNORED_INDEX as it is *implicitly* ignored inside
      # the model.forward that shifts the logits left by 1
      labels = [
          (
              list(map(lambda _: IGNORED_INDEX, instruction_ids))
              + response_ids
          )[: args.max_training_seq_length]
          for instruction_ids, response_ids in zip(
              prompt_id_batches, completion_id_batches
          )
      ]
      # `len` of each returned value must be the same, which is required by `tokenizer.map`
      # After `map`, they are treated as individual pieces of data, not as a batch.
      assert len(input_ids) == len(labels)
      for input_id_batch, label_batch in zip(input_ids, labels):
          assert len(input_id_batch) == len(label_batch)
      print(context.decode(DecodingConfig.default(), input_ids[0:])[0])
      return {
          "input_ids": input_ids,
          "labels": labels,
          "exceeding_length": exceeding_length,
      }
  
file_path: magicoder/experiments/train_comment_code_pairs.py
imports: import random
  from dataclasses import dataclass, field
  from typing import cast
  import torch
  from datasets import Dataset, load_dataset
  from tqdm.auto import tqdm
  from transformers import HfArgumentParser, Trainer, TrainingArguments
  from magicoder.llm_wrapper import DecodingConfig, EncodingConfig, TokenizationContext, get_model_context, pad_sequences
  from magicoder.prompt_template import MAGICODER_PROMPT
  from magicoder.utils import N_CORES, read_jsonl, write_jsonl
label_desc: Use image node_id 3 for calling a global function with example usage: map_dataset(examples, args, context) and returns: dict