loadDataset.py

from datasets import load_dataset
import random

# Set a fixed seed for reproducibility

# Generate a random integer from 0 to 256 (inclusive)
random_number = random.randint(0, 256)

#print(random_number)
random.seed(random_number)

# Load the dataset
dataset = load_dataset("ajsbsd/14400")
train_dataset = dataset['train']

# Get total number of examples
total_examples = len(train_dataset)
print(f"Total examples in dataset: {total_examples}\n")

# Pick 5 unique random indices
random_indices = random.sample(range(total_examples), 5)

# Print the 5 random examples
for idx in random_indices:
    example = train_dataset[idx]
    print(f"--- Example (ID: {idx}) ---")
    print(f"Chunk ID: {example['id']}")
    print(f"Text:\n{example['text']}\n")