qwen.ai helper

breakupText.py

@@ -0,0 +1,42 @@
+# text_to_training_csv.py
+
+import sys
+import csv
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python text_to_training_csv.py <chunk_size>", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        chunk_size = int(sys.argv[1])
+    except ValueError:
+        print("Error: Chunk size must be an integer.", file=sys.stderr)
+        sys.exit(1)
+
+    # CSV writer setup
+    writer = csv.writer(sys.stdout)
+    writer.writerow(["id", "text"])  # Header row
+
+    id_counter = 1
+    buffer = ''
+
+    while True:
+        chunk = sys.stdin.read(chunk_size)
+        if not chunk:
+            break
+        buffer += chunk
+
+        # If we've reached or exceeded chunk_size, write and reset
+        if len(buffer) >= chunk_size:
+            writer.writerow([id_counter, buffer[:chunk_size]])
+            id_counter += 1
+            buffer = buffer[chunk_size:]  # Remaining text
+
+    # Write any leftover text
+    if buffer:
+        writer.writerow([id_counter, buffer])
+        id_counter += 1
+
+if __name__ == "__main__":
+    main()

chunkedCSVOutput.py

@@ -0,0 +1,46 @@
+# text_to_csv_chunks.py
+
+import sys
+import csv
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python text_to_csv_chunks.py <chunk_size>", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        chunk_size = int(sys.argv[1])
+    except ValueError:
+        print("Error: Chunk size must be an integer.", file=sys.stderr)
+        sys.exit(1)
+
+    # Prepare CSV writer
+    writer = csv.writer(sys.stdout)
+    # Write header
+    writer.writerow(["Chunk", "Text"])
+
+    chunk_number = 1
+    line_buffer = []
+    line_count = 0
+
+    for line in sys.stdin:
+        line_buffer.append(line.rstrip('\n'))
+        line_count += 1
+
+        if line_count >= chunk_size:
+            # Join lines and write chunk
+            chunk_text = '\n'.join(line_buffer)
+            writer.writerow([chunk_number, chunk_text])
+
+            # Reset
+            chunk_number += 1
+            line_buffer = []
+            line_count = 0
+
+    # Write any remaining lines
+    if line_buffer:
+        chunk_text = '\n'.join(line_buffer)
+        writer.writerow([chunk_number, chunk_text])
+
+if __name__ == "__main__":
+    main()

csvReader.py

@@ -0,0 +1,15 @@
+# csv_reader.py
+
+import sys
+import csv
+
+def main():
+    if len(sys.argv) > 1:
+        print("Warning: This script ignores any command-line arguments.", file=sys.stderr)
+
+    reader = csv.reader(sys.stdin)
+    for row in reader:
+        print(f"Row: {row}")
+
+if __name__ == "__main__":
+    main()

loadDataset.py

@@ -0,0 +1,28 @@
+from datasets import load_dataset
+import random
+
+# Set a fixed seed for reproducibility
+
+# Generate a random integer from 0 to 256 (inclusive)
+random_number = random.randint(0, 256)
+
+#print(random_number)
+random.seed(random_number)
+
+# Load the dataset
+dataset = load_dataset("ajsbsd/14400")
+train_dataset = dataset['train']
+
+# Get total number of examples
+total_examples = len(train_dataset)
+print(f"Total examples in dataset: {total_examples}\n")
+
+# Pick 5 unique random indices
+random_indices = random.sample(range(total_examples), 5)
+
+# Print the 5 random examples
+for idx in random_indices:
+    example = train_dataset[idx]
+    print(f"--- Example (ID: {idx}) ---")
+    print(f"Chunk ID: {example['id']}")
+    print(f"Text:\n{example['text']}\n")

testToTraining.py

@@ -0,0 +1,39 @@
+# text_to_training_csv.py
+
+import sys
+import csv
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python text_to_training_csv.py <chunk_size>", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        chunk_size = int(sys.argv[1])
+    except ValueError:
+        print("Error: Chunk size must be an integer.", file=sys.stderr)
+        sys.exit(1)
+
+    writer = csv.writer(sys.stdout)
+    writer.writerow(["id", "text"])
+
+    id_counter = 1
+    buffer = ''
+
+    while True:
+        chunk = sys.stdin.read(chunk_size)
+        if not chunk:
+            break
+        buffer += chunk
+
+        if len(buffer) >= chunk_size:
+            writer.writerow([id_counter, buffer[:chunk_size]])
+            id_counter += 1
+            buffer = buffer[chunk_size:]
+
+    # Write remaining buffer
+    if buffer:
+        writer.writerow([id_counter, buffer])
+
+if __name__ == "__main__":
+    main()

trainingText.py

@@ -0,0 +1,75 @@
+#!//home/aaron/gradio_test/bin/python
+### ✅ Example: Pull Random Records Based on Dataset Size
+#
+#Here’s a complete Python example using Hugging Face's `datasets` library:
+#
+from datasets import load_dataset
+import random
+
+# Set seed for reproducibility (optional)
+random.seed(42)
+
+# Load dataset from Hugging Face
+dataset = load_dataset("ajsbsd/14400")
+train_dataset = dataset["train"]
+
+# Get total number of records
+total_records = len(train_dataset)
+print(f"Total records in dataset: {total_records}\n")
+
+# Ask user how many random samples they want
+#num_samples = int(input("How many random records would you like to see? "))
+num_samples = total_records
+
+# Ensure valid input
+if num_samples <= 0 or num_samples > total_records:
+    print(f"Please enter a number between 1 and {total_records}.")
+else:
+    # Generate random indices
+    random_indices = random.sample(range(total_records), num_samples)
+
+    # Print random records
+    for i, idx in enumerate(random_indices, 1):
+        record = train_dataset[idx]
+        print(f"--- Record #{i} (Index: {idx}) ---")
+        print(f"ID: {record['id']}")
+        print(f"Text:\n{record['text']}\n")
+
+### 🧠 What This Does
+#
+# Loads the dataset
+# Gets the total number of records automatically
+# Asks the user how many random entries they want to see
+# Picks that many random rows and prints them
+#
+### 🔁 Example Run
+#
+#Total records in dataset: 256
+#
+#How many random records would you like to see? 5
+#
+#--- Record #1 (Index: 203) ---
+#ID: 204
+#Text:
+#It was the...
+#
+#--- Record #2 (Index: 15) ---
+#ID: 16
+#Text:
+#The period...
+#
+#
+### 📌 Want to Do This Without User Input?
+#
+#You can hardcode the number of samples:
+#
+#num_samples = 5
+#
+#Or make it part of a function:
+#
+#
+#def get_random_samples(dataset, num_samples):
+#    total = len(dataset)
+#    indices = random.sample(range(total), num_samples)
+#    return [dataset[i] for i in indices]
+