File size: 3,984 Bytes
734e414 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 |
pretrain_base_datasets = [
#
# multilingual
#
# 3.17 GB, 2,226,907
*[
{'kind': 'base', 'path': 'ontocord/fineweb-permissive-multilingual-2m', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
for i in range(0, 100, 10)
],
# 1.64 GB, 1,001,000
*[
{'kind': 'base', 'path': 'distily/c4_multilingual_1M', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
for i in range(0, 100, 10)
],
# 3.8 GB, 19,454,996
*[
{'kind': 'base', 'path': 'sentence-transformers/parallel-sentences-wikimatrix', 'data_dir': 'all', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['non_english']}
for i in range(0, 100, 10)
],
#
# general knowledge
#
# 65.1 MB, 7,819
{'kind': 'base', 'path': 'Sketched33/Cities_Wikipedia_Information', 'format': lambda n: n['wikipedia_content']},
# 135 MB, 1,795
{'kind': 'base', 'path': 'open-phi/textbooks', 'format': lambda n: n['markdown']},
# 631 MB, 111,048
{'kind': 'base', 'path': 'open-phi/programming_books_llama', 'format': lambda n: n['markdown']},
#
# misc
#
# 472 KB, 5,034
{'kind': 'base', 'path': 'badrex/llm-emoji-dataset', 'format': '{short description}. {LLM description}. {character}'},
#
# math
#
# 12.6 GB, 14M rows
*[
{'kind': 'base', 'path': 'nvidia/OpenMathInstruct-2', 'split': f'train[{i}%:{i + 10}%]', 'format': '{problem} {generated_solution} {expected_answer}'}
for i in range(0, 100, 10)
],
#
# stem
#
# 1.44 GB, 63,357
*[
{'kind': 'base', 'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['abstract']}
for i in range(0, 100, 10)
],
*[
{'kind': 'base', 'path': 'neuralwork/arxiver', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['markdown']}
for i in range(0, 100, 10)
],
#
# code
#
# 7.81 GB, ~2,804,025
*[
{'kind': 'base', 'path': 'rombodawg/code_bagel_hermes-2.5', 'split': f'train[{i}%:{i + 10}%]', 'format': '{input} {output}'}
for i in range(0, 100, 10)
],
#
# multilingual
#
# 742 MB, 321,697
*[
{'kind': 'base', 'path': 'data-silence/sumnews', 'split': split, 'format': lambda n: n[field]}
for split in ['train', 'test']
for field in ['title', 'resume', 'news']
],
# 193 MB, 1,141,967
*[
{'kind': 'base', 'path': 'xu-song/cc100-samples', 'name': name, 'split': 'train', 'format': lambda n: n['text']}
for name in [
'am', 'ar', 'as', 'az', 'be', 'bg', 'bn', 'bn_rom', 'br',
'bs', 'ca', 'cs', 'cy', 'da', 'de', 'el', 'en', 'eo', 'es',
'et', 'eu', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl',
'gn', 'gu', 'ha', 'he', 'hi', 'hi_rom', 'hr', 'ht', 'hu',
'hy', 'id', 'ig', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km',
'kn', 'ko', 'ku', 'ky', 'la', 'lg', 'li', 'ln', 'lo', 'lt',
'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'my_zaw',
'ne', 'nl', 'no', 'ns', 'om', 'or', 'pa', 'pl', 'ps', 'pt',
'qu', 'rm', 'ro', 'ru', 'sa', 'si', 'sc', 'sd', 'sk', 'sl',
'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'ta_rom',
'te', 'te_rom', 'th', 'tl', 'tn', 'tr', 'ug', 'uk', 'ur',
'ur_rom', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo',
'zh-Hans', 'zh-Hant', 'zu',
]
],
#
# general knowledge
#
# 3.18 GB, 1,010,500 - uncompressed 6GB
*[
{'kind': 'base', 'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
for i in range(0, 100, 10)
],
{'kind': 'base', 'path': 'JeanKaddour/minipile', 'split': 'validation', 'format': lambda n: n['text']},
{'kind': 'base', 'path': 'JeanKaddour/minipile', 'split': 'test', 'format': lambda n: n['text']},
]
|