Hugging Face's logo

pietrolesci
/
tokenisers

Model card Files Community
pietrolesci commited on
Commit
c5527f0
·
verified ·
1 Parent(s): 5673c97

Upload folder using huggingface_hub

Browse files
Files changed (20) hide show
  1. bpe128000minipile/raw_tok_path.txt +1 -0
  2. bpe128000minipile/special_tokens_map.json +3 -0
  3. bpe128000minipile/tokenizer.json +0 -0
  4. bpe128000minipile/tokenizer_config.json +16 -0
  5. bpe2wp32000minipile/raw_tok_path.txt +1 -0
  6. bpe2wp32000minipile/special_tokens_map.json +3 -0
  7. bpe2wp32000minipile/tokenizer.json +0 -0
  8. bpe2wp32000minipile/tokenizer_config.json +17 -0
  9. bpe32000minipile/raw_tok_path.txt +1 -0
  10. bpe32000minipile/special_tokens_map.json +3 -0
  11. bpe32000minipile/tokenizer.json +0 -0
  12. bpe32000minipile/tokenizer_config.json +17 -0
  13. bpe8064minipile/raw_tok_path.txt +1 -0
  14. bpe8064minipile/special_tokens_map.json +3 -0
  15. bpe8064minipile/tokenizer.json +0 -0
  16. bpe8064minipile/tokenizer_config.json +16 -0
  17. wordpiece32000minipile/raw_tok_path.txt +1 -0
  18. wordpiece32000minipile/special_tokens_map.json +4 -0
  19. wordpiece32000minipile/tokenizer.json +0 -0
  20. wordpiece32000minipile/tokenizer_config.json +25 -0
bpe128000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe128000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe128000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe128000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
bpe2wp32000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe2wp32000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe2wp32000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe2wp32000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "extra_special_tokens": {},
15
+ "model_max_length": 1000000000000000019884624838656,
16
+ "tokenizer_class": "PreTrainedTokenizerFast"
17
+ }
bpe32000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe32000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe32000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe32000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "extra_special_tokens": {},
15
+ "model_max_length": 1000000000000000019884624838656,
16
+ "tokenizer_class": "PreTrainedTokenizerFast"
17
+ }
bpe8064minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe8064minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe8064minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe8064minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
wordpiece32000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/wordpiece_minipile_2025-02-03T12-10-57
wordpiece32000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>",
3
+ "unk_token": "<|unk|>"
4
+ }
wordpiece32000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
wordpiece32000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|unk|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ }
19
+ },
20
+ "clean_up_tokenization_spaces": true,
21
+ "eos_token": "<|endoftext|>",
22
+ "model_max_length": 1000000000000000019884624838656,
23
+ "tokenizer_class": "PreTrainedTokenizerFast",
24
+ "unk_token": "<|unk|>"
25
+ }