Hugging Face's logo

pietrolesci
/
tokenisers

Model card Files Community
pietrolesci commited on
Commit
cacd0e5
·
verified ·
1 Parent(s): c5527f0

Upload folder using huggingface_hub

Browse files
Files changed (9) hide show
  1. .gitattributes +3 -0
  2. _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/all_merges.jsonl +0 -0
  3. _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl +3 -0
  4. _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/metadata.yaml +3 -0
  5. _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/tokenizer.json +3 -0
  6. _raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/all_merges.jsonl +0 -0
  7. _raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/implemented_merges.jsonl +3 -0
  8. _raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/metadata.yaml +4 -0
  9. _raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/tokenizer.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
37
+ _raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ _raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
_raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
_raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10c86b1954635fa7975e6f70b69c1bd3c996714dd775c2a0710d490b5e8404ac
3
+ size 35288764
_raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/metadata.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 1000000
3
+ eos_token: <|endoftext|>
_raw_tokenisers/bpe_minipile_2024-10-11T11-23-27/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b597d8e1fa7b685bfda953a15e6981be9ea03e56f008c6cb521caf28d0a72d6
3
+ size 23777543
_raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
_raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:030d3efa446342bfd5dd49d244b3bcbc4f3f13a32b5c7bb864d313c20ed71c72
3
+ size 36200163
_raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/metadata.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 1000000
3
+ eos_token: <|endoftext|>
4
+ unk_token: <|unk|>
_raw_tokenisers/wordpiece_minipile_2025-02-03T12-10-57/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff