Update README.md
Browse files
README.md
CHANGED
@@ -4,6 +4,6 @@ BPE tokenizer based on `tiktoken.cl100k_base`.
|
|
4 |
|
5 |
```
|
6 |
from transformers import AutoTokenizer
|
7 |
-
tokenizer = AutoTokenizer.from_pretrained("
|
8 |
tokenizer("hello, world!", return_tensors='pt')
|
9 |
```
|
|
|
4 |
|
5 |
```
|
6 |
from transformers import AutoTokenizer
|
7 |
+
tokenizer = AutoTokenizer.from_pretrained("stabilityai/arcade100k", trust_remote_code=True)
|
8 |
tokenizer("hello, world!", return_tensors='pt')
|
9 |
```
|