Spaces:
Running
Running
taka-yamakoshi
commited on
Commit
·
f70863b
1
Parent(s):
6353e7e
fix bugs
Browse files
app.py
CHANGED
@@ -55,6 +55,7 @@ if __name__=='__main__':
|
|
55 |
sents[f'sent_{sent_id+1}'] = sentence
|
56 |
if len(sentence)>0:
|
57 |
input_sent = tokenizer(sentence)['input_ids']
|
|
|
58 |
decoded_sent = [tokenizer.decode([token]) for token in input_sent[1:-1]]
|
59 |
num_tokens[f'sent_{sent_id+1}'] = len(decoded_sent)
|
60 |
|
@@ -63,7 +64,7 @@ if __name__=='__main__':
|
|
63 |
#for word_col,word in zip(word_cols,decoded_sent):
|
64 |
#with word_col:
|
65 |
#st.write(word)
|
66 |
-
st.write(' '.join(
|
67 |
st.write(' '.join(decoded_sent))
|
68 |
st.markdown(f"<p style='text-align: center; color: black; font-family:Arial; font-size:20px;'>{len(decoded_sent)} tokens </p>", unsafe_allow_html=True)
|
69 |
|
|
|
55 |
sents[f'sent_{sent_id+1}'] = sentence
|
56 |
if len(sentence)>0:
|
57 |
input_sent = tokenizer(sentence)['input_ids']
|
58 |
+
encoded_sent = [str(token) for token in input_sent]
|
59 |
decoded_sent = [tokenizer.decode([token]) for token in input_sent[1:-1]]
|
60 |
num_tokens[f'sent_{sent_id+1}'] = len(decoded_sent)
|
61 |
|
|
|
64 |
#for word_col,word in zip(word_cols,decoded_sent):
|
65 |
#with word_col:
|
66 |
#st.write(word)
|
67 |
+
st.write(' '.join(encoded_sent))
|
68 |
st.write(' '.join(decoded_sent))
|
69 |
st.markdown(f"<p style='text-align: center; color: black; font-family:Arial; font-size:20px;'>{len(decoded_sent)} tokens </p>", unsafe_allow_html=True)
|
70 |
|