Update app.py

app.py

@@ -7,17 +7,20 @@ import matplotlib.pyplot as plt
 st.set_page_config(layout="wide")
 
 with st.sidebar:
-    subset = st.selectbox('Flores eng_Latn-ukr_Cyrl subset', ('dev', 'devtest'))
+    subset = st.selectbox('subset', ('dev', 'devtest'))
 
-tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
-flores =  load_dataset("facebook/flores", "eng_Latn-ukr_Cyrl")
-dataset = flores[subset]
+with st.echo():
+    tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
+    flores = load_dataset("facebook/flores", "eng_Latn-ukr_Cyrl")
+    dataset = flores[subset]
+    eng_num_tokens = dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_eng_Latn'])['input_ids'])})['num_tokens']
+    ukr_num_tokens = dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_ukr_Cyrl'])['input_ids'])})['num_tokens']
 
 fig, (axl, axr) = plt.subplots(1, 2, figsize=(10,3))
-axl.hist(dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_eng_Latn'])['input_ids'])})['num_tokens'])
-axl.set_title('eng mistral tokens')
-axr.hist(dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_ukr_Cyrl'])['input_ids'])})['num_tokens'])
-axr.set_title('ukr mistral tokens')
+axl.hist(eng_num_tokens)
+axl.set_title(f'eng mistral tokens ({np.sum(eng_num_tokens)} total)')
+axr.hist(ukr_num_tokens)
+axr.set_title(f'ukr mistral tokens ({np.sum(ukr_num_tokens)} total)')
 st.pyplot(fig)
 
 st.dataframe(pd.DataFrame(dataset))