app.py

from transformers import pipeline
import requests
import json

pipe = pipeline("translation", "guymorlan/TokenizerLabeller")

# download json and open
# from https://huggingface.co/guymorlan/TokenizerLabeller/raw/main/playaling_words.json
r = requests.get("https://huggingface.co/guymorlan/TokenizerLabeller/raw/main/playaling_words.json")
data = json.loads(r.text)


# built gradio interface
import gradio as gr

def predict(input):

    out = pipe(input)[0]['translation_text']
    out = [x.strip() for x in out.split("+")]

    output =  ""
    for o in out:
        if o in data:
            output += f"<span style='color: green' title='{data[o]['translation']}\n{data[o]['features']}'>{data[o]['word']}</span> "
        else:
            output += o + " "

    return output


gr.Interface(predict, "textbox", "html", title="Ammiya Tokenizer", description="Tokenize Ammiya text and show Playaling words").launch()