app.py

import gradio as gr
from minbpe.regex import RegexTokenizer
from typing import List, Dict

# Initialize the tokenizer
tokenizer = RegexTokenizer()
tokenizer.load("modelsV3-regex-new-testament-only/regex.model")

def process_text(text: str) -> str:
    # Get token IDs
    token_ids = tokenizer.encode(text)
    
    # Build formatted output
    output = []
    output.append("Token Details:")
    output.append("-" * 50)
    
    for token_id in token_ids:
        # Get the bytes for this token
        token_bytes = tokenizer.vocab[token_id]
        # Convert bytes to string representation, replacing invalid chars
        token_text = token_bytes.decode('utf-8', errors='replace')
        
        output.append(f"Token ID: {token_id}")
        output.append(f"Token Bytes: {str(token_bytes)}")
        output.append(f"Token Text: {token_text}")
        output.append("-" * 50)
    
    # Add decoded text at the end
    decoded_text = tokenizer.decode(token_ids)
    output.append(f"\nFull decoded text: {decoded_text}")
    
    return "\n".join(output)

# Create Gradio interface
demo = gr.Interface(
    fn=process_text,
    inputs=gr.Textbox(label="Enter text to tokenize", lines=3),
    outputs=gr.Textbox(label="Token Information", lines=10),
    title="Hindi Text Tokenizer",
    description="Enter Hindi text to see its token representation using Byte Pair Encoding",
    examples=[
        ["उज्जियाह से योताम उत्पन्न"],
        ["नमस्ते दुनिया"]
    ]
)

if __name__ == "__main__":
    demo.launch()