add ava, bqi, lki, lrc, bar, bal.

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+---
+title: GlotWeb
+emoji: 🕸
+colorFrom: pink
+colorTo: red
+sdk: streamlit
+sdk_version: 1.33.0
+app_file: app.py
+pinned: True
+license: mit
+---
+

app.py

@@ -0,0 +1,111 @@
+import pandas as pd
+import streamlit as st
+from utils import nav_to, df_to_html, render_svg, combine_json_files
+
+data = combine_json_files('./languages')
+
+
+@st.cache_data
+def render_home_table():
+    """Renders home table."""
+    # Compute number of unique domains/urls
+    for key in data.keys():
+        data[key]['Number of Sites'] = len(data[key].get('Sites', []))
+        data[key]["Number of Links"] = sum(len(url_data["Links"]) for url_data in data[key].get('Sites', []))
+
+    # Convert dict to df
+    df_data = pd.DataFrame(data).transpose()
+    df_data['ISO Code'] = df_data.index
+
+    # Sort by ISO Code
+    df_data = df_data.sort_values(by='ISO Code')
+
+    df_data['Number of Sites'] = df_data['Number of Sites'].astype(str)  # Convert to string
+    df_data['ISO Code'] = df_data['ISO Code'].astype(str)  # Convert to string
+    df_data['Number of Sites'] = df_data.apply(lambda row: '<a href="/?isocode={}&site=True" target="_self">{}</a>'.format(row['ISO Code'], row['Number of Sites']), axis=1)
+    df_data['Number of Links'] = df_data.apply(lambda row: '<a href="/?isocode={}&links=True" target="_self">{}</a>'.format(row['ISO Code'], row['Number of Links']), axis=1)
+
+
+    # Display the table
+    df_data = df_data[['ISO Code', 'Language Name', 'Family', 'Subgrouping', 'Number of Sites', 'Number of Links', 'Number of Speakers', 'Supported by allenai/MADLAD-400 or facebook/flores']]
+    st.write(df_to_html(df_data), unsafe_allow_html=True)
+
+
+@st.cache_data
+def render_site_table(isocode):
+
+    # back 
+    back_text = '<a href="/?home=True" target="_self">Back</a>'
+    st.markdown(back_text, unsafe_allow_html=True)
+
+    st.write({'Language Name': data[isocode]['Language Name'], 'ISO Code': isocode})
+
+    # site
+    urls = data[isocode].get('Sites', [])
+    df_urls = pd.DataFrame(urls)
+    df_urls['Number of Links'] = df_urls['Links'].apply(len)
+    df_urls = df_urls.sort_values(by='Number of Links', ascending=False)
+    df_urls = df_urls.reset_index(drop=True)
+    df_urls['Number of Links'] = df_urls.apply(lambda row: '<a href="/?isocode={}&siteurl={}" target="_self">{}</a>'.format(isocode, row['Site URL'], row['Number of Links']) if row['Number of Links'] else row['Number of Links'], axis=1)
+
+    # Display the table
+    df_urls = df_urls[['Site Name', 'Site URL', 'Category', 'Number of Links', 'Possible Parallel Languages', 'Confidence', 'Info']]
+    st.write(df_to_html(df_urls), unsafe_allow_html=True)
+
+
+@st.cache_data
+def render_siteurl_table(isocode, url):
+
+    # back
+    back_text = '<a href="/?isocode={}&site=True" target="_self">Back</a>'.format(isocode)
+    st.markdown(back_text, unsafe_allow_html=True)
+
+    # Find selected domain
+    urls = data[isocode].get('Sites', [])
+    selected_domain = next((d for d in urls if 'Site URL' in d and d['Site URL'] == url), None)
+    
+    if selected_domain:
+        st.write({'Language Name': data[isocode]['Language Name'], 'ISO Code': isocode, 'Site URL': url, 'Links': selected_domain['Links']})
+
+
+
+@st.cache_data
+def render_links_table(isocode):
+
+    # back 
+    back_text = '<a href="/?home=True" target="_self">Back</a>'
+    st.markdown(back_text, unsafe_allow_html=True)
+
+    # output
+    urls = data[isocode].get('Sites', [])
+    lang_name = data[isocode]['Language Name']
+    all_urls = [{'Site URL': du['Site URL'], 'Links': du['Links']} for du in urls] 
+        
+    st.write({'Language Name': lang_name, 'ISO Code': isocode, 'URLs': all_urls})
+
+
+
+# show logo
+render_svg(open("assets/glotweb_logo.svg").read())
+st.text("")
+
+
+def main():
+    params = st.query_params
+
+    if 'isocode' in params:
+        if 'siteurl' in params:
+            render_siteurl_table(params['isocode'], params['siteurl'])
+        if 'site' in params:
+            render_site_table(params['isocode'])
+        if 'links' in params:
+            render_links_table(params['isocode'])
+
+    elif 'home' not in params:
+        # redirect to .space
+        nav_to('https://cis-lmu-glotweb.hf.space/?home=True')
+    else:
+        # show home
+        render_home_table()
+
+main()

languages/ava_Cyrl.json

@@ -0,0 +1,18 @@
+{
+    "Language Name": "Avaric",
+    "Family": "Nakh-Daghestanian",
+    "Subgrouping": "Avar–Andic",
+    "Number of Speakers": "800_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟨",
+    "Sites": [
+        {
+            "Site Name": "radioerkenli.com",
+            "Site URL": "https://www.radioerkenli.com/",
+            "Category": "news",
+            "Confidence": "🟩",
+            "Info": "confirmed by glotlid, some webpage annotation also backed it up.",
+            "Possible Parallel Languages": "",
+            "Links": []
+        }
+    ]
+}

languages/bal_Arab.json

@@ -0,0 +1,45 @@
+{
+    "Language Name": "Balochi",
+    "Family": "Indo-European",
+    "Subgrouping": "Iranian",
+    "Number of Speakers": "8_000_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟥",
+    "Sites": [
+        {
+            "Site Name": "sunnionline.us",
+            "Site URL": "https://sunnionline.us/balochi/",
+            "Category": "news",
+            "Confidence": "🟩",
+            "Info": "confirmed by webpage metadata",
+            "Possible Parallel Languages": "eng_Latn, fas_Arab, urd_Arab, arb_Arab, rus_Cyrl",
+            "Links": []
+        },
+        {
+            "Site Name": "kissah.org",
+            "Site URL": "https://kissah.org/",
+            "Category": "literature",
+            "Confidence": "🟩",
+            "Info": "confirmed by native speakers approval",
+            "Possible Parallel Languages": "",
+            "Links": []
+        },
+        {
+            "Site Name": "baask.com",
+            "Site URL": "https://baask.com/archive/category/balochi/",
+            "Category": "literature",
+            "Confidence": "🟩",
+            "Info": "confirmed by webpage metadata",
+            "Possible Parallel Languages": "",
+            "Links": []
+        },
+        {
+            "Site Name": "facebook.com/BaaskDotCom",
+            "Site URL": "https://www.facebook.com/BaaskDotCom",
+            "Category": "literature, social media",
+            "Confidence": "🟩",
+            "Info": "confirmed by webpage metadata",
+            "Possible Parallel Languages": "",
+            "Links": []
+        }
+    ]
+}

languages/bar_Latn.json

@@ -0,0 +1,27 @@
+{
+    "Language Name": "Bavarian",
+    "Family": "Indo-European",
+    "Subgrouping": "Germanic",
+    "Number of Speakers": "14_000_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟥",
+    "Sites": [
+        {
+            "Site Name": "bar.wikipedia.org",
+            "Site URL": "https://bar.wikipedia.org/wiki/",
+            "Category": "article",
+            "Confidence": "🟩",
+            "Info": "confirmed by webpage metadata",
+            "Possible Parallel Languages": "many",
+            "Links": []
+        },
+        {
+            "Site Name": "twitter.com/bayernuhr",
+            "Site URL": "https://twitter.com/bayernuhr",
+            "Category": "social media",
+            "Confidence": "🟩",
+            "Info": "glotlid search on http://indigenoustweets.com/bar/",
+            "Possible Parallel Languages": "",
+            "Links": []
+        }
+    ]
+}

languages/bqi_Arab.json

@@ -0,0 +1,19 @@
+{
+    "Language Name": "Bakhtiari",
+    "Family": "Indo-European",
+    "Subgrouping": "Iranian",
+    "Number of Speakers": "1_200_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟥",
+    "Sites": [
+        {
+            "Site Name": "lurishacademy.org",
+            "Site URL": "https://lurishacademy.org/",
+            "Category": "literature",
+            "Confidence": "🟩",
+            "Info": "native speaker confirmation with respect to the webpage annotation",
+            "Possible Parallel Languages": "",
+            "Links": ["https://lurishacademy.org/articles/داستان-ل-وری-پؽا-ۉ-پٱری-داستان-لری-مرد-و-پری",
+                    "https://lurishacademy.org/articles/دؽاری-کردن-آمٱئمتقتٱقی-خوݩ-چالٱنگ"]
+        }
+    ]
+}

languages/lki_Arab.json

@@ -0,0 +1,18 @@
+{
+    "Language Name": "Laki",
+    "Family": "Indo-European",
+    "Subgrouping": "Iranian",
+    "Number of Speakers": "600_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟥",
+    "Sites": [
+        {
+            "Site Name": "lurishacademy.org",
+            "Site URL": "https://lurishacademy.org/",
+            "Category": "literature",
+            "Confidence": "🟩",
+            "Info": "native speaker confirmation with respect to the webpage annotation and confirmed by glotlid",
+            "Possible Parallel Languages": "",
+            "Links": ["https://lurishacademy.org/articles/شعر-شیرین-ترازیا-سروده-نجف-آزادبخت"]
+        }
+    ]
+}

languages/lrc_Arab.json

@@ -0,0 +1,21 @@
+{
+    "Language Name": "Northern Luri",
+    "Family": "Indo-European",
+    "Subgrouping": "Iranian",
+    "Number of Speakers": "4_000_000",
+    "Supported by allenai/MADLAD-400 or facebook/flores": "🟨",
+    "Sites": [
+        {
+            "Site Name": "lurishacademy.org",
+            "Site URL": "https://lurishacademy.org/",
+            "Category": "news",
+            "Confidence": "🟩",
+            "Info": "native speaker confirmation with respect to the webpage annotation",
+            "Possible Parallel Languages": "",
+            "Links": ["https://lurishacademy.org/articles/خٱلک-ل-ۏر",
+                    "https://lurishacademy.org/articles/ڤیرشناسی",
+                    "https://lurishacademy.org/articles/یٱهۊدیٱت",
+                    "https://lurishacademy.org/articles/کومرٱ-ڤلات-کولومبیا"]
+        }
+    ]
+}

requirements.txt

@@ -0,0 +1,3 @@
+streamlit
+pandas
+tabulate

utils.py

@@ -0,0 +1,70 @@
+import os
+import pandas as pd
+import streamlit as st
+import base64
+import json
+
+# navigate to url
+def nav_to(url):
+    nav_script = """
+        <meta http-equiv="refresh" content="0; url='%s'">
+    """ % (url)
+    st.write(nav_script, unsafe_allow_html=True)
+
+
+@st.cache_data
+def df_to_html(df):
+
+    df = df.fillna("")
+
+    # Define table styling
+    styles = [
+        {'selector': 'tr', 'props': [('border', 'none')]},  # Hide row borders
+        {'selector': 'td, th', 'props': [('border', 'none'), ("text-align", "center"), ('font-size', 'smaller')]},  # Remove cell borders, reduce font size
+        {'selector': 'tr:hover', 'props': [('background-color', '#f5f5f5')]},
+        {'selector': 'a:hover', 'props': [('color', 'darkblue')]},
+        {'selector': 'table', 'props': [('border-collapse', 'collapse'), ('border', 'none'), ('border-bottom', '1px solid black'), ('width', '50%')]},  # Set table width to 50%
+        {'selector': 'thead', 'props': [('border', 'none')]},  # Hide header border
+        {'selector': 'tbody td', 'props': [('border-left', 'none'), ('border-right', 'none')]}, 
+        {'selector': 'tr:not(:first-child) td', 'props': [('border-left', 'none'), ('border-right', 'none'), ('border-top', 'none')]},
+        {'selector': 'table', 'props': [('table-layout', 'fixed')]},  # Prevent overflow
+    ]
+
+    # Apply table styles and convert DataFrame to HTML
+    styled_html = df.style.hide(axis="index").set_table_styles(styles).to_html(escape=False, index=False, bold_rows=True, justify='center').replace('<td>', '<td align="center">')
+        
+    return styled_html
+
+
+
+
+@st.cache_data
+def render_svg(svg):
+    """Renders the given svg string."""
+    b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
+    html = rf'<p align="center"> <img src="data:image/svg+xml;base64,{b64}", width="40%"/> </p>'
+    c = st.container()
+    c.write(html, unsafe_allow_html=True)
+
+
+
+@st.cache_resource
+def combine_json_files(folder_path):
+    combined_data = {}
+
+    # Iterate through each file in the folder
+    for filename in os.listdir(folder_path):
+        file_path = os.path.join(folder_path, filename)
+
+        # Check if the file is a JSON file
+        if filename.endswith('.json'):
+            with open(file_path, 'r') as file:
+                # Load JSON data from the file
+                data = {filename.replace('.json', ''): json.load(file)}
+
+                # Merge the loaded data into the combined_data dictionary
+                combined_data.update(data)
+
+    return combined_data
+
+