update collection format

README.md

@@ -79,8 +79,13 @@ models:
   - JJhooww/Mistral-7B-v0.2-Base_ptbr
   - JJhooww/MistralReloadBR_v2_ptbr
   - JJhooww/Mistral_Relora_Step2k
+  - JosephusCheung/LL7M
+  - M4-ai/tau-0.5B
+  - M4-ai/tau-0.5B-instruct-DPOP
+  - M4-ai/tau-1.8B
   - MagusCorp/legislinho
   - MaziyarPanahi/Mistral-7B-Instruct-Aya-101
+  - MulaBR/Mula-4x160-v0.1
   - NOVA-vision-language/GlorIA-1.3B
   - Nexusflow/Starling-LM-7B-beta
   - NousResearch/Nous-Hermes-13b
@@ -130,6 +135,7 @@ models:
   - TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
   - Unbabel/TowerBase-7B-v0.1
   - Walmart-the-bag/Misted-v2-7B
+  - Walmart-the-bag/Quintellect-10.7B
   - Walmart-the-bag/WordWoven-2x7B
   - Weni/WeniGPT-2.2.3-Zephyr-7B-LLM_Base_2.0.3_SFT
   - Weni/WeniGPT-2.2.3-Zephyr-7B-merged-LLM_Base_2.0.3_SFT
@@ -143,6 +149,9 @@ models:
   - Weni/ZeroShot-Multilanguage-Zephyr-7B
   - abacusai/Smaug-34B-v0.1
   - abacusai/Smaug-72B-v0.1
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.1
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.2
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.2-fft
   - allenai/OLMo-1B
   - allenai/OLMo-7B
   - allenai/OLMo-7B-Twin-2T
@@ -160,7 +169,9 @@ models:
   - bigscience/bloom-3b
   - bigscience/bloom-560m
   - bigscience/bloom-7b1
+  - botbot-ai/CabraLlama3-8b
   - cnmoro/Mistral-7B-Portuguese
+  - cognitivecomputations/dolphin-2.9-llama3-8b
   - croissantllm/CroissantLLMBase
   - deepseek-ai/deepseek-llm-7b-base
   - deepseek-ai/deepseek-moe-16b-base
@@ -171,6 +182,7 @@ models:
   - dynamofl/dynamo-8B-v0.1
   - eduagarcia/gemma-7b-it_no_chat_template
   - eduagarcia/gemma-7b-it_singleturn_chat_template
+  - ericzzz/falcon-rw-1b-instruct-openorca
   - facebook/opt-1.3b
   - facebook/opt-125m
   - facebook/opt-13b
@@ -206,8 +218,11 @@ models:
   - internlm/internlm2-base-20b
   - internlm/internlm2-base-7b
   - internlm/internlm2-chat-1_8b
+  - internlm/internlm2-chat-1_8b-sft
   - internlm/internlm2-chat-20b
+  - internlm/internlm2-chat-20b-sft
   - internlm/internlm2-chat-7b
+  - internlm/internlm2-chat-7b-sft
   - josu/gpt-neo-pt-1.3B
   - josu/gpt-neo-pt-br
   - lmsys/vicuna-13b-v1.5
@@ -215,6 +230,7 @@ models:
   - lrds-code/boana-7b-instruct
   - lrds-code/samba-1.1B
   - lucianosb/boto-7B
+  - lucianosb/boto-7B-v1.1
   - maritaca-ai/sabia-7b
   - matsuo-lab/weblab-10b
   - meta-llama/Llama-2-13b-chat-hf
@@ -229,6 +245,7 @@ models:
   - microsoft/phi-1_5
   - microsoft/phi-2
   - mistral-community/Mistral-7B-v0.2
+  - mistral-community/Mixtral-8x22B-Instruct-v0.1-4bit
   - mistral-community/Mixtral-8x22B-v0.1-4bit
   - mistralai/Mistral-7B-Instruct-v0.2
   - mistralai/Mistral-7B-v0.1
@@ -271,11 +288,19 @@ models:
   - recogna-nlp/Phi-Bode
   - recogna-nlp/bode-13b-alpaca-pt-br
   - recogna-nlp/bode-7b-alpaca-pt-br
+  - recogna-nlp/gembode-2b-base-ultraalpaca
+  - recogna-nlp/gembode-2b-base-ultraalpaca-qlora
   - recogna-nlp/gembode-2b-ultraalpaca
+  - recogna-nlp/gembode-2b-ultraalpaca-qlora
   - recogna-nlp/internlmbode-7b
   - recogna-nlp/mistral-bode
+  - recogna-nlp/mistralbode_7b_qlora_ultraalpaca
   - recogna-nlp/phi-bode-2-ultraalpaca
+  - recogna-nlp/qwenbode_1_8b_chat_ultraalpaca
+  - recogna-nlp/qwenbode_1_8b_chat_ultraalpaca_qlora
+  - recogna-nlp/zephyr_7b_beta_ultraalpaca
   - rhaymison/Llama-portuguese-13b-Luana-v0.2
+  - rhaymison/Mistral-8x7b-portuguese-luana
   - rhaymison/Mistral-portuguese-luana-7b
   - rhaymison/Mistral-portuguese-luana-7b-Mathematics
   - rhaymison/Mistral-portuguese-luana-7b-chat
@@ -311,6 +336,7 @@ models:
   - tiiuae/falcon-7b
   - togethercomputer/RedPajama-INCITE-7B-Base
   - togethercomputer/RedPajama-INCITE-Base-3B-v1
+  - unsloth/mistral-7b-bnb-4bit
   - upstage/SOLAR-10.7B-Instruct-v1.0
   - upstage/SOLAR-10.7B-v1.0
   - wandgibaut/periquito-3B

model_list.txt

@@ -50,8 +50,13 @@
   - JJhooww/Mistral-7B-v0.2-Base_ptbr
   - JJhooww/MistralReloadBR_v2_ptbr
   - JJhooww/Mistral_Relora_Step2k
+  - JosephusCheung/LL7M
+  - M4-ai/tau-0.5B
+  - M4-ai/tau-0.5B-instruct-DPOP
+  - M4-ai/tau-1.8B
   - MagusCorp/legislinho
   - MaziyarPanahi/Mistral-7B-Instruct-Aya-101
+  - MulaBR/Mula-4x160-v0.1
   - NOVA-vision-language/GlorIA-1.3B
   - Nexusflow/Starling-LM-7B-beta
   - NousResearch/Nous-Hermes-13b
@@ -101,6 +106,7 @@
   - TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
   - Unbabel/TowerBase-7B-v0.1
   - Walmart-the-bag/Misted-v2-7B
+  - Walmart-the-bag/Quintellect-10.7B
   - Walmart-the-bag/WordWoven-2x7B
   - Weni/WeniGPT-2.2.3-Zephyr-7B-LLM_Base_2.0.3_SFT
   - Weni/WeniGPT-2.2.3-Zephyr-7B-merged-LLM_Base_2.0.3_SFT
@@ -114,6 +120,9 @@
   - Weni/ZeroShot-Multilanguage-Zephyr-7B
   - abacusai/Smaug-34B-v0.1
   - abacusai/Smaug-72B-v0.1
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.1
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.2
+  - adalbertojunior/Llama-3-8B-Instruct-Portuguese-v0.2-fft
   - allenai/OLMo-1B
   - allenai/OLMo-7B
   - allenai/OLMo-7B-Twin-2T
@@ -131,7 +140,9 @@
   - bigscience/bloom-3b
   - bigscience/bloom-560m
   - bigscience/bloom-7b1
+  - botbot-ai/CabraLlama3-8b
   - cnmoro/Mistral-7B-Portuguese
+  - cognitivecomputations/dolphin-2.9-llama3-8b
   - croissantllm/CroissantLLMBase
   - deepseek-ai/deepseek-llm-7b-base
   - deepseek-ai/deepseek-moe-16b-base
@@ -142,6 +153,7 @@
   - dynamofl/dynamo-8B-v0.1
   - eduagarcia/gemma-7b-it_no_chat_template
   - eduagarcia/gemma-7b-it_singleturn_chat_template
+  - ericzzz/falcon-rw-1b-instruct-openorca
   - facebook/opt-1.3b
   - facebook/opt-125m
   - facebook/opt-13b
@@ -177,8 +189,11 @@
   - internlm/internlm2-base-20b
   - internlm/internlm2-base-7b
   - internlm/internlm2-chat-1_8b
+  - internlm/internlm2-chat-1_8b-sft
   - internlm/internlm2-chat-20b
+  - internlm/internlm2-chat-20b-sft
   - internlm/internlm2-chat-7b
+  - internlm/internlm2-chat-7b-sft
   - josu/gpt-neo-pt-1.3B
   - josu/gpt-neo-pt-br
   - lmsys/vicuna-13b-v1.5
@@ -186,6 +201,7 @@
   - lrds-code/boana-7b-instruct
   - lrds-code/samba-1.1B
   - lucianosb/boto-7B
+  - lucianosb/boto-7B-v1.1
   - maritaca-ai/sabia-7b
   - matsuo-lab/weblab-10b
   - meta-llama/Llama-2-13b-chat-hf
@@ -200,6 +216,7 @@
   - microsoft/phi-1_5
   - microsoft/phi-2
   - mistral-community/Mistral-7B-v0.2
+  - mistral-community/Mixtral-8x22B-Instruct-v0.1-4bit
   - mistral-community/Mixtral-8x22B-v0.1-4bit
   - mistralai/Mistral-7B-Instruct-v0.2
   - mistralai/Mistral-7B-v0.1
@@ -242,11 +259,19 @@
   - recogna-nlp/Phi-Bode
   - recogna-nlp/bode-13b-alpaca-pt-br
   - recogna-nlp/bode-7b-alpaca-pt-br
+  - recogna-nlp/gembode-2b-base-ultraalpaca
+  - recogna-nlp/gembode-2b-base-ultraalpaca-qlora
   - recogna-nlp/gembode-2b-ultraalpaca
+  - recogna-nlp/gembode-2b-ultraalpaca-qlora
   - recogna-nlp/internlmbode-7b
   - recogna-nlp/mistral-bode
+  - recogna-nlp/mistralbode_7b_qlora_ultraalpaca
   - recogna-nlp/phi-bode-2-ultraalpaca
+  - recogna-nlp/qwenbode_1_8b_chat_ultraalpaca
+  - recogna-nlp/qwenbode_1_8b_chat_ultraalpaca_qlora
+  - recogna-nlp/zephyr_7b_beta_ultraalpaca
   - rhaymison/Llama-portuguese-13b-Luana-v0.2
+  - rhaymison/Mistral-8x7b-portuguese-luana
   - rhaymison/Mistral-portuguese-luana-7b
   - rhaymison/Mistral-portuguese-luana-7b-Mathematics
   - rhaymison/Mistral-portuguese-luana-7b-chat
@@ -282,6 +307,7 @@
   - tiiuae/falcon-7b
   - togethercomputer/RedPajama-INCITE-7B-Base
   - togethercomputer/RedPajama-INCITE-Base-3B-v1
+  - unsloth/mistral-7b-bnb-4bit
   - upstage/SOLAR-10.7B-Instruct-v1.0
   - upstage/SOLAR-10.7B-v1.0
   - wandgibaut/periquito-3B

src/display/utils.py

@@ -193,11 +193,11 @@ class ModelDetails:
 
 class ModelType(Enum):
     PT = ModelDetails(name="pretrained", symbol="🟢")
-    LA = ModelDetails(name="language adapted models (FP, FT, ...)", symbol="🆎")
+    LA = ModelDetails(name="language adapted (FP, FT, ...)", symbol="🆎")
     FT = ModelDetails(name="fine-tuned/fp on domain-specific datasets", symbol="🔶")
-    chat = ModelDetails(name="chat models (RLHF, DPO, IFT, ...)", symbol="💬")
+    chat = ModelDetails(name="chat (RLHF, DPO, IFT, ...)", symbol="💬")
     merges = ModelDetails(name="base merges and moerges", symbol="🤝")
-    proprietary = ModelDetails(name="proprietary models (closed)", symbol="🔒")
+    proprietary = ModelDetails(name="proprietary (closed)", symbol="🔒")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):

src/tools/collections.py

@@ -4,6 +4,7 @@ import pandas as pd
 from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
 from huggingface_hub.utils._errors import HfHubHTTPError
 from pandas import DataFrame
+import numpy as np
 
 from src.display.utils import AutoEvalColumn, ModelType, NUMERIC_INTERVALS
 from src.envs import H4_TOKEN, PATH_TO_COLLECTION
@@ -29,50 +30,120 @@ def update_collections(df: DataFrame):
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
 
     cur_best_models = []
+    cur_best_scores = []
+    scores_per_type = {'pretrained': 0, 'other': 0, 'language': 0}
 
-    ix = 0
-    for type in ModelType:
-        if type.value.name == "":
+    types_to_consider = [('pretrained', [ModelType.PT]), ('other', [ModelType.LA, ModelType.FT, ModelType.chat])]
+
+    for item in collection.items:
+        try:
+            delete_collection_item(
+                collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
+            )
+        except HfHubHTTPError:
             continue
-        for size in intervals:
-            # We filter the df to gather the relevant models
-            type_emoji = [t[0] for t in type.value.symbol]
-            filtered_df = df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
 
-            numeric_interval = pd.IntervalIndex([intervals[size]])
-            mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-            filtered_df = filtered_df.loc[mask]
+    ix = 0
+    for size in intervals:
+        interval_scores = []
+        interval_itens_languages = []
+        interval_itens = []
 
-            best_models = list(
-                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name]
-            )
-            print(type.value.symbol, size, best_models[:10])
+        numeric_interval = pd.IntervalIndex([intervals[size]])
+        mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+        size_df = df.loc[mask]
 
+        for model_type, types in types_to_consider:
+            type_emojis = []
+            for type in types:
+                if type.value.name == "":
+                    continue
+                type_emoji = [t[0] for t in type.value.symbol]
+                type_emojis.extend(type_emoji)
+            filtered_df = size_df[size_df[AutoEvalColumn.model_type_symbol.name].isin(type_emojis)]
+            filtered_df = filtered_df[filtered_df[AutoEvalColumn.average.name].astype(float) > scores_per_type[model_type]]
+
+            best_models = filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)
+            print(type_emojis, size, list(best_models[AutoEvalColumn.dummy.name])[:10])
             # We add them one by one to the leaderboard
-            for model in best_models:
-                ix += 1
-                cur_len_collection = len(collection.items)
+            for i, row in best_models.iterrows():
+                model = row[AutoEvalColumn.dummy.name]
+                score = row[AutoEvalColumn.average.name]
+                language = row[AutoEvalColumn.main_language.name]
+                if language == 'Portuguese':
+                    note = f"Best Portuguese {type.to_str(' ')} model of around {size} on the leaderboard today! (Score: {score})"
+                else:
+                    note = f"Best {type.to_str(' ')} model of around {size} on the leaderboard today! (Score: {score})"
                 try:
                     collection = add_collection_item(
                         PATH_TO_COLLECTION,
                         item_id=model,
                         item_type="model",
                         exists_ok=True,
-                        note=f"Best {type.to_str(' ')} model of around {size} on the leaderboard today!",
+                        note=note,
                         token=H4_TOKEN,
                     )
-                    if (
-                        len(collection.items) > cur_len_collection
-                    ):  # we added an item - we make sure its position is correct
-                        item_object_id = collection.items[-1].item_object_id
-                        update_collection_item(
-                            collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix
-                        )
-                        cur_len_collection = len(collection.items)
+                    ix += 1
+                    item_object_id = collection.items[-1].item_object_id
                     cur_best_models.append(model)
+                    interval_scores.append(float(score))
+                    interval_itens_languages.append(language)
+                    interval_itens.append(item_object_id)
+                    scores_per_type[model_type] = float(score)
                     break
                 except HfHubHTTPError:
                     continue
+        if 'Portuguese' not in interval_itens_languages:
+            language = ['Portuguese']
+            model_type = 'language'
+            filtered_df = size_df[size_df[AutoEvalColumn.main_language.name].isin(language)]
+            filtered_df = filtered_df[filtered_df[AutoEvalColumn.average.name].astype(float) > scores_per_type[model_type]]
+
+            best_models = filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)
+            print(language, size, list(best_models[AutoEvalColumn.dummy.name])[:10])
+            # We add them one by one to the leaderboard
+            for i, row in best_models.iterrows():
+                model = row[AutoEvalColumn.dummy.name]
+                score = row[AutoEvalColumn.average.name]
+                language = row[AutoEvalColumn.main_language.name]
+                
+                if language == 'Portuguese':
+                    note = f"Best Portuguese {type.to_str(' ')} model of around {size} on the leaderboard today! (Score: {score})"
+                else:
+                    note = f"Best {type.to_str(' ')} model of around {size} on the leaderboard today! (Score: {score})"
+                try:
+                    collection = add_collection_item(
+                        PATH_TO_COLLECTION,
+                        item_id=model,
+                        item_type="model",
+                        exists_ok=True,
+                        note=note,
+                        token=H4_TOKEN,
+                    )
+                    ix += 1
+                    item_object_id = collection.items[-1].item_object_id
+                    cur_best_models.append(model)
+                    interval_scores.append(float(score))
+                    interval_itens_languages.append(language)
+                    interval_itens.append(item_object_id)
+                    scores_per_type[model_type] = float(score)
+                    break
+                except HfHubHTTPError:
+                    continue
+        # fix order:
+        starting_idx = len(cur_best_models)
+        k = 0
+        for i in np.argsort(interval_scores):
+            if i == k:
+                continue
+            else:
+                try:
+                    update_collection_item(
+                        collection_slug=PATH_TO_COLLECTION, item_object_id=interval_itens[i], position=starting_idx+k
+                    )
+                except:
+                    pass
+            k += 1
 
     collection = get_collection(PATH_TO_COLLECTION, token=H4_TOKEN)
     for item in collection.items: