Update app.py

app.py

@@ -301,74 +301,75 @@ def get_models_data(progress=gr.Progress()):
     try:
         progress(0, desc="Fetching global rankings...")
         
-        # 전체 모델 목록 가져오기 (다운로드 순으로 정렬)
-        global_url = "https://huggingface.co/api/models"
-        global_params = {
-            'full': 'true',
-            'limit': 10000,  # 충분히 큰 수로 설정
-            'sort': 'downloads',
-            'direction': -1
-        }
-        
-        global_response = requests.get(global_url, headers={'Accept': 'application/json'}, params=global_params)
-        if global_response.status_code != 200:
-            print(f"Failed to fetch global rankings: {global_response.status_code}")
-            return create_error_plot(), "<div>전체 순위 데이터를 가져오는데 실패했습니다.</div>", pd.DataFrame()
-        
-        global_models = global_response.json()
+        # 전체 모델 목록을 여러 번 나눠서 가져오기
+        all_global_models = []
+        page_size = 10000
+        max_models = 50000  # 충분히 큰 수로 설정
+        
+        for offset in range(0, max_models, page_size):
+            global_params = {
+                'full': 'true',
+                'limit': page_size,
+                'offset': offset,
+                'sort': 'downloads',
+                'direction': -1
+            }
+            
+            global_response = requests.get(
+                "https://huggingface.co/api/models",
+                headers={'Accept': 'application/json'},
+                params=global_params
+            )
+            
+            if global_response.status_code != 200:
+                print(f"Failed to fetch global rankings page {offset//page_size + 1}")
+                continue
+                
+            page_models = global_response.json()
+            if not page_models:  # 더 이상 모델이 없으면 중단
+                break
+                
+            all_global_models.extend(page_models)
+            print(f"Fetched {len(all_global_models)} models so far...")
         
         # 전체 순위 맵 생성
-        global_ranks = {normalize_model_id(model['id']): idx + 1 
-                       for idx, model in enumerate(global_models)}
+        global_ranks = {}
+        for idx, model in enumerate(all_global_models, 1):
+            model_id = normalize_model_id(model.get('id', ''))
+            global_ranks[model_id] = {
+                'rank': idx,
+                'downloads': model.get('downloads', 0),
+                'likes': model.get('likes', 0)
+            }
         
-        # 각 모델의 상세 정보를 개별적으로 가져오기
-        filtered_models = []
-        total_models = len(target_models)
+        print(f"Total models fetched: {len(global_ranks)}")
         
-        for idx, (model_id, model_url) in enumerate(target_models.items()):
-            progress((idx + 1) / total_models, desc=f"Fetching model {idx + 1}/{total_models}...")
-            
-            try:
-                # 개별 모델 API 호출
-                model_url_api = f"https://huggingface.co/api/models/{model_id}"
-                response = requests.get(model_url_api, headers={'Accept': 'application/json'})
-                
-                if response.status_code == 200:
-                    model_data = response.json()
-                    normalized_id = normalize_model_id(model_id)
-                    global_rank = global_ranks.get(normalized_id, 'Not in top 10000')
-                    
-                    model_info = {
-                        'id': model_id,
-                        'downloads': model_data.get('downloads', 0),
-                        'likes': model_data.get('likes', 0),
-                        'title': model_data.get('title', 'No Title'),
-                        'global_rank': global_rank
-                    }
-                    filtered_models.append(model_info)
-                    print(f"Model {model_id}: Global Rank={global_rank}, Downloads={model_info['downloads']}, Likes={model_info['likes']}")
-                else:
-                    print(f"Failed to fetch data for {model_id}: {response.status_code}")
-                    model_info = {
-                        'id': model_id,
-                        'downloads': 0,
-                        'likes': 0,
-                        'title': 'No Title',
-                        'global_rank': 'Not in top 10000'
-                    }
-                    filtered_models.append(model_info)
-            except Exception as e:
-                print(f"Error fetching data for {model_id}: {str(e)}")
+        # target_models의 순위 확인 및 저장
+        filtered_models = []
+        for model_id in target_models.keys():
+            normalized_id = normalize_model_id(model_id)
+            if normalized_id in global_ranks:
+                rank_info = global_ranks[normalized_id]
+                model_info = {
+                    'id': model_id,
+                    'global_rank': rank_info['rank'],
+                    'downloads': rank_info['downloads'],
+                    'likes': rank_info['likes'],
+                    'title': 'No Title'  # 필요한 경우 개별 API 호출로 가져올 수 있음
+                }
+                print(f"Found {model_id} at rank {rank_info['rank']}")
+            else:
                 model_info = {
                     'id': model_id,
+                    'global_rank': 'Not found',
                     'downloads': 0,
                     'likes': 0,
-                    'title': 'No Title',
-                    'global_rank': 'Not in top 10000'
+                    'title': 'No Title'
                 }
-                filtered_models.append(model_info)
+                print(f"Model {model_id} not found in rankings")
+            filtered_models.append(model_info)
         
-        # 전체 순위로 정렬
+        # 순위로 정렬
         filtered_models.sort(key=lambda x: float('inf') if isinstance(x['global_rank'], str) else x['global_rank'])
         
         if not filtered_models: