Upload app.py with huggingface_hub

app.py

@@ -69,6 +69,24 @@ PROVIDERS = [
     "nscale",
 ]
 
+# Mapping from display provider names to inference provider API names
+PROVIDER_TO_INFERENCE_NAME = {
+    "togethercomputer": "together",
+    "fal": "fal-ai", 
+    "sambanovasystems": "sambanova",
+    "Hyperbolic": "hyperbolic",
+    "CohereLabs": "cohere",
+    # Other providers may not have inference provider support or use different names
+    "fireworks-ai": "fireworks-ai",
+    "nebius": "nebius",
+    "groq": "groq", 
+    "cerebras": "cerebras",
+    "replicate": "replicate",
+    "novita": "novita",
+    "featherless-ai": "featherless-ai",
+    "nscale": "nscale",
+}
+
 templates = Jinja2Templates(directory="templates")
 
 async def get_monthly_requests(session: aiohttp.ClientSession, provider: str) -> Dict[str, str]:
@@ -99,6 +117,33 @@ async def get_monthly_requests(session: aiohttp.ClientSession, provider: str) ->
             "monthly_requests_int": 0
         }
 
+async def get_provider_models(session: aiohttp.ClientSession, provider: str) -> List[str]:
+    """Get supported models for a provider from HuggingFace API"""
+    if not HF_TOKEN:
+        return []
+    
+    # Map display provider name to inference provider API name
+    inference_provider = PROVIDER_TO_INFERENCE_NAME.get(provider)
+    if not inference_provider:
+        logger.warning(f"No inference provider mapping found for {provider}")
+        return []
+    
+    url = f"https://huggingface.co/api/models?inference_provider={inference_provider}&limit=50&sort=downloads&direction=-1"
+    headers = {"Authorization": f"Bearer {HF_TOKEN}"}
+    
+    try:
+        async with session.get(url, headers=headers) as response:
+            if response.status == 200:
+                models_data = await response.json()
+                model_ids = [model.get('id', '') for model in models_data if model.get('id')]
+                return model_ids
+            else:
+                logger.warning(f"Failed to fetch models for {provider} (inference_provider={inference_provider}): {response.status}")
+                return []
+    except Exception as e:
+        logger.error(f"Error fetching models for {provider} (inference_provider={inference_provider}): {e}")
+        return []
+
 async def collect_and_store_data():
     """Collect current data and store it in the dataset"""
     if not HF_TOKEN:
@@ -309,6 +354,61 @@ async def get_historical_data():
             "message": "Historical data temporarily unavailable"
         }
 
+@app.get("/api/models")
+async def get_provider_models_data():
+    """API endpoint to get supported models matrix for all providers"""
+    if not HF_TOKEN:
+        return {"error": "HF_TOKEN required for models data", "matrix": [], "providers": PROVIDERS}
+    
+    async with aiohttp.ClientSession() as session:
+        tasks = [get_provider_models(session, provider) for provider in PROVIDERS]
+        results = await asyncio.gather(*tasks)
+    
+    # Create provider -> models mapping
+    provider_models = {}
+    all_models = set()
+    
+    for provider, models in zip(PROVIDERS, results):
+        provider_models[provider] = set(models)
+        all_models.update(models)
+    
+    # Convert to list and sort by popularity (number of providers supporting each model)
+    model_popularity = []
+    for model in all_models:
+        provider_count = sum(1 for provider in PROVIDERS if model in provider_models.get(provider, set()))
+        model_popularity.append((model, provider_count))
+    
+    # Sort by popularity (descending) then by model name
+    model_popularity.sort(key=lambda x: (-x[1], x[0]))
+    
+    # Build matrix data
+    matrix = []
+    for model_id, popularity in model_popularity:
+        row = {
+            "model_id": model_id,
+            "total_providers": popularity,
+            "providers": {}
+        }
+        
+        for provider in PROVIDERS:
+            row["providers"][provider] = model_id in provider_models.get(provider, set())
+        
+        matrix.append(row)
+    
+    # Calculate totals per provider
+    provider_totals = {}
+    for provider in PROVIDERS:
+        provider_totals[provider] = len(provider_models.get(provider, set()))
+    
+    return {
+        "matrix": matrix,
+        "providers": PROVIDERS,
+        "provider_totals": provider_totals,
+        "provider_mapping": PROVIDER_TO_INFERENCE_NAME,
+        "total_models": len(all_models),
+        "last_updated": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    }
+
 @app.post("/api/collect-now")
 async def trigger_data_collection(background_tasks: BackgroundTasks):
     """Manual trigger for data collection"""