Spaces:
Sleeping
Sleeping
Commit ·
06b3485
1
Parent(s): 7ad5273
deploy backend
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +24 -0
- README.md +7 -5
- __init__.py +1 -0
- __pycache__/database.cpython-312.pyc +0 -0
- __pycache__/database.cpython-314.pyc +0 -0
- __pycache__/main.cpython-312.pyc +0 -0
- __pycache__/main.cpython-314.pyc +0 -0
- agents/__init__.py +23 -0
- agents/__pycache__/__init__.cpython-312.pyc +0 -0
- agents/__pycache__/__init__.cpython-314.pyc +0 -0
- agents/__pycache__/agent1_chatbot.cpython-312.pyc +0 -0
- agents/__pycache__/agent1_chatbot.cpython-314.pyc +0 -0
- agents/__pycache__/agent2_diagnostic.cpython-312.pyc +0 -0
- agents/__pycache__/agent2_diagnostic.cpython-314.pyc +0 -0
- agents/__pycache__/alert_agent.cpython-312.pyc +0 -0
- agents/__pycache__/alert_agent.cpython-314.pyc +0 -0
- agents/__pycache__/detection_agent.cpython-312.pyc +0 -0
- agents/__pycache__/detection_agent.cpython-314.pyc +0 -0
- agents/__pycache__/prediction_agent.cpython-312.pyc +0 -0
- agents/__pycache__/prediction_agent.cpython-314.pyc +0 -0
- agents/agent1_chatbot.py +630 -0
- agents/agent2_diagnostic.py +1053 -0
- agents/alert_agent.py +192 -0
- agents/detection_agent.py +124 -0
- agents/prediction_agent.py +164 -0
- core/__init__.py +1 -0
- core/__pycache__/__init__.cpython-312.pyc +0 -0
- core/__pycache__/__init__.cpython-314.pyc +0 -0
- core/__pycache__/config.cpython-312.pyc +0 -0
- core/__pycache__/config.cpython-314.pyc +0 -0
- core/__pycache__/filled_transform.cpython-312.pyc +0 -0
- core/__pycache__/filled_transform.cpython-314.pyc +0 -0
- core/__pycache__/schemas.cpython-312.pyc +0 -0
- core/__pycache__/schemas.cpython-314.pyc +0 -0
- core/config.py +88 -0
- core/filled_transform.py +169 -0
- core/schemas.py +254 -0
- data/processed/network_kpis_full.csv +0 -0
- database.py +110 -0
- generate_data.py +278 -0
- main.py +160 -0
- qos_buddy.db +0 -0
- requirements.txt +158 -0
- requirements_hf.txt +48 -0
- routes/__init__.py +1 -0
- routes/__pycache__/__init__.cpython-312.pyc +0 -0
- routes/__pycache__/__init__.cpython-314.pyc +0 -0
- routes/__pycache__/auth.cpython-312.pyc +0 -0
- routes/__pycache__/auth.cpython-314.pyc +0 -0
- routes/__pycache__/chat.cpython-312.pyc +0 -0
Dockerfile
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
# System dependencies
|
| 6 |
+
RUN apt-get update && apt-get install -y \
|
| 7 |
+
gcc \
|
| 8 |
+
g++ \
|
| 9 |
+
git \
|
| 10 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 11 |
+
|
| 12 |
+
# Install CPU-only torch FIRST (évite de télécharger la version GPU 2GB)
|
| 13 |
+
RUN pip install --no-cache-dir torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu
|
| 14 |
+
|
| 15 |
+
# Install requirements
|
| 16 |
+
COPY requirements_hf.txt .
|
| 17 |
+
RUN pip install --no-cache-dir -r requirements_hf.txt
|
| 18 |
+
|
| 19 |
+
COPY . .
|
| 20 |
+
|
| 21 |
+
# HF Spaces utilise le port 7860 obligatoirement
|
| 22 |
+
EXPOSE 7860
|
| 23 |
+
|
| 24 |
+
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,10 +1,12 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: QosBuddy Backend
|
| 3 |
+
emoji: 📡
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# QoS Buddy Backend API
|
| 11 |
+
|
| 12 |
+
Multi-Agent Telecom Network Diagnostic System built with FastAPI.
|
__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# QoS Buddy Backend
|
__pycache__/database.cpython-312.pyc
ADDED
|
Binary file (4.96 kB). View file
|
|
|
__pycache__/database.cpython-314.pyc
ADDED
|
Binary file (5.2 kB). View file
|
|
|
__pycache__/main.cpython-312.pyc
ADDED
|
Binary file (7.1 kB). View file
|
|
|
__pycache__/main.cpython-314.pyc
ADDED
|
Binary file (7.37 kB). View file
|
|
|
agents/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
QoS Buddy — Agents Package
|
| 3 |
+
All 5 agents of the multi-agent system:
|
| 4 |
+
- Agent 1: TelecomAdvisorAgent (RAG Chatbot)
|
| 5 |
+
- Agent 2: DiagnosticAgent (Autonomous Diagnostic Worker)
|
| 6 |
+
- DetectionAgent (Statistical Anomaly Detection)
|
| 7 |
+
- PredictionAgent (Future Risk Prediction)
|
| 8 |
+
- AlertAgent (Alert Fusion & Deduplication)
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from agents.agent1_chatbot import TelecomAdvisorAgent
|
| 12 |
+
from agents.agent2_diagnostic import DiagnosticAgent
|
| 13 |
+
from agents.detection_agent import DetectionAgent
|
| 14 |
+
from agents.prediction_agent import PredictionAgent
|
| 15 |
+
from agents.alert_agent import AlertAgent
|
| 16 |
+
|
| 17 |
+
__all__ = [
|
| 18 |
+
"TelecomAdvisorAgent",
|
| 19 |
+
"DiagnosticAgent",
|
| 20 |
+
"DetectionAgent",
|
| 21 |
+
"PredictionAgent",
|
| 22 |
+
"AlertAgent",
|
| 23 |
+
]
|
agents/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (859 Bytes). View file
|
|
|
agents/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (939 Bytes). View file
|
|
|
agents/__pycache__/agent1_chatbot.cpython-312.pyc
ADDED
|
Binary file (30.1 kB). View file
|
|
|
agents/__pycache__/agent1_chatbot.cpython-314.pyc
ADDED
|
Binary file (34.3 kB). View file
|
|
|
agents/__pycache__/agent2_diagnostic.cpython-312.pyc
ADDED
|
Binary file (53.2 kB). View file
|
|
|
agents/__pycache__/agent2_diagnostic.cpython-314.pyc
ADDED
|
Binary file (62.1 kB). View file
|
|
|
agents/__pycache__/alert_agent.cpython-312.pyc
ADDED
|
Binary file (8.45 kB). View file
|
|
|
agents/__pycache__/alert_agent.cpython-314.pyc
ADDED
|
Binary file (9.81 kB). View file
|
|
|
agents/__pycache__/detection_agent.cpython-312.pyc
ADDED
|
Binary file (6.22 kB). View file
|
|
|
agents/__pycache__/detection_agent.cpython-314.pyc
ADDED
|
Binary file (7.29 kB). View file
|
|
|
agents/__pycache__/prediction_agent.cpython-312.pyc
ADDED
|
Binary file (9.25 kB). View file
|
|
|
agents/__pycache__/prediction_agent.cpython-314.pyc
ADDED
|
Binary file (10.3 kB). View file
|
|
|
agents/agent1_chatbot.py
ADDED
|
@@ -0,0 +1,630 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Agent 1: RAG-Powered Chatbot Advisor
|
| 3 |
+
Uses ChromaDB + Google Gemini for telecom network diagnostics
|
| 4 |
+
|
| 5 |
+
Refinements v2:
|
| 6 |
+
- Conversation memory (sliding window of last N turns)
|
| 7 |
+
- Intent classification before RAG
|
| 8 |
+
- Adaptive chunk retrieval (query complexity)
|
| 9 |
+
- Cell-specific context injection
|
| 10 |
+
- Response confidence score from ChromaDB distances
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
import re
|
| 15 |
+
from collections import deque
|
| 16 |
+
from typing import List, Dict, Optional, Tuple
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
import chromadb
|
| 19 |
+
from chromadb.config import Settings
|
| 20 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 21 |
+
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 22 |
+
from dotenv import load_dotenv
|
| 23 |
+
import json
|
| 24 |
+
import warnings
|
| 25 |
+
import requests
|
| 26 |
+
import pandas as pd
|
| 27 |
+
from datetime import datetime, timedelta
|
| 28 |
+
|
| 29 |
+
warnings.filterwarnings('ignore', category=FutureWarning)
|
| 30 |
+
warnings.filterwarnings('ignore', category=DeprecationWarning)
|
| 31 |
+
|
| 32 |
+
load_dotenv()
|
| 33 |
+
|
| 34 |
+
from core.filled_transform import apply_filled_column_mapping, finalize_pipeline_dataframe
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ── Intent Classification ──
|
| 38 |
+
|
| 39 |
+
class QueryIntent:
|
| 40 |
+
DATA_QUERY = "data_query" # user asks about their dataset / statistics
|
| 41 |
+
CELL_DIAGNOSTIC = "cell_diagnostic" # user asks about a specific cell
|
| 42 |
+
ISSUE_DIAGNOSTIC = "issue_diagnostic" # user asks about a network problem type
|
| 43 |
+
COMPARISON = "comparison" # user compares KPIs, cells, time periods
|
| 44 |
+
GENERAL_KNOWLEDGE = "general_knowledge" # general telecom question
|
| 45 |
+
HOWTO = "howto" # how to fix / optimize something
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
_INTENT_PATTERNS = [
|
| 49 |
+
(QueryIntent.CELL_DIAGNOSTIC, [
|
| 50 |
+
r'\bcell[_ ]?\w{2,}', r'\bsite[_ ]\w+', r'\bcell_id\b', r'\bspecific cell\b',
|
| 51 |
+
r'\bthis cell\b', r'\bthat cell\b', r'\bcell \w+\b',
|
| 52 |
+
]),
|
| 53 |
+
(QueryIntent.DATA_QUERY, [
|
| 54 |
+
r'\bmy data\b', r'\bdataset\b', r'\bhow many\b', r'\bstatistics\b',
|
| 55 |
+
r'\bshow me\b', r'\bsample\b', r'\brows?\b', r'\bdistribution\b',
|
| 56 |
+
r'\bcurrent network\b', r'\bcells?\b.*\bmonitored\b',
|
| 57 |
+
]),
|
| 58 |
+
(QueryIntent.COMPARISON, [
|
| 59 |
+
r'\bcompare\b', r'\bvs\.?\b', r'\bbetter\b.*\bworse\b',
|
| 60 |
+
r'\bdifference\b', r'\btrend\b', r'\bover time\b',
|
| 61 |
+
]),
|
| 62 |
+
(QueryIntent.HOWTO, [
|
| 63 |
+
r'\bhow to\b', r'\bhow do i\b', r'\bfix\b', r'\boptimize\b',
|
| 64 |
+
r'\bimprove\b', r'\breduce\b', r'\bincrease\b', r'\bwhat should\b',
|
| 65 |
+
]),
|
| 66 |
+
(QueryIntent.ISSUE_DIAGNOSTIC, [
|
| 67 |
+
r'\bcall drop\b', r'\bhandover\b', r'\blatency\b', r'\bthroughput\b',
|
| 68 |
+
r'\bmos\b', r'\bcongestion\b', r'\bprb\b', r'\bbler\b', r'\brsrp\b',
|
| 69 |
+
r'\bsinr\b', r'\bsla\b', r'\banomaly\b', r'\bcapacity\b',
|
| 70 |
+
]),
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def classify_intent(query: str) -> str:
|
| 75 |
+
"""Rule-based intent classification. Returns the most specific matching intent."""
|
| 76 |
+
q = query.lower()
|
| 77 |
+
for intent, patterns in _INTENT_PATTERNS:
|
| 78 |
+
for pat in patterns:
|
| 79 |
+
if re.search(pat, q):
|
| 80 |
+
return intent
|
| 81 |
+
return QueryIntent.GENERAL_KNOWLEDGE
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _extract_cell_id(query: str) -> Optional[str]:
|
| 85 |
+
"""Try to extract a cell_id from a user query."""
|
| 86 |
+
m = re.search(r'\b(cell[_ ]?\w{3,})\b', query, re.IGNORECASE)
|
| 87 |
+
if m:
|
| 88 |
+
return m.group(1).replace(' ', '_')
|
| 89 |
+
m = re.search(r'\b([A-Z]{2,4}[_-]\d{3,})\b', query)
|
| 90 |
+
if m:
|
| 91 |
+
return m.group(1)
|
| 92 |
+
return None
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class TelecomAdvisorAgent:
|
| 96 |
+
"""
|
| 97 |
+
Agent 1: Chatbot Advisor
|
| 98 |
+
Answers telecom diagnostic questions using RAG
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
# Maximum conversation turns to remember per conversation
|
| 102 |
+
MEMORY_WINDOW = 6
|
| 103 |
+
|
| 104 |
+
def __init__(self, knowledge_base_path: str = None, data_path: str = None):
|
| 105 |
+
repo_root = Path(__file__).resolve().parent.parent.parent
|
| 106 |
+
default_kb = repo_root / "knowledge_base" / "telecom_causes_solutions.txt"
|
| 107 |
+
default_filled = Path(__file__).resolve().parent.parent / "unified_telecom_dataset_FILLED.csv"
|
| 108 |
+
|
| 109 |
+
self.knowledge_base_path = str(knowledge_base_path) if knowledge_base_path else str(default_kb)
|
| 110 |
+
self.data_path = str(data_path) if data_path else str(default_filled)
|
| 111 |
+
self.collection_name = "telecom_knowledge"
|
| 112 |
+
self.df = None
|
| 113 |
+
|
| 114 |
+
self.chroma_client = chromadb.Client(Settings(
|
| 115 |
+
persist_directory=str(repo_root / "data" / "chroma_db"),
|
| 116 |
+
anonymized_telemetry=False
|
| 117 |
+
))
|
| 118 |
+
|
| 119 |
+
self.embeddings = HuggingFaceEmbeddings(
|
| 120 |
+
model_name="sentence-transformers/all-MiniLM-L6-v2",
|
| 121 |
+
model_kwargs={'device': 'cpu'}
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
# Check for OpenRouter API key (preferred)
|
| 125 |
+
openrouter_api_key = os.getenv('OPENROUTER_API_KEY')
|
| 126 |
+
gemini_api_key = os.getenv('GEMINI_API_KEY')
|
| 127 |
+
|
| 128 |
+
if openrouter_api_key:
|
| 129 |
+
self.api_key = openrouter_api_key
|
| 130 |
+
self.api_type = 'openrouter'
|
| 131 |
+
self.model_name = 'openai/gpt-4o-mini'
|
| 132 |
+
self.api_url = 'https://openrouter.ai/api/v1/chat/completions'
|
| 133 |
+
print(f" OpenRouter API configured ({self.model_name})")
|
| 134 |
+
elif gemini_api_key:
|
| 135 |
+
self.api_key = gemini_api_key
|
| 136 |
+
self.api_type = 'gemini'
|
| 137 |
+
self.model_name = 'gemini-1.5-flash'
|
| 138 |
+
self.api_url = f'https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent'
|
| 139 |
+
print(" Gemini API configured (gemini-1.5-flash)")
|
| 140 |
+
else:
|
| 141 |
+
self.api_key = None
|
| 142 |
+
self.api_type = None
|
| 143 |
+
self.model_name = None
|
| 144 |
+
print(" No API key found. Set OPENROUTER_API_KEY or GEMINI_API_KEY in .env file")
|
| 145 |
+
|
| 146 |
+
self.collection = None
|
| 147 |
+
self.is_initialized = False
|
| 148 |
+
self._insights_provider = None
|
| 149 |
+
self._diagnostic_provider = None # Agent 2 feeds live diagnostics
|
| 150 |
+
|
| 151 |
+
# Conversation memory: conversation_id -> deque of (role, text) tuples
|
| 152 |
+
self._conversations: Dict[str, deque] = {}
|
| 153 |
+
|
| 154 |
+
self.load_network_data()
|
| 155 |
+
|
| 156 |
+
def set_insights_provider(self, fn):
|
| 157 |
+
"""Callable returning a short string of live KPI / ML snapshot for RAG context."""
|
| 158 |
+
self._insights_provider = fn
|
| 159 |
+
|
| 160 |
+
def set_diagnostic_provider(self, fn):
|
| 161 |
+
"""Callable(cell_id=None) returning live diagnostic context from Agent 2."""
|
| 162 |
+
self._diagnostic_provider = fn
|
| 163 |
+
|
| 164 |
+
# ── Conversation Memory ──
|
| 165 |
+
|
| 166 |
+
def _get_memory(self, conversation_id: Optional[str]) -> deque:
|
| 167 |
+
cid = conversation_id or "__default__"
|
| 168 |
+
if cid not in self._conversations:
|
| 169 |
+
self._conversations[cid] = deque(maxlen=self.MEMORY_WINDOW * 2)
|
| 170 |
+
return self._conversations[cid]
|
| 171 |
+
|
| 172 |
+
def _add_to_memory(self, conversation_id: Optional[str], role: str, text: str):
|
| 173 |
+
mem = self._get_memory(conversation_id)
|
| 174 |
+
mem.append((role, text[:500])) # truncate to save context space
|
| 175 |
+
|
| 176 |
+
def _format_memory(self, conversation_id: Optional[str]) -> str:
|
| 177 |
+
mem = self._get_memory(conversation_id)
|
| 178 |
+
if not mem:
|
| 179 |
+
return ""
|
| 180 |
+
lines = []
|
| 181 |
+
for role, text in mem:
|
| 182 |
+
prefix = "User" if role == "user" else "Assistant"
|
| 183 |
+
lines.append(f"{prefix}: {text}")
|
| 184 |
+
return "\n".join(lines)
|
| 185 |
+
|
| 186 |
+
# ── Data Loading ──
|
| 187 |
+
|
| 188 |
+
def load_network_data(self):
|
| 189 |
+
"""Load the actual network KPI dataset"""
|
| 190 |
+
try:
|
| 191 |
+
df = pd.read_csv(self.data_path)
|
| 192 |
+
filled_cols = set(df.columns)
|
| 193 |
+
is_filled = (
|
| 194 |
+
"risk_tier" in filled_cols
|
| 195 |
+
or "RSRP_dBm" in filled_cols
|
| 196 |
+
or "signal_strength_dbm" in filled_cols
|
| 197 |
+
)
|
| 198 |
+
if is_filled:
|
| 199 |
+
df = self._transform_filled_dataset(df)
|
| 200 |
+
else:
|
| 201 |
+
df = df.copy()
|
| 202 |
+
if "timestamp" in df.columns:
|
| 203 |
+
df["timestamp"] = pd.to_datetime(df["timestamp"], errors="coerce")
|
| 204 |
+
self.df = df
|
| 205 |
+
n_cells = self.df["cell_id"].nunique() if "cell_id" in self.df.columns else 0
|
| 206 |
+
print(f" Loaded network data: {len(self.df)} records from {n_cells} cells")
|
| 207 |
+
except Exception as e:
|
| 208 |
+
print(f" Could not load network data: {e}")
|
| 209 |
+
self.df = None
|
| 210 |
+
|
| 211 |
+
def _transform_filled_dataset(self, df: pd.DataFrame) -> pd.DataFrame:
|
| 212 |
+
df = apply_filled_column_mapping(df)
|
| 213 |
+
out = finalize_pipeline_dataframe(df)
|
| 214 |
+
return out.sort_values("timestamp").reset_index(drop=True)
|
| 215 |
+
|
| 216 |
+
# ── Cell-Specific Context ──
|
| 217 |
+
|
| 218 |
+
def _get_cell_context(self, cell_id: str) -> str:
|
| 219 |
+
"""Pull live KPIs for a specific cell from the dataset + Agent 2 diagnostics."""
|
| 220 |
+
parts = []
|
| 221 |
+
|
| 222 |
+
# From local dataset
|
| 223 |
+
if self.df is not None and "cell_id" in self.df.columns:
|
| 224 |
+
cell_data = self.df[self.df["cell_id"] == cell_id]
|
| 225 |
+
if not cell_data.empty:
|
| 226 |
+
latest = cell_data.iloc[-1]
|
| 227 |
+
kpi_cols = ["RSRP", "SINR", "PRB_DL", "PRB_UL", "throughput_DL",
|
| 228 |
+
"latency_ms", "MOS", "handover_success_rate"]
|
| 229 |
+
kpis = {c: float(latest[c]) for c in kpi_cols if c in latest and pd.notna(latest[c])}
|
| 230 |
+
parts.append(f"CELL {cell_id} LIVE KPIs: {json.dumps(kpis, indent=2)}")
|
| 231 |
+
|
| 232 |
+
# Recent history (last 10 records)
|
| 233 |
+
recent = cell_data.tail(10)
|
| 234 |
+
if "issue_type" in recent.columns:
|
| 235 |
+
issues = recent["issue_type"].value_counts().to_dict()
|
| 236 |
+
parts.append(f"Recent issues on this cell: {issues}")
|
| 237 |
+
if "severity" in recent.columns:
|
| 238 |
+
sevs = recent["severity"].value_counts().to_dict()
|
| 239 |
+
parts.append(f"Severity distribution: {sevs}")
|
| 240 |
+
|
| 241 |
+
# From Agent 2 live diagnostics
|
| 242 |
+
if self._diagnostic_provider:
|
| 243 |
+
try:
|
| 244 |
+
diag = self._diagnostic_provider(cell_id=cell_id)
|
| 245 |
+
if diag:
|
| 246 |
+
parts.append(f"AGENT 2 DIAGNOSTICS: {diag}")
|
| 247 |
+
except Exception:
|
| 248 |
+
pass
|
| 249 |
+
|
| 250 |
+
return "\n".join(parts) if parts else ""
|
| 251 |
+
|
| 252 |
+
# ── Dataset Analysis ──
|
| 253 |
+
|
| 254 |
+
def get_dataset_sample(self, n: int = 10, issue_type: str = None) -> str:
|
| 255 |
+
if self.df is None:
|
| 256 |
+
return ""
|
| 257 |
+
if issue_type:
|
| 258 |
+
sample_df = self.df[self.df['issue_type'] == issue_type].head(n)
|
| 259 |
+
else:
|
| 260 |
+
sample_df = self.df.tail(n)
|
| 261 |
+
display_cols = ['timestamp', 'cell_id', 'RSRP', 'SINR', 'PRB_DL', 'throughput_DL',
|
| 262 |
+
'latency_ms', 'MOS', 'call_drop_rate', 'issue_type', 'severity']
|
| 263 |
+
display_cols = [c for c in display_cols if c in sample_df.columns]
|
| 264 |
+
sample_df = sample_df[display_cols]
|
| 265 |
+
result = f"\nSAMPLE DATA ({len(sample_df)} rows):\n"
|
| 266 |
+
result += sample_df.to_string(index=False)
|
| 267 |
+
return result
|
| 268 |
+
|
| 269 |
+
def analyze_dataset(self) -> str:
|
| 270 |
+
if self.df is None:
|
| 271 |
+
return "No dataset available for analysis."
|
| 272 |
+
total_records = len(self.df)
|
| 273 |
+
num_cells = self.df['cell_id'].nunique()
|
| 274 |
+
date_range = f"{self.df['timestamp'].min()} to {self.df['timestamp'].max()}"
|
| 275 |
+
issue_dist = self.df['issue_type'].value_counts().to_dict() if 'issue_type' in self.df.columns else {}
|
| 276 |
+
severity_dist = self.df['severity'].value_counts().to_dict() if 'severity' in self.df.columns else {}
|
| 277 |
+
latest_time = self.df['timestamp'].max()
|
| 278 |
+
cutoff_time = latest_time - timedelta(hours=24)
|
| 279 |
+
recent_df = self.df[self.df['timestamp'] >= cutoff_time]
|
| 280 |
+
recent_issues = recent_df[recent_df.get('issue_type', pd.Series()) != 'Normal'] if 'issue_type' in recent_df.columns else recent_df.head(0)
|
| 281 |
+
problem_df = self.df[self.df['severity'].isin(['CRITICAL', 'HIGH'])] if 'severity' in self.df.columns else self.df.head(0)
|
| 282 |
+
problem_cells = problem_df['cell_id'].unique() if not problem_df.empty else []
|
| 283 |
+
|
| 284 |
+
analysis = f"""DATASET ANALYSIS:
|
| 285 |
+
Total Records: {total_records:,}
|
| 286 |
+
Cells Monitored: {num_cells}
|
| 287 |
+
Date Range: {date_range}
|
| 288 |
+
|
| 289 |
+
ISSUE DISTRIBUTION:
|
| 290 |
+
{json.dumps(issue_dist, indent=2)}
|
| 291 |
+
|
| 292 |
+
SEVERITY BREAKDOWN:
|
| 293 |
+
{json.dumps(severity_dist, indent=2)}
|
| 294 |
+
|
| 295 |
+
RECENT 24H ISSUES: {len(recent_issues)} problems detected
|
| 296 |
+
- Critical: {len(recent_df[recent_df['severity'] == 'CRITICAL']) if 'severity' in recent_df.columns else 0}
|
| 297 |
+
- High: {len(recent_df[recent_df['severity'] == 'HIGH']) if 'severity' in recent_df.columns else 0}
|
| 298 |
+
- Medium: {len(recent_df[recent_df['severity'] == 'MEDIUM']) if 'severity' in recent_df.columns else 0}
|
| 299 |
+
|
| 300 |
+
CELLS WITH CRITICAL/HIGH ISSUES: {len(problem_cells)} cells
|
| 301 |
+
"""
|
| 302 |
+
if len(problem_cells) > 0:
|
| 303 |
+
analysis += f"Top problem cells: {', '.join(list(problem_cells)[:5])}\n"
|
| 304 |
+
if not problem_df.empty:
|
| 305 |
+
analysis += f"""
|
| 306 |
+
AVERAGE KPIs (Problem Records):
|
| 307 |
+
- RSRP: {problem_df['RSRP'].mean():.1f} dBm
|
| 308 |
+
- SINR: {problem_df['SINR'].mean():.1f} dB
|
| 309 |
+
- PRB_DL: {problem_df['PRB_DL'].mean():.1f}%
|
| 310 |
+
- Latency: {problem_df['latency_ms'].mean():.1f} ms
|
| 311 |
+
- MOS: {problem_df['MOS'].mean():.2f}
|
| 312 |
+
"""
|
| 313 |
+
return analysis
|
| 314 |
+
|
| 315 |
+
# ── Vector DB ──
|
| 316 |
+
|
| 317 |
+
def load_knowledge_base(self) -> str:
|
| 318 |
+
try:
|
| 319 |
+
with open(self.knowledge_base_path, 'r', encoding='utf-8') as f:
|
| 320 |
+
content = f.read()
|
| 321 |
+
print(f" Loaded knowledge base: {len(content)} characters")
|
| 322 |
+
return content
|
| 323 |
+
except FileNotFoundError:
|
| 324 |
+
print(f" Knowledge base not found: {self.knowledge_base_path}")
|
| 325 |
+
return ""
|
| 326 |
+
|
| 327 |
+
def chunk_documents(self, text: str) -> List[Dict[str, str]]:
|
| 328 |
+
text_splitter = RecursiveCharacterTextSplitter(
|
| 329 |
+
chunk_size=1000,
|
| 330 |
+
chunk_overlap=200,
|
| 331 |
+
length_function=len,
|
| 332 |
+
separators=["\n## ", "\n### ", "\n\n", "\n", " ", ""]
|
| 333 |
+
)
|
| 334 |
+
chunks = text_splitter.split_text(text)
|
| 335 |
+
documents = []
|
| 336 |
+
for idx, chunk in enumerate(chunks):
|
| 337 |
+
documents.append({
|
| 338 |
+
'id': f'chunk_{idx}',
|
| 339 |
+
'text': chunk,
|
| 340 |
+
'metadata': {'chunk_id': idx, 'source': 'knowledge_base'}
|
| 341 |
+
})
|
| 342 |
+
print(f" Created {len(documents)} chunks")
|
| 343 |
+
return documents
|
| 344 |
+
|
| 345 |
+
def initialize_vector_db(self):
|
| 346 |
+
print("\nInitializing vector database...")
|
| 347 |
+
try:
|
| 348 |
+
self.collection = self.chroma_client.get_collection(self.collection_name)
|
| 349 |
+
print(f" Using existing collection: {self.collection_name}")
|
| 350 |
+
self.is_initialized = True
|
| 351 |
+
return
|
| 352 |
+
except (ValueError, Exception) as e:
|
| 353 |
+
if 'InvalidCollectionException' in type(e).__name__ or 'InvalidCollection' in str(type(e)):
|
| 354 |
+
pass
|
| 355 |
+
else:
|
| 356 |
+
print(f" Collection lookup failed, recreating: {e}")
|
| 357 |
+
|
| 358 |
+
knowledge_text = self.load_knowledge_base()
|
| 359 |
+
if not knowledge_text:
|
| 360 |
+
print(" Cannot initialize without knowledge base")
|
| 361 |
+
return
|
| 362 |
+
|
| 363 |
+
documents = self.chunk_documents(knowledge_text)
|
| 364 |
+
self.collection = self.chroma_client.create_collection(
|
| 365 |
+
name=self.collection_name,
|
| 366 |
+
metadata={"description": "Telecom network diagnostics knowledge base"}
|
| 367 |
+
)
|
| 368 |
+
texts = [doc['text'] for doc in documents]
|
| 369 |
+
ids = [doc['id'] for doc in documents]
|
| 370 |
+
metadatas = [doc['metadata'] for doc in documents]
|
| 371 |
+
embeddings_list = self.embeddings.embed_documents(texts)
|
| 372 |
+
self.collection.add(
|
| 373 |
+
embeddings=embeddings_list,
|
| 374 |
+
documents=texts,
|
| 375 |
+
ids=ids,
|
| 376 |
+
metadatas=metadatas
|
| 377 |
+
)
|
| 378 |
+
print(f" ChromaDB initialized with {len(documents)} chunks")
|
| 379 |
+
self.is_initialized = True
|
| 380 |
+
|
| 381 |
+
# ── Adaptive Retrieval ──
|
| 382 |
+
|
| 383 |
+
def get_relevant_context(self, query: str, intent: str) -> Tuple[List[str], float]:
|
| 384 |
+
"""
|
| 385 |
+
Retrieve relevant context with adaptive top_k based on intent/complexity.
|
| 386 |
+
Returns (context_chunks, confidence_score).
|
| 387 |
+
"""
|
| 388 |
+
if not self.is_initialized:
|
| 389 |
+
return [], 0.0
|
| 390 |
+
|
| 391 |
+
# Adaptive top_k: broad questions get more chunks
|
| 392 |
+
if intent in (QueryIntent.GENERAL_KNOWLEDGE, QueryIntent.COMPARISON):
|
| 393 |
+
top_k = 5
|
| 394 |
+
elif intent == QueryIntent.DATA_QUERY:
|
| 395 |
+
top_k = 2 # data queries rely on dataset, not KB
|
| 396 |
+
else:
|
| 397 |
+
top_k = 3
|
| 398 |
+
|
| 399 |
+
query_embedding = self.embeddings.embed_query(query)
|
| 400 |
+
results = self.collection.query(
|
| 401 |
+
query_embeddings=[query_embedding],
|
| 402 |
+
n_results=top_k,
|
| 403 |
+
include=["documents", "distances"],
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
if not results or not results['documents']:
|
| 407 |
+
return [], 0.0
|
| 408 |
+
|
| 409 |
+
docs = results['documents'][0]
|
| 410 |
+
distances = results['distances'][0] if results.get('distances') else []
|
| 411 |
+
|
| 412 |
+
# Confidence: ChromaDB returns L2 distances. Lower = more similar.
|
| 413 |
+
# Map to 0-1 confidence score. Typical good match < 1.0, poor > 1.5
|
| 414 |
+
if distances:
|
| 415 |
+
avg_dist = sum(distances) / len(distances)
|
| 416 |
+
confidence = max(0.0, min(1.0, 1.0 - (avg_dist - 0.3) / 1.2))
|
| 417 |
+
else:
|
| 418 |
+
confidence = 0.5
|
| 419 |
+
|
| 420 |
+
return docs, round(confidence, 3)
|
| 421 |
+
|
| 422 |
+
# ── Response Generation ──
|
| 423 |
+
|
| 424 |
+
def generate_response(
|
| 425 |
+
self,
|
| 426 |
+
query: str,
|
| 427 |
+
context: List[str],
|
| 428 |
+
intent: str,
|
| 429 |
+
cell_context: str = "",
|
| 430 |
+
live_insights: str = "",
|
| 431 |
+
conversation_history: str = "",
|
| 432 |
+
) -> str:
|
| 433 |
+
if not self.api_key:
|
| 434 |
+
return "Error: No API key configured. Please add OPENROUTER_API_KEY or GEMINI_API_KEY to .env file."
|
| 435 |
+
|
| 436 |
+
context_text = "\n\n".join(context)
|
| 437 |
+
|
| 438 |
+
# Intent-driven dataset injection
|
| 439 |
+
dataset_context = ""
|
| 440 |
+
if intent == QueryIntent.DATA_QUERY and self.df is not None:
|
| 441 |
+
dataset_context = f"\n\nACTUAL NETWORK DATA ANALYSIS:\n{self.analyze_dataset()}\n"
|
| 442 |
+
if any(w in query.lower() for w in ['top', 'rows', 'sample', 'show', 'display']):
|
| 443 |
+
dataset_context += f"\n{self.get_dataset_sample(n=10)}\n"
|
| 444 |
+
|
| 445 |
+
cell_block = ""
|
| 446 |
+
if cell_context:
|
| 447 |
+
cell_block = f"\n\nCELL-SPECIFIC DATA:\n{cell_context}\n"
|
| 448 |
+
|
| 449 |
+
insights_block = ""
|
| 450 |
+
if live_insights:
|
| 451 |
+
insights_block = f"\n\nLIVE SYSTEM INSIGHTS:\n{live_insights}\n"
|
| 452 |
+
|
| 453 |
+
history_block = ""
|
| 454 |
+
if conversation_history:
|
| 455 |
+
history_block = f"\n\nCONVERSATION HISTORY (for follow-up context):\n{conversation_history}\n"
|
| 456 |
+
|
| 457 |
+
system_prompt = """You are an expert telecom network engineer assistant specializing in 4G/5G network diagnostics and optimization. You have access to telecom knowledge, actual network data, and live diagnostic results from monitoring agents.
|
| 458 |
+
|
| 459 |
+
When cell-specific data is provided, use it to give precise answers about that cell.
|
| 460 |
+
When conversation history is provided, use it to understand follow-up questions and maintain context."""
|
| 461 |
+
|
| 462 |
+
user_prompt = f"""CONTEXT FROM KNOWLEDGE BASE:
|
| 463 |
+
{context_text}
|
| 464 |
+
{dataset_context}{cell_block}{insights_block}{history_block}
|
| 465 |
+
USER QUESTION:
|
| 466 |
+
{query}
|
| 467 |
+
|
| 468 |
+
INSTRUCTIONS:
|
| 469 |
+
1. Use the provided context to answer the question accurately
|
| 470 |
+
2. If cell-specific data is provided, reference the actual KPI values
|
| 471 |
+
3. If asked about data/dataset, use the ACTUAL NETWORK DATA ANALYSIS section
|
| 472 |
+
4. If the question is about network issues, identify:
|
| 473 |
+
- The root causes
|
| 474 |
+
- The relevant KPI thresholds
|
| 475 |
+
- Specific actionable solutions
|
| 476 |
+
5. Be concise but thorough
|
| 477 |
+
6. If multiple solutions exist, prioritize them (quick wins first, then infrastructure changes)
|
| 478 |
+
7. Use technical terminology but explain when necessary
|
| 479 |
+
8. If this is a follow-up question, use conversation history for context
|
| 480 |
+
9. If the context doesn't contain the answer, say so clearly
|
| 481 |
+
|
| 482 |
+
ANSWER:"""
|
| 483 |
+
|
| 484 |
+
try:
|
| 485 |
+
if self.api_type == 'openrouter':
|
| 486 |
+
headers = {
|
| 487 |
+
'Authorization': f'Bearer {self.api_key}',
|
| 488 |
+
'Content-Type': 'application/json',
|
| 489 |
+
'HTTP-Referer': 'http://localhost:5173',
|
| 490 |
+
'X-Title': 'QoS Buddy'
|
| 491 |
+
}
|
| 492 |
+
data = {
|
| 493 |
+
'model': self.model_name,
|
| 494 |
+
'messages': [
|
| 495 |
+
{'role': 'system', 'content': system_prompt},
|
| 496 |
+
{'role': 'user', 'content': user_prompt}
|
| 497 |
+
],
|
| 498 |
+
'temperature': 0.7,
|
| 499 |
+
'max_tokens': 1000
|
| 500 |
+
}
|
| 501 |
+
response = requests.post(self.api_url, headers=headers, json=data, timeout=30)
|
| 502 |
+
response.raise_for_status()
|
| 503 |
+
result = response.json()
|
| 504 |
+
return result['choices'][0]['message']['content']
|
| 505 |
+
|
| 506 |
+
else:
|
| 507 |
+
headers = {'Content-Type': 'application/json'}
|
| 508 |
+
data = {
|
| 509 |
+
'contents': [{
|
| 510 |
+
'parts': [{'text': f"{system_prompt}\n\n{user_prompt}"}]
|
| 511 |
+
}]
|
| 512 |
+
}
|
| 513 |
+
url = f'{self.api_url}?key={self.api_key}'
|
| 514 |
+
response = requests.post(url, headers=headers, json=data, timeout=30)
|
| 515 |
+
response.raise_for_status()
|
| 516 |
+
return response.json()['candidates'][0]['content']['parts'][0]['text']
|
| 517 |
+
|
| 518 |
+
except requests.exceptions.HTTPError as e:
|
| 519 |
+
return f"HTTP Error: {e.response.status_code} - {e.response.text}"
|
| 520 |
+
except Exception as e:
|
| 521 |
+
return f"Error generating response: {str(e)}"
|
| 522 |
+
|
| 523 |
+
# ── Main Query Pipeline ──
|
| 524 |
+
|
| 525 |
+
def query(
|
| 526 |
+
self,
|
| 527 |
+
user_question: str,
|
| 528 |
+
include_live_insights: bool = True,
|
| 529 |
+
conversation_id: Optional[str] = None,
|
| 530 |
+
) -> Dict[str, any]:
|
| 531 |
+
"""
|
| 532 |
+
Main query method — enhanced RAG pipeline with:
|
| 533 |
+
- Intent classification
|
| 534 |
+
- Conversation memory
|
| 535 |
+
- Cell-specific context
|
| 536 |
+
- Adaptive retrieval
|
| 537 |
+
- Confidence scoring
|
| 538 |
+
"""
|
| 539 |
+
if not self.is_initialized:
|
| 540 |
+
self.initialize_vector_db()
|
| 541 |
+
|
| 542 |
+
# 1. Classify intent
|
| 543 |
+
intent = classify_intent(user_question)
|
| 544 |
+
|
| 545 |
+
# 2. Adaptive retrieval + confidence
|
| 546 |
+
relevant_context, confidence = self.get_relevant_context(user_question, intent)
|
| 547 |
+
|
| 548 |
+
if not relevant_context and intent not in (QueryIntent.DATA_QUERY, QueryIntent.CELL_DIAGNOSTIC):
|
| 549 |
+
return {
|
| 550 |
+
'question': user_question,
|
| 551 |
+
'answer': "I don't have enough information to answer this question. Please check the knowledge base.",
|
| 552 |
+
'context': [],
|
| 553 |
+
'success': False,
|
| 554 |
+
'intent': intent,
|
| 555 |
+
'confidence': 0.0,
|
| 556 |
+
}
|
| 557 |
+
|
| 558 |
+
# 3. Cell-specific context injection
|
| 559 |
+
cell_context = ""
|
| 560 |
+
cell_id = _extract_cell_id(user_question)
|
| 561 |
+
if cell_id or intent == QueryIntent.CELL_DIAGNOSTIC:
|
| 562 |
+
if cell_id:
|
| 563 |
+
cell_context = self._get_cell_context(cell_id)
|
| 564 |
+
|
| 565 |
+
# 4. Live insights from orchestrator
|
| 566 |
+
live = ""
|
| 567 |
+
if include_live_insights and self._insights_provider:
|
| 568 |
+
try:
|
| 569 |
+
live = self._insights_provider()
|
| 570 |
+
except Exception:
|
| 571 |
+
live = ""
|
| 572 |
+
|
| 573 |
+
# 5. Conversation history
|
| 574 |
+
history = self._format_memory(conversation_id)
|
| 575 |
+
|
| 576 |
+
# 6. Generate response
|
| 577 |
+
answer = self.generate_response(
|
| 578 |
+
user_question,
|
| 579 |
+
relevant_context,
|
| 580 |
+
intent=intent,
|
| 581 |
+
cell_context=cell_context,
|
| 582 |
+
live_insights=live,
|
| 583 |
+
conversation_history=history,
|
| 584 |
+
)
|
| 585 |
+
|
| 586 |
+
# 7. Update conversation memory
|
| 587 |
+
self._add_to_memory(conversation_id, "user", user_question)
|
| 588 |
+
self._add_to_memory(conversation_id, "assistant", answer)
|
| 589 |
+
|
| 590 |
+
return {
|
| 591 |
+
'question': user_question,
|
| 592 |
+
'answer': answer,
|
| 593 |
+
'context': relevant_context,
|
| 594 |
+
'success': True,
|
| 595 |
+
'intent': intent,
|
| 596 |
+
'confidence': confidence,
|
| 597 |
+
}
|
| 598 |
+
|
| 599 |
+
def chat(self, user_input: str) -> str:
|
| 600 |
+
result = self.query(user_input)
|
| 601 |
+
return result['answer']
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
def test_agent():
|
| 605 |
+
print("=" * 70)
|
| 606 |
+
print("TESTING AGENT 1: TELECOM ADVISOR CHATBOT")
|
| 607 |
+
print("=" * 70)
|
| 608 |
+
|
| 609 |
+
agent = TelecomAdvisorAgent()
|
| 610 |
+
agent.initialize_vector_db()
|
| 611 |
+
|
| 612 |
+
test_queries = [
|
| 613 |
+
"Why is my MOS low on a cell?",
|
| 614 |
+
"How to fix high call drop rate?",
|
| 615 |
+
"What causes handover failures?",
|
| 616 |
+
"My PRB is over 85%, what should I do?",
|
| 617 |
+
"How to improve throughput?"
|
| 618 |
+
]
|
| 619 |
+
|
| 620 |
+
for query in test_queries:
|
| 621 |
+
print(f"\n{'=' * 70}")
|
| 622 |
+
print(f"Q: {query}")
|
| 623 |
+
print(f"{'=' * 70}")
|
| 624 |
+
result = agent.query(query)
|
| 625 |
+
print(f"\nA: {result['answer']}")
|
| 626 |
+
print(f"\n[Intent: {result['intent']} | Confidence: {result['confidence']} | Context: {len(result['context'])} chunks]")
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
if __name__ == "__main__":
|
| 630 |
+
test_agent()
|
agents/agent2_diagnostic.py
ADDED
|
@@ -0,0 +1,1053 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Agent 2: Autonomous Diagnostic Worker
|
| 3 |
+
Real-time KPI monitoring, anomaly detection, and alert generation
|
| 4 |
+
|
| 5 |
+
Refinements v2:
|
| 6 |
+
- Compound issue detection (correlated alerts on same cell)
|
| 7 |
+
- Severity escalation over time (persistent issues auto-escalate)
|
| 8 |
+
- Cell health score (0-100 composite score)
|
| 9 |
+
- Root cause analysis chain (symptom vs root cause)
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import pandas as pd
|
| 13 |
+
import numpy as np
|
| 14 |
+
from datetime import datetime, timedelta
|
| 15 |
+
from typing import List, Dict, Optional, Tuple
|
| 16 |
+
from collections import defaultdict
|
| 17 |
+
import joblib
|
| 18 |
+
import os
|
| 19 |
+
import json
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
from core.filled_transform import apply_filled_column_mapping, finalize_diagnostic_dataframe
|
| 23 |
+
from core.config import KPI_THRESHOLDS, BUSY_HOURS
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# ── Root Cause Graph ──
|
| 27 |
+
# Maps issue_type -> list of issues it commonly *causes* (downstream symptoms).
|
| 28 |
+
# If root cause X is present alongside symptom Y, Y is demoted.
|
| 29 |
+
|
| 30 |
+
ROOT_CAUSE_GRAPH = {
|
| 31 |
+
"Capacity Problem": ["Low Throughput", "High Latency", "Low MOS (Voice Quality)"],
|
| 32 |
+
"Call Drop": ["Session Drop"],
|
| 33 |
+
"Handover Failure": ["Call Drop", "Session Drop"],
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
# Compound issue patterns: when these co-occur on one cell, emit a compound alert
|
| 37 |
+
COMPOUND_PATTERNS = {
|
| 38 |
+
"Coverage Hole": {
|
| 39 |
+
"requires": ["Call Drop", "Session Drop"],
|
| 40 |
+
"optional": ["Handover Failure"],
|
| 41 |
+
"severity": "CRITICAL",
|
| 42 |
+
"description": "Coverage hole detected — multiple signal-related failures co-occurring",
|
| 43 |
+
"recommendations": [
|
| 44 |
+
"Analyse drive test pour confirmer trou de couverture",
|
| 45 |
+
"Considerer ajout site / small cell 800 MHz",
|
| 46 |
+
"Verifier tilt et azimut des cellules voisines",
|
| 47 |
+
"Optimiser seuils handover inter-frequence",
|
| 48 |
+
],
|
| 49 |
+
},
|
| 50 |
+
"Congestion Cascade": {
|
| 51 |
+
"requires": ["Capacity Problem", "Low Throughput"],
|
| 52 |
+
"optional": ["High Latency", "Low MOS (Voice Quality)"],
|
| 53 |
+
"severity": "CRITICAL",
|
| 54 |
+
"description": "Congestion cascade — PRB saturation causing throughput/latency/voice degradation",
|
| 55 |
+
"recommendations": [
|
| 56 |
+
"URGENT: Ajouter carrier ou activer CA",
|
| 57 |
+
"Activer Massive MIMO",
|
| 58 |
+
"Implementer load balancing inter-cellule",
|
| 59 |
+
"Considerer sectorisation 3 -> 6",
|
| 60 |
+
"Prioriser trafic VoLTE dans scheduler",
|
| 61 |
+
],
|
| 62 |
+
},
|
| 63 |
+
"Interference Storm": {
|
| 64 |
+
"requires": ["Handover Failure", "Low Throughput"],
|
| 65 |
+
"optional": ["Low MOS (Voice Quality)"],
|
| 66 |
+
"severity": "HIGH",
|
| 67 |
+
"description": "Interference pattern — handover failures + throughput degradation suggest PCI/overshooting issues",
|
| 68 |
+
"recommendations": [
|
| 69 |
+
"Verifier et nettoyer plan PCI",
|
| 70 |
+
"Analyser overshooting cells",
|
| 71 |
+
"Optimiser A3 offset et TTT",
|
| 72 |
+
"Verifier interferences co-canal et canal adjacent",
|
| 73 |
+
],
|
| 74 |
+
},
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
# ── Health Score Weights ──
|
| 78 |
+
# Each KPI contributes to a 0-100 cell health score
|
| 79 |
+
|
| 80 |
+
HEALTH_WEIGHTS = {
|
| 81 |
+
"RSRP": {"weight": 0.15, "good": -80, "bad": -115, "higher_is_better": True},
|
| 82 |
+
"SINR": {"weight": 0.12, "good": 20, "bad": 0, "higher_is_better": True},
|
| 83 |
+
"PRB_DL": {"weight": 0.12, "good": 50, "bad": 95, "higher_is_better": False},
|
| 84 |
+
"throughput_DL": {"weight": 0.12, "good": 80, "bad": 5, "higher_is_better": True},
|
| 85 |
+
"latency_ms": {"weight": 0.12, "good": 20, "bad": 200, "higher_is_better": False},
|
| 86 |
+
"MOS": {"weight": 0.12, "good": 4.5, "bad": 1.5, "higher_is_better": True},
|
| 87 |
+
"handover_success_rate": {"weight": 0.15, "good": 99, "bad": 80, "higher_is_better": True},
|
| 88 |
+
"call_drop_rate": {"weight": 0.10, "good": 0, "bad": 8, "higher_is_better": False},
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _compute_health_score(row: pd.Series) -> float:
|
| 93 |
+
"""Compute a 0-100 composite health score for a single record."""
|
| 94 |
+
total = 0.0
|
| 95 |
+
weight_sum = 0.0
|
| 96 |
+
for col, cfg in HEALTH_WEIGHTS.items():
|
| 97 |
+
if col not in row or pd.isna(row[col]):
|
| 98 |
+
continue
|
| 99 |
+
val = float(row[col])
|
| 100 |
+
good = cfg["good"]
|
| 101 |
+
bad = cfg["bad"]
|
| 102 |
+
w = cfg["weight"]
|
| 103 |
+
|
| 104 |
+
if cfg["higher_is_better"]:
|
| 105 |
+
# Score: 0 at bad, 100 at good
|
| 106 |
+
if good == bad:
|
| 107 |
+
score = 50.0
|
| 108 |
+
else:
|
| 109 |
+
score = (val - bad) / (good - bad) * 100.0
|
| 110 |
+
else:
|
| 111 |
+
# Score: 100 at good, 0 at bad
|
| 112 |
+
if good == bad:
|
| 113 |
+
score = 50.0
|
| 114 |
+
else:
|
| 115 |
+
score = (bad - val) / (bad - good) * 100.0
|
| 116 |
+
|
| 117 |
+
score = max(0.0, min(100.0, score))
|
| 118 |
+
total += score * w
|
| 119 |
+
weight_sum += w
|
| 120 |
+
|
| 121 |
+
return round(total / weight_sum, 1) if weight_sum > 0 else 50.0
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class DiagnosticAgent:
|
| 125 |
+
COUNTRY_COORDINATES = {
|
| 126 |
+
"united states": (39.8283, -98.5795),
|
| 127 |
+
"usa": (39.8283, -98.5795),
|
| 128 |
+
"france": (46.2276, 2.2137),
|
| 129 |
+
"united kingdom": (55.3781, -3.4360),
|
| 130 |
+
"uk": (55.3781, -3.4360),
|
| 131 |
+
"netherlands": (52.1326, 5.2913),
|
| 132 |
+
"japan": (36.2048, 138.2529),
|
| 133 |
+
"south korea": (35.9078, 127.7669),
|
| 134 |
+
"singapore": (1.3521, 103.8198),
|
| 135 |
+
"hong kong": (22.3193, 114.1694),
|
| 136 |
+
"united arab emirates": (23.4241, 53.8478),
|
| 137 |
+
"uae": (23.4241, 53.8478),
|
| 138 |
+
"india": (20.5937, 78.9629),
|
| 139 |
+
"brazil": (-14.2350, -51.9253),
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
"""
|
| 143 |
+
Agent 2: Diagnostic Worker
|
| 144 |
+
Monitors KPIs, detects issues, generates alerts automatically
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
def __init__(self, data_path: str = None):
|
| 148 |
+
repo_root = Path(__file__).resolve().parent.parent.parent
|
| 149 |
+
default_processed = repo_root / "data" / "processed" / "network_kpis_full.csv"
|
| 150 |
+
default_filled = Path(__file__).resolve().parent.parent / "unified_telecom_dataset_FILLED.csv"
|
| 151 |
+
self.data_path = str(data_path) if data_path else str(default_filled if default_filled.exists() else default_processed)
|
| 152 |
+
self.alerts = []
|
| 153 |
+
self.models = {}
|
| 154 |
+
|
| 155 |
+
# Thresholds loaded from core.config (externalized for easy tuning)
|
| 156 |
+
self.thresholds = KPI_THRESHOLDS
|
| 157 |
+
self.busy_hours = BUSY_HOURS
|
| 158 |
+
|
| 159 |
+
# Severity escalation: track persistent issues per cell
|
| 160 |
+
# key = (cell_id, issue_type), value = list of timestamps
|
| 161 |
+
self._issue_history: Dict[Tuple[str, str], List[datetime]] = defaultdict(list)
|
| 162 |
+
|
| 163 |
+
self.load_models()
|
| 164 |
+
|
| 165 |
+
def load_models(self):
|
| 166 |
+
# Use absolute path relative to this file
|
| 167 |
+
current_dir = Path(__file__).resolve().parent
|
| 168 |
+
model_dir = current_dir.parent.parent / 'models'
|
| 169 |
+
|
| 170 |
+
# Load unified scaler
|
| 171 |
+
scaler_path = os.path.join(model_dir, 'unified_scaler.joblib')
|
| 172 |
+
if os.path.exists(scaler_path):
|
| 173 |
+
try:
|
| 174 |
+
self.models['scaler'] = joblib.load(scaler_path)
|
| 175 |
+
print(" Loaded unified scaler")
|
| 176 |
+
except Exception as e:
|
| 177 |
+
print(f" Error loading scaler: {e}")
|
| 178 |
+
|
| 179 |
+
# Load BO2 and BO3
|
| 180 |
+
model_files = {
|
| 181 |
+
'sla_predictor': 'bo2_sla_risk.joblib',
|
| 182 |
+
'zone_scorer': 'bo3_capex_score.joblib'
|
| 183 |
+
}
|
| 184 |
+
for model_name, filename in model_files.items():
|
| 185 |
+
model_path = os.path.join(model_dir, filename)
|
| 186 |
+
if os.path.exists(model_path):
|
| 187 |
+
try:
|
| 188 |
+
self.models[model_name] = joblib.load(model_path)
|
| 189 |
+
print(f" Loaded model: {model_name}")
|
| 190 |
+
except Exception as e:
|
| 191 |
+
print(f" Error loading {model_name}: {e}")
|
| 192 |
+
|
| 193 |
+
# Load BO1 Autoencoder
|
| 194 |
+
ae_path = os.path.join(model_dir, 'bo1_ae_anomaly.keras')
|
| 195 |
+
if os.path.exists(ae_path):
|
| 196 |
+
try:
|
| 197 |
+
from tensorflow.keras.models import load_model
|
| 198 |
+
self.models['anomaly_detector'] = load_model(ae_path)
|
| 199 |
+
print(" Loaded model: Keras Autoencoder (BO1)")
|
| 200 |
+
except ImportError:
|
| 201 |
+
print(" TensorFlow not installed. Cannot load BO1 Autoencoder. Run: pip install tensorflow")
|
| 202 |
+
except Exception as e:
|
| 203 |
+
print(f" Error loading Autoencoder: {e}")
|
| 204 |
+
|
| 205 |
+
def load_data(self) -> Optional[pd.DataFrame]:
|
| 206 |
+
# Check if simulator is running and has a current batch
|
| 207 |
+
try:
|
| 208 |
+
from routes.simulator import _simulator_state
|
| 209 |
+
if _simulator_state.get("running") and _simulator_state.get("current_batch_df") is not None:
|
| 210 |
+
print("[Agent2] Using simulator's current batch")
|
| 211 |
+
df = _simulator_state["current_batch_df"].copy()
|
| 212 |
+
# Apply transformations if needed
|
| 213 |
+
filled_cols = set(df.columns)
|
| 214 |
+
is_filled = (
|
| 215 |
+
"risk_tier" in filled_cols
|
| 216 |
+
or "RSRP_dBm" in filled_cols
|
| 217 |
+
or "signal_strength_dbm" in filled_cols
|
| 218 |
+
)
|
| 219 |
+
if is_filled:
|
| 220 |
+
df = self._transform_filled_dataset(df)
|
| 221 |
+
return df
|
| 222 |
+
except Exception as e:
|
| 223 |
+
print(f"[Agent2] Could not use simulator batch: {e}")
|
| 224 |
+
|
| 225 |
+
# Fallback to loading from file
|
| 226 |
+
try:
|
| 227 |
+
df = pd.read_csv(self.data_path)
|
| 228 |
+
filled_cols = set(df.columns)
|
| 229 |
+
is_filled = (
|
| 230 |
+
"risk_tier" in filled_cols
|
| 231 |
+
or "RSRP_dBm" in filled_cols
|
| 232 |
+
or "signal_strength_dbm" in filled_cols
|
| 233 |
+
)
|
| 234 |
+
if is_filled:
|
| 235 |
+
df = self._transform_filled_dataset(df)
|
| 236 |
+
else:
|
| 237 |
+
if "timestamp" in df.columns:
|
| 238 |
+
df["timestamp"] = pd.to_datetime(df["timestamp"], errors="coerce")
|
| 239 |
+
return df
|
| 240 |
+
except FileNotFoundError:
|
| 241 |
+
print(f" Data file not found: {self.data_path}")
|
| 242 |
+
return None
|
| 243 |
+
|
| 244 |
+
def _transform_filled_dataset(self, df: pd.DataFrame) -> pd.DataFrame:
|
| 245 |
+
df = apply_filled_column_mapping(df)
|
| 246 |
+
return finalize_diagnostic_dataframe(df)
|
| 247 |
+
|
| 248 |
+
def _city_to_coordinates(self, city: str) -> Optional[Dict[str, float]]:
|
| 249 |
+
"""Return known coordinates for city-level map plotting."""
|
| 250 |
+
known = {
|
| 251 |
+
"New York": (40.7128, -74.0060),
|
| 252 |
+
"Los Angeles": (34.0522, -118.2437),
|
| 253 |
+
"Chicago": (41.8781, -87.6298),
|
| 254 |
+
"Houston": (29.7604, -95.3698),
|
| 255 |
+
"Phoenix": (33.4484, -112.0740),
|
| 256 |
+
"Dallas": (32.7767, -96.7970),
|
| 257 |
+
"San Diego": (32.7157, -117.1611),
|
| 258 |
+
"San Antonio": (29.4241, -98.4936),
|
| 259 |
+
"London": (51.5074, -0.1278),
|
| 260 |
+
"Paris": (48.8566, 2.3522),
|
| 261 |
+
"Amsterdam": (52.3676, 4.9041),
|
| 262 |
+
"Tokyo": (35.6762, 139.6503),
|
| 263 |
+
"Seoul": (37.5665, 126.9780),
|
| 264 |
+
"Singapore": (1.3521, 103.8198),
|
| 265 |
+
"Hong Kong": (22.3193, 114.1694),
|
| 266 |
+
"Dubai": (25.2048, 55.2708),
|
| 267 |
+
"Mumbai": (19.0760, 72.8777),
|
| 268 |
+
"Sao Paulo": (-23.5505, -46.6333),
|
| 269 |
+
"Bangalore": (12.9716, 77.5946),
|
| 270 |
+
"Sydney": (-33.8688, 151.2093),
|
| 271 |
+
"Toronto": (43.6532, -79.3832),
|
| 272 |
+
"Bangkok": (13.7563, 100.5018),
|
| 273 |
+
"Delhi": (28.6139, 77.2090),
|
| 274 |
+
"Jacksonville": (30.3322, -81.6557),
|
| 275 |
+
"Mexico City": (19.4326, -99.1332),
|
| 276 |
+
"Berlin": (52.5200, 13.4050),
|
| 277 |
+
"Madrid": (40.4168, -3.7038),
|
| 278 |
+
"Buenos Aires": (-34.6037, -58.3816),
|
| 279 |
+
"San Jose": (37.3382, -121.8863),
|
| 280 |
+
"Austin": (30.2672, -97.7431),
|
| 281 |
+
"Philadelphia": (39.9526, -75.1652),
|
| 282 |
+
# Nigeria / Africa
|
| 283 |
+
"Lagos": (6.5244, 3.3792),
|
| 284 |
+
"Abuja": (9.0765, 7.3986),
|
| 285 |
+
"Kano": (12.0022, 8.5920),
|
| 286 |
+
"Ibadan": (7.3775, 3.9470),
|
| 287 |
+
"Port Harcourt": (4.8156, 7.0498),
|
| 288 |
+
"Benin City": (6.3350, 5.6037),
|
| 289 |
+
"Kaduna": (10.5222, 7.4383),
|
| 290 |
+
"Enugu": (6.4402, 7.4943),
|
| 291 |
+
"Ilorin": (8.4966, 4.5421),
|
| 292 |
+
"Aba": (5.1066, 7.3667),
|
| 293 |
+
"Jos": (9.8965, 8.8583),
|
| 294 |
+
"Calabar": (4.9517, 8.3220),
|
| 295 |
+
"Abeokuta": (7.1608, 3.3483),
|
| 296 |
+
"Onitsha": (6.1498, 6.7857),
|
| 297 |
+
"Warri": (5.5544, 5.7932),
|
| 298 |
+
}
|
| 299 |
+
if city in known:
|
| 300 |
+
lat, lon = known[city]
|
| 301 |
+
return {"lat": lat, "lon": lon}
|
| 302 |
+
|
| 303 |
+
# Unknown city: prefer hiding point rather than wrong placement.
|
| 304 |
+
return None
|
| 305 |
+
|
| 306 |
+
def _country_to_coordinates(self, country: str) -> Optional[Dict[str, float]]:
|
| 307 |
+
if not country:
|
| 308 |
+
return None
|
| 309 |
+
key = str(country).strip().lower()
|
| 310 |
+
coords = self.COUNTRY_COORDINATES.get(key)
|
| 311 |
+
if not coords:
|
| 312 |
+
return None
|
| 313 |
+
lat, lon = coords
|
| 314 |
+
return {"lat": float(lat), "lon": float(lon)}
|
| 315 |
+
|
| 316 |
+
def _city_to_country(self, city: str) -> Optional[str]:
|
| 317 |
+
if not city:
|
| 318 |
+
return None
|
| 319 |
+
city_map = {
|
| 320 |
+
"New York": "United States",
|
| 321 |
+
"Los Angeles": "United States",
|
| 322 |
+
"Chicago": "United States",
|
| 323 |
+
"Houston": "United States",
|
| 324 |
+
"Phoenix": "United States",
|
| 325 |
+
"Dallas": "United States",
|
| 326 |
+
"San Diego": "United States",
|
| 327 |
+
"San Antonio": "United States",
|
| 328 |
+
"San Jose": "United States",
|
| 329 |
+
"Austin": "United States",
|
| 330 |
+
"Philadelphia": "United States",
|
| 331 |
+
"Jacksonville": "United States",
|
| 332 |
+
"London": "United Kingdom",
|
| 333 |
+
"Paris": "France",
|
| 334 |
+
"Amsterdam": "Netherlands",
|
| 335 |
+
"Tokyo": "Japan",
|
| 336 |
+
"Seoul": "South Korea",
|
| 337 |
+
"Singapore": "Singapore",
|
| 338 |
+
"Hong Kong": "Hong Kong",
|
| 339 |
+
"Dubai": "United Arab Emirates",
|
| 340 |
+
"Mumbai": "India",
|
| 341 |
+
"Delhi": "India",
|
| 342 |
+
"Bangalore": "India",
|
| 343 |
+
"Sao Paulo": "Brazil",
|
| 344 |
+
"Toronto": "Canada",
|
| 345 |
+
"Sydney": "Australia",
|
| 346 |
+
"Bangkok": "Thailand",
|
| 347 |
+
"Mexico City": "Mexico",
|
| 348 |
+
"Berlin": "Germany",
|
| 349 |
+
"Madrid": "Spain",
|
| 350 |
+
"Buenos Aires": "Argentina",
|
| 351 |
+
"Lagos": "Nigeria",
|
| 352 |
+
"Abuja": "Nigeria",
|
| 353 |
+
"Kano": "Nigeria",
|
| 354 |
+
"Ibadan": "Nigeria",
|
| 355 |
+
"Port Harcourt": "Nigeria",
|
| 356 |
+
"Benin City": "Nigeria",
|
| 357 |
+
"Kaduna": "Nigeria",
|
| 358 |
+
"Enugu": "Nigeria",
|
| 359 |
+
"Ilorin": "Nigeria",
|
| 360 |
+
"Aba": "Nigeria",
|
| 361 |
+
"Jos": "Nigeria",
|
| 362 |
+
"Calabar": "Nigeria",
|
| 363 |
+
"Abeokuta": "Nigeria",
|
| 364 |
+
"Onitsha": "Nigeria",
|
| 365 |
+
"Warri": "Nigeria",
|
| 366 |
+
}
|
| 367 |
+
return city_map.get(str(city).strip())
|
| 368 |
+
|
| 369 |
+
def _resolve_zone_coordinates(self, row: pd.Series) -> Optional[Dict[str, float]]:
|
| 370 |
+
"""
|
| 371 |
+
Prefer dataset geo columns; fallback to known country; then known city.
|
| 372 |
+
"""
|
| 373 |
+
lat = pd.to_numeric(row.get("avg_lat", pd.NA), errors="coerce")
|
| 374 |
+
lon = pd.to_numeric(row.get("avg_lon", pd.NA), errors="coerce")
|
| 375 |
+
if pd.notna(lat) and pd.notna(lon):
|
| 376 |
+
return {"lat": float(lat), "lon": float(lon)}
|
| 377 |
+
|
| 378 |
+
country_coords = self._country_to_coordinates(str(row.get("country", "")).strip())
|
| 379 |
+
if country_coords:
|
| 380 |
+
return country_coords
|
| 381 |
+
|
| 382 |
+
return self._city_to_coordinates(str(row.get("zone", "")))
|
| 383 |
+
|
| 384 |
+
def _jitter_coordinates(self, lat: float, lon: float, key: str, scale_deg: float = 0.35) -> Dict[str, float]:
|
| 385 |
+
"""Deterministic micro-offset so multiple bubbles are visible per city/country."""
|
| 386 |
+
h = abs(hash(str(key)))
|
| 387 |
+
dlat = ((h % 1000) / 1000.0 - 0.5) * 2 * scale_deg
|
| 388 |
+
dlon = (((h // 1000) % 1000) / 1000.0 - 0.5) * 2 * scale_deg
|
| 389 |
+
out_lat = max(min(lat + dlat, 89.9), -89.9)
|
| 390 |
+
out_lon = lon + dlon
|
| 391 |
+
if out_lon > 180:
|
| 392 |
+
out_lon -= 360
|
| 393 |
+
if out_lon < -180:
|
| 394 |
+
out_lon += 360
|
| 395 |
+
return {"lat": round(out_lat, 6), "lon": round(out_lon, 6)}
|
| 396 |
+
|
| 397 |
+
# ── Cell Health Score ──
|
| 398 |
+
|
| 399 |
+
def compute_cell_health(self, df: pd.DataFrame) -> Dict[str, Dict]:
|
| 400 |
+
"""Compute composite health score for each cell."""
|
| 401 |
+
if df is None or df.empty or "cell_id" not in df.columns:
|
| 402 |
+
return {}
|
| 403 |
+
|
| 404 |
+
results = {}
|
| 405 |
+
for cell_id, group in df.groupby("cell_id"):
|
| 406 |
+
latest = group.iloc[-1]
|
| 407 |
+
score = _compute_health_score(latest)
|
| 408 |
+
|
| 409 |
+
# Status based on score
|
| 410 |
+
if score >= 80:
|
| 411 |
+
status = "HEALTHY"
|
| 412 |
+
elif score >= 60:
|
| 413 |
+
status = "DEGRADED"
|
| 414 |
+
elif score >= 40:
|
| 415 |
+
status = "POOR"
|
| 416 |
+
else:
|
| 417 |
+
status = "CRITICAL"
|
| 418 |
+
|
| 419 |
+
results[cell_id] = {
|
| 420 |
+
"health_score": score,
|
| 421 |
+
"status": status,
|
| 422 |
+
"kpis": {
|
| 423 |
+
col: float(latest[col])
|
| 424 |
+
for col in HEALTH_WEIGHTS
|
| 425 |
+
if col in latest and pd.notna(latest[col])
|
| 426 |
+
},
|
| 427 |
+
}
|
| 428 |
+
return results
|
| 429 |
+
|
| 430 |
+
# ── Severity Escalation ──
|
| 431 |
+
|
| 432 |
+
def _check_escalation(self, cell_id: str, issue_type: str, current_severity: str) -> str:
|
| 433 |
+
"""Escalate severity if the same issue persists on the same cell over time."""
|
| 434 |
+
key = (str(cell_id), issue_type)
|
| 435 |
+
now = datetime.now()
|
| 436 |
+
self._issue_history[key].append(now)
|
| 437 |
+
|
| 438 |
+
# Keep only last 24h of history
|
| 439 |
+
cutoff = now - timedelta(hours=24)
|
| 440 |
+
self._issue_history[key] = [t for t in self._issue_history[key] if t > cutoff]
|
| 441 |
+
|
| 442 |
+
count = len(self._issue_history[key])
|
| 443 |
+
|
| 444 |
+
# Escalation rules
|
| 445 |
+
if current_severity == "MEDIUM" and count >= 5:
|
| 446 |
+
return "HIGH"
|
| 447 |
+
if current_severity == "HIGH" and count >= 10:
|
| 448 |
+
return "CRITICAL"
|
| 449 |
+
|
| 450 |
+
return current_severity
|
| 451 |
+
|
| 452 |
+
# ── Root Cause Analysis ──
|
| 453 |
+
|
| 454 |
+
def _analyze_root_causes(self, cell_alerts: List[Dict]) -> List[Dict]:
|
| 455 |
+
"""
|
| 456 |
+
Given alerts for a single cell, identify root causes vs symptoms.
|
| 457 |
+
Demote symptoms and tag root causes.
|
| 458 |
+
"""
|
| 459 |
+
if len(cell_alerts) <= 1:
|
| 460 |
+
return cell_alerts
|
| 461 |
+
|
| 462 |
+
issue_types = {a["issue_type"] for a in cell_alerts}
|
| 463 |
+
|
| 464 |
+
for alert in cell_alerts:
|
| 465 |
+
it = alert["issue_type"]
|
| 466 |
+
# Check if this issue is a known root cause
|
| 467 |
+
downstream = ROOT_CAUSE_GRAPH.get(it, [])
|
| 468 |
+
if downstream:
|
| 469 |
+
symptoms_present = [s for s in downstream if s in issue_types]
|
| 470 |
+
if symptoms_present:
|
| 471 |
+
alert["is_root_cause"] = True
|
| 472 |
+
alert["explains"] = symptoms_present
|
| 473 |
+
|
| 474 |
+
# Check if this issue is a symptom of something else
|
| 475 |
+
for root, symptoms in ROOT_CAUSE_GRAPH.items():
|
| 476 |
+
if it in symptoms and root in issue_types:
|
| 477 |
+
alert["is_symptom_of"] = root
|
| 478 |
+
# Don't lower priority if it's already low
|
| 479 |
+
alert["priority"] = max(alert.get("priority", 2), 3)
|
| 480 |
+
|
| 481 |
+
return cell_alerts
|
| 482 |
+
|
| 483 |
+
# ── Compound Issue Detection ──
|
| 484 |
+
|
| 485 |
+
def _detect_compound_issues(self, cell_alerts: List[Dict], cell_id: str) -> List[Dict]:
|
| 486 |
+
"""Check if alerts on a single cell match a known compound pattern."""
|
| 487 |
+
issue_types = {a["issue_type"] for a in cell_alerts}
|
| 488 |
+
compounds = []
|
| 489 |
+
|
| 490 |
+
for pattern_name, pattern in COMPOUND_PATTERNS.items():
|
| 491 |
+
required = set(pattern["requires"])
|
| 492 |
+
if required.issubset(issue_types):
|
| 493 |
+
optional_present = [o for o in pattern.get("optional", []) if o in issue_types]
|
| 494 |
+
compounds.append({
|
| 495 |
+
"issue_type": f"Compound: {pattern_name}",
|
| 496 |
+
"severity": pattern["severity"],
|
| 497 |
+
"cell_id": cell_id,
|
| 498 |
+
"timestamp": cell_alerts[0]["timestamp"] if cell_alerts else str(datetime.now()),
|
| 499 |
+
"kpis": {},
|
| 500 |
+
"causes": [
|
| 501 |
+
pattern["description"],
|
| 502 |
+
f"Co-occurring issues: {', '.join(required)}",
|
| 503 |
+
] + ([f"Also present: {', '.join(optional_present)}"] if optional_present else []),
|
| 504 |
+
"recommendations": pattern["recommendations"],
|
| 505 |
+
"priority": 0,
|
| 506 |
+
"is_compound": True,
|
| 507 |
+
"component_issues": list(required | set(optional_present)),
|
| 508 |
+
})
|
| 509 |
+
|
| 510 |
+
return compounds
|
| 511 |
+
|
| 512 |
+
# ── Individual Detectors ──
|
| 513 |
+
|
| 514 |
+
def detect_call_drop(self, row: pd.Series) -> Optional[Dict]:
|
| 515 |
+
if (row['RSRP'] < self.thresholds['RSRP']['critical'] and
|
| 516 |
+
row['handover_success_rate'] < self.thresholds['handover_success_rate']['critical']):
|
| 517 |
+
return {
|
| 518 |
+
'issue_type': 'Call Drop', 'severity': 'HIGH',
|
| 519 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 520 |
+
'kpis': {'RSRP': row['RSRP'], 'SINR': row['SINR'], 'HO_success_rate': row['handover_success_rate']},
|
| 521 |
+
'causes': ['Mauvais niveau signal (RSRP < -110 dBm)', 'Taux de succes handover faible (< 85%)', 'Possible overshooting ou mauvais voisinage'],
|
| 522 |
+
'recommendations': ['Optimiser tilt mecanique/electrique', 'Ajuster puissance cellule', 'Verifier et optimiser liste voisins', 'Analyser parametres handover'],
|
| 523 |
+
'priority': 1
|
| 524 |
+
}
|
| 525 |
+
return None
|
| 526 |
+
|
| 527 |
+
def detect_low_mos(self, row: pd.Series) -> Optional[Dict]:
|
| 528 |
+
if (row['MOS'] < self.thresholds['MOS']['poor'] and
|
| 529 |
+
row['BLER_UL'] > self.thresholds['BLER']['moderate']):
|
| 530 |
+
return {
|
| 531 |
+
'issue_type': 'Low MOS (Voice Quality)', 'severity': 'MEDIUM',
|
| 532 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 533 |
+
'kpis': {'MOS': row['MOS'], 'BLER_UL': row['BLER_UL'], 'BLER_DL': row['BLER_DL'], 'PRB_DL': row['PRB_DL']},
|
| 534 |
+
'causes': [f'MOS faible ({row["MOS"]:.2f} < 3.0)', f'BLER eleve ({row["BLER_UL"]:.1f}% uplink)', 'Possibles interferences ou congestion PRB'],
|
| 535 |
+
'recommendations': ['Activer AMR adaptatif', 'Optimiser puissance UL', 'Ajuster scheduler priorite VoLTE', 'Verifier couverture indoor'],
|
| 536 |
+
'priority': 2
|
| 537 |
+
}
|
| 538 |
+
return None
|
| 539 |
+
|
| 540 |
+
def detect_low_throughput(self, row: pd.Series) -> Optional[Dict]:
|
| 541 |
+
if (row['throughput_DL'] < self.thresholds['throughput_DL']['poor'] and
|
| 542 |
+
row['SINR'] < self.thresholds['SINR']['poor']):
|
| 543 |
+
return {
|
| 544 |
+
'issue_type': 'Low Throughput', 'severity': 'MEDIUM',
|
| 545 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 546 |
+
'kpis': {'throughput_DL': row['throughput_DL'], 'SINR': row['SINR'], 'CQI': row['CQI'], 'PRB_DL': row['PRB_DL']},
|
| 547 |
+
'causes': [f'Debit DL faible ({row["throughput_DL"]:.1f} Mbps)', f'SINR faible ({row["SINR"]:.1f} dB)', 'Possibles interferences ou congestion'],
|
| 548 |
+
'recommendations': ['Optimiser tilt antenne', 'Activer Carrier Aggregation (CA)', 'Verifier configuration MIMO', 'Optimiser parametres CQI'],
|
| 549 |
+
'priority': 2
|
| 550 |
+
}
|
| 551 |
+
return None
|
| 552 |
+
|
| 553 |
+
def detect_high_latency(self, row: pd.Series) -> Optional[Dict]:
|
| 554 |
+
if (row['latency_ms'] > self.thresholds['latency_ms']['poor'] and
|
| 555 |
+
row['RLC_retransmissions'] > 10):
|
| 556 |
+
return {
|
| 557 |
+
'issue_type': 'High Latency', 'severity': 'MEDIUM',
|
| 558 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 559 |
+
'kpis': {'latency_ms': row['latency_ms'], 'RLC_retransmissions': row['RLC_retransmissions']},
|
| 560 |
+
'causes': [f'Latence elevee ({row["latency_ms"]:.0f} ms)', 'Retransmissions RLC excessives', 'Possible congestion transport ou bufferbloat'],
|
| 561 |
+
'recommendations': ['Optimiser buffers reseau', 'Verifier routage IP', 'Verifier configuration MTU', 'Optimiser scheduling', 'Considerer local breakout'],
|
| 562 |
+
'priority': 2
|
| 563 |
+
}
|
| 564 |
+
return None
|
| 565 |
+
|
| 566 |
+
def detect_session_drop(self, row: pd.Series) -> Optional[Dict]:
|
| 567 |
+
if row['RSRP'] < -115 and row['handover_success_rate'] < 90:
|
| 568 |
+
return {
|
| 569 |
+
'issue_type': 'Session Drop', 'severity': 'HIGH',
|
| 570 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 571 |
+
'kpis': {'RSRP': row['RSRP'], 'handover_success_rate': row['handover_success_rate']},
|
| 572 |
+
'causes': ['Coverage hole detecte (RSRP < -115 dBm)', 'Handover inter-frequence problematique', 'Possible ERAB drop'],
|
| 573 |
+
'recommendations': ['Optimiser seuils handover', 'Ajuster timer T310', 'Considerer ajout cellule 800 MHz', 'Analyser drive test logs (evenements L3)'],
|
| 574 |
+
'priority': 1
|
| 575 |
+
}
|
| 576 |
+
return None
|
| 577 |
+
|
| 578 |
+
def detect_handover_failure(self, row: pd.Series) -> Optional[Dict]:
|
| 579 |
+
if row['handover_success_rate'] < self.thresholds['handover_success_rate']['critical']:
|
| 580 |
+
return {
|
| 581 |
+
'issue_type': 'Handover Failure', 'severity': 'HIGH',
|
| 582 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 583 |
+
'kpis': {'handover_success_rate': row['handover_success_rate'], 'RSRP': row['RSRP']},
|
| 584 |
+
'causes': [f'Taux succes HO tres faible ({row["handover_success_rate"]:.1f}%)', 'Possible mauvaise definition voisins', 'Possible overshooting cell ou confusion PCI'],
|
| 585 |
+
'recommendations': ['Verifier et nettoyer plan PCI', 'Optimiser A3 offset', 'Ajuster TTT (Time-To-Trigger)', 'Auditer relations de voisinage'],
|
| 586 |
+
'priority': 1
|
| 587 |
+
}
|
| 588 |
+
return None
|
| 589 |
+
|
| 590 |
+
def detect_capacity_issue(self, row: pd.Series) -> Optional[Dict]:
|
| 591 |
+
prb_max = max(row['PRB_UL'], row['PRB_DL'])
|
| 592 |
+
is_busy_hour = row['hour'] in self.busy_hours
|
| 593 |
+
if prb_max > self.thresholds['PRB']['moderate']:
|
| 594 |
+
severity = 'CRITICAL' if (is_busy_hour and prb_max > 90) else 'HIGH'
|
| 595 |
+
return {
|
| 596 |
+
'issue_type': 'Capacity Problem', 'severity': severity,
|
| 597 |
+
'cell_id': row['cell_id'], 'timestamp': row['timestamp'],
|
| 598 |
+
'kpis': {'PRB_UL': row['PRB_UL'], 'PRB_DL': row['PRB_DL'], 'throughput_DL': row['throughput_DL'], 'hour': row['hour']},
|
| 599 |
+
'causes': [f'PRB congestion severe (DL: {row["PRB_DL"]:.1f}%, UL: {row["PRB_UL"]:.1f}%)',
|
| 600 |
+
'Capacite insuffisante' + (' pendant busy hour' if is_busy_hour else ''), 'Degradation debit probable'],
|
| 601 |
+
'recommendations': ['URGENT: Ajouter carrier', 'Activer Carrier Aggregation (CA)', 'Considerer sectorisation 3 -> 6', 'Activer Massive MIMO', 'Implementer load balancing inter-cellule'],
|
| 602 |
+
'priority': 0 if severity == 'CRITICAL' else 1
|
| 603 |
+
}
|
| 604 |
+
return None
|
| 605 |
+
|
| 606 |
+
# ── Analysis Pipeline ──
|
| 607 |
+
|
| 608 |
+
def analyze_record(self, row: pd.Series) -> List[Dict]:
|
| 609 |
+
alerts = []
|
| 610 |
+
|
| 611 |
+
# ── Fast path: use pre-computed anomaly_flag if available ─────────
|
| 612 |
+
# This avoids slow per-row TF inference when CLEAN dataset is loaded
|
| 613 |
+
precomputed_flag = pd.to_numeric(row.get('anomaly_flag', pd.NA), errors='coerce')
|
| 614 |
+
if pd.notna(precomputed_flag) and int(precomputed_flag) == 1:
|
| 615 |
+
alerts.append({
|
| 616 |
+
'issue_type': 'ML Anomaly', 'severity': 'HIGH',
|
| 617 |
+
'cell_id': row.get('cell_id', 'unknown'),
|
| 618 |
+
'timestamp': str(row.get('timestamp', '')),
|
| 619 |
+
'kpis': {},
|
| 620 |
+
'causes': ['Pre-computed Autoencoder Anomaly Flag'],
|
| 621 |
+
'recommendations': ['Deep KPI analysis required', 'Check radio hardware and interference'],
|
| 622 |
+
'priority': 1,
|
| 623 |
+
'is_ml': True
|
| 624 |
+
})
|
| 625 |
+
elif not pd.notna(precomputed_flag):
|
| 626 |
+
# Only run live TF inference if NO pre-computed flag exists
|
| 627 |
+
scaler = self.models.get('scaler')
|
| 628 |
+
anomaly_model = self.models.get('anomaly_detector')
|
| 629 |
+
|
| 630 |
+
if scaler and anomaly_model:
|
| 631 |
+
try:
|
| 632 |
+
feat_cols = ['PRBUsageUL', 'MIMO_Rank', 'HO_SuccessRate_pct', 'ERAB_DropRate_pct', 'CQI', 'day_of_week', 'latency_ms', 'unified_risk_score', 'signal_strength_dbm', 'throughput_mbps', 'meanThr_UL', 'jitter_ms', 'maxThr_DL', 'is_peak', 'active_users', 'meanUE_UL', 'meanUE_DL', 'packet_loss_rate', 'error_rate', 'RSRP_dBm', 'PRBUsageDL', 'CSSR_pct', 'hour', 'SINR_dB', 'radio_health_score', 'meanThr_DL']
|
| 633 |
+
input_data = row.reindex(feat_cols).fillna(0).values.reshape(1, -1)
|
| 634 |
+
scaled_data = scaler.transform(input_data)
|
| 635 |
+
reconstructed = anomaly_model.predict(scaled_data, verbose=0)
|
| 636 |
+
mse = np.mean(np.power(scaled_data - reconstructed, 2), axis=1)[0]
|
| 637 |
+
if mse > 261.5:
|
| 638 |
+
alerts.append({
|
| 639 |
+
'issue_type': 'ML Anomaly', 'severity': 'HIGH',
|
| 640 |
+
'cell_id': row.get('cell_id', 'unknown'),
|
| 641 |
+
'timestamp': str(row.get('timestamp', '')),
|
| 642 |
+
'kpis': {'MSE': round(float(mse), 2)},
|
| 643 |
+
'causes': ['Autoencoder Mathematical Anomaly', f'MSE Error ({mse:.1f}) Exceeded limit'],
|
| 644 |
+
'recommendations': ['Deep KPI analysis required', 'Check radio hardware and interference'],
|
| 645 |
+
'priority': 1,
|
| 646 |
+
'is_ml': True
|
| 647 |
+
})
|
| 648 |
+
except Exception:
|
| 649 |
+
pass
|
| 650 |
+
|
| 651 |
+
detectors = [
|
| 652 |
+
self.detect_call_drop, self.detect_low_mos, self.detect_low_throughput,
|
| 653 |
+
self.detect_high_latency, self.detect_session_drop,
|
| 654 |
+
self.detect_handover_failure, self.detect_capacity_issue,
|
| 655 |
+
]
|
| 656 |
+
for detector in detectors:
|
| 657 |
+
alert = detector(row)
|
| 658 |
+
if alert:
|
| 659 |
+
alert['severity'] = self._check_escalation(
|
| 660 |
+
alert['cell_id'], alert['issue_type'], alert['severity']
|
| 661 |
+
)
|
| 662 |
+
alerts.append(alert)
|
| 663 |
+
return alerts
|
| 664 |
+
|
| 665 |
+
def analyze_batch(self, df: pd.DataFrame, time_window_hours: int = 1) -> List[Dict]:
|
| 666 |
+
"""Analyze recent records with compound detection + root cause analysis."""
|
| 667 |
+
if 'timestamp' not in df.columns:
|
| 668 |
+
return []
|
| 669 |
+
|
| 670 |
+
latest_time = df['timestamp'].max()
|
| 671 |
+
cutoff_time = latest_time - timedelta(hours=time_window_hours)
|
| 672 |
+
recent_df = df[df['timestamp'] >= cutoff_time]
|
| 673 |
+
|
| 674 |
+
# Hard cap to prevent event-loop blocking (PERFORMANCE CRITICAL)
|
| 675 |
+
MAX_ROWS_FOR_ANALYSIS = 1500
|
| 676 |
+
if len(recent_df) > MAX_ROWS_FOR_ANALYSIS:
|
| 677 |
+
recent_df = recent_df.tail(MAX_ROWS_FOR_ANALYSIS)
|
| 678 |
+
|
| 679 |
+
# Step 1: Run individual detectors on each row
|
| 680 |
+
all_alerts = []
|
| 681 |
+
for idx, row in recent_df.iterrows():
|
| 682 |
+
alerts = self.analyze_record(row)
|
| 683 |
+
all_alerts.extend(alerts)
|
| 684 |
+
|
| 685 |
+
# Step 2: Group by cell for compound + root cause analysis
|
| 686 |
+
cell_groups: Dict[str, List[Dict]] = defaultdict(list)
|
| 687 |
+
for alert in all_alerts:
|
| 688 |
+
cell_groups[str(alert['cell_id'])].append(alert)
|
| 689 |
+
|
| 690 |
+
enhanced_alerts = []
|
| 691 |
+
for cell_id, cell_alerts in cell_groups.items():
|
| 692 |
+
cell_alerts = self._analyze_root_causes(cell_alerts)
|
| 693 |
+
compounds = self._detect_compound_issues(cell_alerts, cell_id)
|
| 694 |
+
enhanced_alerts.extend(compounds)
|
| 695 |
+
enhanced_alerts.extend(cell_alerts)
|
| 696 |
+
|
| 697 |
+
enhanced_alerts.sort(key=lambda x: x.get('priority', 9))
|
| 698 |
+
return enhanced_alerts
|
| 699 |
+
|
| 700 |
+
# ── Public API ──
|
| 701 |
+
|
| 702 |
+
def get_latest_alerts(self, n: int = 20) -> List[Dict]:
|
| 703 |
+
df = self.load_data()
|
| 704 |
+
if df is None:
|
| 705 |
+
return []
|
| 706 |
+
alerts = self.analyze_batch(df, time_window_hours=1)
|
| 707 |
+
return alerts[:n]
|
| 708 |
+
|
| 709 |
+
def get_cell_status(self, cell_id: str) -> Dict:
|
| 710 |
+
df = self.load_data()
|
| 711 |
+
if df is None:
|
| 712 |
+
return {'error': 'Data not available'}
|
| 713 |
+
cell_data = df[df['cell_id'] == cell_id]
|
| 714 |
+
if cell_data.empty:
|
| 715 |
+
return {'error': f'Cell {cell_id} not found'}
|
| 716 |
+
|
| 717 |
+
latest_record = cell_data.iloc[-1]
|
| 718 |
+
alerts = self.analyze_record(latest_record)
|
| 719 |
+
health = _compute_health_score(latest_record)
|
| 720 |
+
status = 'NORMAL' if not alerts else alerts[0]['severity']
|
| 721 |
+
|
| 722 |
+
return {
|
| 723 |
+
'cell_id': cell_id,
|
| 724 |
+
'status': status,
|
| 725 |
+
'health_score': health,
|
| 726 |
+
'last_update': str(latest_record['timestamp']),
|
| 727 |
+
'kpis': {
|
| 728 |
+
col: float(latest_record[col])
|
| 729 |
+
for col in ['RSRP', 'SINR', 'PRB_DL', 'PRB_UL', 'throughput_DL', 'MOS', 'latency_ms', 'handover_success_rate']
|
| 730 |
+
if col in latest_record and pd.notna(latest_record[col])
|
| 731 |
+
},
|
| 732 |
+
'alerts': alerts,
|
| 733 |
+
}
|
| 734 |
+
|
| 735 |
+
def get_cell_diagnostic_context(self, cell_id: str) -> str:
|
| 736 |
+
"""Produce a compact text summary for Agent 1 chat context."""
|
| 737 |
+
status = self.get_cell_status(cell_id)
|
| 738 |
+
if "error" in status:
|
| 739 |
+
return f"Cell {cell_id}: {status['error']}"
|
| 740 |
+
|
| 741 |
+
lines = [
|
| 742 |
+
f"Cell {cell_id} — Status: {status['status']} | Health: {status['health_score']}/100",
|
| 743 |
+
]
|
| 744 |
+
kpis = status.get("kpis", {})
|
| 745 |
+
if kpis:
|
| 746 |
+
kpi_str = ", ".join(f"{k}={v:.1f}" for k, v in kpis.items())
|
| 747 |
+
lines.append(f"KPIs: {kpi_str}")
|
| 748 |
+
for a in status.get("alerts", [])[:3]:
|
| 749 |
+
tag = "[ROOT CAUSE] " if a.get("is_root_cause") else ""
|
| 750 |
+
lines.append(f" Alert: {tag}{a['issue_type']} ({a['severity']}) — {a['causes'][0]}")
|
| 751 |
+
return "\n".join(lines)
|
| 752 |
+
|
| 753 |
+
def get_realtime_kpis(self, limit: int = 100) -> List[Dict]:
|
| 754 |
+
df = self.load_data()
|
| 755 |
+
if df is None:
|
| 756 |
+
return []
|
| 757 |
+
recent = df.tail(limit)
|
| 758 |
+
kpis = []
|
| 759 |
+
for _, row in recent.iterrows():
|
| 760 |
+
entry = {
|
| 761 |
+
'timestamp': str(row['timestamp']),
|
| 762 |
+
'cell_id': row['cell_id'],
|
| 763 |
+
'RSRP': float(row['RSRP']) if pd.notna(row.get('RSRP')) else 0.0,
|
| 764 |
+
'SINR': float(row['SINR']) if pd.notna(row.get('SINR')) else 0.0,
|
| 765 |
+
'PRB_DL': float(row['PRB_DL']) if pd.notna(row.get('PRB_DL')) else 0.0,
|
| 766 |
+
'throughput_DL': float(row['throughput_DL']) if pd.notna(row.get('throughput_DL')) else 0.0,
|
| 767 |
+
'MOS': float(row['MOS']) if pd.notna(row.get('MOS')) else 0.0,
|
| 768 |
+
'latency_ms': float(row['latency_ms']) if pd.notna(row.get('latency_ms')) else 0.0,
|
| 769 |
+
}
|
| 770 |
+
# Add health score
|
| 771 |
+
entry['health_score'] = _compute_health_score(row)
|
| 772 |
+
kpis.append(entry)
|
| 773 |
+
return kpis
|
| 774 |
+
|
| 775 |
+
def generate_summary_report(self) -> Dict:
|
| 776 |
+
df = self.load_data()
|
| 777 |
+
if df is None:
|
| 778 |
+
return {'error': 'Data not available'}
|
| 779 |
+
|
| 780 |
+
alerts = self.analyze_batch(df, time_window_hours=24)
|
| 781 |
+
critical_alerts = [a for a in alerts if a['severity'] == 'CRITICAL']
|
| 782 |
+
high_alerts = [a for a in alerts if a['severity'] == 'HIGH']
|
| 783 |
+
medium_alerts = [a for a in alerts if a['severity'] == 'MEDIUM']
|
| 784 |
+
compound_alerts = [a for a in alerts if a.get('is_compound')]
|
| 785 |
+
|
| 786 |
+
issue_counts = {}
|
| 787 |
+
for alert in alerts:
|
| 788 |
+
issue_type = alert['issue_type']
|
| 789 |
+
issue_counts[issue_type] = issue_counts.get(issue_type, 0) + 1
|
| 790 |
+
|
| 791 |
+
cells_with_issues = set(a['cell_id'] for a in alerts)
|
| 792 |
+
|
| 793 |
+
# Cell health summary
|
| 794 |
+
health_data = self.compute_cell_health(df)
|
| 795 |
+
health_dist = defaultdict(int)
|
| 796 |
+
for ch in health_data.values():
|
| 797 |
+
health_dist[ch["status"]] += 1
|
| 798 |
+
|
| 799 |
+
return {
|
| 800 |
+
'timestamp': str(datetime.now()),
|
| 801 |
+
'total_alerts': len(alerts),
|
| 802 |
+
'critical': len(critical_alerts),
|
| 803 |
+
'high': len(high_alerts),
|
| 804 |
+
'medium': len(medium_alerts),
|
| 805 |
+
'compound_issues': len(compound_alerts),
|
| 806 |
+
'cells_monitored': int(df['cell_id'].nunique()),
|
| 807 |
+
'cells_with_issues': len(cells_with_issues),
|
| 808 |
+
'issue_breakdown': issue_counts,
|
| 809 |
+
'health_distribution': dict(health_dist),
|
| 810 |
+
'top_alerts': alerts[:10],
|
| 811 |
+
}
|
| 812 |
+
|
| 813 |
+
def get_bo_integrated_overview(
|
| 814 |
+
self,
|
| 815 |
+
time_window_hours: int = 24,
|
| 816 |
+
max_zones: int = 120,
|
| 817 |
+
bo_filter: Optional[str] = None,
|
| 818 |
+
) -> Dict:
|
| 819 |
+
df = self.load_data()
|
| 820 |
+
if df is None or df.empty:
|
| 821 |
+
return {"error": "Data not available"}
|
| 822 |
+
|
| 823 |
+
if bo_filter and "source_bo" in df.columns:
|
| 824 |
+
df = df[df["source_bo"].astype(str) == str(bo_filter)]
|
| 825 |
+
if df.empty:
|
| 826 |
+
return {"error": "No data for selected filter"}
|
| 827 |
+
|
| 828 |
+
if "timestamp" in df.columns:
|
| 829 |
+
latest_time = df["timestamp"].max()
|
| 830 |
+
cutoff_time = latest_time - timedelta(hours=time_window_hours)
|
| 831 |
+
recent_df = df[df["timestamp"] >= cutoff_time].copy()
|
| 832 |
+
else:
|
| 833 |
+
recent_df = df.copy()
|
| 834 |
+
|
| 835 |
+
# BO1 - anomaly monitoring
|
| 836 |
+
if "anomaly_flag" in recent_df.columns:
|
| 837 |
+
anomaly_flag = pd.to_numeric(recent_df["anomaly_flag"], errors="coerce").fillna(0)
|
| 838 |
+
anomaly_count = int((anomaly_flag > 0).sum())
|
| 839 |
+
else:
|
| 840 |
+
anomaly_flag = pd.Series([0] * len(recent_df))
|
| 841 |
+
anomaly_count = 0
|
| 842 |
+
|
| 843 |
+
bo1 = {
|
| 844 |
+
"records": int(len(recent_df)),
|
| 845 |
+
"anomalies": anomaly_count,
|
| 846 |
+
"anomaly_rate_pct": round((anomaly_count / max(len(recent_df), 1)) * 100, 2),
|
| 847 |
+
"high_risk_cells": int(recent_df[recent_df.get("risk_tier", pd.Series(dtype=object)).astype(str) == "CRITICAL"]["cell_id"].nunique()) if "risk_tier" in recent_df.columns else 0,
|
| 848 |
+
}
|
| 849 |
+
|
| 850 |
+
# BO2 - SLA/MOS
|
| 851 |
+
if "sla_breach_risk" in recent_df.columns:
|
| 852 |
+
sla = pd.to_numeric(recent_df["sla_breach_risk"], errors="coerce").fillna(0)
|
| 853 |
+
else:
|
| 854 |
+
sla = pd.Series([0.0] * len(recent_df))
|
| 855 |
+
|
| 856 |
+
if "MOS" in recent_df.columns:
|
| 857 |
+
mos = pd.to_numeric(recent_df["MOS"], errors="coerce").fillna(0)
|
| 858 |
+
else:
|
| 859 |
+
mos = pd.Series([0.0] * len(recent_df))
|
| 860 |
+
|
| 861 |
+
bo2 = {
|
| 862 |
+
"avg_sla_breach_risk": round(float(sla.mean()), 4) if not sla.empty else None,
|
| 863 |
+
"high_sla_risk_count": int((sla >= 0.7).sum()),
|
| 864 |
+
"avg_mos": round(float(mos.mean()), 3) if not mos.empty else None,
|
| 865 |
+
"low_mos_count": int((mos < 3.0).sum()),
|
| 866 |
+
}
|
| 867 |
+
|
| 868 |
+
# BO3 - CAPEX zone prioritization
|
| 869 |
+
if "capex_score" in recent_df.columns:
|
| 870 |
+
capex = pd.to_numeric(recent_df["capex_score"], errors="coerce").fillna(0)
|
| 871 |
+
else:
|
| 872 |
+
capex = pd.Series([0.0] * len(recent_df))
|
| 873 |
+
|
| 874 |
+
bo3 = {
|
| 875 |
+
"avg_capex_score": round(float(capex.mean()), 2) if not capex.empty else 0.0,
|
| 876 |
+
"urgent_count": int((recent_df.get("capex_priority", pd.Series(dtype=object, index=recent_df.index)).astype(str).isin(["5", "URGENT", "Urgent", "urgent"])).sum()) if "capex_priority" in recent_df.columns else 0,
|
| 877 |
+
"critical_risk_count": int((recent_df.get("risk_tier", pd.Series(dtype=object, index=recent_df.index)).astype(str) == "CRITICAL").sum()) if "risk_tier" in recent_df.columns else 0,
|
| 878 |
+
}
|
| 879 |
+
|
| 880 |
+
# Trend samples
|
| 881 |
+
trend_cols = ["timestamp", "RSRP", "SINR", "PRB_DL", "throughput_DL", "latency_ms", "MOS", "sla_breach_risk", "capex_score"]
|
| 882 |
+
trend_cols = [c for c in trend_cols if c in recent_df.columns]
|
| 883 |
+
trends = recent_df[trend_cols].tail(180).copy()
|
| 884 |
+
if "timestamp" in trends.columns:
|
| 885 |
+
trends["timestamp"] = trends["timestamp"].astype(str)
|
| 886 |
+
# Replace NaN/inf with None for JSON serialization
|
| 887 |
+
trends = trends.where(trends.notna(), other=None)
|
| 888 |
+
trend_data = trends.to_dict("records")
|
| 889 |
+
# Clean None values in each record
|
| 890 |
+
trend_data = [
|
| 891 |
+
{k: (None if (v is not None and isinstance(v, float) and (v != v or v == float('inf') or v == float('-inf'))) else v)
|
| 892 |
+
for k, v in row.items()}
|
| 893 |
+
for row in trend_data
|
| 894 |
+
]
|
| 895 |
+
|
| 896 |
+
# Zone map: use ALL data (not time-filtered) so every city always appears.
|
| 897 |
+
# We group by city and take aggregate stats — this ensures 46/46 cities show on map.
|
| 898 |
+
zone_df = df.copy()
|
| 899 |
+
|
| 900 |
+
if "country" in zone_df.columns:
|
| 901 |
+
raw_country = zone_df["country"]
|
| 902 |
+
normalized = raw_country.where(raw_country.notna(), None).astype("object")
|
| 903 |
+
normalized = normalized.map(lambda x: str(x).strip() if x is not None else None)
|
| 904 |
+
normalized = normalized.replace({"": None, "nan": None, "None": None})
|
| 905 |
+
if "city" in zone_df.columns:
|
| 906 |
+
normalized = normalized.fillna(zone_df["city"].astype(str).map(self._city_to_country))
|
| 907 |
+
zone_df["country"] = normalized.fillna("Unknown")
|
| 908 |
+
elif "city" in zone_df.columns:
|
| 909 |
+
zone_df["country"] = zone_df["city"].astype(str).map(self._city_to_country).fillna("Unknown")
|
| 910 |
+
|
| 911 |
+
zone_col = "city" if "city" in zone_df.columns else ("country" if "country" in zone_df.columns else None)
|
| 912 |
+
lat_col = next((c for c in ["lat", "latitude", "Latitude", "LAT"] if c in zone_df.columns), None)
|
| 913 |
+
lon_col = next((c for c in ["lon", "lng", "longitude", "Longitude", "LON", "LNG"] if c in zone_df.columns), None)
|
| 914 |
+
|
| 915 |
+
if "cell_id" in zone_df.columns and zone_col:
|
| 916 |
+
# 10x Faster Vectorized Aggregation
|
| 917 |
+
df_g = zone_df.assign(
|
| 918 |
+
zone_str=zone_df[zone_col].astype(str)
|
| 919 |
+
)
|
| 920 |
+
|
| 921 |
+
# Numeric values
|
| 922 |
+
for c in ["capex_score", "sla_breach_risk", "MOS"]:
|
| 923 |
+
if c in df_g.columns:
|
| 924 |
+
df_g[c] = pd.to_numeric(df_g[c], errors="coerce")
|
| 925 |
+
|
| 926 |
+
if lat_col: df_g["_lat"] = pd.to_numeric(df_g[lat_col], errors="coerce")
|
| 927 |
+
if lon_col: df_g["_lon"] = pd.to_numeric(df_g[lon_col], errors="coerce")
|
| 928 |
+
|
| 929 |
+
agg_map = {
|
| 930 |
+
"records": ("zone_str", "count"),
|
| 931 |
+
"avg_capex_score": ("capex_score", "mean"),
|
| 932 |
+
"avg_sla_breach_risk": ("sla_breach_risk", "mean"),
|
| 933 |
+
"avg_mos": ("MOS", "mean")
|
| 934 |
+
}
|
| 935 |
+
if lat_col: agg_map["avg_lat"] = ("_lat", "mean")
|
| 936 |
+
if lon_col: agg_map["avg_lon"] = ("_lon", "mean")
|
| 937 |
+
if "country" in df_g.columns:
|
| 938 |
+
agg_map["country"] = ("country", "first")
|
| 939 |
+
|
| 940 |
+
# Final high-speed aggregation
|
| 941 |
+
grouped = df_g.groupby("zone_str").agg(**agg_map).reset_index()
|
| 942 |
+
|
| 943 |
+
# Handle Critical Counts
|
| 944 |
+
if "risk_tier" in df_g.columns:
|
| 945 |
+
crit = df_g[df_g["risk_tier"].astype(str) == "CRITICAL"].groupby("zone_str").size()
|
| 946 |
+
grouped["critical_count"] = grouped["zone_str"].map(crit).fillna(0).astype(int)
|
| 947 |
+
else:
|
| 948 |
+
grouped["critical_count"] = 0
|
| 949 |
+
|
| 950 |
+
grouped = grouped.sort_values(by=["critical_count", "avg_capex_score"], ascending=[False, False]).head(max_zones)
|
| 951 |
+
|
| 952 |
+
zones = []
|
| 953 |
+
for _, row in grouped.iterrows():
|
| 954 |
+
zname = row["zone_str"]
|
| 955 |
+
rec = {
|
| 956 |
+
"zone": zname,
|
| 957 |
+
"country": row.get("country", "Unknown"),
|
| 958 |
+
"cell_id": "Various",
|
| 959 |
+
"lat": row.get("avg_lat"),
|
| 960 |
+
"lon": row.get("avg_lon"),
|
| 961 |
+
"records": int(row["records"]),
|
| 962 |
+
"avg_capex_score": round(float(row["avg_capex_score"]), 2) if pd.notna(row["avg_capex_score"]) else 0,
|
| 963 |
+
"avg_sla_breach_risk": round(float(row["avg_sla_breach_risk"]), 4) if pd.notna(row["avg_sla_breach_risk"]) else 0,
|
| 964 |
+
"avg_mos": round(float(row["avg_mos"]), 3) if pd.notna(row["avg_mos"]) else 0,
|
| 965 |
+
"critical_count": int(row["critical_count"]),
|
| 966 |
+
"capex_priority": "AUTO"
|
| 967 |
+
}
|
| 968 |
+
|
| 969 |
+
# Resolve coordinates if missing
|
| 970 |
+
if pd.isna(rec["lat"]) or pd.isna(rec["lon"]):
|
| 971 |
+
coords = self._resolve_zone_coordinates({"zone": zname, "country": rec["country"]})
|
| 972 |
+
if coords:
|
| 973 |
+
rec["lat"], rec["lon"] = coords["lat"], coords["lon"]
|
| 974 |
+
|
| 975 |
+
# Apply jitter for map visibility
|
| 976 |
+
if pd.notna(rec["lat"]):
|
| 977 |
+
jitter = self._jitter_coordinates(rec["lat"], rec["lon"], zname)
|
| 978 |
+
rec["lat"], rec["lon"] = jitter["lat"], jitter["lon"]
|
| 979 |
+
|
| 980 |
+
zones.append(rec)
|
| 981 |
+
else:
|
| 982 |
+
zones = []
|
| 983 |
+
|
| 984 |
+
models_status = {k: True for k in self.models.keys()}
|
| 985 |
+
for key in ["anomaly_detector", "sla_predictor", "zone_scorer"]:
|
| 986 |
+
models_status.setdefault(key, False)
|
| 987 |
+
|
| 988 |
+
return {
|
| 989 |
+
"timestamp": str(datetime.now()),
|
| 990 |
+
"time_window_hours": int(time_window_hours),
|
| 991 |
+
"bo1": bo1, "bo2": bo2, "bo3": bo3,
|
| 992 |
+
"zones": zones, "trends": trend_data,
|
| 993 |
+
"models": models_status,
|
| 994 |
+
}
|
| 995 |
+
|
| 996 |
+
def run_diagnostics(self):
|
| 997 |
+
print("=" * 70)
|
| 998 |
+
print("AGENT 2: AUTONOMOUS DIAGNOSTIC SCAN")
|
| 999 |
+
print("=" * 70)
|
| 1000 |
+
print(f"Scan time: {datetime.now()}")
|
| 1001 |
+
df = self.load_data()
|
| 1002 |
+
if df is None:
|
| 1003 |
+
print(" Cannot run diagnostics without data")
|
| 1004 |
+
return
|
| 1005 |
+
alerts = self.analyze_batch(df, time_window_hours=1)
|
| 1006 |
+
if not alerts:
|
| 1007 |
+
print("\n No issues detected - All systems nominal")
|
| 1008 |
+
return
|
| 1009 |
+
|
| 1010 |
+
compounds = [a for a in alerts if a.get("is_compound")]
|
| 1011 |
+
roots = [a for a in alerts if a.get("is_root_cause")]
|
| 1012 |
+
print(f"\n Detected {len(alerts)} alerts ({len(compounds)} compound, {len(roots)} root causes):\n")
|
| 1013 |
+
|
| 1014 |
+
for i, alert in enumerate(alerts[:5], 1):
|
| 1015 |
+
tag = ""
|
| 1016 |
+
if alert.get("is_compound"):
|
| 1017 |
+
tag = "[COMPOUND] "
|
| 1018 |
+
elif alert.get("is_root_cause"):
|
| 1019 |
+
tag = "[ROOT CAUSE] "
|
| 1020 |
+
elif alert.get("is_symptom_of"):
|
| 1021 |
+
tag = f"[SYMPTOM of {alert['is_symptom_of']}] "
|
| 1022 |
+
print(f"{i}. {tag}[{alert['severity']}] {alert['issue_type']}")
|
| 1023 |
+
print(f" Cell: {alert['cell_id']} | Primary action: {alert['recommendations'][0]}")
|
| 1024 |
+
|
| 1025 |
+
self.alerts = alerts
|
| 1026 |
+
return alerts
|
| 1027 |
+
|
| 1028 |
+
|
| 1029 |
+
def test_agent():
|
| 1030 |
+
print("=" * 70)
|
| 1031 |
+
print("TESTING AGENT 2: DIAGNOSTIC WORKER")
|
| 1032 |
+
print("=" * 70)
|
| 1033 |
+
agent = DiagnosticAgent()
|
| 1034 |
+
alerts = agent.run_diagnostics()
|
| 1035 |
+
print("\n" + "=" * 70)
|
| 1036 |
+
print("SUMMARY REPORT")
|
| 1037 |
+
print("=" * 70)
|
| 1038 |
+
summary = agent.generate_summary_report()
|
| 1039 |
+
print(json.dumps(summary, indent=2, default=str))
|
| 1040 |
+
|
| 1041 |
+
print("\n" + "=" * 70)
|
| 1042 |
+
print("CELL HEALTH SCORES (top 10)")
|
| 1043 |
+
print("=" * 70)
|
| 1044 |
+
df = agent.load_data()
|
| 1045 |
+
if df is not None:
|
| 1046 |
+
health = agent.compute_cell_health(df)
|
| 1047 |
+
sorted_cells = sorted(health.items(), key=lambda x: x[1]["health_score"])
|
| 1048 |
+
for cell_id, info in sorted_cells[:10]:
|
| 1049 |
+
print(f" {cell_id}: {info['health_score']}/100 ({info['status']})")
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
if __name__ == "__main__":
|
| 1053 |
+
test_agent()
|
agents/alert_agent.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Alert Agent: merge ML detection, predictions, and rule alerts; attach recommendations.
|
| 2 |
+
|
| 3 |
+
Refinement v2: Time-window deduplication — suppress repeat alerts for the same
|
| 4 |
+
(cell_id, issue_type) within a configurable window (default 30 min).
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from datetime import datetime, timedelta
|
| 10 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 11 |
+
|
| 12 |
+
import pandas as pd
|
| 13 |
+
from sqlalchemy.orm import Session
|
| 14 |
+
|
| 15 |
+
from services.recommendation_service import recommendations_for_issue, recommendations_from_kpis
|
| 16 |
+
|
| 17 |
+
# Default dedup window in minutes
|
| 18 |
+
DEDUP_WINDOW_MINUTES = 30
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _key(a: Dict[str, Any]) -> Tuple:
|
| 22 |
+
return (
|
| 23 |
+
str(a.get("cell_id", "")),
|
| 24 |
+
str(a.get("issue_type", a.get("source", ""))),
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _parse_timestamp(ts) -> Optional[datetime]:
|
| 29 |
+
if ts is None:
|
| 30 |
+
return None
|
| 31 |
+
if isinstance(ts, datetime):
|
| 32 |
+
return ts
|
| 33 |
+
try:
|
| 34 |
+
return pd.to_datetime(ts)
|
| 35 |
+
except Exception:
|
| 36 |
+
return None
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class AlertAgent:
|
| 40 |
+
|
| 41 |
+
def __init__(self):
|
| 42 |
+
# Track last alert time per (cell_id, issue_type) for dedup
|
| 43 |
+
self._last_alert_time: Dict[Tuple[str, str], datetime] = {}
|
| 44 |
+
|
| 45 |
+
def _is_duplicate(self, alert: Dict[str, Any], window_minutes: int = DEDUP_WINDOW_MINUTES) -> bool:
|
| 46 |
+
"""Check if an identical alert was already emitted within the time window."""
|
| 47 |
+
key = _key(alert)
|
| 48 |
+
ts = _parse_timestamp(alert.get("timestamp"))
|
| 49 |
+
now = ts or datetime.utcnow()
|
| 50 |
+
|
| 51 |
+
last = self._last_alert_time.get(key)
|
| 52 |
+
if last and (now - last) < timedelta(minutes=window_minutes):
|
| 53 |
+
return True
|
| 54 |
+
|
| 55 |
+
self._last_alert_time[key] = now
|
| 56 |
+
return False
|
| 57 |
+
|
| 58 |
+
def _cleanup_old_entries(self, max_age_hours: int = 24):
|
| 59 |
+
"""Remove stale dedup entries older than max_age_hours."""
|
| 60 |
+
cutoff = datetime.utcnow() - timedelta(hours=max_age_hours)
|
| 61 |
+
stale = [k for k, v in self._last_alert_time.items() if v < cutoff]
|
| 62 |
+
for k in stale:
|
| 63 |
+
del self._last_alert_time[k]
|
| 64 |
+
|
| 65 |
+
def merge_alerts(
|
| 66 |
+
self,
|
| 67 |
+
rule_alerts: List[Dict[str, Any]],
|
| 68 |
+
if_records: List[Dict[str, Any]],
|
| 69 |
+
prediction_high_risk: Optional[List[Dict[str, Any]]] = None,
|
| 70 |
+
dedup_window_minutes: int = DEDUP_WINDOW_MINUTES,
|
| 71 |
+
) -> List[Dict[str, Any]]:
|
| 72 |
+
merged: List[Dict[str, Any]] = []
|
| 73 |
+
seen_exact = set() # exact dedup within this batch
|
| 74 |
+
|
| 75 |
+
self._cleanup_old_entries()
|
| 76 |
+
|
| 77 |
+
# ── Rule-based alerts ──
|
| 78 |
+
for a in rule_alerts:
|
| 79 |
+
enriched = dict(a)
|
| 80 |
+
it = enriched.get("issue_type")
|
| 81 |
+
recs = enriched.get("recommendations") or []
|
| 82 |
+
if not recs and it:
|
| 83 |
+
recs = recommendations_for_issue(it)
|
| 84 |
+
if recs:
|
| 85 |
+
enriched["recommendations"] = recs
|
| 86 |
+
|
| 87 |
+
# Exact dedup within batch
|
| 88 |
+
exact_key = (
|
| 89 |
+
str(enriched.get("cell_id", "")),
|
| 90 |
+
str(enriched.get("issue_type", "")),
|
| 91 |
+
str(enriched.get("timestamp", "")),
|
| 92 |
+
)
|
| 93 |
+
if exact_key in seen_exact:
|
| 94 |
+
continue
|
| 95 |
+
seen_exact.add(exact_key)
|
| 96 |
+
|
| 97 |
+
# Time-window dedup across calls
|
| 98 |
+
if self._is_duplicate(enriched, dedup_window_minutes):
|
| 99 |
+
continue
|
| 100 |
+
|
| 101 |
+
merged.append(enriched)
|
| 102 |
+
|
| 103 |
+
# ── Isolation Forest ML alerts ──
|
| 104 |
+
for r in if_records:
|
| 105 |
+
kpis = r.get("kpis") or {}
|
| 106 |
+
lat = kpis.get("latency_ms")
|
| 107 |
+
prb = kpis.get("PRB_DL") or kpis.get("prb_dl")
|
| 108 |
+
prbu = kpis.get("PRB_UL") or kpis.get("prb_ul")
|
| 109 |
+
extra_recs = recommendations_from_kpis(
|
| 110 |
+
prb_dl=prb, prb_ul=prbu, latency_ms=lat,
|
| 111 |
+
packet_loss_rate=kpis.get("packet_loss_rate"),
|
| 112 |
+
)
|
| 113 |
+
alert = {
|
| 114 |
+
"issue_type": "ML Anomaly (Isolation Forest)",
|
| 115 |
+
"severity": "HIGH" if r.get("if_score", 0) < -0.05 else "MEDIUM",
|
| 116 |
+
"cell_id": r.get("cell_id"),
|
| 117 |
+
"timestamp": r.get("timestamp"),
|
| 118 |
+
"kpis": {**kpis, "if_score": r.get("if_score", kpis.get("if_score"))},
|
| 119 |
+
"causes": [
|
| 120 |
+
"Score anomalie Isolation Forest sous le seuil",
|
| 121 |
+
"Combinaison KPI inhabituelle vs distribution recente",
|
| 122 |
+
],
|
| 123 |
+
"recommendations": extra_recs or [
|
| 124 |
+
"Analyser correlation PRB / debit / latence sur la cellule",
|
| 125 |
+
"Verifier evenements reseau concurrents (maintenance, transport)",
|
| 126 |
+
],
|
| 127 |
+
"priority": 2,
|
| 128 |
+
"source": "detection_ml",
|
| 129 |
+
}
|
| 130 |
+
if "source_bo" in r:
|
| 131 |
+
alert["source_bo"] = r["source_bo"]
|
| 132 |
+
|
| 133 |
+
if self._is_duplicate(alert, dedup_window_minutes):
|
| 134 |
+
continue
|
| 135 |
+
merged.append(alert)
|
| 136 |
+
|
| 137 |
+
# ── Prediction risk alerts ──
|
| 138 |
+
if prediction_high_risk:
|
| 139 |
+
for p in prediction_high_risk:
|
| 140 |
+
if (p.get("error_pct") or 0) < 25:
|
| 141 |
+
continue
|
| 142 |
+
alert = {
|
| 143 |
+
"issue_type": "Prediction Risk (latency)",
|
| 144 |
+
"severity": "MEDIUM",
|
| 145 |
+
"cell_id": p.get("cell_id"),
|
| 146 |
+
"timestamp": p.get("timestamp"),
|
| 147 |
+
"kpis": {
|
| 148 |
+
"predicted_latency_ms": p.get("predicted"),
|
| 149 |
+
"actual_latency_ms": p.get("actual"),
|
| 150 |
+
"error_pct": p.get("error_pct"),
|
| 151 |
+
},
|
| 152 |
+
"causes": ["Ecart important entre latence predite et observee"],
|
| 153 |
+
"recommendations": recommendations_from_kpis(
|
| 154 |
+
latency_ms=p.get("actual") or p.get("predicted")
|
| 155 |
+
),
|
| 156 |
+
"priority": 3,
|
| 157 |
+
"source": "prediction_ml",
|
| 158 |
+
}
|
| 159 |
+
if self._is_duplicate(alert, dedup_window_minutes):
|
| 160 |
+
continue
|
| 161 |
+
merged.append(alert)
|
| 162 |
+
|
| 163 |
+
merged.sort(key=lambda x: x.get("priority", 9))
|
| 164 |
+
return merged
|
| 165 |
+
|
| 166 |
+
def persist_alerts(self, db: Session, alerts: List[Dict[str, Any]], limit: int = 500) -> int:
|
| 167 |
+
from database import Alert as AlertORM
|
| 168 |
+
|
| 169 |
+
n = 0
|
| 170 |
+
for a in alerts[:limit]:
|
| 171 |
+
ts = a.get("timestamp")
|
| 172 |
+
if isinstance(ts, str):
|
| 173 |
+
try:
|
| 174 |
+
ts_parsed = pd.to_datetime(ts)
|
| 175 |
+
except Exception:
|
| 176 |
+
ts_parsed = datetime.utcnow()
|
| 177 |
+
else:
|
| 178 |
+
ts_parsed = datetime.utcnow()
|
| 179 |
+
row = AlertORM(
|
| 180 |
+
cell_id=str(a.get("cell_id", "")),
|
| 181 |
+
timestamp=ts_parsed,
|
| 182 |
+
issue_type=str(a.get("issue_type", "")),
|
| 183 |
+
severity=str(a.get("severity", "")),
|
| 184 |
+
kpis=a.get("kpis"),
|
| 185 |
+
causes=a.get("causes"),
|
| 186 |
+
recommendations=a.get("recommendations"),
|
| 187 |
+
priority=int(a.get("priority", 0)),
|
| 188 |
+
)
|
| 189 |
+
db.add(row)
|
| 190 |
+
n += 1
|
| 191 |
+
db.commit()
|
| 192 |
+
return n
|
agents/detection_agent.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Detection Agent: Isolation Forest + simple statistical flags on QoS features."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import pandas as pd
|
| 9 |
+
from sklearn.ensemble import IsolationForest
|
| 10 |
+
|
| 11 |
+
IF_FEATURES = [
|
| 12 |
+
"RSRP",
|
| 13 |
+
"SINR",
|
| 14 |
+
"PRB_UL",
|
| 15 |
+
"PRB_DL",
|
| 16 |
+
"throughput_DL",
|
| 17 |
+
"throughput_UL",
|
| 18 |
+
"latency_ms",
|
| 19 |
+
"MOS",
|
| 20 |
+
"call_drop_rate",
|
| 21 |
+
"handover_success_rate",
|
| 22 |
+
"BLER_UL",
|
| 23 |
+
"BLER_DL",
|
| 24 |
+
"CQI",
|
| 25 |
+
"RLC_retransmissions",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class DetectionAgent:
|
| 30 |
+
def __init__(self, contamination: float = 0.06, random_state: int = 42):
|
| 31 |
+
self.contamination = contamination
|
| 32 |
+
self.random_state = random_state
|
| 33 |
+
|
| 34 |
+
def _build_feature_matrix(self, df: pd.DataFrame) -> Tuple[pd.DataFrame, List[str]]:
|
| 35 |
+
# Deduplicate columns
|
| 36 |
+
df = df.loc[:, ~df.columns.duplicated()]
|
| 37 |
+
# Only keep IF features that exist in df
|
| 38 |
+
cols = [c for c in IF_FEATURES if c in df.columns]
|
| 39 |
+
if not cols:
|
| 40 |
+
return pd.DataFrame(), []
|
| 41 |
+
# Convert to numeric, fill NaN with column median (or 0 if all NaN)
|
| 42 |
+
X_list = []
|
| 43 |
+
valid_cols = []
|
| 44 |
+
for c in cols:
|
| 45 |
+
s = pd.to_numeric(df[c], errors="coerce")
|
| 46 |
+
median = s.median()
|
| 47 |
+
if np.isnan(median):
|
| 48 |
+
median = 0.0
|
| 49 |
+
X_list.append(s.fillna(median).values)
|
| 50 |
+
valid_cols.append(c)
|
| 51 |
+
if not valid_cols:
|
| 52 |
+
return pd.DataFrame(), []
|
| 53 |
+
X_vals = np.column_stack(X_list) # shape: (n_rows, len(valid_cols))
|
| 54 |
+
return pd.DataFrame(X_vals, columns=valid_cols), valid_cols
|
| 55 |
+
|
| 56 |
+
def detect(
|
| 57 |
+
self, df: pd.DataFrame, max_anomaly_records: int = 200
|
| 58 |
+
) -> Tuple[Optional[IsolationForest], Dict[str, Any], List[Dict[str, Any]]]:
|
| 59 |
+
"""
|
| 60 |
+
Fit IsolationForest on df rows; return model meta + top anomaly records with score.
|
| 61 |
+
"""
|
| 62 |
+
if df is None or df.empty:
|
| 63 |
+
return None, {"error": "empty_frame"}, []
|
| 64 |
+
|
| 65 |
+
X, cols = self._build_feature_matrix(df)
|
| 66 |
+
if X.empty or not cols:
|
| 67 |
+
return None, {"error": "no_feature_columns"}, []
|
| 68 |
+
|
| 69 |
+
import warnings
|
| 70 |
+
with warnings.catch_warnings():
|
| 71 |
+
warnings.filterwarnings("ignore", category=UserWarning)
|
| 72 |
+
clf = IsolationForest(
|
| 73 |
+
contamination=min(self.contamination, 0.5),
|
| 74 |
+
random_state=self.random_state,
|
| 75 |
+
n_estimators=120,
|
| 76 |
+
)
|
| 77 |
+
clf.fit(X.values)
|
| 78 |
+
scores = clf.decision_function(X)
|
| 79 |
+
pred = clf.predict(X)
|
| 80 |
+
|
| 81 |
+
# Work on a reset-index copy to align with X
|
| 82 |
+
df_out = df.reset_index(drop=True).copy()
|
| 83 |
+
df_out["_if_score"] = scores
|
| 84 |
+
df_out["_if_outlier"] = pred == -1
|
| 85 |
+
|
| 86 |
+
outliers = df_out[df_out["_if_outlier"]].copy()
|
| 87 |
+
outliers = outliers.sort_values("_if_score")
|
| 88 |
+
|
| 89 |
+
meta = {
|
| 90 |
+
"n_samples": int(len(df_out)),
|
| 91 |
+
"n_outliers": int(df_out["_if_outlier"].sum()),
|
| 92 |
+
"outlier_ratio": float(df_out["_if_outlier"].mean()),
|
| 93 |
+
"feature_columns": cols,
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
records: List[Dict[str, Any]] = []
|
| 97 |
+
tail = outliers.head(max_anomaly_records)
|
| 98 |
+
for _, row in tail.iterrows():
|
| 99 |
+
rec = {
|
| 100 |
+
"cell_id": row.get("cell_id"),
|
| 101 |
+
"timestamp": str(row["timestamp"]) if "timestamp" in row and pd.notna(row["timestamp"]) else None,
|
| 102 |
+
"if_score": float(row["_if_score"]),
|
| 103 |
+
"kpis": {
|
| 104 |
+
k: float(row[k])
|
| 105 |
+
for k in cols
|
| 106 |
+
if k in row and pd.notna(row[k])
|
| 107 |
+
},
|
| 108 |
+
}
|
| 109 |
+
if "source_bo" in row and pd.notna(row["source_bo"]):
|
| 110 |
+
rec["source_bo"] = str(row["source_bo"])
|
| 111 |
+
records.append(rec)
|
| 112 |
+
|
| 113 |
+
return clf, meta, records
|
| 114 |
+
|
| 115 |
+
@staticmethod
|
| 116 |
+
def zscore_flags(df: pd.DataFrame, col: str, z: float = 3.5) -> pd.Series:
|
| 117 |
+
if col not in df.columns:
|
| 118 |
+
return pd.Series(False, index=df.index)
|
| 119 |
+
s = pd.to_numeric(df[col], errors="coerce")
|
| 120 |
+
m = s.mean()
|
| 121 |
+
std = s.std()
|
| 122 |
+
if std is None or std == 0 or np.isnan(std):
|
| 123 |
+
return pd.Series(False, index=df.index)
|
| 124 |
+
return (s - m).abs() > z * std
|
agents/prediction_agent.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Prediction Agent: Champion Model Loader for BO2/BO3.
|
| 3 |
+
Supports offline inference with pre-trained Joblib models and unified scaler.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import joblib
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any, Dict, List, Optional
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
from sklearn.impute import SimpleImputer
|
| 17 |
+
from sklearn.metrics import mean_absolute_error, r2_score, mean_squared_error
|
| 18 |
+
|
| 19 |
+
from core.config import ML_RUNTIME_META_JSON, MODEL_VERSIONS_DIR
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class PredictionAgent:
|
| 23 |
+
def __init__(self):
|
| 24 |
+
self._model: Optional[Any] = None
|
| 25 |
+
self._scaler: Optional[Any] = None
|
| 26 |
+
self._imputer: Optional[SimpleImputer] = SimpleImputer(strategy="median")
|
| 27 |
+
self._feature_cols: List[str] = []
|
| 28 |
+
self._target: Optional[str] = None
|
| 29 |
+
self._last_metrics: Dict[str, Optional[float]] = {}
|
| 30 |
+
self._last_fit_at: Optional[str] = None
|
| 31 |
+
self._version: int = 73 # Unified V7 compatible
|
| 32 |
+
self._training_stats: Optional[Dict[str, Any]] = None
|
| 33 |
+
|
| 34 |
+
# Load scaler immediately if possible
|
| 35 |
+
self._load_predefined_scaler()
|
| 36 |
+
|
| 37 |
+
def _load_predefined_scaler(self):
|
| 38 |
+
model_dir = Path(__file__).resolve().parent.parent.parent / 'models'
|
| 39 |
+
scaler_path = model_dir / 'unified_scaler.joblib'
|
| 40 |
+
if scaler_path.exists():
|
| 41 |
+
try:
|
| 42 |
+
self._scaler = joblib.load(scaler_path)
|
| 43 |
+
if hasattr(self._scaler, 'feature_names_in_'):
|
| 44 |
+
self._feature_cols = list(self._scaler.feature_names_in_)
|
| 45 |
+
print(f" [PredictionAgent] Scaler loaded: {len(self._feature_cols)} features")
|
| 46 |
+
except Exception as e:
|
| 47 |
+
print(f" [PredictionAgent] Scaler load failed: {e}")
|
| 48 |
+
|
| 49 |
+
# Also load runtime metadata if available
|
| 50 |
+
meta_path = model_dir / 'ml_runtime_metadata.json'
|
| 51 |
+
if meta_path.exists():
|
| 52 |
+
try:
|
| 53 |
+
import json
|
| 54 |
+
with open(meta_path) as f:
|
| 55 |
+
meta = json.load(f)
|
| 56 |
+
self._last_metrics = meta.get('metrics', {})
|
| 57 |
+
self._training_stats = meta.get('training_stats', {})
|
| 58 |
+
self._target = meta.get('target', 'sla_breach_risk')
|
| 59 |
+
self._last_fit_at = meta.get('fit_at')
|
| 60 |
+
# Mark as trained if metadata says so
|
| 61 |
+
if meta.get('trained') and not self._model:
|
| 62 |
+
# Try to load the champion model
|
| 63 |
+
champion_path = model_dir / 'bo2_sla_risk.joblib'
|
| 64 |
+
if champion_path.exists():
|
| 65 |
+
self._model = joblib.load(champion_path)
|
| 66 |
+
print(f" [PredictionAgent] Champion model loaded from metadata")
|
| 67 |
+
print(f" [PredictionAgent] Runtime metadata loaded: R²={self._last_metrics.get('r2', '?')}")
|
| 68 |
+
except Exception as e:
|
| 69 |
+
print(f" [PredictionAgent] Metadata load failed: {e}")
|
| 70 |
+
|
| 71 |
+
def fit(
|
| 72 |
+
self,
|
| 73 |
+
df: pd.DataFrame,
|
| 74 |
+
target: str = "sla_breach_risk",
|
| 75 |
+
max_train_rows: int = 20000,
|
| 76 |
+
cv_folds: int = 5,
|
| 77 |
+
) -> Dict[str, Optional[float]]:
|
| 78 |
+
"""
|
| 79 |
+
In V7 mode, fit() preferentially loads the champion model rather than training.
|
| 80 |
+
"""
|
| 81 |
+
model_dir = Path(__file__).resolve().parent.parent.parent / 'models'
|
| 82 |
+
champion_map = {
|
| 83 |
+
'sla_breach_risk': 'bo2_sla_risk.joblib',
|
| 84 |
+
'latency_ms': 'bo2_sla_risk.joblib',
|
| 85 |
+
'capex_score': 'bo3_capex_score.joblib'
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
target_file = champion_map.get(target)
|
| 89 |
+
if target_file and (model_dir / target_file).exists():
|
| 90 |
+
print(f" [PredictionAgent] Loading champion model for {target}")
|
| 91 |
+
try:
|
| 92 |
+
self._model = joblib.load(model_dir / target_file)
|
| 93 |
+
self._target = target
|
| 94 |
+
self._last_fit_at = datetime.utcnow().isoformat() + "Z"
|
| 95 |
+
self._last_metrics = {"status": "champion_loaded"}
|
| 96 |
+
return self._last_metrics
|
| 97 |
+
except Exception as e:
|
| 98 |
+
print(f" Failed to load champion model: {e}")
|
| 99 |
+
|
| 100 |
+
# Fallback (Very limited training if no champion found)
|
| 101 |
+
print(f" No champion found for {target}. Backend training skipped for safety.")
|
| 102 |
+
return {"error": "no_champion_model"}
|
| 103 |
+
|
| 104 |
+
def predict_latest(
|
| 105 |
+
self, df: pd.DataFrame, n: int = 50
|
| 106 |
+
) -> List[Dict[str, Any]]:
|
| 107 |
+
if self._model is None:
|
| 108 |
+
# Try auto-load default
|
| 109 |
+
print(" Attempting auto-load for Latency/SLA model...")
|
| 110 |
+
self.fit(df, target="sla_breach_risk")
|
| 111 |
+
|
| 112 |
+
if self._model is None or not self._feature_cols:
|
| 113 |
+
raise RuntimeError("Model or feature set not loaded. Ensure unified_scaler.joblib and bo2_*.joblib exist.")
|
| 114 |
+
|
| 115 |
+
tail = df.sort_values("timestamp").tail(n) if "timestamp" in df.columns else df.tail(n)
|
| 116 |
+
tail = tail.reset_index(drop=True)
|
| 117 |
+
|
| 118 |
+
# Prepare feature matrix
|
| 119 |
+
X_raw = tail.reindex(columns=self._feature_cols).fillna(0)
|
| 120 |
+
|
| 121 |
+
# Scale if scaler available
|
| 122 |
+
if self._scaler:
|
| 123 |
+
import warnings
|
| 124 |
+
with warnings.catch_warnings():
|
| 125 |
+
warnings.filterwarnings("ignore", category=UserWarning)
|
| 126 |
+
X_in = self._scaler.transform(X_raw.values)
|
| 127 |
+
else:
|
| 128 |
+
X_in = X_raw.values
|
| 129 |
+
|
| 130 |
+
import warnings
|
| 131 |
+
with warnings.catch_warnings():
|
| 132 |
+
warnings.filterwarnings("ignore", category=UserWarning)
|
| 133 |
+
y_hat = self._model.predict(X_in)
|
| 134 |
+
|
| 135 |
+
# Actual values for error tracking
|
| 136 |
+
y_true = pd.to_numeric(tail.get(self._target, pd.NA), errors="coerce").values if self._target in tail.columns else None
|
| 137 |
+
|
| 138 |
+
out: List[Dict[str, Any]] = []
|
| 139 |
+
for i in range(len(tail)):
|
| 140 |
+
row = tail.iloc[i]
|
| 141 |
+
pred = float(y_hat[i])
|
| 142 |
+
actual = float(y_true[i]) if y_true is not None and i < len(y_true) and pd.notna(y_true[i]) else None
|
| 143 |
+
|
| 144 |
+
rec: Dict[str, Any] = {
|
| 145 |
+
"cell_id": row.get("cell_id"),
|
| 146 |
+
"timestamp": str(row["timestamp"]) if "timestamp" in row else None,
|
| 147 |
+
"predicted": pred,
|
| 148 |
+
"actual": actual,
|
| 149 |
+
"error_pct": round(abs(pred-actual)/max(actual, 1e-6)*100, 2) if actual else None
|
| 150 |
+
}
|
| 151 |
+
out.append(rec)
|
| 152 |
+
return out
|
| 153 |
+
|
| 154 |
+
def status(self) -> Dict[str, Any]:
|
| 155 |
+
return {
|
| 156 |
+
"trained": self._model is not None,
|
| 157 |
+
"target": self._target,
|
| 158 |
+
"features": self._feature_cols,
|
| 159 |
+
"metrics": self._last_metrics,
|
| 160 |
+
"training_stats": self._training_stats,
|
| 161 |
+
"fit_at": self._last_fit_at,
|
| 162 |
+
"version": self._version,
|
| 163 |
+
"is_v7_champion": True
|
| 164 |
+
}
|
core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Core configuration, schemas, and shared data transforms for QoS Buddy."""
|
core/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (244 Bytes). View file
|
|
|
core/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (329 Bytes). View file
|
|
|
core/__pycache__/config.cpython-312.pyc
ADDED
|
Binary file (4.01 kB). View file
|
|
|
core/__pycache__/config.cpython-314.pyc
ADDED
|
Binary file (4.22 kB). View file
|
|
|
core/__pycache__/filled_transform.cpython-312.pyc
ADDED
|
Binary file (9.67 kB). View file
|
|
|
core/__pycache__/filled_transform.cpython-314.pyc
ADDED
|
Binary file (11.2 kB). View file
|
|
|
core/__pycache__/schemas.cpython-312.pyc
ADDED
|
Binary file (13.8 kB). View file
|
|
|
core/__pycache__/schemas.cpython-314.pyc
ADDED
|
Binary file (15.4 kB). View file
|
|
|
core/config.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Paths and tunables for QoS Buddy backend."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from dotenv import load_dotenv
|
| 8 |
+
|
| 9 |
+
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
| 10 |
+
REPO_ROOT = BACKEND_DIR.parent
|
| 11 |
+
|
| 12 |
+
# Load backend .env automatically so local dev works without exporting vars manually.
|
| 13 |
+
load_dotenv(Path(__file__).resolve().parent.parent / ".env")
|
| 14 |
+
|
| 15 |
+
UNIFIED_FILLED_CSV = REPO_ROOT / "data" / "unified_telecom_dataset_FILLED.csv"
|
| 16 |
+
PROCESSED_FALLBACK_CSV = REPO_ROOT / "data" / "processed" / "network_kpis_full.csv"
|
| 17 |
+
|
| 18 |
+
# Optional per-BO slices: backend/data/bo/BO1.csv, BO2.json, etc. — same schema as FILLED when possible.
|
| 19 |
+
BO_DATA_DIR = BACKEND_DIR / "data" / "bo"
|
| 20 |
+
|
| 21 |
+
ENRICHMENT_DIR = BACKEND_DIR / "data" / "enrichment"
|
| 22 |
+
SCRAPED_CATALOG_JSON = ENRICHMENT_DIR / "scraped_catalog.json"
|
| 23 |
+
SCRAPER_SEEDS_JSON = ENRICHMENT_DIR / "scraper_seeds.json"
|
| 24 |
+
UCI_SEEDS_JSON = ENRICHMENT_DIR / "uci_telecom_seeds.json"
|
| 25 |
+
|
| 26 |
+
MODELS_DIR = REPO_ROOT / "models"
|
| 27 |
+
MODEL_VERSIONS_DIR = REPO_ROOT / "outputs" / "versions"
|
| 28 |
+
ML_RUNTIME_META_JSON = MODELS_DIR / "ml_runtime_metadata.json"
|
| 29 |
+
|
| 30 |
+
# Scraping
|
| 31 |
+
SCRAPER_REQUEST_TIMEOUT_SEC = 25
|
| 32 |
+
SCRAPER_MIN_DELAY_SEC = float(os.getenv("SCRAPER_MIN_DELAY_SEC", "0.6"))
|
| 33 |
+
TELECOM_RELEVANCE_THRESHOLD = float(os.getenv("TELECOM_RELEVANCE_THRESHOLD", "0.35"))
|
| 34 |
+
|
| 35 |
+
# data.gov CKAN
|
| 36 |
+
DATAGOV_CKAN_API = os.getenv(
|
| 37 |
+
"DATAGOV_CKAN_API",
|
| 38 |
+
"https://catalog.data.gov/api/3/action/package_search",
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
# EU data discovery (REST search)
|
| 42 |
+
EU_DATA_API = os.getenv(
|
| 43 |
+
"EU_DATA_API",
|
| 44 |
+
"https://data.europa.eu/api/hub/search/datasets",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
# Alerts persist to SQLite by default
|
| 48 |
+
ENABLE_DB_ALERTS = os.getenv("ENABLE_DB_ALERTS", "true").lower() == "true"
|
| 49 |
+
|
| 50 |
+
# CORS
|
| 51 |
+
CORS_ORIGINS = [
|
| 52 |
+
o.strip()
|
| 53 |
+
for o in os.getenv("CORS_ORIGINS", "http://localhost:5173,http://localhost:5174,http://localhost:3000").split(",")
|
| 54 |
+
if o.strip()
|
| 55 |
+
]
|
| 56 |
+
|
| 57 |
+
# JWT Authentication
|
| 58 |
+
JWT_SECRET_KEY = os.getenv("JWT_SECRET_KEY", "change-me-to-a-random-secret-string")
|
| 59 |
+
JWT_ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256")
|
| 60 |
+
JWT_ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("JWT_ACCESS_TOKEN_EXPIRE_MINUTES", "60"))
|
| 61 |
+
|
| 62 |
+
# OTP / Email
|
| 63 |
+
BREVO_API_KEY = os.getenv("BREVO_API_KEY", "")
|
| 64 |
+
SENDER_EMAIL = os.getenv("SENDER_EMAIL", "noreply@qosbuddy.com")
|
| 65 |
+
SENDER_NAME = os.getenv("SENDER_NAME", "QoS Buddy")
|
| 66 |
+
OTP_EXPIRE_MINUTES = int(os.getenv("OTP_EXPIRE_MINUTES", "10"))
|
| 67 |
+
OTP_MAX_ATTEMPTS = int(os.getenv("OTP_MAX_ATTEMPTS", "5"))
|
| 68 |
+
OTP_EMAIL_REQUIRED = os.getenv("OTP_EMAIL_REQUIRED", "true").lower() == "true"
|
| 69 |
+
|
| 70 |
+
# ──────────────────────────────────────────────
|
| 71 |
+
# KPI Thresholds (used by DiagnosticAgent)
|
| 72 |
+
# Externalized here so they can be tuned without
|
| 73 |
+
# editing detection logic.
|
| 74 |
+
# ──────────────────────────────────────────────
|
| 75 |
+
KPI_THRESHOLDS = {
|
| 76 |
+
"RSRP": {"critical": -110, "poor": -100, "fair": -90, "good": -80},
|
| 77 |
+
"SINR": {"critical": 5, "poor": 10, "fair": 15, "good": 20},
|
| 78 |
+
"PRB": {"normal": 70, "moderate": 85, "high": 95},
|
| 79 |
+
"MOS": {"bad": 2.5, "poor": 3.0, "fair": 3.5, "good": 4.0},
|
| 80 |
+
"throughput_DL": {"critical": 10, "poor": 25, "fair": 50},
|
| 81 |
+
"latency_ms": {"good": 40, "fair": 80, "poor": 150},
|
| 82 |
+
"call_drop_rate": {"normal": 1, "moderate": 3, "high": 5},
|
| 83 |
+
"handover_success_rate": {"critical": 85, "poor": 90, "fair": 95, "good": 98},
|
| 84 |
+
"BLER": {"normal": 2, "moderate": 5, "high": 10},
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
# Busy-hour range used by capacity issue detector
|
| 88 |
+
BUSY_HOURS = list(range(17, 22)) # 17h–21h
|
core/filled_transform.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Single source of truth for unified FILLED CSV -> KPI column mapping.
|
| 3 |
+
Used by DiagnosticAgent, TelecomDataPipeline, and DataManagerAgent.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pandas as pd
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def apply_filled_column_mapping(df: pd.DataFrame) -> pd.DataFrame:
|
| 13 |
+
"""
|
| 14 |
+
Map unified_telecom_dataset_FILLED.csv columns to the internal KPI schema.
|
| 15 |
+
Preserves BO and rawQoS helper columns; does not drop columns (wide frame).
|
| 16 |
+
"""
|
| 17 |
+
df = df.copy()
|
| 18 |
+
if "timestamp" in df.columns:
|
| 19 |
+
df["timestamp"] = pd.to_datetime(df["timestamp"], errors="coerce")
|
| 20 |
+
df["hour"] = df["timestamp"].dt.hour
|
| 21 |
+
df["day_of_week"] = df["timestamp"].dt.dayofweek
|
| 22 |
+
df["is_peak"] = df["hour"].apply(lambda h: 1 if 8 <= h <= 22 else 0)
|
| 23 |
+
|
| 24 |
+
if "cell_id" not in df.columns and "entity_id" in df.columns:
|
| 25 |
+
df["cell_id"] = df["entity_id"]
|
| 26 |
+
|
| 27 |
+
# Mapping for Internal Dashboard compatibility (Agent 2 style)
|
| 28 |
+
df["RSRP"] = pd.to_numeric(
|
| 29 |
+
df.get("RSRP_dBm", df.get("signal_strength_dbm")), errors="coerce"
|
| 30 |
+
)
|
| 31 |
+
df["SINR"] = pd.to_numeric(df.get("SINR_dB", pd.NA), errors="coerce")
|
| 32 |
+
df["PRB_DL"] = pd.to_numeric(df.get("PRBUsageDL", pd.NA), errors="coerce")
|
| 33 |
+
df["PRB_UL"] = pd.to_numeric(df.get("PRBUsageUL", pd.NA), errors="coerce")
|
| 34 |
+
df["throughput_DL"] = pd.to_numeric(df.get("meanThr_DL", pd.NA), errors="coerce")
|
| 35 |
+
df["throughput_UL"] = pd.to_numeric(df.get("meanThr_UL", pd.NA), errors="coerce")
|
| 36 |
+
df["latency_ms"] = pd.to_numeric(df.get("latency_ms", pd.NA), errors="coerce")
|
| 37 |
+
|
| 38 |
+
sla_breach_risk = pd.to_numeric(df.get("sla_breach_risk", pd.NA), errors="coerce")
|
| 39 |
+
df["MOS"] = (5 - 4 * sla_breach_risk).clip(1.0, 5.0)
|
| 40 |
+
|
| 41 |
+
error_rate = pd.to_numeric(df.get("error_rate", pd.NA), errors="coerce")
|
| 42 |
+
df["BLER_DL"] = (error_rate * 5.0).clip(0.5, 20.0)
|
| 43 |
+
df["BLER_UL"] = (error_rate * 4.5).clip(0.5, 20.0)
|
| 44 |
+
|
| 45 |
+
df["CQI"] = pd.to_numeric(df.get("CQI", pd.NA), errors="coerce")
|
| 46 |
+
df["RLC_retransmissions"] = pd.to_numeric(df.get("RLC_RetxRate_pct", pd.NA), errors="coerce")
|
| 47 |
+
|
| 48 |
+
df["call_drop_rate"] = pd.to_numeric(df.get("ERAB_DropRate_pct", pd.NA), errors="coerce")
|
| 49 |
+
df["handover_success_rate"] = pd.to_numeric(df.get("HO_SuccessRate_pct", pd.NA), errors="coerce")
|
| 50 |
+
|
| 51 |
+
df["jitter_ms"] = pd.to_numeric(df.get("jitter_ms", pd.NA), errors="coerce")
|
| 52 |
+
df["packet_loss_rate"] = pd.to_numeric(df.get("packet_loss_rate", pd.NA), errors="coerce")
|
| 53 |
+
|
| 54 |
+
df["source_bo"] = df.get("source_bo", pd.NA)
|
| 55 |
+
df["city"] = df.get("city", pd.NA)
|
| 56 |
+
df["anomaly_flag"] = pd.to_numeric(df.get("anomaly_flag", pd.NA), errors="coerce")
|
| 57 |
+
df["sla_breach_risk"] = pd.to_numeric(df.get("sla_breach_risk", pd.NA), errors="coerce")
|
| 58 |
+
df["capex_score"] = pd.to_numeric(df.get("capex_score", pd.NA), errors="coerce")
|
| 59 |
+
df["radio_health_score"] = pd.to_numeric(df.get("radio_health_score", pd.NA), errors="coerce")
|
| 60 |
+
df["unified_risk_score"] = pd.to_numeric(df.get("unified_risk_score", pd.NA), errors="coerce")
|
| 61 |
+
df["capex_priority"] = df.get("capex_priority", pd.NA)
|
| 62 |
+
df["risk_tier"] = df.get("risk_tier", pd.NA)
|
| 63 |
+
|
| 64 |
+
return df
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
DIAGNOSTIC_KEEP_COLS = [
|
| 68 |
+
"timestamp",
|
| 69 |
+
"cell_id",
|
| 70 |
+
# Mapped Names
|
| 71 |
+
"RSRP", "SINR", "PRB_UL", "PRB_DL", "throughput_DL", "throughput_UL",
|
| 72 |
+
"latency_ms", "MOS", "call_drop_rate", "handover_success_rate",
|
| 73 |
+
"BLER_UL", "BLER_DL", "CQI", "RLC_retransmissions",
|
| 74 |
+
"hour", "day_of_week", "is_peak",
|
| 75 |
+
# Original Names for Scaler/V7 Models
|
| 76 |
+
"PRBUsageUL", "MIMO_Rank", "HO_SuccessRate_pct", "ERAB_DropRate_pct",
|
| 77 |
+
"unified_risk_score", "signal_strength_dbm", "throughput_mbps",
|
| 78 |
+
"meanThr_UL", "jitter_ms", "maxThr_DL", "active_users", "meanUE_UL", "meanUE_DL",
|
| 79 |
+
"packet_loss_rate", "error_rate", "RSRP_dBm", "PRBUsageDL", "CSSR_pct",
|
| 80 |
+
"SINR_dB", "radio_health_score", "meanThr_DL",
|
| 81 |
+
# Target / Metadata
|
| 82 |
+
"source_bo", "city", "anomaly_flag", "sla_breach_risk", "capex_score",
|
| 83 |
+
"capex_priority", "risk_tier",
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def finalize_diagnostic_dataframe(df: pd.DataFrame) -> pd.DataFrame:
|
| 88 |
+
# Deduplicate columns first (handles any duplicate column names)
|
| 89 |
+
df = df.loc[:, ~df.columns.duplicated()]
|
| 90 |
+
keep = []
|
| 91 |
+
for c in DIAGNOSTIC_KEEP_COLS:
|
| 92 |
+
if c in df.columns and c not in keep:
|
| 93 |
+
keep.append(c)
|
| 94 |
+
return df[keep].sort_values("timestamp").reset_index(drop=True)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _add_pipeline_issue_labels(df: pd.DataFrame) -> pd.DataFrame:
|
| 98 |
+
df = df.copy()
|
| 99 |
+
risk_tier = df.get("risk_tier", pd.Series(index=df.index, data="LOW"))
|
| 100 |
+
unified_risk_score = pd.to_numeric(df.get("unified_risk_score", pd.NA), errors="coerce").fillna(0)
|
| 101 |
+
anomaly_flag = pd.to_numeric(df.get("anomaly_flag", 0), errors="coerce").fillna(0)
|
| 102 |
+
|
| 103 |
+
is_normal = (anomaly_flag == 0) & (risk_tier == "LOW")
|
| 104 |
+
|
| 105 |
+
is_capacity = (df.get("PRB_DL", 0) >= 85) | (df.get("PRB_UL", 0) >= 85)
|
| 106 |
+
is_high_latency = df.get("latency_ms", 0) >= 110
|
| 107 |
+
is_call_drop = df.get("call_drop_rate", 0) >= 7
|
| 108 |
+
is_handover_failure = df.get("handover_success_rate", 100) <= 80
|
| 109 |
+
is_low_throughput = df.get("throughput_DL", 100) <= 12
|
| 110 |
+
is_low_mos = df.get("MOS", 5) <= 2.5
|
| 111 |
+
pl = df.get("packet_loss_rate", pd.Series(0.0, index=df.index))
|
| 112 |
+
jt = df.get("jitter_ms", pd.Series(0.0, index=df.index))
|
| 113 |
+
is_session_drop = (pl >= 2.5) | (jt >= 20)
|
| 114 |
+
|
| 115 |
+
conditions = [
|
| 116 |
+
(~is_normal) & is_capacity,
|
| 117 |
+
(~is_normal) & is_high_latency,
|
| 118 |
+
(~is_normal) & is_call_drop,
|
| 119 |
+
(~is_normal) & is_handover_failure,
|
| 120 |
+
(~is_normal) & is_low_throughput,
|
| 121 |
+
(~is_normal) & is_low_mos,
|
| 122 |
+
(~is_normal) & is_session_drop,
|
| 123 |
+
]
|
| 124 |
+
choices = [
|
| 125 |
+
"Capacity Problem",
|
| 126 |
+
"High Latency",
|
| 127 |
+
"Call Drop",
|
| 128 |
+
"Handover Failure",
|
| 129 |
+
"Low Throughput",
|
| 130 |
+
"Low MOS",
|
| 131 |
+
"Session Drop",
|
| 132 |
+
]
|
| 133 |
+
df["issue_type"] = np.select(conditions, choices, default="Normal")
|
| 134 |
+
|
| 135 |
+
df["severity"] = "NONE"
|
| 136 |
+
df.loc[risk_tier == "CRITICAL", "severity"] = "CRITICAL"
|
| 137 |
+
df.loc[risk_tier == "MEDIUM", "severity"] = "MEDIUM"
|
| 138 |
+
df.loc[(risk_tier == "MEDIUM") & (unified_risk_score >= 40), "severity"] = "HIGH"
|
| 139 |
+
df.loc[df["issue_type"] == "Normal", "severity"] = "NONE"
|
| 140 |
+
return df
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
PIPELINE_KEEP_COLS = [
|
| 144 |
+
"timestamp",
|
| 145 |
+
"cell_id",
|
| 146 |
+
"RSRP",
|
| 147 |
+
"SINR",
|
| 148 |
+
"PRB_UL",
|
| 149 |
+
"PRB_DL",
|
| 150 |
+
"throughput_DL",
|
| 151 |
+
"throughput_UL",
|
| 152 |
+
"latency_ms",
|
| 153 |
+
"call_drop_rate",
|
| 154 |
+
"handover_success_rate",
|
| 155 |
+
"MOS",
|
| 156 |
+
"BLER_UL",
|
| 157 |
+
"BLER_DL",
|
| 158 |
+
"CQI",
|
| 159 |
+
"RLC_retransmissions",
|
| 160 |
+
"hour",
|
| 161 |
+
"issue_type",
|
| 162 |
+
"severity",
|
| 163 |
+
]
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def finalize_pipeline_dataframe(df: pd.DataFrame) -> pd.DataFrame:
|
| 167 |
+
df = _add_pipeline_issue_labels(df)
|
| 168 |
+
keep = [c for c in PIPELINE_KEEP_COLS if c in df.columns]
|
| 169 |
+
return df[keep]
|
core/schemas.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pydantic DTOs for ML, enrichment, and orchestration APIs."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from typing import Any, Dict, List, Optional
|
| 7 |
+
|
| 8 |
+
from pydantic import BaseModel, Field, field_validator
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# ── Chat ──
|
| 12 |
+
|
| 13 |
+
class ChatRequest(BaseModel):
|
| 14 |
+
message: str = Field(..., min_length=1, max_length=2000)
|
| 15 |
+
conversation_id: Optional[str] = None
|
| 16 |
+
include_live_insights: bool = True
|
| 17 |
+
|
| 18 |
+
@field_validator("message")
|
| 19 |
+
@classmethod
|
| 20 |
+
def strip_message(cls, v: str) -> str:
|
| 21 |
+
return v.strip()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class ChatResponse(BaseModel):
|
| 25 |
+
answer: str
|
| 26 |
+
context: List[str]
|
| 27 |
+
success: bool
|
| 28 |
+
timestamp: str
|
| 29 |
+
intent: Optional[str] = None
|
| 30 |
+
confidence: Optional[float] = None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class AnalyzeRequest(BaseModel):
|
| 34 |
+
time_window_hours: int = Field(default=1, ge=1, le=168)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class BOOverviewRequest(BaseModel):
|
| 38 |
+
time_window_hours: int = Field(default=24, ge=1, le=168)
|
| 39 |
+
max_zones: int = Field(default=120, ge=1, le=500)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# ── Auth ──
|
| 43 |
+
|
| 44 |
+
class UserCreate(BaseModel):
|
| 45 |
+
username: str = Field(..., min_length=3, max_length=50)
|
| 46 |
+
password: str = Field(..., min_length=6, max_length=128)
|
| 47 |
+
role: str = Field(..., description="User role: noc_engineer, network_planner, or executive")
|
| 48 |
+
|
| 49 |
+
@field_validator("role")
|
| 50 |
+
@classmethod
|
| 51 |
+
def validate_role(cls, v: str) -> str:
|
| 52 |
+
valid_roles = ["noc_engineer", "network_planner", "executive"]
|
| 53 |
+
if v.lower() not in valid_roles:
|
| 54 |
+
raise ValueError(f"Role must be one of {valid_roles}")
|
| 55 |
+
return v.lower()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class UserLogin(BaseModel):
|
| 59 |
+
username: str
|
| 60 |
+
password: str
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class TokenResponse(BaseModel):
|
| 64 |
+
access_token: str
|
| 65 |
+
token_type: str = "bearer"
|
| 66 |
+
username: str
|
| 67 |
+
role: str
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class RegisterSendOtpRequest(BaseModel):
|
| 71 |
+
username: str = Field(..., min_length=3, max_length=50)
|
| 72 |
+
email: str = Field(..., min_length=5, max_length=255)
|
| 73 |
+
password: str = Field(..., min_length=6, max_length=128)
|
| 74 |
+
role: str = Field(..., description="User role: noc_engineer, network_planner, or executive")
|
| 75 |
+
|
| 76 |
+
@field_validator("role")
|
| 77 |
+
@classmethod
|
| 78 |
+
def validate_role_for_otp(cls, v: str) -> str:
|
| 79 |
+
valid_roles = ["noc_engineer", "network_planner", "executive"]
|
| 80 |
+
if v.lower() not in valid_roles:
|
| 81 |
+
raise ValueError(f"Role must be one of {valid_roles}")
|
| 82 |
+
return v.lower()
|
| 83 |
+
|
| 84 |
+
@field_validator("email")
|
| 85 |
+
@classmethod
|
| 86 |
+
def normalize_email(cls, v: str) -> str:
|
| 87 |
+
value = v.strip().lower()
|
| 88 |
+
if "@" not in value or "." not in value.split("@")[-1]:
|
| 89 |
+
raise ValueError("Invalid email format")
|
| 90 |
+
return value
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class VerifyOtpRequest(BaseModel):
|
| 94 |
+
email: str = Field(..., min_length=5, max_length=255)
|
| 95 |
+
otp_code: str = Field(..., min_length=6, max_length=6)
|
| 96 |
+
|
| 97 |
+
@field_validator("email")
|
| 98 |
+
@classmethod
|
| 99 |
+
def normalize_verify_email(cls, v: str) -> str:
|
| 100 |
+
return v.strip().lower()
|
| 101 |
+
|
| 102 |
+
@field_validator("otp_code")
|
| 103 |
+
@classmethod
|
| 104 |
+
def validate_otp_code(cls, v: str) -> str:
|
| 105 |
+
if not v.isdigit():
|
| 106 |
+
raise ValueError("OTP code must contain 6 digits")
|
| 107 |
+
return v
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
class OtpActionResponse(BaseModel):
|
| 111 |
+
success: bool
|
| 112 |
+
message: str
|
| 113 |
+
access_token: Optional[str] = None
|
| 114 |
+
token_type: str = "bearer"
|
| 115 |
+
username: Optional[str] = None
|
| 116 |
+
role: Optional[str] = None
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# ── Simulator ──
|
| 120 |
+
|
| 121 |
+
class SimulatorControlRequest(BaseModel):
|
| 122 |
+
interval_minutes: int = Field(default=5, ge=1, le=3600, description="Interval value")
|
| 123 |
+
interval_unit: str = Field(default="min", description="Unit: 'sec' or 'min'")
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class SimulatorStatusResponse(BaseModel):
|
| 127 |
+
running: bool
|
| 128 |
+
interval_minutes: Optional[float] = None
|
| 129 |
+
last_run: Optional[str] = None
|
| 130 |
+
total_runs: int = 0
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class MLDetectRequest(BaseModel):
|
| 134 |
+
time_window_hours: int = Field(default=24, ge=1, le=168)
|
| 135 |
+
bo_filter: Optional[str] = Field(
|
| 136 |
+
default=None,
|
| 137 |
+
description="Filter rows where source_bo equals this value (e.g. BO1)",
|
| 138 |
+
)
|
| 139 |
+
include_rule_alerts: bool = True
|
| 140 |
+
max_rows: int = Field(default=5000, ge=100, le=100000)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
class IsolationForestResult(BaseModel):
|
| 144 |
+
n_samples: int
|
| 145 |
+
n_outliers: int
|
| 146 |
+
outlier_ratio: float
|
| 147 |
+
feature_columns: List[str]
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class MLDetectResponse(BaseModel):
|
| 151 |
+
success: bool
|
| 152 |
+
timestamp: str
|
| 153 |
+
time_window_hours: int
|
| 154 |
+
isolation_forest: Optional[IsolationForestResult] = None
|
| 155 |
+
anomaly_records: List[Dict[str, Any]] = Field(default_factory=list)
|
| 156 |
+
rule_alerts: List[Dict[str, Any]] = Field(default_factory=list)
|
| 157 |
+
merged_alerts: List[Dict[str, Any]] = Field(default_factory=list)
|
| 158 |
+
message: Optional[str] = None
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class MLPredictRequest(BaseModel):
|
| 162 |
+
target: str = Field(default="latency_ms", description="Column to predict")
|
| 163 |
+
time_window_hours: int = Field(default=48, ge=1, le=336)
|
| 164 |
+
bo_filter: Optional[str] = None
|
| 165 |
+
max_train_rows: int = Field(default=20000, ge=500, le=200000)
|
| 166 |
+
retrain: bool = Field(default=False, description="Force refit on this request")
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
class MLPredictResponse(BaseModel):
|
| 170 |
+
success: bool
|
| 171 |
+
timestamp: str
|
| 172 |
+
target: str
|
| 173 |
+
model: str
|
| 174 |
+
metrics: Dict[str, Any] = Field(default_factory=dict)
|
| 175 |
+
predictions: List[Dict[str, Any]] = Field(default_factory=list)
|
| 176 |
+
message: Optional[str] = None
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class MLOrchestrateRequest(BaseModel):
|
| 180 |
+
prompt: str = Field(..., min_length=2, max_length=2000)
|
| 181 |
+
bo_filter: Optional[str] = Field(
|
| 182 |
+
default=None,
|
| 183 |
+
description="Filter rows where source_bo equals this value (e.g. BO1)",
|
| 184 |
+
)
|
| 185 |
+
detect_time_window_hours: int = Field(default=24, ge=1, le=168)
|
| 186 |
+
predict_time_window_hours: int = Field(default=72, ge=1, le=336)
|
| 187 |
+
include_rule_alerts: bool = True
|
| 188 |
+
max_detect_rows: int = Field(default=8000, ge=100, le=100000)
|
| 189 |
+
max_train_rows: int = Field(default=25000, ge=500, le=200000)
|
| 190 |
+
retrain: bool = Field(default=True, description="Force model refit for prediction")
|
| 191 |
+
confirm: bool = Field(
|
| 192 |
+
default=False,
|
| 193 |
+
description="When true, execute proposed action; otherwise return only routing proposal",
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
@field_validator("prompt")
|
| 197 |
+
@classmethod
|
| 198 |
+
def strip_prompt(cls, v: str) -> str:
|
| 199 |
+
return v.strip()
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class MLOrchestrateResponse(BaseModel):
|
| 203 |
+
success: bool
|
| 204 |
+
timestamp: str
|
| 205 |
+
action: Optional[str] = Field(default=None, description="Selected action: detect or predict")
|
| 206 |
+
intent: str = Field(description="Intent inferred from prompt")
|
| 207 |
+
confidence: float = Field(default=0.0, ge=0.0, le=1.0)
|
| 208 |
+
executed: bool = False
|
| 209 |
+
needs_confirmation: bool = False
|
| 210 |
+
message: Optional[str] = None
|
| 211 |
+
detect_result: Optional[MLDetectResponse] = None
|
| 212 |
+
predict_result: Optional[MLPredictResponse] = None
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
class MLModelsStatusResponse(BaseModel):
|
| 216 |
+
timestamp: str
|
| 217 |
+
diagnostic_joblib: Dict[str, bool] = Field(default_factory=dict)
|
| 218 |
+
runtime: Dict[str, Any] = Field(default_factory=dict)
|
| 219 |
+
data_manager: Dict[str, Any] = Field(default_factory=dict)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class DataRefreshResponse(BaseModel):
|
| 223 |
+
success: bool
|
| 224 |
+
timestamp: str
|
| 225 |
+
rows: int
|
| 226 |
+
source_path: str
|
| 227 |
+
mtime: Optional[str] = None
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
class ExternalDatasetRecord(BaseModel):
|
| 231 |
+
title: str
|
| 232 |
+
source_portal: str
|
| 233 |
+
url: str
|
| 234 |
+
description: Optional[str] = None
|
| 235 |
+
license: Optional[str] = None
|
| 236 |
+
telecom_tags: List[str] = Field(default_factory=list)
|
| 237 |
+
relevance_score: float = 0.0
|
| 238 |
+
fetched_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat() + "Z")
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class EnrichmentCatalogResponse(BaseModel):
|
| 242 |
+
success: bool
|
| 243 |
+
total: int
|
| 244 |
+
offset: int
|
| 245 |
+
limit: int
|
| 246 |
+
records: List[ExternalDatasetRecord] = Field(default_factory=list)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
class EnrichmentScrapeResponse(BaseModel):
|
| 250 |
+
success: bool
|
| 251 |
+
timestamp: str
|
| 252 |
+
added: int
|
| 253 |
+
total_in_catalog: int
|
| 254 |
+
errors: List[str] = Field(default_factory=list)
|
data/processed/network_kpis_full.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
database.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Database schema and utilities for QoS Buddy
|
| 3 |
+
Uses SQLite for simplicity
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, Text, JSON
|
| 7 |
+
from sqlalchemy.ext.declarative import declarative_base
|
| 8 |
+
from sqlalchemy.orm import sessionmaker
|
| 9 |
+
from datetime import datetime
|
| 10 |
+
import os
|
| 11 |
+
|
| 12 |
+
DATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///qos_buddy.db')
|
| 13 |
+
|
| 14 |
+
engine = create_engine(DATABASE_URL, echo=False)
|
| 15 |
+
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
| 16 |
+
Base = declarative_base()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class Alert(Base):
|
| 20 |
+
"""Store diagnostic alerts from Agent 2"""
|
| 21 |
+
__tablename__ = "alerts"
|
| 22 |
+
|
| 23 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 24 |
+
cell_id = Column(String, index=True)
|
| 25 |
+
timestamp = Column(DateTime, default=datetime.utcnow)
|
| 26 |
+
issue_type = Column(String)
|
| 27 |
+
severity = Column(String)
|
| 28 |
+
kpis = Column(JSON)
|
| 29 |
+
causes = Column(JSON)
|
| 30 |
+
recommendations = Column(JSON)
|
| 31 |
+
priority = Column(Integer)
|
| 32 |
+
resolved = Column(Integer, default=0)
|
| 33 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class KPIHistory(Base):
|
| 37 |
+
"""Store time-series KPI data"""
|
| 38 |
+
__tablename__ = "kpi_history"
|
| 39 |
+
|
| 40 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 41 |
+
timestamp = Column(DateTime, index=True)
|
| 42 |
+
cell_id = Column(String, index=True)
|
| 43 |
+
rsrp = Column(Float)
|
| 44 |
+
sinr = Column(Float)
|
| 45 |
+
prb_ul = Column(Float)
|
| 46 |
+
prb_dl = Column(Float)
|
| 47 |
+
throughput_dl = Column(Float)
|
| 48 |
+
throughput_ul = Column(Float)
|
| 49 |
+
latency_ms = Column(Float)
|
| 50 |
+
mos = Column(Float)
|
| 51 |
+
handover_success_rate = Column(Float)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class ChatLog(Base):
|
| 55 |
+
"""Store chatbot conversation history"""
|
| 56 |
+
__tablename__ = "chat_logs"
|
| 57 |
+
|
| 58 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 59 |
+
conversation_id = Column(String, index=True)
|
| 60 |
+
user_message = Column(Text)
|
| 61 |
+
assistant_response = Column(Text)
|
| 62 |
+
timestamp = Column(DateTime, default=datetime.utcnow)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class User(Base):
|
| 66 |
+
"""Registered users for JWT authentication"""
|
| 67 |
+
__tablename__ = "users"
|
| 68 |
+
|
| 69 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 70 |
+
username = Column(String, unique=True, index=True, nullable=False)
|
| 71 |
+
email = Column(String, unique=True, index=True, nullable=True)
|
| 72 |
+
hashed_password = Column(String, nullable=False)
|
| 73 |
+
role = Column(String, default="viewer")
|
| 74 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class PendingOTPRegistration(Base):
|
| 78 |
+
"""Pending user registration waiting for OTP verification."""
|
| 79 |
+
__tablename__ = "pending_otp_registrations"
|
| 80 |
+
|
| 81 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 82 |
+
email = Column(String, unique=True, index=True, nullable=False)
|
| 83 |
+
username = Column(String, nullable=False)
|
| 84 |
+
hashed_password = Column(String, nullable=False)
|
| 85 |
+
role = Column(String, nullable=False)
|
| 86 |
+
otp_code = Column(String, nullable=False)
|
| 87 |
+
otp_expires_at = Column(DateTime, nullable=False)
|
| 88 |
+
attempts = Column(Integer, default=0)
|
| 89 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def init_db():
|
| 93 |
+
"""Initialize database tables"""
|
| 94 |
+
Base.metadata.create_all(bind=engine)
|
| 95 |
+
print("Database initialized")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get_db():
|
| 99 |
+
"""Get database session"""
|
| 100 |
+
db = SessionLocal()
|
| 101 |
+
try:
|
| 102 |
+
yield db
|
| 103 |
+
finally:
|
| 104 |
+
db.close()
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
if __name__ == "__main__":
|
| 108 |
+
print("Initializing QoS Buddy database...")
|
| 109 |
+
init_db()
|
| 110 |
+
print("Done!")
|
generate_data.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Quick script to generate telecom dataset and place it where the backend expects it.
|
| 3 |
+
Run: python generate_data.py
|
| 4 |
+
"""
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# Add backend to path
|
| 9 |
+
sys.path.insert(0, str(Path(__file__).parent))
|
| 10 |
+
|
| 11 |
+
import pandas as pd
|
| 12 |
+
import numpy as np
|
| 13 |
+
from datetime import datetime, timedelta
|
| 14 |
+
import random
|
| 15 |
+
|
| 16 |
+
np.random.seed(42)
|
| 17 |
+
random.seed(42)
|
| 18 |
+
|
| 19 |
+
CITIES = [
|
| 20 |
+
("Lagos", "Nigeria", 6.5244, 3.3792),
|
| 21 |
+
("Abuja", "Nigeria", 9.0765, 7.3986),
|
| 22 |
+
("Kano", "Nigeria", 12.0022, 8.5920),
|
| 23 |
+
("Port Harcourt", "Nigeria", 4.8156, 7.0498),
|
| 24 |
+
("Ibadan", "Nigeria", 7.3775, 3.9470),
|
| 25 |
+
("Benin City", "Nigeria", 6.3350, 5.6037),
|
| 26 |
+
("Kaduna", "Nigeria", 10.5222, 7.4383),
|
| 27 |
+
("Enugu", "Nigeria", 6.4402, 7.4943),
|
| 28 |
+
("Jos", "Nigeria", 9.8965, 8.8583),
|
| 29 |
+
("Ilorin", "Nigeria", 8.4966, 4.5421),
|
| 30 |
+
("New York", "USA", 40.7128, -74.0060),
|
| 31 |
+
("Los Angeles", "USA", 34.0522, -118.2437),
|
| 32 |
+
("Chicago", "USA", 41.8781, -87.6298),
|
| 33 |
+
("Houston", "USA", 29.7604, -95.3698),
|
| 34 |
+
("London", "UK", 51.5074, -0.1278),
|
| 35 |
+
("Paris", "France", 48.8566, 2.3522),
|
| 36 |
+
("Berlin", "Germany", 52.5200, 13.4050),
|
| 37 |
+
("Amsterdam", "Netherlands", 52.3676, 4.9041),
|
| 38 |
+
("Tokyo", "Japan", 35.6762, 139.6503),
|
| 39 |
+
("Seoul", "South Korea", 37.5665, 126.9780),
|
| 40 |
+
("Mumbai", "India", 19.0760, 72.8777),
|
| 41 |
+
("Dubai", "UAE", 25.2048, 55.2708),
|
| 42 |
+
("Singapore", "Singapore", 1.3521, 103.8198),
|
| 43 |
+
("Sydney", "Australia", -33.8688, 151.2093),
|
| 44 |
+
("Toronto", "Canada", 43.6532, -79.3832),
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
NUM_CELLS = 50
|
| 48 |
+
NUM_RECORDS = 20000
|
| 49 |
+
|
| 50 |
+
print("Generating telecom dataset...")
|
| 51 |
+
|
| 52 |
+
cells = []
|
| 53 |
+
# Assign cities with varying "health profiles" for realistic map colors
|
| 54 |
+
CITY_PROFILES = {
|
| 55 |
+
# Good zones (low SLA risk → green on map)
|
| 56 |
+
"Lagos": "good", "Abuja": "good", "Toronto": "good",
|
| 57 |
+
"Singapore": "good", "Sydney": "good", "Amsterdam": "good",
|
| 58 |
+
# Medium zones (moderate SLA risk → yellow on map)
|
| 59 |
+
"London": "medium", "Paris": "medium", "Tokyo": "medium",
|
| 60 |
+
"Seoul": "medium", "Dubai": "medium", "Mumbai": "medium",
|
| 61 |
+
"Berlin": "medium", "Houston": "medium",
|
| 62 |
+
# Poor zones (high SLA risk → orange/red on map)
|
| 63 |
+
"Kano": "poor", "Port Harcourt": "poor", "Ibadan": "poor",
|
| 64 |
+
"Benin City": "poor", "Kaduna": "poor", "Enugu": "poor",
|
| 65 |
+
"Jos": "poor", "Ilorin": "poor",
|
| 66 |
+
# Critical zones
|
| 67 |
+
"New York": "critical", "Los Angeles": "critical", "Chicago": "critical",
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
for i in range(NUM_CELLS):
|
| 71 |
+
city_data = CITIES[i % len(CITIES)]
|
| 72 |
+
cells.append({
|
| 73 |
+
"cell_id": f"CELL_{i+1:03d}",
|
| 74 |
+
"city": city_data[0],
|
| 75 |
+
"country": city_data[1],
|
| 76 |
+
"lat": city_data[2] + np.random.uniform(-0.1, 0.1),
|
| 77 |
+
"lon": city_data[3] + np.random.uniform(-0.1, 0.1),
|
| 78 |
+
"profile": CITY_PROFILES.get(city_data[0], "medium"),
|
| 79 |
+
})
|
| 80 |
+
|
| 81 |
+
end_time = datetime.now()
|
| 82 |
+
start_time = end_time - timedelta(hours=72)
|
| 83 |
+
|
| 84 |
+
records = []
|
| 85 |
+
for i in range(NUM_RECORDS):
|
| 86 |
+
cell = cells[i % NUM_CELLS]
|
| 87 |
+
ts = start_time + timedelta(minutes=i * (72*60/NUM_RECORDS))
|
| 88 |
+
hour = ts.hour
|
| 89 |
+
is_busy = hour in range(17, 22)
|
| 90 |
+
profile = cell.get("profile", "medium")
|
| 91 |
+
|
| 92 |
+
# Adjust scenario probability based on city profile
|
| 93 |
+
if profile == "good":
|
| 94 |
+
r = random.random() * 0.55 # mostly normal (0-0.55 → normal)
|
| 95 |
+
elif profile == "poor":
|
| 96 |
+
r = 0.40 + random.random() * 0.60 # mostly issues
|
| 97 |
+
elif profile == "critical":
|
| 98 |
+
r = 0.55 + random.random() * 0.45 # mostly critical issues
|
| 99 |
+
else: # medium
|
| 100 |
+
r = random.random()
|
| 101 |
+
if r < 0.40:
|
| 102 |
+
scenario = "Normal"
|
| 103 |
+
rsrp = np.random.uniform(-90, -70)
|
| 104 |
+
sinr = np.random.uniform(15, 25)
|
| 105 |
+
prb_dl = np.random.uniform(30, 55)
|
| 106 |
+
prb_ul = np.random.uniform(25, 50)
|
| 107 |
+
thr_dl = np.random.uniform(50, 150)
|
| 108 |
+
thr_ul = np.random.uniform(10, 30)
|
| 109 |
+
lat = np.random.uniform(10, 35)
|
| 110 |
+
mos = np.random.uniform(3.8, 4.5)
|
| 111 |
+
cdr = np.random.uniform(0, 0.8)
|
| 112 |
+
ho = np.random.uniform(95, 99.5)
|
| 113 |
+
bler = np.random.uniform(0.5, 2)
|
| 114 |
+
cqi = np.random.uniform(12, 15)
|
| 115 |
+
rlc = np.random.uniform(0.5, 2)
|
| 116 |
+
sev = "NONE"
|
| 117 |
+
sla_risk = np.random.uniform(0.0, 0.15) # LOW risk → green on map
|
| 118 |
+
capex = np.random.uniform(5, 25)
|
| 119 |
+
elif r < 0.55:
|
| 120 |
+
scenario = "Capacity Problem"
|
| 121 |
+
rsrp = np.random.uniform(-90, -75)
|
| 122 |
+
sinr = np.random.uniform(10, 18)
|
| 123 |
+
prb_dl = np.random.uniform(87, 100)
|
| 124 |
+
prb_ul = np.random.uniform(85, 100)
|
| 125 |
+
thr_dl = np.random.uniform(10, 30)
|
| 126 |
+
thr_ul = np.random.uniform(2, 8)
|
| 127 |
+
lat = np.random.uniform(40, 90)
|
| 128 |
+
mos = np.random.uniform(2.5, 3.5)
|
| 129 |
+
cdr = np.random.uniform(1.5, 4)
|
| 130 |
+
ho = np.random.uniform(85, 92)
|
| 131 |
+
bler = np.random.uniform(3, 8)
|
| 132 |
+
cqi = np.random.uniform(6, 10)
|
| 133 |
+
rlc = np.random.uniform(3, 8)
|
| 134 |
+
sev = "CRITICAL" if is_busy else "HIGH"
|
| 135 |
+
sla_risk = np.random.uniform(0.6, 0.9)
|
| 136 |
+
capex = np.random.uniform(70, 95)
|
| 137 |
+
elif r < 0.67:
|
| 138 |
+
scenario = "High Latency"
|
| 139 |
+
rsrp = np.random.uniform(-95, -75)
|
| 140 |
+
sinr = np.random.uniform(10, 18)
|
| 141 |
+
prb_dl = np.random.uniform(55, 80)
|
| 142 |
+
prb_ul = np.random.uniform(50, 75)
|
| 143 |
+
thr_dl = np.random.uniform(30, 60)
|
| 144 |
+
thr_ul = np.random.uniform(5, 15)
|
| 145 |
+
lat = np.random.uniform(80, 200)
|
| 146 |
+
mos = np.random.uniform(2.8, 3.5)
|
| 147 |
+
cdr = np.random.uniform(0.5, 2)
|
| 148 |
+
ho = np.random.uniform(90, 96)
|
| 149 |
+
bler = np.random.uniform(2, 5)
|
| 150 |
+
cqi = np.random.uniform(8, 12)
|
| 151 |
+
rlc = np.random.uniform(10, 25)
|
| 152 |
+
sev = "MEDIUM"
|
| 153 |
+
sla_risk = np.random.uniform(0.4, 0.7)
|
| 154 |
+
capex = np.random.uniform(40, 65)
|
| 155 |
+
elif r < 0.77:
|
| 156 |
+
scenario = "Call Drop"
|
| 157 |
+
rsrp = np.random.uniform(-115, -105)
|
| 158 |
+
sinr = np.random.uniform(2, 8)
|
| 159 |
+
prb_dl = np.random.uniform(40, 70)
|
| 160 |
+
prb_ul = np.random.uniform(35, 65)
|
| 161 |
+
thr_dl = np.random.uniform(5, 20)
|
| 162 |
+
thr_ul = np.random.uniform(1, 5)
|
| 163 |
+
lat = np.random.uniform(50, 120)
|
| 164 |
+
mos = np.random.uniform(1.5, 2.5)
|
| 165 |
+
cdr = np.random.uniform(5, 12)
|
| 166 |
+
ho = np.random.uniform(70, 84)
|
| 167 |
+
bler = np.random.uniform(5, 12)
|
| 168 |
+
cqi = np.random.uniform(3, 7)
|
| 169 |
+
rlc = np.random.uniform(5, 15)
|
| 170 |
+
sev = "HIGH"
|
| 171 |
+
sla_risk = np.random.uniform(0.7, 0.95)
|
| 172 |
+
capex = np.random.uniform(60, 85)
|
| 173 |
+
elif r < 0.87:
|
| 174 |
+
scenario = "Handover Failure"
|
| 175 |
+
rsrp = np.random.uniform(-98, -80)
|
| 176 |
+
sinr = np.random.uniform(8, 15)
|
| 177 |
+
prb_dl = np.random.uniform(40, 65)
|
| 178 |
+
prb_ul = np.random.uniform(35, 60)
|
| 179 |
+
thr_dl = np.random.uniform(25, 55)
|
| 180 |
+
thr_ul = np.random.uniform(5, 15)
|
| 181 |
+
lat = np.random.uniform(30, 60)
|
| 182 |
+
mos = np.random.uniform(2.5, 3.5)
|
| 183 |
+
cdr = np.random.uniform(2, 6)
|
| 184 |
+
ho = np.random.uniform(65, 80)
|
| 185 |
+
bler = np.random.uniform(2, 6)
|
| 186 |
+
cqi = np.random.uniform(7, 11)
|
| 187 |
+
rlc = np.random.uniform(3, 7)
|
| 188 |
+
sev = "HIGH"
|
| 189 |
+
sla_risk = np.random.uniform(0.5, 0.75)
|
| 190 |
+
capex = np.random.uniform(45, 70)
|
| 191 |
+
else:
|
| 192 |
+
scenario = "Low MOS"
|
| 193 |
+
rsrp = np.random.uniform(-100, -85)
|
| 194 |
+
sinr = np.random.uniform(5, 12)
|
| 195 |
+
prb_dl = np.random.uniform(60, 85)
|
| 196 |
+
prb_ul = np.random.uniform(55, 80)
|
| 197 |
+
thr_dl = np.random.uniform(15, 40)
|
| 198 |
+
thr_ul = np.random.uniform(3, 10)
|
| 199 |
+
lat = np.random.uniform(40, 80)
|
| 200 |
+
mos = np.random.uniform(1.5, 2.8)
|
| 201 |
+
cdr = np.random.uniform(1, 4)
|
| 202 |
+
ho = np.random.uniform(82, 92)
|
| 203 |
+
bler = np.random.uniform(4, 10)
|
| 204 |
+
cqi = np.random.uniform(5, 9)
|
| 205 |
+
rlc = np.random.uniform(4, 10)
|
| 206 |
+
sev = "MEDIUM"
|
| 207 |
+
sla_risk = np.random.uniform(0.45, 0.70)
|
| 208 |
+
capex = np.random.uniform(35, 60)
|
| 209 |
+
|
| 210 |
+
anomaly_flag = 1 if sev in ("CRITICAL", "HIGH") else 0
|
| 211 |
+
|
| 212 |
+
# Assign critical_count based on severity for map coloring
|
| 213 |
+
if sev == "CRITICAL":
|
| 214 |
+
critical_count = random.randint(3, 8)
|
| 215 |
+
elif sev == "HIGH":
|
| 216 |
+
critical_count = random.randint(1, 3)
|
| 217 |
+
else:
|
| 218 |
+
critical_count = 0
|
| 219 |
+
|
| 220 |
+
records.append({
|
| 221 |
+
"timestamp": ts.strftime("%Y-%m-%d %H:%M:%S"),
|
| 222 |
+
"cell_id": cell["cell_id"],
|
| 223 |
+
"city": cell["city"],
|
| 224 |
+
"country": cell["country"],
|
| 225 |
+
"lat": round(cell["lat"], 6),
|
| 226 |
+
"lon": round(cell["lon"], 6),
|
| 227 |
+
"RSRP": round(rsrp, 2),
|
| 228 |
+
"SINR": round(sinr, 2),
|
| 229 |
+
"PRB_UL": round(prb_ul, 2),
|
| 230 |
+
"PRB_DL": round(prb_dl, 2),
|
| 231 |
+
"throughput_DL": round(thr_dl, 2),
|
| 232 |
+
"throughput_UL": round(thr_ul, 2),
|
| 233 |
+
"latency_ms": round(lat, 2),
|
| 234 |
+
"MOS": round(mos, 3),
|
| 235 |
+
"call_drop_rate": round(cdr, 3),
|
| 236 |
+
"handover_success_rate": round(ho, 2),
|
| 237 |
+
"BLER_UL": round(bler, 2),
|
| 238 |
+
"BLER_DL": round(bler * 0.9, 2),
|
| 239 |
+
"CQI": round(cqi, 1),
|
| 240 |
+
"RLC_retransmissions": round(rlc, 2),
|
| 241 |
+
"hour": hour,
|
| 242 |
+
"issue_type": scenario,
|
| 243 |
+
"severity": sev,
|
| 244 |
+
"anomaly_flag": anomaly_flag,
|
| 245 |
+
"sla_breach_risk": round(sla_risk, 4),
|
| 246 |
+
"capex_score": round(capex, 2),
|
| 247 |
+
"critical_count": critical_count,
|
| 248 |
+
"source_bo": "BO1" if anomaly_flag else ("BO2" if sla_risk > 0.4 else "BO3"),
|
| 249 |
+
"risk_tier": sev if sev != "NONE" else "LOW",
|
| 250 |
+
})
|
| 251 |
+
|
| 252 |
+
df = pd.DataFrame(records)
|
| 253 |
+
# Replace any NaN/inf values with 0
|
| 254 |
+
df = df.fillna(0)
|
| 255 |
+
df = df.replace([float('inf'), float('-inf')], 0)
|
| 256 |
+
|
| 257 |
+
# Save to all expected locations
|
| 258 |
+
import os
|
| 259 |
+
|
| 260 |
+
paths = [
|
| 261 |
+
Path(__file__).parent / "data" / "processed" / "network_kpis_full.csv",
|
| 262 |
+
Path(__file__).parent / "unified_telecom_dataset_FILLED.csv",
|
| 263 |
+
Path(__file__).parent.parent / "data" / "unified_telecom_dataset_FILLED.csv",
|
| 264 |
+
]
|
| 265 |
+
|
| 266 |
+
for p in paths:
|
| 267 |
+
p.parent.mkdir(parents=True, exist_ok=True)
|
| 268 |
+
df.to_csv(p, index=False)
|
| 269 |
+
print(f"Saved {len(df)} records to: {p}")
|
| 270 |
+
|
| 271 |
+
print(f"\nDataset summary:")
|
| 272 |
+
print(f" Records: {len(df):,}")
|
| 273 |
+
print(f" Cells: {df['cell_id'].nunique()}")
|
| 274 |
+
print(f" Cities: {df['city'].nunique()}")
|
| 275 |
+
print(f" Date range: {df['timestamp'].min()} → {df['timestamp'].max()}")
|
| 276 |
+
print(f" Issues: {df['issue_type'].value_counts().to_dict()}")
|
| 277 |
+
print(f" Severities: {df['severity'].value_counts().to_dict()}")
|
| 278 |
+
print("\nDone!")
|
main.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FastAPI Backend for QoS Buddy
|
| 3 |
+
Thin app factory — business logic lives in routes/ and services/.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import warnings
|
| 7 |
+
warnings.filterwarnings("ignore", category=UserWarning)
|
| 8 |
+
warnings.filterwarnings("ignore", category=FutureWarning)
|
| 9 |
+
import os
|
| 10 |
+
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
|
| 11 |
+
|
| 12 |
+
from fastapi import FastAPI
|
| 13 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
+
from datetime import datetime
|
| 15 |
+
import uvicorn
|
| 16 |
+
import os
|
| 17 |
+
|
| 18 |
+
from core.config import CORS_ORIGINS, ENABLE_DB_ALERTS
|
| 19 |
+
from database import init_db
|
| 20 |
+
from routes import deps
|
| 21 |
+
from routes.chat import router as chat_router
|
| 22 |
+
from routes.diagnostics import router as diagnostics_router
|
| 23 |
+
from routes.ml import router as ml_router
|
| 24 |
+
from routes.data import router as data_router
|
| 25 |
+
from routes.auth import router as auth_router
|
| 26 |
+
from routes.simulator import router as simulator_router
|
| 27 |
+
from routes.live_data import router as live_data_router
|
| 28 |
+
|
| 29 |
+
app = FastAPI(
|
| 30 |
+
title="QoS Buddy API",
|
| 31 |
+
description="Multi-Agent Telecom Network Diagnostic System",
|
| 32 |
+
version="2.0.0",
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
app.add_middleware(
|
| 36 |
+
CORSMiddleware,
|
| 37 |
+
allow_origins=CORS_ORIGINS,
|
| 38 |
+
allow_credentials=True,
|
| 39 |
+
allow_methods=["*"],
|
| 40 |
+
allow_headers=["*"],
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
# Register route modules
|
| 44 |
+
app.include_router(auth_router)
|
| 45 |
+
app.include_router(chat_router)
|
| 46 |
+
app.include_router(diagnostics_router)
|
| 47 |
+
app.include_router(ml_router)
|
| 48 |
+
app.include_router(data_router)
|
| 49 |
+
app.include_router(simulator_router)
|
| 50 |
+
app.include_router(live_data_router)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@app.on_event("startup")
|
| 54 |
+
async def startup_event():
|
| 55 |
+
"""Initialize agents on startup."""
|
| 56 |
+
print("=" * 70)
|
| 57 |
+
print("INITIALIZING QoS BUDDY BACKEND v2.0")
|
| 58 |
+
print("=" * 70)
|
| 59 |
+
|
| 60 |
+
agent1 = None
|
| 61 |
+
agent2 = None
|
| 62 |
+
orchestrator = None
|
| 63 |
+
|
| 64 |
+
try:
|
| 65 |
+
print("\n[0/3] Initializing database tables...")
|
| 66 |
+
init_db()
|
| 67 |
+
print(" Database schema ready")
|
| 68 |
+
except Exception as e:
|
| 69 |
+
print(f" Database init warning: {e}")
|
| 70 |
+
|
| 71 |
+
try:
|
| 72 |
+
print("\n[1/3] Initializing orchestrator + Data Manager...")
|
| 73 |
+
from services.orchestrator import QoSOrchestrator
|
| 74 |
+
|
| 75 |
+
orchestrator = QoSOrchestrator()
|
| 76 |
+
orchestrator.refresh_data()
|
| 77 |
+
print(" Orchestrator ready")
|
| 78 |
+
except Exception as e:
|
| 79 |
+
print(f" Orchestrator initialization failed: {e}")
|
| 80 |
+
|
| 81 |
+
try:
|
| 82 |
+
print("\n[2/3] Initializing Agent 1 (Chatbot Advisor)...")
|
| 83 |
+
from agents.agent1_chatbot import TelecomAdvisorAgent
|
| 84 |
+
|
| 85 |
+
agent1 = TelecomAdvisorAgent()
|
| 86 |
+
# Heavy init moved to thread/background manually here for simplicity in this specific app structure
|
| 87 |
+
import threading
|
| 88 |
+
def bg_agent1_init(a1, orch):
|
| 89 |
+
try:
|
| 90 |
+
a1.initialize_vector_db()
|
| 91 |
+
if orch:
|
| 92 |
+
a1.set_insights_provider(orch.live_insights_snapshot)
|
| 93 |
+
orch.wire_agent1(a1)
|
| 94 |
+
print(" [BG] Agent 1 ready")
|
| 95 |
+
except Exception as e_bg:
|
| 96 |
+
print(f" [BG] Agent 1 init failed: {e_bg}")
|
| 97 |
+
|
| 98 |
+
threading.Thread(target=bg_agent1_init, args=(agent1, orchestrator), daemon=True).start()
|
| 99 |
+
print(" Agent 1 initialization started in background")
|
| 100 |
+
except Exception as e:
|
| 101 |
+
print(f" Agent 1 setup failed: {e}")
|
| 102 |
+
|
| 103 |
+
try:
|
| 104 |
+
print("\n[3/3] Initializing Agent 2 (Diagnostic Worker)...")
|
| 105 |
+
from agents.agent2_diagnostic import DiagnosticAgent
|
| 106 |
+
|
| 107 |
+
agent2 = DiagnosticAgent()
|
| 108 |
+
print(" Agent 2 ready")
|
| 109 |
+
except Exception as e:
|
| 110 |
+
print(f" Agent 2 initialization failed: {e}")
|
| 111 |
+
|
| 112 |
+
# Wire globals so route modules can access agents
|
| 113 |
+
deps.set_globals(agent1, agent2, orchestrator)
|
| 114 |
+
|
| 115 |
+
print("\n" + "=" * 70)
|
| 116 |
+
print("QoS BUDDY BACKEND READY")
|
| 117 |
+
print(f" DB alerts: {'enabled' if ENABLE_DB_ALERTS else 'disabled'}")
|
| 118 |
+
print(f" CORS origins: {CORS_ORIGINS}")
|
| 119 |
+
print(f" API docs: http://localhost:8000/docs")
|
| 120 |
+
print("=" * 70 + "\n")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@app.get("/")
|
| 124 |
+
async def root():
|
| 125 |
+
return {
|
| 126 |
+
"service": "QoS Buddy API",
|
| 127 |
+
"version": "2.0.0",
|
| 128 |
+
"status": "online",
|
| 129 |
+
"agents": {
|
| 130 |
+
"agent1_chatbot": deps.get_agent1() is not None,
|
| 131 |
+
"agent2_diagnostic": deps.get_agent2() is not None,
|
| 132 |
+
"orchestrator": deps.get_orchestrator() is not None,
|
| 133 |
+
},
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@app.get("/health")
|
| 138 |
+
async def health_check():
|
| 139 |
+
return {
|
| 140 |
+
"status": "healthy",
|
| 141 |
+
"timestamp": datetime.now().isoformat(),
|
| 142 |
+
"agents": {
|
| 143 |
+
"agent1": "online" if deps.get_agent1() else "offline",
|
| 144 |
+
"agent2": "online" if deps.get_agent2() else "offline",
|
| 145 |
+
"orchestrator": "online" if deps.get_orchestrator() else "offline",
|
| 146 |
+
},
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if __name__ == "__main__":
|
| 151 |
+
print("\nStarting QoS Buddy Backend Server...")
|
| 152 |
+
use_reload = os.getenv("DEV_MODE", "false").lower() == "true"
|
| 153 |
+
uvicorn.run(
|
| 154 |
+
"main:app",
|
| 155 |
+
host="0.0.0.0",
|
| 156 |
+
port=8000,
|
| 157 |
+
reload=use_reload,
|
| 158 |
+
reload_excludes=["../data/*", "*.db", "*.sqlite"] if use_reload else None,
|
| 159 |
+
log_level="info",
|
| 160 |
+
)
|
qos_buddy.db
ADDED
|
Binary file (69.6 kB). View file
|
|
|
requirements.txt
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
aiofiles==25.1.0
|
| 2 |
+
aiohappyeyeballs==2.6.1
|
| 3 |
+
aiohttp==3.13.5
|
| 4 |
+
aiosignal==1.4.0
|
| 5 |
+
annotated-doc==0.0.4
|
| 6 |
+
annotated-types==0.7.0
|
| 7 |
+
anyio==4.13.0
|
| 8 |
+
attrs==26.1.0
|
| 9 |
+
bcrypt==5.0.0
|
| 10 |
+
beautifulsoup4==4.14.3
|
| 11 |
+
build==1.5.0
|
| 12 |
+
certifi==2026.4.22
|
| 13 |
+
cffi==2.0.0
|
| 14 |
+
charset-normalizer==3.4.7
|
| 15 |
+
chromadb==1.5.8
|
| 16 |
+
click==8.3.3
|
| 17 |
+
cryptography==47.0.0
|
| 18 |
+
cuda-bindings==13.2.0
|
| 19 |
+
cuda-pathfinder==1.5.4
|
| 20 |
+
cuda-toolkit==13.0.2
|
| 21 |
+
dataclasses-json==0.6.7
|
| 22 |
+
durationpy==0.10
|
| 23 |
+
ecdsa==0.19.2
|
| 24 |
+
fastapi==0.136.1
|
| 25 |
+
filelock==3.29.0
|
| 26 |
+
flatbuffers==25.12.19
|
| 27 |
+
frozenlist==1.8.0
|
| 28 |
+
fsspec==2026.4.0
|
| 29 |
+
googleapis-common-protos==1.74.0
|
| 30 |
+
greenlet==3.5.0
|
| 31 |
+
grpcio==1.80.0
|
| 32 |
+
h11==0.16.0
|
| 33 |
+
hf-xet==1.4.3
|
| 34 |
+
httpcore==1.0.9
|
| 35 |
+
httptools==0.7.1
|
| 36 |
+
httpx==0.28.1
|
| 37 |
+
httpx-sse==0.4.3
|
| 38 |
+
huggingface_hub==1.13.0
|
| 39 |
+
idna==3.13
|
| 40 |
+
importlib_metadata==8.7.1
|
| 41 |
+
importlib_resources==7.1.0
|
| 42 |
+
Jinja2==3.1.6
|
| 43 |
+
joblib==1.5.3
|
| 44 |
+
jsonpatch==1.33
|
| 45 |
+
jsonpointer==3.1.1
|
| 46 |
+
jsonschema==4.26.0
|
| 47 |
+
jsonschema-specifications==2025.9.1
|
| 48 |
+
kubernetes==35.0.0
|
| 49 |
+
langchain==1.2.17
|
| 50 |
+
langchain-classic==1.0.4
|
| 51 |
+
langchain-community==0.4.1
|
| 52 |
+
langchain-core==1.3.2
|
| 53 |
+
langchain-protocol==0.0.15
|
| 54 |
+
langchain-text-splitters==1.1.2
|
| 55 |
+
langgraph==1.1.10
|
| 56 |
+
langgraph-checkpoint==4.0.3
|
| 57 |
+
langgraph-prebuilt==1.0.13
|
| 58 |
+
langgraph-sdk==0.3.13
|
| 59 |
+
langsmith==0.8.0
|
| 60 |
+
lxml==6.1.0
|
| 61 |
+
markdown-it-py==4.0.0
|
| 62 |
+
MarkupSafe==3.0.3
|
| 63 |
+
marshmallow==3.26.2
|
| 64 |
+
mdurl==0.1.2
|
| 65 |
+
mmh3==5.2.1
|
| 66 |
+
mpmath==1.3.0
|
| 67 |
+
multidict==6.7.1
|
| 68 |
+
mypy_extensions==1.1.0
|
| 69 |
+
networkx==3.6.1
|
| 70 |
+
numpy==2.4.4
|
| 71 |
+
nvidia-cublas==13.1.0.3
|
| 72 |
+
nvidia-cuda-cupti==13.0.85
|
| 73 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 74 |
+
nvidia-cuda-runtime==13.0.96
|
| 75 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 76 |
+
nvidia-cufft==12.0.0.61
|
| 77 |
+
nvidia-cufile==1.15.1.6
|
| 78 |
+
nvidia-curand==10.4.0.35
|
| 79 |
+
nvidia-cusolver==12.0.4.66
|
| 80 |
+
nvidia-cusparse==12.6.3.3
|
| 81 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 82 |
+
nvidia-nccl-cu13==2.28.9
|
| 83 |
+
nvidia-nvjitlink==13.0.88
|
| 84 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 85 |
+
nvidia-nvtx==13.0.85
|
| 86 |
+
oauthlib==3.3.1
|
| 87 |
+
onnxruntime==1.25.1
|
| 88 |
+
opentelemetry-api==1.41.1
|
| 89 |
+
opentelemetry-exporter-otlp-proto-common==1.41.1
|
| 90 |
+
opentelemetry-exporter-otlp-proto-grpc==1.41.1
|
| 91 |
+
opentelemetry-proto==1.41.1
|
| 92 |
+
opentelemetry-sdk==1.41.1
|
| 93 |
+
opentelemetry-semantic-conventions==0.62b1
|
| 94 |
+
orjson==3.11.8
|
| 95 |
+
ormsgpack==1.12.2
|
| 96 |
+
overrides==7.7.0
|
| 97 |
+
packaging==26.2
|
| 98 |
+
pandas==3.0.2
|
| 99 |
+
passlib==1.7.4
|
| 100 |
+
propcache==0.4.1
|
| 101 |
+
protobuf==6.33.6
|
| 102 |
+
psycopg2-binary==2.9.12
|
| 103 |
+
pyasn1==0.6.3
|
| 104 |
+
pybase64==1.4.3
|
| 105 |
+
pycparser==3.0
|
| 106 |
+
pydantic==2.13.3
|
| 107 |
+
pydantic-settings==2.14.0
|
| 108 |
+
pydantic_core==2.46.3
|
| 109 |
+
Pygments==2.20.0
|
| 110 |
+
PyPika==0.51.1
|
| 111 |
+
pyproject_hooks==1.2.0
|
| 112 |
+
python-dateutil==2.9.0.post0
|
| 113 |
+
python-dotenv==1.2.2
|
| 114 |
+
python-jose==3.5.0
|
| 115 |
+
python-multipart==0.0.27
|
| 116 |
+
PyYAML==6.0.3
|
| 117 |
+
referencing==0.37.0
|
| 118 |
+
regex==2026.4.4
|
| 119 |
+
requests==2.33.1
|
| 120 |
+
requests-oauthlib==2.0.0
|
| 121 |
+
requests-toolbelt==1.0.0
|
| 122 |
+
rich==15.0.0
|
| 123 |
+
rpds-py==0.30.0
|
| 124 |
+
rsa==4.9.1
|
| 125 |
+
safetensors==0.7.0
|
| 126 |
+
scikit-learn==1.8.0
|
| 127 |
+
scipy==1.17.1
|
| 128 |
+
sentence-transformers==5.4.1
|
| 129 |
+
setuptools==81.0.0
|
| 130 |
+
shellingham==1.5.4
|
| 131 |
+
sib-api-v3-sdk==7.6.0
|
| 132 |
+
six==1.17.0
|
| 133 |
+
soupsieve==2.8.3
|
| 134 |
+
SQLAlchemy==2.0.49
|
| 135 |
+
starlette==1.0.0
|
| 136 |
+
sympy==1.14.0
|
| 137 |
+
tenacity==9.1.4
|
| 138 |
+
threadpoolctl==3.6.0
|
| 139 |
+
tokenizers==0.22.2
|
| 140 |
+
torch==2.11.0
|
| 141 |
+
tqdm==4.67.3
|
| 142 |
+
transformers==5.7.0
|
| 143 |
+
triton==3.6.0
|
| 144 |
+
typer==0.25.1
|
| 145 |
+
typing-inspect==0.9.0
|
| 146 |
+
typing-inspection==0.4.2
|
| 147 |
+
typing_extensions==4.15.0
|
| 148 |
+
urllib3==2.6.3
|
| 149 |
+
uuid_utils==0.14.1
|
| 150 |
+
uvicorn==0.46.0
|
| 151 |
+
uvloop==0.22.1
|
| 152 |
+
watchfiles==1.1.1
|
| 153 |
+
websocket-client==1.9.0
|
| 154 |
+
websockets==16.0
|
| 155 |
+
xxhash==3.7.0
|
| 156 |
+
yarl==1.23.0
|
| 157 |
+
zipp==3.23.1
|
| 158 |
+
zstandard==0.25.0
|
requirements_hf.txt
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# QoS Buddy - HuggingFace Spaces (CPU only, no CUDA)
|
| 2 |
+
|
| 3 |
+
# Core API
|
| 4 |
+
fastapi==0.109.2
|
| 5 |
+
uvicorn[standard]==0.27.1
|
| 6 |
+
python-multipart==0.0.9
|
| 7 |
+
python-dotenv==1.0.1
|
| 8 |
+
|
| 9 |
+
# Data Science
|
| 10 |
+
pandas==2.2.0
|
| 11 |
+
numpy==1.26.4
|
| 12 |
+
scikit-learn==1.4.0
|
| 13 |
+
joblib==1.3.2
|
| 14 |
+
scipy
|
| 15 |
+
|
| 16 |
+
# ML
|
| 17 |
+
lightgbm==4.3.0
|
| 18 |
+
imbalanced-learn==0.12.0
|
| 19 |
+
|
| 20 |
+
# RAG / Chatbot (torch CPU installé séparément dans Dockerfile)
|
| 21 |
+
sentence-transformers==2.3.1
|
| 22 |
+
langchain==0.1.9
|
| 23 |
+
langchain-community==0.0.21
|
| 24 |
+
|
| 25 |
+
# ChromaDB (version légère)
|
| 26 |
+
chromadb==0.4.22
|
| 27 |
+
|
| 28 |
+
# Google AI
|
| 29 |
+
google-genai>=1.0.0
|
| 30 |
+
|
| 31 |
+
# Database
|
| 32 |
+
sqlalchemy==2.0.27
|
| 33 |
+
psycopg2-binary==2.9.9
|
| 34 |
+
|
| 35 |
+
# Auth
|
| 36 |
+
python-jose[cryptography]==3.3.0
|
| 37 |
+
passlib==1.7.4
|
| 38 |
+
bcrypt==4.1.2
|
| 39 |
+
|
| 40 |
+
# Email
|
| 41 |
+
sib-api-v3-sdk==7.6.0
|
| 42 |
+
|
| 43 |
+
# Utils
|
| 44 |
+
apscheduler==3.10.4
|
| 45 |
+
requests==2.31.0
|
| 46 |
+
beautifulsoup4==4.12.3
|
| 47 |
+
pydantic==2.6.1
|
| 48 |
+
aiofiles
|
routes/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Route modules for QoS Buddy API."""
|
routes/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (208 Bytes). View file
|
|
|
routes/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (293 Bytes). View file
|
|
|
routes/__pycache__/auth.cpython-312.pyc
ADDED
|
Binary file (11.9 kB). View file
|
|
|
routes/__pycache__/auth.cpython-314.pyc
ADDED
|
Binary file (13.4 kB). View file
|
|
|
routes/__pycache__/chat.cpython-312.pyc
ADDED
|
Binary file (1.77 kB). View file
|
|
|