Commit
·
b8dbd20
1
Parent(s):
3b1904c
refactor(judges): extract shared title-extraction helper
Browse filesAddress CodeRabbit nitpicks:
- Add language tag to fenced code block in docs
- Extract _extract_titles_from_evidence() to reduce duplication
between HFInferenceJudgeHandler and MockJudgeHandler
All 136 tests pass.
docs/bugs/INVESTIGATION_QUOTA_BLOCKER.md
CHANGED
|
@@ -15,7 +15,7 @@ This results in a "Synthesis" step that has 0 candidates and 0 findings, renderi
|
|
| 15 |
|
| 16 |
## Evidence
|
| 17 |
Output provided:
|
| 18 |
-
```
|
| 19 |
### Citations (20 sources)
|
| 20 |
...
|
| 21 |
### Reasoning
|
|
|
|
| 15 |
|
| 16 |
## Evidence
|
| 17 |
Output provided:
|
| 18 |
+
```text
|
| 19 |
### Citations (20 sources)
|
| 20 |
...
|
| 21 |
### Reasoning
|
src/agent_factory/judges.py
CHANGED
|
@@ -26,6 +26,31 @@ from src.utils.models import AssessmentDetails, Evidence, JudgeAssessment
|
|
| 26 |
logger = structlog.get_logger()
|
| 27 |
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
def get_model() -> Any:
|
| 30 |
"""Get the LLM model based on configuration.
|
| 31 |
|
|
@@ -346,16 +371,11 @@ IMPORTANT: Respond with ONLY valid JSON matching this schema:
|
|
| 346 |
self, question: str, evidence: list[Evidence]
|
| 347 |
) -> JudgeAssessment:
|
| 348 |
"""Create an assessment that stops the loop when quota is exhausted."""
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
title = title[:147] + "..."
|
| 355 |
-
findings.append(title)
|
| 356 |
-
|
| 357 |
-
if not findings:
|
| 358 |
-
findings = ["No findings available (Quota exceeded and no search results)."]
|
| 359 |
|
| 360 |
return JudgeAssessment(
|
| 361 |
details=AssessmentDetails(
|
|
@@ -431,13 +451,11 @@ class MockJudgeHandler:
|
|
| 431 |
|
| 432 |
def _extract_key_findings(self, evidence: list[Evidence], max_findings: int = 5) -> list[str]:
|
| 433 |
"""Extract key findings from evidence titles."""
|
| 434 |
-
findings =
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
title = title[:147] + "..."
|
| 440 |
-
findings.append(title)
|
| 441 |
return findings if findings else ["No specific findings extracted (demo mode)"]
|
| 442 |
|
| 443 |
def _extract_drug_candidates(self, question: str, evidence: list[Evidence]) -> list[str]:
|
|
|
|
| 26 |
logger = structlog.get_logger()
|
| 27 |
|
| 28 |
|
| 29 |
+
def _extract_titles_from_evidence(
|
| 30 |
+
evidence: list[Evidence], max_items: int = 5, fallback_message: str | None = None
|
| 31 |
+
) -> list[str]:
|
| 32 |
+
"""Extract truncated titles from evidence for fallback display.
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
evidence: List of evidence items
|
| 36 |
+
max_items: Maximum number of titles to extract
|
| 37 |
+
fallback_message: Message to return if no evidence provided
|
| 38 |
+
|
| 39 |
+
Returns:
|
| 40 |
+
List of truncated titles (max 150 chars each)
|
| 41 |
+
"""
|
| 42 |
+
findings = []
|
| 43 |
+
for e in evidence[:max_items]:
|
| 44 |
+
title = e.citation.title
|
| 45 |
+
if len(title) > 150:
|
| 46 |
+
title = title[:147] + "..."
|
| 47 |
+
findings.append(title)
|
| 48 |
+
|
| 49 |
+
if not findings and fallback_message:
|
| 50 |
+
return [fallback_message]
|
| 51 |
+
return findings
|
| 52 |
+
|
| 53 |
+
|
| 54 |
def get_model() -> Any:
|
| 55 |
"""Get the LLM model based on configuration.
|
| 56 |
|
|
|
|
| 371 |
self, question: str, evidence: list[Evidence]
|
| 372 |
) -> JudgeAssessment:
|
| 373 |
"""Create an assessment that stops the loop when quota is exhausted."""
|
| 374 |
+
findings = _extract_titles_from_evidence(
|
| 375 |
+
evidence,
|
| 376 |
+
max_items=5,
|
| 377 |
+
fallback_message="No findings available (Quota exceeded and no search results).",
|
| 378 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
|
| 380 |
return JudgeAssessment(
|
| 381 |
details=AssessmentDetails(
|
|
|
|
| 451 |
|
| 452 |
def _extract_key_findings(self, evidence: list[Evidence], max_findings: int = 5) -> list[str]:
|
| 453 |
"""Extract key findings from evidence titles."""
|
| 454 |
+
findings = _extract_titles_from_evidence(
|
| 455 |
+
evidence,
|
| 456 |
+
max_items=max_findings,
|
| 457 |
+
fallback_message="No specific findings extracted (demo mode)",
|
| 458 |
+
)
|
|
|
|
|
|
|
| 459 |
return findings if findings else ["No specific findings extracted (demo mode)"]
|
| 460 |
|
| 461 |
def _extract_drug_candidates(self, question: str, evidence: list[Evidence]) -> list[str]:
|