admin/OpenFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: robust category matching — normalize dashes/accents + partial match

Browse Source 
The AI often returns 'Guerre - Moyen-Orient' (ASCII dash) instead of
'Guerre — Moyen-Orient' (em-dash), causing exact match to fail silently.
Fix: normalize unicode, collapse all dash variants, then try
exact → normalized → partial substring matching.
Also log the raw AI response for debugging, increase max_tokens 1200→2000.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-25 22:07:56 +02:00
parent 7eb8c24719
commit 91a448e8c9
1 changed files with 34 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
backend/services/impact_service.py
View File

@@ -186,7 +186,7 @@ def evaluate_event_impacts(
messages=[{"role": "user", "content": prompt}],
response_format={"type": "json_object"},
temperature=0.25,
max_tokens=1200,
max_tokens=2000,
)
raw = json.loads(response.choices[0].message.content)
except Exception as e:
@@ -198,25 +198,48 @@ def evaluate_event_impacts(
key_driver = raw.get("key_driver", "")
# ── Use AI-chosen category ────────────────────────────────────────────────
import unicodedata, re as _re
def _normalize(s: str) -> str:
"""Lowercase, strip accents, collapse all dash/space variants."""
s = unicodedata.normalize("NFD", s)
s = "".join(c for c in s if unicodedata.category(c) != "Mn") # strip accents
s = _re.sub(r"[–—―\-–—]", "-", s) # normalize dashes
return s.lower().strip()
ai_cat_name = (raw.get("matched_category") or "").strip()
if ai_cat_name and ai_cat_name.lower() != "none":
# Validate against known categories (case-insensitive)
matched = next((c for c in all_cats if c["name"].lower() == ai_cat_name.lower()), None)
logger.info(f"[impact_service] AI returned matched_category='{ai_cat_name}' for event {event_id}")
if ai_cat_name and ai_cat_name.lower() not in ("none", "null", ""):
# Exact match first, then normalized fuzzy match
matched = (
next((c for c in all_cats if c["name"].lower() == ai_cat_name.lower()), None)
or next((c for c in all_cats if _normalize(c["name"]) == _normalize(ai_cat_name)), None)
# Partial: AI name is contained in category name or vice versa
or next((c for c in all_cats
if _normalize(ai_cat_name) in _normalize(c["name"])
or _normalize(c["name"]) in _normalize(ai_cat_name)), None)
)
if matched:
matched_cat_name = matched["name"]
logger.info(f"[impact_service] AI matched event {event_id} → category '{matched_cat_name}'")
# Auto-update event sub_type if not already set
logger.info(f"[impact_service] Matched event {event_id} → '{matched_cat_name}'")
current_sub = (event.get("sub_type") or "").strip()
if not current_sub:
try:
from services.database import update_market_event
update_market_event(event_id, {**event, "sub_type": matched_cat_name,
"source_refs": event.get("source_refs", [])})
logger.info(f"[impact_service] Auto-set sub_type='{matched_cat_name}' on event {event_id}")
from services.database import update_market_event, get_all_market_events
# Re-fetch full event to get proper JSON fields
full_ev = next((e for e in get_all_market_events() if e["id"] == event_id), None)
if full_ev:
refs = full_ev.get("source_refs") or "[]"
if isinstance(refs, str):
refs = json.loads(refs)
update_market_event(event_id, {**full_ev, "sub_type": matched_cat_name,
"source_refs": refs})
logger.info(f"[impact_service] Auto-set sub_type='{matched_cat_name}' on event {event_id}")
except Exception as _e:
logger.warning(f"[impact_service] Could not auto-set sub_type: {_e}")
else:
logger.warning(f"[impact_service] AI returned unknown category '{ai_cat_name}' for event {event_id}")
logger.warning(f"[impact_service] No category match for AI response '{ai_cat_name}' (event {event_id})")
cat_label = matched_cat_name or (event.get("sub_type") or "").strip() or (event.get("category") or "")