Code / scripts/ml_training/essay_scoring/training/evaluation.py
scripts/ml_training/essay_scoring/training/evaluation.py
1
"""Evaluation utilities for essay scoring models."""
2
3
from __future__ import annotations
4
5
from dataclasses import dataclass
6
7
import numpy as np
8
import pandas as pd
9
10
from ..training.qwk import (
11
clip_bands,
12
qwk_score,
13
round_to_half_band,
14
)
15
16
17
@dataclass(frozen=True)
18
class EvaluationResult:
19
"""Structured evaluation output."""
20
21
qwk: float
22
adjacent_accuracy: float
23
mean_absolute_error: float
24
per_band_mae: dict[str, float]
25
confusion_matrix: list[list[int]]
26
band_labels: list[float]
27
28
29
def evaluate_predictions(
30
y_true: np.ndarray,
31
y_pred: np.ndarray,
32
*,
33
min_band: float,
34
max_band: float,
35
) -> EvaluationResult:
36
"""Evaluate predictions with QWK and supporting diagnostics."""
37
38
rounded = clip_bands(round_to_half_band(y_pred), min_band=min_band, max_band=max_band)
39
qwk = qwk_score(y_true, y_pred, min_band=min_band, max_band=max_band)
40
adjacent_accuracy = float(np.mean(np.abs(rounded - y_true) <= 0.5))
41
mae = float(np.mean(np.abs(rounded - y_true)))
42
43
per_band_mae = _per_band_mae(y_true, rounded)
44
confusion_df = _confusion_matrix(y_true, rounded)
45
46
return EvaluationResult(
47
qwk=qwk,
48
adjacent_accuracy=adjacent_accuracy,
49
mean_absolute_error=mae,
50
per_band_mae=per_band_mae,
51
confusion_matrix=confusion_df.values.tolist(),
52
band_labels=[float(label) for label in confusion_df.index.tolist()],
53
)
54
55
56
def _per_band_mae(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
57
"""Compute MAE per band label."""
58
59
results: dict[str, float] = {}
60
for band in sorted(set(y_true)):
61
mask = y_true == band
62
if not np.any(mask):
63
continue
64
results[str(band)] = float(np.mean(np.abs(y_pred[mask] - y_true[mask])))
65
return results
66
67
68
def _confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray) -> pd.DataFrame:
69
"""Build a confusion matrix for rounded band predictions."""
70
71
labels = sorted(set(y_true))
72
return pd.crosstab(
73
pd.Series(y_true, name="Actual"),
74
pd.Series(y_pred, name="Predicted"),
75
rownames=["Actual"],
76
colnames=["Predicted"],
77
dropna=False,
78
).reindex(index=labels, columns=labels, fill_value=0)