Adding evaluation metrics¶
The metrics are defined in icu_benchmarks/models/constants.py, separately for classical models (MLMetrics, using
scikit-learn functions) and deep learning models (DLMetrics, using PyTorch Ignite and
TorchMetrics). Each class has one dictionary per kind of task:
| Dictionary | Used for |
|---|---|
BINARY_CLASSIFICATION |
Two classes |
MULTICLASS_CLASSIFICATION |
More than two classes |
REGRESSION |
Regression, in native target units |
IMPUTATION (DLMetrics only) |
Imputation models |
The current definitions:
from ignite.contrib.metrics import (
AveragePrecision,
ROC_AUC,
RocCurve,
PrecisionRecallCurve,
)
from ignite.metrics import Accuracy, MeanSquaredError, RootMeanSquaredError
from sklearn.calibration import calibration_curve
from sklearn.metrics import (
average_precision_score,
roc_auc_score,
accuracy_score,
balanced_accuracy_score,
mean_absolute_error,
mean_squared_error,
precision_recall_curve,
roc_curve,
r2_score,
root_mean_squared_error,
)
from torchmetrics.classification import (
AUROC,
AveragePrecision as TorchMetricsAveragePrecision,
PrecisionRecallCurve as TorchMetricsPrecisionRecallCurve,
CalibrationError,
F1Score,
)
from enum import Enum
from icu_benchmarks.models.custom_metrics import (
CalibrationCurve,
BalancedAccuracy,
MAE,
JSD,
BinaryFairnessWrapper,
confusion_matrix,
)
class MLMetrics:
BINARY_CLASSIFICATION = {
"AUC": roc_auc_score,
"Calibration_Curve": calibration_curve,
"PR": average_precision_score,
"PR_Curve": precision_recall_curve,
"RO_Curve": roc_curve,
"Confusion_Matrix": confusion_matrix,
}
MULTICLASS_CLASSIFICATION = {
"Accuracy": accuracy_score,
"AUC": roc_auc_score,
"Balanced_Accuracy": balanced_accuracy_score,
# "PR": average_precision_score,
"Confusion_Matrix": confusion_matrix,
}
REGRESSION = {
"MAE_native": mean_absolute_error,
"MSE_native": mean_squared_error,
"R2": r2_score,
"RMSE_native": root_mean_squared_error,
}
# TODO: add support for confusion matrix
class DLMetrics:
BINARY_CLASSIFICATION = {
"AUC": ROC_AUC,
"Calibration_Curve": CalibrationCurve,
"PR": AveragePrecision,
"PR_Curve": PrecisionRecallCurve,
"RO_Curve": RocCurve,
}
BINARY_CLASSIFICATION_TORCHMETRICS = {
"AUC": AUROC(task="binary"),
"PR": TorchMetricsAveragePrecision(task="binary"),
"PrecisionRecallCurve": TorchMetricsPrecisionRecallCurve(task="binary"),
"Calibration_Error": CalibrationError(task="binary", n_bins=10),
"F1": F1Score(task="binary", num_classes=2),
"Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task="demographic_parity", group_name="sex"),
}
MULTICLASS_CLASSIFICATION = {
"Accuracy": Accuracy,
"BalancedAccuracy": BalancedAccuracy,
}
REGRESSION = {
"MAE_native": MAE,
"MSE_native": MeanSquaredError,
}
IMPUTATION = {
"rmse": RootMeanSquaredError,
"mae": MAE,
"jsd": JSD,
}
class ImputationInit(str, Enum):
"""Type of initialization to use for the imputation model."""
NORMAL = "normal"
UNIFORM = "uniform"
XAVIER = "xavier"
KAIMING = "kaiming"
ORTHOGONAL = "orthogonal"
Adding a metric¶
To add a metric, add an entry to the dictionaries of the tasks it applies to. The key is the name under which the
metric is logged and saved in test_metrics.json.
- For classical models, the value is a function
metric(y_true, y_pred), as in scikit-learn. - For deep learning models, the value is an Ignite metric class or a TorchMetrics metric instance.
Custom metrics go in icu_benchmarks/models/custom_metrics.py. For example, the Jensen-Shannon divergence used for
imputation is an Ignite EpochMetric that wraps SciPy's jensenshannon:
class JSD(EpochMetric):
def __init__(
self,
output_transform: Callable = lambda x: x,
check_compute_fn: bool = False,
) -> None:
super(JSD, self).__init__(
lambda x, y: JSD_fn(x, y),
output_transform=output_transform,
check_compute_fn=check_compute_fn,
)
def JSD_fn(y_preds: torch.Tensor, y_targets: torch.Tensor):
return jensenshannon(abs(y_preds).flatten(), abs(y_targets).flatten()) ** 2
It is then listed in DLMetrics.IMPUTATION as "jsd": JSD.
Only metrics that return a single number are logged and averaged across folds. Curve metrics (PR_Curve, RO_Curve,
Calibration_Curve) are currently skipped when logging. For classical models, the confusion matrix is saved as one
count per cell, class_<true>_pred_<predicted>.
Fairness metrics¶
custom_metrics.py also contains BinaryFairnessWrapper, which computes TorchMetrics' group fairness metrics
(for example demographic parity) for a group variable such as sex. It is defined in
DLMetrics.BINARY_CLASSIFICATION_TORCHMETRICS, which the wrappers do not use by default.