Skip to content

Adding evaluation metrics

The metrics are defined in icu_benchmarks/models/constants.py, separately for classical models (MLMetrics, using scikit-learn functions) and deep learning models (DLMetrics, using PyTorch Ignite and TorchMetrics). Each class has one dictionary per kind of task:

Dictionary Used for
BINARY_CLASSIFICATION Two classes
MULTICLASS_CLASSIFICATION More than two classes
REGRESSION Regression, in native target units
IMPUTATION (DLMetrics only) Imputation models

The current definitions:

icu_benchmarks/models/constants.py
from ignite.contrib.metrics import (
    AveragePrecision,
    ROC_AUC,
    RocCurve,
    PrecisionRecallCurve,
)
from ignite.metrics import Accuracy, MeanSquaredError, RootMeanSquaredError
from sklearn.calibration import calibration_curve
from sklearn.metrics import (
    average_precision_score,
    roc_auc_score,
    accuracy_score,
    balanced_accuracy_score,
    mean_absolute_error,
    mean_squared_error,
    precision_recall_curve,
    roc_curve,
    r2_score,
    root_mean_squared_error,
)
from torchmetrics.classification import (
    AUROC,
    AveragePrecision as TorchMetricsAveragePrecision,
    PrecisionRecallCurve as TorchMetricsPrecisionRecallCurve,
    CalibrationError,
    F1Score,
)
from enum import Enum
from icu_benchmarks.models.custom_metrics import (
    CalibrationCurve,
    BalancedAccuracy,
    MAE,
    JSD,
    BinaryFairnessWrapper,
    confusion_matrix,
)


class MLMetrics:
    BINARY_CLASSIFICATION = {
        "AUC": roc_auc_score,
        "Calibration_Curve": calibration_curve,
        "PR": average_precision_score,
        "PR_Curve": precision_recall_curve,
        "RO_Curve": roc_curve,
        "Confusion_Matrix": confusion_matrix,
    }

    MULTICLASS_CLASSIFICATION = {
        "Accuracy": accuracy_score,
        "AUC": roc_auc_score,
        "Balanced_Accuracy": balanced_accuracy_score,
        # "PR": average_precision_score,
        "Confusion_Matrix": confusion_matrix,
    }

    REGRESSION = {
        "MAE_native": mean_absolute_error,
        "MSE_native": mean_squared_error,
        "R2": r2_score,
        "RMSE_native": root_mean_squared_error,
    }


# TODO: add support for confusion matrix
class DLMetrics:
    BINARY_CLASSIFICATION = {
        "AUC": ROC_AUC,
        "Calibration_Curve": CalibrationCurve,
        "PR": AveragePrecision,
        "PR_Curve": PrecisionRecallCurve,
        "RO_Curve": RocCurve,
    }

    BINARY_CLASSIFICATION_TORCHMETRICS = {
        "AUC": AUROC(task="binary"),
        "PR": TorchMetricsAveragePrecision(task="binary"),
        "PrecisionRecallCurve": TorchMetricsPrecisionRecallCurve(task="binary"),
        "Calibration_Error": CalibrationError(task="binary", n_bins=10),
        "F1": F1Score(task="binary", num_classes=2),
        "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task="demographic_parity", group_name="sex"),
    }

    MULTICLASS_CLASSIFICATION = {
        "Accuracy": Accuracy,
        "BalancedAccuracy": BalancedAccuracy,
    }

    REGRESSION = {
        "MAE_native": MAE,
        "MSE_native": MeanSquaredError,
    }

    IMPUTATION = {
        "rmse": RootMeanSquaredError,
        "mae": MAE,
        "jsd": JSD,
    }


class ImputationInit(str, Enum):
    """Type of initialization to use for the imputation model."""

    NORMAL = "normal"
    UNIFORM = "uniform"
    XAVIER = "xavier"
    KAIMING = "kaiming"
    ORTHOGONAL = "orthogonal"

Adding a metric

To add a metric, add an entry to the dictionaries of the tasks it applies to. The key is the name under which the metric is logged and saved in test_metrics.json.

  • For classical models, the value is a function metric(y_true, y_pred), as in scikit-learn.
  • For deep learning models, the value is an Ignite metric class or a TorchMetrics metric instance.

Custom metrics go in icu_benchmarks/models/custom_metrics.py. For example, the Jensen-Shannon divergence used for imputation is an Ignite EpochMetric that wraps SciPy's jensenshannon:

icu_benchmarks/models/custom_metrics.py
class JSD(EpochMetric):
    def __init__(
        self,
        output_transform: Callable = lambda x: x,
        check_compute_fn: bool = False,
    ) -> None:
        super(JSD, self).__init__(
            lambda x, y: JSD_fn(x, y),
            output_transform=output_transform,
            check_compute_fn=check_compute_fn,
        )

        def JSD_fn(y_preds: torch.Tensor, y_targets: torch.Tensor):
            return jensenshannon(abs(y_preds).flatten(), abs(y_targets).flatten()) ** 2

It is then listed in DLMetrics.IMPUTATION as "jsd": JSD.

Only metrics that return a single number are logged and averaged across folds. Curve metrics (PR_Curve, RO_Curve, Calibration_Curve) are currently skipped when logging. For classical models, the confusion matrix is saved as one count per cell, class_<true>_pred_<predicted>.

Fairness metrics

custom_metrics.py also contains BinaryFairnessWrapper, which computes TorchMetrics' group fairness metrics (for example demographic parity) for a group variable such as sex. It is defined in DLMetrics.BINARY_CLASSIFICATION_TORCHMETRICS, which the wrappers do not use by default.