Skip to content

Classical machine learning models

Wrappers around scikit-learn, LightGBM, XGBoost, CatBoost and imbalanced-learn estimators.

icu_benchmarks.models.ml_models.sklearn

icu_benchmarks.models.ml_models.lgbm

Classes:

Name Description
LGBMWrapper
LGBMClassifier

LGBMWrapper

LGBMWrapper(*args, run_mode=classification, loss=log_loss, patience=10, mps=False, **kwargs)

Bases: MLWrapper

Methods:

Name Description
fit_model

Fitting function for LGBM models.

Source code in icu_benchmarks/models/wrappers.py
def __init__(
    self,
    *args,
    run_mode=RunMode.classification,
    loss=log_loss,
    patience=10,
    mps=False,
    **kwargs,
):
    super().__init__()
    self.save_hyperparameters()
    self.check_supported_runmode(run_mode)
    self.run_mode = run_mode
    self.loss = loss
    self.patience = patience
    self.mps = mps
    self.loss_weight = None

fit_model

fit_model(train_data, train_labels, val_data, val_labels)

Fitting function for LGBM models.

Source code in icu_benchmarks/models/ml_models/lgbm.py
def fit_model(self, train_data, train_labels, val_data, val_labels):
    """Fitting function for LGBM models."""
    self.model.set_params(random_state=np.random.get_state()[1][0])
    callbacks = [
        lgbm.early_stopping(self.hparams.patience, verbose=False),
        lgbm.log_evaluation(period=-1),
    ]

    if wandb.run is not None:
        callbacks.append(wandb_lgbm())

    self.model = self.model.fit(
        train_data,
        train_labels,
        eval_set=(val_data, val_labels),
        callbacks=callbacks,
    )
    val_loss = list(self.model.best_score_["valid_0"].values())[0]
    return val_loss

LGBMClassifier

LGBMClassifier(*args, verbosity: int = -1, **kwargs)

Bases: LGBMWrapper

Methods:

Name Description
predict

Predicts class probabilities for the given features.

Source code in icu_benchmarks/models/ml_models/lgbm.py
def __init__(self, *args, verbosity: int = -1, **kwargs):
    self.model = self.set_model_args(lgbm.LGBMClassifier, *args, **kwargs)
    # verbosity is not in the LightGBM signature, so set_model_args would drop it.
    self.model.set_params(verbosity=verbosity)
    super().__init__(*args, **kwargs)

predict

predict(features)

Predicts class probabilities for the given features.

Parameters:

Name Type Description Default
features

Input features for prediction.

required

Returns:

Type Description

numpy.ndarray: Predicted probabilities for each class.

Source code in icu_benchmarks/models/ml_models/lgbm.py
def predict(self, features):
    """
    Predicts class probabilities for the given features.

    Args:
        features: Input features for prediction.

    Returns:
        numpy.ndarray: Predicted probabilities for each class.
    """
    return self.model.predict_proba(features)

icu_benchmarks.models.ml_models.xgboost

Classes:

Name Description
XGBClassifier

XGBClassifier

XGBClassifier(*args, device: str = 'cpu', verbosity: int = 0, **kwargs)

Bases: MLWrapper

Methods:

Name Description
resolve_device

Returns the XGBoost device; "auto" picks CUDA when available, and the --cpu flag always forces the CPU.

predict

Predicts class probabilities for the given features.

fit_model

Fit the model to the training data (default SKlearn syntax)

set_model_args

Pass every argument except the wrapper's own to XGBoost, which takes its hyperparameters as **kwargs.

Source code in icu_benchmarks/models/ml_models/xgboost.py
def __init__(self, *args, device: str = "cpu", verbosity: int = 0, **kwargs):
    device = self.resolve_device(device, cpu=kwargs.get("cpu", False))
    self.model = self.set_model_args(xgb.XGBClassifier, *args, **kwargs, device=device, verbosity=verbosity)
    super().__init__(*args, **kwargs)

resolve_device staticmethod

resolve_device(device: str, cpu: bool = False) -> str

Returns the XGBoost device; "auto" picks CUDA when available, and the --cpu flag always forces the CPU.

Source code in icu_benchmarks/models/ml_models/xgboost.py
@staticmethod
def resolve_device(device: str, cpu: bool = False) -> str:
    """Returns the XGBoost device; "auto" picks CUDA when available, and the --cpu flag always forces the CPU."""
    if cpu:
        if device not in ("cpu", "auto"):
            logging.warning(f"Ignoring XGBoost device={device!r} because the CPU was requested.")
        return "cpu"
    if device == "auto":
        return "cuda" if torch.cuda.is_available() else "cpu"
    return device

predict

predict(features)

Predicts class probabilities for the given features.

Parameters:

Name Type Description Default
features

Input features for prediction.

required

Returns:

Type Description

numpy.ndarray: Predicted probabilities for each class.

Source code in icu_benchmarks/models/ml_models/xgboost.py
def predict(self, features):
    """
    Predicts class probabilities for the given features.

    Args:
        features: Input features for prediction.

    Returns:
        numpy.ndarray: Predicted probabilities for each class.
    """
    return self.model.predict_proba(features)

fit_model

fit_model(train_data, train_labels, val_data, val_labels)

Fit the model to the training data (default SKlearn syntax)

Source code in icu_benchmarks/models/ml_models/xgboost.py
def fit_model(self, train_data, train_labels, val_data, val_labels):
    """Fit the model to the training data (default SKlearn syntax)"""
    callbacks = [EarlyStopping(self.hparams.patience)]

    if wandb.run is not None:
        # XGBoost >= 2.0 only accepts TrainingCallback instances; wandb_callback() returns a legacy function
        # gblinear only defines the "weight" feature importance that the callback logs after training
        importance_type = "weight" if self.model.get_params().get("booster") == "gblinear" else "gain"
        callbacks.append(WandbCallback(importance_type=importance_type))
    logging.debug(f"train_data: {train_data.shape}, train_labels: {train_labels.shape}")
    logging.debug(train_labels)
    # XGBoost >= 2.0 takes callbacks as an estimator parameter, not as a fit() argument.
    self.model.set_params(callbacks=callbacks)
    self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], verbose=False)
    # shap.summary_plot(shap_values, X_test, feature_names=features)
    # logging.info(self.model.get_booster().get_score(importance_type='weight'))
    # self.log_dict(self.model.get_booster().get_score(importance_type='weight'))
    # Return the first metric we use for validation
    eval_score = mean(next(iter(self.model.evals_result_["validation_0"].values())))
    return eval_score

set_model_args

set_model_args(model, *args, **kwargs)

Pass every argument except the wrapper's own to XGBoost, which takes its hyperparameters as **kwargs.

Booster-specific parameters (such as DART's rate_drop) are not in get_params(), so filtering against it would drop them. XGBoost warns about parameters it does not use.

Source code in icu_benchmarks/models/ml_models/xgboost.py
def set_model_args(self, model, *args, **kwargs):
    """Pass every argument except the wrapper's own to XGBoost, which takes its hyperparameters as **kwargs.

    Booster-specific parameters (such as DART's rate_drop) are not in get_params(), so filtering against it would
    drop them. XGBoost warns about parameters it does not use.
    """
    xgb_kwargs = {k: v for k, v in kwargs.items() if k not in self._wrapper_args}
    logging.debug(f"Creating model with: {xgb_kwargs}.")
    return model(**xgb_kwargs)

icu_benchmarks.models.ml_models.catboost

Classes:

Name Description
CBClassifier

CBClassifier

CBClassifier(task_type='CPU', *args, **kwargs)

Bases: MLWrapper

Methods:

Name Description
predict

Predicts class probabilities for the given features.

Source code in icu_benchmarks/models/ml_models/catboost.py
def __init__(self, task_type="CPU", *args, **kwargs):
    model_kwargs = {"task_type": task_type, **kwargs}
    self.model = self.set_model_args(cb.CatBoostClassifier, *args, **model_kwargs)
    super().__init__(*args, **kwargs)

predict

predict(features)

Predicts class probabilities for the given features.

Parameters:

Name Type Description Default
features

Input features for prediction.

required

Returns:

Type Description

numpy.ndarray: Predicted probabilities for each class.

Source code in icu_benchmarks/models/ml_models/catboost.py
def predict(self, features):
    """
    Predicts class probabilities for the given features.

    Args:
        features: Input features for prediction.

    Returns:
        numpy.ndarray: Predicted probabilities for each class.
    """
    return self.model.predict_proba(features)

icu_benchmarks.models.ml_models.imblearn