Skip to content

Data

Loading, splitting and preprocessing the cohorts, and the datasets the models train on.

icu_benchmarks.data.split_process_data

Functions:

Name Description
select_label

The label to train on: the requested one, which must be configured, else the first of several configured labels.

preprocess_data

Perform loading, splitting, imputing and normalising of task data.

check_sanitize_data

Check for duplicates in the loaded data and remove them.

make_train_val_pandas

Randomly splits the data into training and validation sets for fitting a full model,

join_id_dtype

The dtype all segments' IDs are cast to for the split joins: integer IDs are widened, others (e.g. strings) kept.

make_train_val_polars

Randomly splits the data into training and validation sets for fitting a full model,

make_train_val

Randomly splits the data into training and validation sets for fitting a full model.

make_single_split_pandas

Randomly splits the data into training, validation, and test sets,

make_single_split_polars

Randomly splits the data into training, validation, and test set,

make_single_split

Randomly splits the data into training, validation, and test sets.

check_required_keys

Checks if all required keys are present in the vars dictionary.

select_label

select_label(vars: dict[str, Any], label: Optional[str] = None) -> Union[str, list[str]]

The label to train on: the requested one, which must be configured, else the first of several configured labels.

Source code in icu_benchmarks/data/split_process_data.py
def select_label(vars: dict[str, Any], label: Optional[str] = None) -> Union[str, list[str]]:
    """The label to train on: the requested one, which must be configured, else the first of several configured labels."""
    configured_labels = vars[VarType.label] if isinstance(vars[VarType.label], list) else [vars[VarType.label]]
    if label is not None:
        if label not in configured_labels:
            raise ValueError(f"Label {label} is not one of the configured labels: {configured_labels}.")
        return label
    if len(configured_labels) > 1:
        logging.debug(f"Multiple labels found and no value provided. Using first label: {configured_labels}")
        return configured_labels[0]
    return vars[VarType.label]

preprocess_data

preprocess_data(data_dir: Path, file_names: dict[str, str] | Any = REQUIRED, preprocessor: type[PolarsClassificationPreprocessor | PolarsRegressionPreprocessor] = PolarsClassificationPreprocessor, use_static: bool = True, vars: dict[str, str | list[str]] | Any = REQUIRED, modality_mapping: Optional[dict[str, list[str]]] = None, selected_modalities: Optional[list[str]] = None, seed: int = 42, debug: bool = False, cv_repetitions: int = 5, repetition_index: int = 0, cv_folds: int = 5, train_size: Optional[float] = None, load_cache: bool = False, generate_cache: bool = False, fold_index: int = 0, pretrained_imputation_model: Optional[str] = None, complete_train: bool = False, runmode: RunMode = classification, label: Optional[str] = None, required_var_types: Optional[list[str]] = None, required_segments: Optional[list[str]] = None, target_transform: Optional[TargetTransform] = None) -> tuple[dict[str, dict[str, DataFrame]], Optional[TargetTransform]]

Perform loading, splitting, imputing and normalising of task data.

Parameters:

Name Type Description Default
use_static bool

Whether to use static features (for DL models).

True
complete_train bool

Whether to use all data for training/validation.

False
runmode RunMode

Run mode. Can be one of the values of RunMode

classification
preprocessor type[PolarsClassificationPreprocessor | PolarsRegressionPreprocessor]

Define the preprocessor.

PolarsClassificationPreprocessor
data_dir Path

Path to the directory holding the data.

required
file_names dict[str, str] | Any

Contains the parquet file names in data_dir.

REQUIRED
vars dict[str, str | list[str]] | Any

Contains the names of columns in the data.

REQUIRED
seed int

Random seed.

42
debug bool

Load less data if true.

False
cv_repetitions int

Number of times to repeat cross validation.

5
repetition_index int

Index of the repetition to return.

0
cv_folds int

Number of folds to use for cross validation.

5
train_size Optional[float]

Fixed size of train split (including validation data).

None
load_cache bool

Use cached preprocessed data if true.

False
generate_cache bool

Generate cached preprocessed data if true.

False
fold_index int

Index of the fold to return.

0
pretrained_imputation_model Optional[str]

pretrained imputation model to use. if None, standard imputation is used.

None

Returns:

Type Description
dict[str, dict[str, DataFrame]]

(Preprocessed split data, fitted regression target transform or None).

Optional[TargetTransform]

A supplied target_transform is reused without fitting (evaluation/fine-tuning).

Source code in icu_benchmarks/data/split_process_data.py
@gin.configurable("preprocess")
def preprocess_data(
    data_dir: Path,
    file_names: dict[str, str] | Any = gin.REQUIRED,
    preprocessor: type[PolarsClassificationPreprocessor | PolarsRegressionPreprocessor] = PolarsClassificationPreprocessor,
    use_static: bool = True,
    vars: dict[str, str | list[str]] | Any = gin.REQUIRED,
    modality_mapping: Optional[dict[str, list[str]]] = None,
    selected_modalities: Optional[list[str]] = None,
    seed: int = 42,
    debug: bool = False,
    cv_repetitions: int = 5,
    repetition_index: int = 0,
    cv_folds: int = 5,
    train_size: Optional[float] = None,
    load_cache: bool = False,
    generate_cache: bool = False,
    fold_index: int = 0,
    pretrained_imputation_model: Optional[str] = None,
    complete_train: bool = False,
    runmode: RunMode = RunMode.classification,
    label: Optional[str] = None,
    required_var_types: Optional[list[str]] = None,
    required_segments: Optional[list[str]] = None,
    target_transform: Optional[TargetTransform] = None,
) -> tuple[dict[str, dict[str, pl.DataFrame]], Optional[TargetTransform]]:
    """
    Perform loading, splitting, imputing and normalising of task data.

    Args:
        use_static: Whether to use static features (for DL models).
        complete_train: Whether to use all data for training/validation.
        runmode: Run mode. Can be one of the values of RunMode
        preprocessor: Define the preprocessor.
        data_dir: Path to the directory holding the data.
        file_names: Contains the parquet file names in data_dir.
        vars: Contains the names of columns in the data.
        seed: Random seed.
        debug: Load less data if true.
        cv_repetitions: Number of times to repeat cross validation.
        repetition_index: Index of the repetition to return.
        cv_folds: Number of folds to use for cross validation.
        train_size: Fixed size of train split (including validation data).
        load_cache: Use cached preprocessed data if true.
        generate_cache: Generate cached preprocessed data if true.
        fold_index: Index of the fold to return.
        pretrained_imputation_model: pretrained imputation model to use. if None, standard imputation is used.

    Returns:
        (Preprocessed split data, fitted regression target transform or None).
        A supplied target_transform is reused without fitting (evaluation/fine-tuning).
    """
    if modality_mapping is None:
        modality_mapping = {}
    if selected_modalities is None:
        selected_modalities = ["all"]
    if required_var_types is None:
        required_var_types = ["GROUP", "SEQUENCE", "LABEL"]
    if required_segments is None:
        required_segments = [
            DataSegment.static,
            DataSegment.dynamic,
            DataSegment.outcome,
        ]

    check_required_keys(vars, required_var_types)
    check_required_keys(file_names, required_segments)

    if not use_static:
        file_names.pop(DataSegment.static)
        vars.pop(DataSegment.static)

    selected_label = select_label(vars, label)
    if selected_label != vars[VarType.label]:
        vars[VarType.label] = selected_label
        logging.info(f"Using label: {vars[VarType.label]}")

    if not vars[VarType.label]:
        raise ValueError("No label selected after filtering.")
    if runmode == RunMode.regression and isinstance(vars[VarType.label], list):
        vars[VarType.label] = vars[VarType.label][0]

    dumped_file_names = json.dumps(file_names, sort_keys=True)
    dumped_vars = json.dumps(vars, sort_keys=True)
    # The modality selection changes vars and the excluded variables later on, so it must be part of the cache key
    dumped_modalities = json.dumps({"mapping": modality_mapping, "selected": selected_modalities}, sort_keys=True)

    logging.info(f"Using preprocessor: {preprocessor.__name__}")

    cat_clinical_notes = modality_mapping.get("cat_clinical_notes")
    cat_med_embeddings_map = modality_mapping.get("cat_med_embeddings_map")
    if cat_clinical_notes is not None and cat_med_embeddings_map is not None:
        vars_to_exclude = cat_clinical_notes + cat_med_embeddings_map
    else:
        vars_to_exclude = None

    cache_dir = data_dir / "cache"
    cache_filename = (
        f"v2_s_{seed}_r_{repetition_index}of{cv_repetitions}_f_{fold_index}of{cv_folds}"
        f"_t_{train_size}_d_{debug}_full_{complete_train}"
    )
    preprocessor_instance: Preprocessor = preprocessor(
        use_static_features=use_static,
        vars_to_exclude=vars_to_exclude,
    )
    if isinstance(preprocessor_instance, PandasClassificationPreprocessor):
        preprocessor_instance.set_imputation_model(pretrained_imputation_model)

    preprocessor_instance.target_transform = target_transform
    hash_config = hashlib.md5(
        f"{preprocessor_instance.to_cache_string()}{dumped_file_names}{dumped_vars}{dumped_modalities}{runmode}"
        f"{target_transform}".encode("utf-8")
    )
    cache_filename += f"_{hash_config.hexdigest()}"
    cache_file = cache_dir / cache_filename
    if generate_cache:
        preprocessor_instance.save_cache = data_dir / "preproc" / (cache_filename + "_recipe")

    if load_cache:
        if cache_file.exists():
            with open(cache_file, "rb") as f:
                logging.info(f"Loading cached data from {cache_file}.")
                return pickle.load(f)
        else:
            logging.info(f"No cached data found in {cache_file}, loading raw features.")

    # Read parquet files into dataframes and remove the parquet file from memory
    logging.info(f"Loading data from directory {data_dir.absolute()}")
    data: dict[str, pl.DataFrame] = {
        f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys() if os.path.exists(data_dir / file_names[f])
    }

    logging.info(f"Loaded data: {list(data.keys())}")
    sanitized_data = check_sanitize_data(data, vars)

    if DataSegment.dynamic not in sanitized_data.keys():
        logging.warning("No dynamic data found, using only static data.")

    logging.debug(f"Modality mapping: {modality_mapping}")
    if len(modality_mapping) > 0:
        # Optional modality selection
        if selected_modalities not in [None, "all", ["all"]]:
            data, vars = modality_selection(sanitized_data, modality_mapping, selected_modalities, vars)
        else:
            logging.info("Selecting all modalities.")

    # Generate the splits
    logging.info("Generating splits.")
    if not complete_train:
        sanitized_data = make_single_split_polars(
            sanitized_data,
            vars,
            cv_repetitions,
            repetition_index,
            cv_folds,
            fold_index,
            train_size=train_size,
            seed=seed,
            debug=debug,
            runmode=runmode,
        )
    else:
        # If full train is set, we use all data for training/validation
        sanitized_data = make_train_val_polars(
            sanitized_data,
            vars,
            train_size=train_size,
            seed=seed,
            debug=debug,
            runmode=runmode,
        )

    # Apply preprocessing
    start = timer()
    sanitized_data = preprocessor_instance.apply(sanitized_data, vars)
    end = timer()
    logging.info(f"Preprocessing took {end - start:.2f} seconds.")
    logging.info(f"Checking for NaNs and nulls in {data.keys()}.")
    for _dict in sanitized_data.values():
        for key, val in _dict.items():
            # Preserve missing labels for dataset masks/filtering, rather than inventing zero targets or negatives.
            if key == DataSegment.outcome:
                continue
            logging.debug(f"Data type: {key}")
            logging.debug("Is NaN:")
            sel = _dict[key].select(pl.selectors.numeric().is_nan().max())
            logging.debug(sel.select(col.name for col in sel if col.item(0)))
            logging.debug("Has nulls:")
            sel = _dict[key].select(pl.all().has_nulls())
            logging.debug(sel.select(col.name for col in sel if col.item(0)))
            _dict[key] = val.fill_null(strategy="zero")
            _dict[key] = _dict[key].fill_nan(0)
            logging.debug("Dropping columns with nulls")
            sel = _dict[key].select(pl.all().has_nulls())
            logging.debug(sel.select(col.name for col in sel if col.item(0)))

    # Generate cache
    if generate_cache:
        caching(cache_dir, cache_file, (sanitized_data, preprocessor_instance.target_transform))
    else:
        logging.info("Cache will not be saved.")

    logging.info("Finished preprocessing.")

    return sanitized_data, preprocessor_instance.target_transform

check_sanitize_data

check_sanitize_data(data: dict[str, DataFrame], vars: dict[str, str | list[str]]) -> dict[str, DataFrame]

Check for duplicates in the loaded data and remove them.

Source code in icu_benchmarks/data/split_process_data.py
def check_sanitize_data(data: dict[str, pl.DataFrame], vars: dict[str, str | list[str]]) -> dict[str, pl.DataFrame]:
    """Check for duplicates in the loaded data and remove them."""
    group: Optional[Union[str, list[str]]] = vars.get(VarType.group)
    sequence: Optional[Union[str, list[str]]] = vars.get(VarType.sequence)
    keep = "last"
    if DataSegment.static in data.keys():
        old_len = len(data[DataSegment.static])
        data[DataSegment.static] = data[DataSegment.static].unique(subset=group, keep=keep, maintain_order=True)
        if old_len != len(data[DataSegment.static]):
            logging.warning(f"Removed {old_len - len(data[DataSegment.static])} duplicates from static data.")
    if DataSegment.dynamic in data.keys():
        old_len = len(data[DataSegment.dynamic])

        data[DataSegment.dynamic] = data[DataSegment.dynamic].unique(
            subset=flatten_column_names(group, sequence), keep=keep, maintain_order=True
        )
        if old_len != len(data[DataSegment.dynamic]):
            logging.warning(f"Removed {old_len - len(data[DataSegment.dynamic])} duplicates from dynamic data.")
    if DataSegment.outcome in data.keys():
        old_len = len(data[DataSegment.outcome])
        if sequence in data[DataSegment.outcome].columns:
            # We have a dynamic outcome with group and sequence
            data[DataSegment.outcome] = data[DataSegment.outcome].unique(
                subset=flatten_column_names(group, sequence),
                keep=keep,
                maintain_order=True,
            )
        else:
            data[DataSegment.outcome] = data[DataSegment.outcome].unique(subset=group, keep=keep, maintain_order=True)
        if old_len != len(data[DataSegment.outcome]):
            logging.warning(f"Removed {old_len - len(data[DataSegment.outcome])} duplicates from outcome data.")
    return data

make_train_val_pandas

make_train_val_pandas(data: dict[str, DataFrame], vars: dict[str, Union[str, list[str]]], train_size: Optional[float] = 0.8, seed: int = 42, debug: bool = False, runmode: RunMode = classification) -> dict[str, dict[str, DataFrame]]

Randomly splits the data into training and validation sets for fitting a full model, specifically designed for Pandas DataFrames.

For a more detailed documentation refer to make_train_val(...)

Source code in icu_benchmarks/data/split_process_data.py
def make_train_val_pandas(
    data: dict[str, pd.DataFrame],
    vars: dict[str, Union[str, list[str]]],
    train_size: Optional[float] = 0.8,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
) -> dict[str, dict[str, pd.DataFrame]]:
    """
    Randomly splits the data into training and validation sets for fitting a full model,
    specifically designed for Pandas DataFrames.

    For a more detailed documentation refer to make_train_val(...)
    """
    _id = vars[VarType.group]
    label = vars[VarType.label]
    if not (isinstance(_id, str) and isinstance(label, str)):
        raise TypeError(
            f'Expected keys "{VarType.group}" and "{VarType.label}" to be of type str, '
            f"got {type(_id)} and {type(label)} instead."
        )

    if debug:
        logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.")
        data[DataSegment.outcome] = data[DataSegment.outcome].sample(frac=0.01, random_state=seed)

    stays = data[DataSegment.outcome][_id].unique()

    if VarType.label in vars and runmode is RunMode.classification:
        labels = data[DataSegment.outcome].groupby(_id)[label].max()
        train_val_splitter = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1)
        train_indices, val_indices = list(train_val_splitter.split(stays, labels))[0]
    else:
        train_val_splitter = ShuffleSplit(train_size=train_size, random_state=seed, n_splits=1)
        train_indices, val_indices = list(train_val_splitter.split(stays))[0]

    split_ids = {
        DataSplit.train: pd.DataFrame({_id: stays[train_indices]}),
        DataSplit.val: pd.DataFrame({_id: stays[val_indices]}),
    }

    data_split: dict[str, dict[str, pd.DataFrame]] = {}

    for fold in split_ids.keys():
        data_split[fold] = {}
        for data_type in data.keys():
            merged_df = data[data_type].merge(split_ids[fold], on=_id, how="right", sort=True)
            data_split[fold][data_type] = merged_df

    data_split[DataSplit.test] = copy.deepcopy(data_split[DataSplit.val])
    return data_split

join_id_dtype

join_id_dtype(stays: Series, data: dict[str, DataFrame], id_col: str) -> DataType

The dtype all segments' IDs are cast to for the split joins: integer IDs are widened, others (e.g. strings) kept.

Integer IDs in one file and string IDs in another would not match after the cast ("001" is not "1"), so the join would silently drop those stays' features; float and string IDs can also collapse ("1" and "01" are both 1.0). Such mixes are rejected instead.

Source code in icu_benchmarks/data/split_process_data.py
def join_id_dtype(stays: pl.Series, data: dict[str, pl.DataFrame], id_col: str) -> pl.DataType:
    """The dtype all segments' IDs are cast to for the split joins: integer IDs are widened, others (e.g. strings) kept.

    Integer IDs in one file and string IDs in another would not match after the cast ("001" is not "1"), so the join
    would silently drop those stays' features; float and string IDs can also collapse ("1" and "01" are both 1.0).
    Such mixes are rejected instead.
    """
    for segment, frame in data.items():
        dtype = frame.schema[id_col]
        if dtype.is_integer() != stays.dtype.is_integer() or dtype.is_numeric() != stays.dtype.is_numeric():
            raise ValueError(
                f"Stay IDs ({id_col}) are {stays.dtype} in the outcome data but {dtype} in the {segment} "
                "data; use the same ID type in every file."
            )
    return pl.datatypes.Int64 if stays.dtype.is_integer() else stays.dtype

make_train_val_polars

make_train_val_polars(data: dict[str, DataFrame], vars: dict[str, Union[str, list[str]]], train_size: Optional[float] = 0.8, seed: int = 42, debug: bool = False, runmode: RunMode = classification) -> dict[str, dict[str, DataFrame]]

Randomly splits the data into training and validation sets for fitting a full model, specifically designed for Polars DataFrames.

For a more detailed documentation refer to make_train_val(...)

Source code in icu_benchmarks/data/split_process_data.py
def make_train_val_polars(
    data: dict[str, pl.DataFrame],
    vars: dict[str, Union[str, list[str]]],
    train_size: Optional[float] = 0.8,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
) -> dict[str, dict[str, pl.DataFrame]]:
    """
    Randomly splits the data into training and validation sets for fitting a full model,
    specifically designed for Polars DataFrames.

    For a more detailed documentation refer to make_train_val(...)
    """
    _id = vars[VarType.group]
    label = vars[VarType.label]
    if not (isinstance(_id, str) and isinstance(label, str)):
        raise TypeError(
            f'Expected keys "{VarType.group}" and "{VarType.label}" to be of type str, '
            f"got {type(_id)} and {type(label)} instead."
        )

    if debug:
        logging.info("Using only 1% of the stay_id's for debugging. Note that this might lead to errors for small datasets.")
        sampled_ids = data[DataSegment.outcome][_id].unique().sample(fraction=0.01, seed=seed)
        data[DataSegment.outcome] = data[DataSegment.outcome].filter(pl.col(_id).is_in(sampled_ids))
        if DataSegment.dynamic in data:
            data[DataSegment.dynamic] = data[DataSegment.dynamic].filter(pl.col(_id).is_in(sampled_ids))
        if DataSegment.static in data:
            data[DataSegment.static] = data[DataSegment.static].filter(pl.col(_id).is_in(sampled_ids))

    # Sort the stays and their labels the same way, so the split is reproducible and the labels line up with the stays
    stays = pl.Series(name=_id, values=data[DataSegment.outcome][_id].unique()).sort()

    if VarType.label in vars and runmode is RunMode.classification:
        labels = data[DataSegment.outcome].group_by(_id).max().sort(_id)[label]
        train_val_splitter = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1)
        train_indices, val_indices = list(train_val_splitter.split(stays, labels))[0]
    else:
        train_val_splitter = ShuffleSplit(train_size=train_size, random_state=seed, n_splits=1)
        train_indices, val_indices = list(train_val_splitter.split(stays))[0]

    id_dtype = join_id_dtype(stays, data, _id)
    split = {
        DataSplit.train: stays[train_indices].cast(id_dtype).to_frame(),
        DataSplit.val: stays[val_indices].cast(id_dtype).to_frame(),
    }

    data_split: dict[str, dict[str, pl.DataFrame]] = {}

    for fold in split.keys():
        data_split[fold] = {
            data_type: split[fold]
            .join(
                data[data_type].with_columns(pl.col(_id).cast(id_dtype)),
                on=_id,
                how="left",
            )
            # Within a stay, keep the rows in time order: the recipe steps (fills, historical features) are row-ordered
            .sort(by=[_id, *_time_sort_key(data[data_type], vars)], maintain_order=True)
            for data_type in data.keys()
        }

    data_split[DataSplit.test] = copy.deepcopy(data_split[DataSplit.val])
    return data_split

make_train_val

make_train_val(data: dict[str, Union[DataFrame, DataFrame]], vars: dict[str, Union[str, list[str]]], train_size: Optional[float] = 0.8, seed: int = 42, debug: bool = False, runmode: RunMode = classification, polars: bool = True) -> dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

Randomly splits the data into training and validation sets for fitting a full model. Dispatches to either a Polars or Pandas backend based on the 'polars' flag.

Parameters:

Name Type Description Default
data dict[str, Union[DataFrame, DataFrame]]

A dictionary containing DataFrames (either Pandas or Polars), divided into segments like OUTCOME, STATIC, and DYNAMIC.

required
vars dict[str, Union[str, list[str]]]

A dictionary containing the names of columns (variables) in the data.

required
train_size Optional[float]

The proportion of the dataset to include in the train split.

0.8
seed int

Random seed for reproducibility.

42
debug bool

If True, uses only a small fraction (1%) of the data for debugging.

False
runmode RunMode

The type of machine learning task (e.g., classification, regression).

classification
polars bool

If True, uses the Polars backend; otherwise, uses the Pandas backend.

True

Returns:

Type Description
dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

A dictionary containing the input data divided into 'train', 'val', and 'test'

dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

splits. Each split is itself a dictionary of DataFrames (Pandas or Polars)

dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

corresponding to the original data segments.

Source code in icu_benchmarks/data/split_process_data.py
def make_train_val(
    data: dict[str, Union[pd.DataFrame, pl.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
    train_size: Optional[float] = 0.8,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
    polars: bool = True,
) -> dict[str, dict[str, pl.DataFrame]] | dict[str, dict[str, pd.DataFrame]]:
    """
    Randomly splits the data into training and validation sets for fitting a full model.
    Dispatches to either a Polars or Pandas backend based on the 'polars' flag.

    Args:
        data: A dictionary containing DataFrames (either Pandas or Polars),
              divided into segments like OUTCOME, STATIC, and DYNAMIC.
        vars: A dictionary containing the names of columns (variables) in the data.
        train_size: The proportion of the dataset to include in the train split.
        seed: Random seed for reproducibility.
        debug: If True, uses only a small fraction (1%) of the data for debugging.
        runmode: The type of machine learning task (e.g., classification, regression).
        polars: If True, uses the Polars backend; otherwise, uses the Pandas backend.

    Returns:
        A dictionary containing the input data divided into 'train', 'val', and 'test'
        splits. Each split is itself a dictionary of DataFrames (Pandas or Polars)
        corresponding to the original data segments.
    """
    if polars:
        polars_data = {k: v if isinstance(v, pl.DataFrame) else pl.DataFrame(v) for k, v in data.items()}
        return make_train_val_polars(polars_data, vars, train_size, seed, debug, runmode)
    else:
        pandas_data = {k: v if isinstance(v, pd.DataFrame) else v.to_pandas() for k, v in data.items()}
        return make_train_val_pandas(pandas_data, vars, train_size, seed, debug, runmode)

make_single_split_pandas

make_single_split_pandas(data: dict[str, DataFrame], vars: dict[str, Union[str, list[str]]], cv_repetitions: int, repetition_index: int, cv_folds: int, fold_index: int, train_size: Optional[float] = None, seed: int = 42, debug: bool = False, runmode: RunMode = classification) -> dict[str, dict[str, DataFrame]]

Randomly splits the data into training, validation, and test sets, specifically designed for Pandas DataFrames.

For a more detailed documentation refer to make_single_splits(...)

Source code in icu_benchmarks/data/split_process_data.py
def make_single_split_pandas(
    data: dict[str, pd.DataFrame],
    vars: dict[str, Union[str, list[str]]],
    cv_repetitions: int,
    repetition_index: int,
    cv_folds: int,
    fold_index: int,
    train_size: Optional[float] = None,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
) -> dict[str, dict[str, pd.DataFrame]]:
    """
    Randomly splits the data into training, validation, and test sets,
    specifically designed for Pandas DataFrames.

    For a more detailed documentation refer to make_single_splits(...)
    """
    _id = vars[VarType.group]
    label = vars[VarType.label]

    if not (isinstance(_id, str) and isinstance(label, str)):
        raise TypeError(
            f'Expected keys "{VarType.group}" and "{VarType.label}" to be of type str, '
            f"got {type(_id)} and {type(label)} instead."
        )

    if debug:
        logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.")
        data[DataSegment.outcome] = data[DataSegment.outcome].sample(frac=0.01, random_state=seed)

    stays = data[DataSegment.outcome][_id].unique()

    if VarType.label in vars and runmode is RunMode.classification:
        labels = data[DataSegment.outcome].groupby(_id)[label].max().reset_index(drop=True)
        if labels.value_counts().min() < cv_folds:
            raise Exception(
                f"The smallest amount of samples in a class is: {labels.value_counts().min()}, "
                f"but {cv_folds} folds are requested. Reduce the number of folds or use more data."
            )

        if train_size:
            outer_cv = StratifiedShuffleSplit(cv_repetitions, train_size=train_size, random_state=seed)
        else:
            outer_cv = StratifiedKFold(cv_repetitions, shuffle=True, random_state=seed)
        inner_cv = StratifiedKFold(cv_folds, shuffle=True, random_state=seed)

        dev_indices, test_indices = list(outer_cv.split(stays, labels))[repetition_index]
        dev_stays = stays[dev_indices]
        train_indices, val_indices = list(inner_cv.split(dev_stays, labels[dev_indices]))[fold_index]
    else:
        if train_size:
            outer_cv = ShuffleSplit(cv_repetitions, train_size=train_size, random_state=seed)
        else:
            outer_cv = KFold(cv_repetitions, shuffle=True, random_state=seed)
        inner_cv = KFold(cv_folds, shuffle=True, random_state=seed)

        dev_indices, test_indices = list(outer_cv.split(stays))[repetition_index]
        dev_stays = stays[dev_indices]
        train_indices, val_indices = list(inner_cv.split(dev_stays))[fold_index]

    split_ids = {
        DataSplit.train: pd.DataFrame({_id: dev_stays[train_indices]}),
        DataSplit.val: pd.DataFrame({_id: dev_stays[val_indices]}),
        DataSplit.test: pd.DataFrame({_id: stays[test_indices]}),
    }

    data_split: dict[str, dict[str, pd.DataFrame]] = {}

    for fold in split_ids.keys():
        data_split[fold] = {}
        for data_type in data.keys():
            merged_df = data[data_type].merge(split_ids[fold], on=_id, how="right", sort=True)
            data_split[fold][data_type] = merged_df

    logging.debug(f"Data split: {data_split}")
    return data_split

make_single_split_polars

make_single_split_polars(data: dict[str, DataFrame], vars: dict[str, Union[str, list[str]]], cv_repetitions: int, repetition_index: int, cv_folds: int, fold_index: int, train_size: Optional[float] = None, seed: int = 42, debug: bool = False, runmode: RunMode = classification) -> dict[str, dict[str, DataFrame]]

Randomly splits the data into training, validation, and test set, specifically designed for Polars DataFrames.

For a more detailed documentation refer to make_single_splits(...)

Source code in icu_benchmarks/data/split_process_data.py
def make_single_split_polars(
    data: dict[str, pl.DataFrame],
    vars: dict[str, Union[str, list[str]]],
    cv_repetitions: int,
    repetition_index: int,
    cv_folds: int,
    fold_index: int,
    train_size: Optional[float] = None,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
) -> dict[str, dict[str, pl.DataFrame]]:
    """
    Randomly splits the data into training, validation, and test set,
    specifically designed for Polars DataFrames.

    For a more detailed documentation refer to make_single_splits(...)
    """
    # ID variable
    _id = vars[VarType.group]
    if debug:
        logging.info("Using only 1% of the stay_id's for debugging. Note that this might lead to errors for small datasets.")
        sampled_ids = data[DataSegment.outcome][_id].unique().sample(fraction=0.01, seed=seed)
        data[DataSegment.outcome] = data[DataSegment.outcome].filter(pl.col(_id).is_in(sampled_ids))
        if DataSegment.dynamic in data:
            data[DataSegment.dynamic] = data[DataSegment.dynamic].filter(pl.col(_id).is_in(sampled_ids))
        if DataSegment.static in data:
            data[DataSegment.static] = data[DataSegment.static].filter(pl.col(_id).is_in(sampled_ids))

    # Get stay IDs from outcome segment
    stays = pl.Series(name=_id, values=data[DataSegment.outcome][_id].unique()).sort()
    # If there are labels, and the task is classification, use stratified k-fold
    if VarType.label in vars and runmode is RunMode.classification:
        # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification)
        labels: pl.Series = data[DataSegment.outcome].group_by(_id).max().sort(_id)[vars[VarType.label]]
        if labels.value_counts().min().item(0, 1) < cv_folds:
            raise Exception(
                f"The smallest amount of samples in a class is: {labels.value_counts().min()}, "
                f"but {cv_folds} folds are requested. Reduce the number of folds or use more data."
            )

        if train_size:
            outer_cv = StratifiedShuffleSplit(cv_repetitions, train_size=train_size, random_state=seed)
        else:
            outer_cv = StratifiedKFold(cv_repetitions, shuffle=True, random_state=seed)

        inner_cv = StratifiedKFold(cv_folds, shuffle=True, random_state=seed)
        dev, test = list(outer_cv.split(stays, labels))[repetition_index]
        dev_stays = stays[dev]
        train, val = list(inner_cv.split(dev_stays, labels[dev]))[fold_index]

    else:
        # If there are no labels, or the task is regression, use regular k-fold.
        if train_size:
            outer_cv = ShuffleSplit(cv_repetitions, train_size=train_size, random_state=seed)
        else:
            outer_cv = KFold(cv_repetitions, shuffle=True, random_state=seed)
        inner_cv = KFold(cv_folds, shuffle=True, random_state=seed)
        dev, test = list(outer_cv.split(stays))[repetition_index]
        dev_stays = stays[dev]
        train, val = list(inner_cv.split(dev_stays))[fold_index]

    id_dtype = join_id_dtype(stays, data, _id)
    split = {
        DataSplit.train: dev_stays[train].cast(id_dtype).to_frame(),
        DataSplit.val: dev_stays[val].cast(id_dtype).to_frame(),
        DataSplit.test: stays[test].cast(id_dtype).to_frame(),
    }

    data_split = {}
    for fold in split.keys():  # Loop through splits (train / val / test)
        # Loop through segments (DYNAMIC / STATIC / OUTCOME)
        # set sort to true to make sure that IDs are reordered after scrambling earlier
        data_split[fold] = {
            data_type: split[fold]
            .join(
                data[data_type].with_columns(pl.col(_id).cast(id_dtype)),
                on=_id,
                how="left",
            )
            # Within a stay, keep the rows in time order: the recipe steps (fills, historical features) are row-ordered
            .sort(by=[_id, *_time_sort_key(data[data_type], vars)], maintain_order=True)
            for data_type in data.keys()
        }

    logging.debug(f"Data split: {data_split}")

    return data_split

make_single_split

make_single_split(data: dict[str, Union[DataFrame, DataFrame]], vars: dict[str, Union[str, list[str]]], cv_repetitions: int, repetition_index: int, cv_folds: int, fold_index: int, train_size: Optional[float] = None, seed: int = 42, debug: bool = False, runmode: RunMode = classification, polars: bool = True) -> dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

Randomly splits the data into training, validation, and test sets. Dispatches to either a Polars or Pandas backend based on the 'polars' flag.

Parameters:

Name Type Description Default
data dict[str, Union[DataFrame, DataFrame]]

A dictionary containing DataFrames (either Pandas or Polars), divided into segments like OUTCOME, STATIC, and DYNAMIC.

required
vars dict[str, Union[str, list[str]]]

Contains the names of columns in the data.

required
cv_repetitions int

Number of times to repeat cross validation.

required
repetition_index int

Index of the repetition to return.

required
cv_folds int

Number of folds for cross validation.

required
fold_index int

Index of the fold to return.

required
train_size Optional[float]

Fixed size of train split (including validation data).

None
seed int

Random seed.

42
debug bool

Load less data if true.

False
runmode RunMode

Run mode. Can be one of the values of RunMode

classification
polars bool

If True, uses the Polars backend; otherwise, uses the Pandas backend.

True

Returns:

Type Description
dict[str, dict[str, DataFrame]] | dict[str, dict[str, DataFrame]]

Input data divided into 'train', 'val', and 'test'.

Source code in icu_benchmarks/data/split_process_data.py
def make_single_split(
    data: dict[str, Union[pd.DataFrame, pl.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
    cv_repetitions: int,
    repetition_index: int,
    cv_folds: int,
    fold_index: int,
    train_size: Optional[float] = None,
    seed: int = 42,
    debug: bool = False,
    runmode: RunMode = RunMode.classification,
    polars: bool = True,
) -> dict[str, dict[str, pl.DataFrame]] | dict[str, dict[str, pd.DataFrame]]:
    """
    Randomly splits the data into training, validation, and test sets.
    Dispatches to either a Polars or Pandas backend based on the 'polars' flag.

    Args:
        data: A dictionary containing DataFrames (either Pandas or Polars),
              divided into segments like OUTCOME, STATIC, and DYNAMIC.
        vars: Contains the names of columns in the data.
        cv_repetitions: Number of times to repeat cross validation.
        repetition_index: Index of the repetition to return.
        cv_folds: Number of folds for cross validation.
        fold_index: Index of the fold to return.
        train_size: Fixed size of train split (including validation data).
        seed: Random seed.
        debug: Load less data if true.
        runmode: Run mode. Can be one of the values of RunMode
        polars: If True, uses the Polars backend; otherwise, uses the Pandas backend.

    Returns:
        Input data divided into 'train', 'val', and 'test'.
    """
    if polars:
        polars_data = {k: v if isinstance(v, pl.DataFrame) else pl.DataFrame(v) for k, v in data.items()}
        return make_single_split_polars(
            polars_data,
            vars,
            cv_repetitions,
            repetition_index,
            cv_folds,
            fold_index,
            train_size,
            seed,
            debug,
            runmode,
        )
    else:
        pandas_data = {k: v if isinstance(v, pd.DataFrame) else v.to_pandas() for k, v in data.items()}
        return make_single_split_pandas(
            pandas_data,
            vars,
            cv_repetitions,
            repetition_index,
            cv_folds,
            fold_index,
            train_size,
            seed,
            debug,
            runmode,
        )

check_required_keys

check_required_keys(vars, required_keys)

Checks if all required keys are present in the vars dictionary.

Parameters:

Name Type Description Default
vars dict

The dictionary to check.

required
required_keys list

The list of required keys.

required

Raises:

Type Description
KeyError

If any required key is missing.

Source code in icu_benchmarks/data/split_process_data.py
def check_required_keys(vars, required_keys):
    """
    Checks if all required keys are present in the vars dictionary.

    Args:
        vars (dict): The dictionary to check.
        required_keys (list): The list of required keys.

    Raises:
        KeyError: If any required key is missing.
    """
    missing_keys = [key for key in required_keys if key not in vars]
    if missing_keys:
        raise KeyError(f"Missing required keys in vars: {', '.join(missing_keys)}")

icu_benchmarks.data.preprocessor

Classes:

Name Description
PolarsClassificationPreprocessor
PolarsRegressionPreprocessor
PandasClassificationPreprocessor
PandasRegressionPreprocessor
PandasImputationPreprocessor

Preprocesses data for imputation.

Functions:

Name Description
apply_recipe_to_splits

Fits and transforms the training features, then transforms the validation and test features with the recipe.

cache_recipe

Cache recipe to make it available for e.g. transfer learning.

restore_recipe

Restore recipe from cache to use for e.g. transfer learning.

PolarsClassificationPreprocessor

PolarsClassificationPreprocessor(generate_features: bool = False, scaling: bool = True, use_static_features: bool = True, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None, vars_to_exclude: Optional[list[str]] = None)

Bases: Preprocessor

Parameters:

Name Type Description Default
generate_features bool

Generate features for dynamic data.

False
scaling bool

Scaling of dynamic and static data.

True
use_static_features bool

Use static features.

True
save_cache Optional[Union[str, Path]]

Save recipe cache from this path.

None
load_cache Optional[Union[str, Path]]

Load recipe cache from this path.

None
vars_to_exclude Optional[list[str]]

Variables to exclude from missing indicator/ feature generation.

None

Returns: Preprocessed data.

Methods:

Name Description
apply

Args:

Source code in icu_benchmarks/data/preprocessor.py
def __init__(
    self,
    generate_features: bool = False,
    scaling: bool = True,
    use_static_features: bool = True,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
    vars_to_exclude: Optional[list[str]] = None,
):
    """
    Args:
        generate_features: Generate features for dynamic data.
        scaling: Scaling of dynamic and static data.
        use_static_features: Use static features.
        save_cache: Save recipe cache from this path.
        load_cache: Load recipe cache from this path.
        vars_to_exclude: Variables to exclude from missing indicator/ feature generation.
    Returns:
        Preprocessed data.
    """
    super().__init__(
        generate_features=generate_features,
        scaling=scaling,
        use_static_features=use_static_features,
        save_cache=save_cache,
        load_cache=load_cache,
        vars_to_exclude=vars_to_exclude,
    )
    self.imputation_model = None

apply

apply(data: dict[str, dict[str, DataFrame]], vars: dict[str, Union[str, list[str]]]) -> dict[str, dict[str, DataFrame]]

Parameters:

Name Type Description Default
data dict[str, dict[str, DataFrame]]

Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.

required
vars dict[str, Union[str, list[str]]]

Variables for static, dynamic, outcome.

required

Returns: Preprocessed data.

Source code in icu_benchmarks/data/preprocessor.py
def apply(
    self,
    data: dict[str, dict[str, pl.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
) -> dict[str, dict[str, pl.DataFrame]]:
    """
    Args:
        data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.
        vars: Variables for static, dynamic, outcome.
    Returns:
        Preprocessed data.
    """
    # Check if dynamic features are present
    if (
        self.use_static_features
        and all(DataSegment.static in value for value in data.values())
        and len(vars[DataSegment.static]) > 0
    ):
        logging.info("Preprocessing static features.")
        data = self._process_static(data, vars)
    else:
        self.use_static_features = False

    if all(DataSegment.dynamic in value for value in data.values()) and len(vars[DataSegment.dynamic]) > 0:
        logging.info("Preprocessing dynamic features.")
        logging.info(data.keys())
        data = self._process_dynamic(data, vars)
        if self.use_static_features:
            # Join static and dynamic data.
            data[DataSplit.train][DataSegment.dynamic] = data[DataSplit.train][DataSegment.dynamic].join(
                data[DataSplit.train][DataSegment.static], on=vars["GROUP"]
            )
            data[DataSplit.val][DataSegment.dynamic] = data[DataSplit.val][DataSegment.dynamic].join(
                data[DataSplit.val][DataSegment.static], on=vars["GROUP"]
            )
            data[DataSplit.test][DataSegment.dynamic] = data[DataSplit.test][DataSegment.dynamic].join(
                data[DataSplit.test][DataSegment.static], on=vars["GROUP"]
            )

            # Remove static features from splits
            data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.static)
            data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.static)
            data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.static)

        # Create feature splits
        data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.dynamic)
        data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.dynamic)
        data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.dynamic)
    elif self.use_static_features:
        data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.static)
        data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.static)
        data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.static)
    else:
        raise Exception(f"No recognized data segments data to preprocess. Available: {data.keys()}")
    logging.debug("Data head")
    logging.debug(data[DataSplit.train][DataSegment.features].head())
    logging.debug(data[DataSplit.train][DataSegment.outcome])

    if not isinstance(vars["SEQUENCE"], str):
        raise TypeError(f'Expected key "SEQUENCE" to be of type str, got {type(vars["SEQUENCE"])} instead')

    for split in [DataSplit.train, DataSplit.val, DataSplit.test]:
        if vars["SEQUENCE"] in data[split][DataSegment.outcome] and len(data[split][DataSegment.features]) != len(
            data[split][DataSegment.outcome]
        ):
            raise Exception(
                f"Data and outcome length mismatch in {split} split: "
                f"features: {len(data[split][DataSegment.features])}, outcome: {len(data[split][DataSegment.outcome])}"
            )
    data[DataSplit.train][DataSegment.features] = data[DataSplit.train][DataSegment.features].unique()
    data[DataSplit.val][DataSegment.features] = data[DataSplit.val][DataSegment.features].unique()
    data[DataSplit.test][DataSegment.features] = data[DataSplit.test][DataSegment.features].unique()

    logging.info(f"Generate features: {self.generate_features}")
    return data

PolarsRegressionPreprocessor

PolarsRegressionPreprocessor(generate_features: bool = False, scaling: bool = True, use_static_features: bool = True, outcome_max=None, outcome_min=None, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None, vars_to_exclude: Optional[list[str]] = None, target_scaling=None)

Bases: PolarsClassificationPreprocessor

Parameters:

Name Type Description Default
generate_features bool

Generate features for dynamic data.

False
scaling bool

Scaling of dynamic and static data.

True
use_static_features bool

Use static features.

True
max_range

Maximum value in outcome.

required
min_range

Minimum value in outcome.

required
save_cache Optional[Union[str, Path]]

Save recipe cache.

None
load_cache Optional[Union[str, Path]]

Load recipe cache.

None

Returns: Preprocessed data.

Methods:

Name Description
apply

Args:

Source code in icu_benchmarks/data/preprocessor.py
def __init__(
    self,
    generate_features: bool = False,
    scaling: bool = True,
    use_static_features: bool = True,
    outcome_max=None,
    outcome_min=None,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
    vars_to_exclude: Optional[list[str]] = None,
    target_scaling=None,
):
    """
    Args:
        generate_features: Generate features for dynamic data.
        scaling: Scaling of dynamic and static data.
        use_static_features: Use static features.
        max_range: Maximum value in outcome.
        min_range: Minimum value in outcome.
        save_cache: Save recipe cache.
        load_cache: Load recipe cache.
    Returns:
        Preprocessed data.
    """
    super().__init__(
        generate_features=generate_features,
        scaling=scaling,
        use_static_features=use_static_features,
        save_cache=save_cache,
        load_cache=load_cache,
        vars_to_exclude=vars_to_exclude,
    )
    self.outcome_max = outcome_max
    self.outcome_min = outcome_min
    self.target_scaling = target_scaling_policy(target_scaling, outcome_min, outcome_max)

apply

apply(data: dict[str, dict[str, DataFrame]], vars: dict[str, Union[str, list[str]]]) -> dict[str, dict[str, DataFrame]]

Parameters:

Name Type Description Default
data dict[str, dict[str, DataFrame]]

Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.

required
vars dict[str, Union[str, list[str]]]

Variables for static, dynamic, outcome.

required

Returns: Preprocessed data.

Source code in icu_benchmarks/data/preprocessor.py
def apply(
    self,
    data: dict[str, dict[str, pl.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
) -> dict[str, dict[str, pl.DataFrame]]:
    """
    Args:
        data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.
        vars: Variables for static, dynamic, outcome.
    Returns:
        Preprocessed data.
    """
    self.target_transform = transform_outcomes(
        data, vars["LABEL"], self.target_scaling, self.outcome_min, self.outcome_max, self.target_transform
    )
    return super().apply(data, vars)

PandasClassificationPreprocessor

PandasClassificationPreprocessor(generate_features: bool = True, scaling: bool = True, use_static_features: bool = True, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None)

Bases: Preprocessor

Parameters:

Name Type Description Default
generate_features bool

Generate features for dynamic data.

True
scaling bool

Scaling of dynamic and static data.

True
use_static_features bool

Use static features.

True
save_cache Optional[Union[str, Path]]

Save recipe cache from this path.

None
load_cache Optional[Union[str, Path]]

Load recipe cache from this path.

None

Returns: Preprocessed data.

Methods:

Name Description
apply

Args:

Source code in icu_benchmarks/data/preprocessor.py
def __init__(
    self,
    generate_features: bool = True,
    scaling: bool = True,
    use_static_features: bool = True,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
):
    """
    Args:
        generate_features: Generate features for dynamic data.
        scaling: Scaling of dynamic and static data.
        use_static_features: Use static features.
        save_cache: Save recipe cache from this path.
        load_cache: Load recipe cache from this path.
    Returns:
        Preprocessed data.
    """
    self.generate_features = generate_features
    self.scaling = scaling
    self.use_static_features = use_static_features
    self.imputation_model = None
    self.save_cache = save_cache
    self.load_cache = load_cache

apply

apply(data: dict[str, dict[str, DataFrame]], vars: dict[str, Union[str, list[str]]]) -> dict[str, dict[str, DataFrame]]

Parameters:

Name Type Description Default
data dict[str, dict[str, DataFrame]]

Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.

required
vars dict[str, Union[str, list[str]]]

Variables for static, dynamic, outcome.

required

Returns: Preprocessed data.

Source code in icu_benchmarks/data/preprocessor.py
def apply(
    self,
    data: dict[str, dict[str, pd.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
) -> dict[str, dict[str, pd.DataFrame]]:
    """
    Args:
        data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.
        vars: Variables for static, dynamic, outcome.
    Returns:
        Preprocessed data.
    """
    logging.info("Preprocessing dynamic features.")

    data = self._process_dynamic(data, vars)
    if self.use_static_features:
        logging.info("Preprocessing static features.")
        data = self._process_static(data, vars)

        # Set index to grouping variable
        data[DataSplit.train][DataSegment.static] = data[DataSplit.train][DataSegment.static].set_index(vars["GROUP"])
        data[DataSplit.val][DataSegment.static] = data[DataSplit.val][DataSegment.static].set_index(vars["GROUP"])
        data[DataSplit.test][DataSegment.static] = data[DataSplit.test][DataSegment.static].set_index(vars["GROUP"])

        # Join static and dynamic data.
        data[DataSplit.train][DataSegment.dynamic] = data[DataSplit.train][DataSegment.dynamic].join(
            data[DataSplit.train][DataSegment.static], on=vars["GROUP"]
        )
        data[DataSplit.val][DataSegment.dynamic] = data[DataSplit.val][DataSegment.dynamic].join(
            data[DataSplit.val][DataSegment.static], on=vars["GROUP"]
        )
        data[DataSplit.test][DataSegment.dynamic] = data[DataSplit.test][DataSegment.dynamic].join(
            data[DataSplit.test][DataSegment.static], on=vars["GROUP"]
        )

        # Remove static features from splits
        data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.static)
        data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.static)
        data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.static)

    # Create feature splits
    data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.dynamic)
    data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.dynamic)
    data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.dynamic)

    logging.debug("Data head")
    logging.debug(data[DataSplit.train][DataSegment.features].head())
    logging.debug(data[DataSplit.train][DataSegment.outcome].head())
    logging.info(f"Generate features: {self.generate_features}")
    return data

PandasRegressionPreprocessor

PandasRegressionPreprocessor(generate_features: bool = True, scaling: bool = True, use_static_features: bool = True, outcome_max=None, outcome_min=None, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None, target_scaling=None)

Bases: PandasClassificationPreprocessor

Parameters:

Name Type Description Default
generate_features bool

Generate features for dynamic data.

True
scaling bool

Scaling of dynamic and static data.

True
use_static_features bool

Use static features.

True
max_range

Maximum value in outcome.

required
min_range

Minimum value in outcome.

required
save_cache Optional[Union[str, Path]]

Save recipe cache.

None
load_cache Optional[Union[str, Path]]

Load recipe cache.

None

Returns: Preprocessed data.

Methods:

Name Description
apply

Args:

Source code in icu_benchmarks/data/preprocessor.py
def __init__(
    self,
    generate_features: bool = True,
    scaling: bool = True,
    use_static_features: bool = True,
    outcome_max=None,
    outcome_min=None,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
    target_scaling=None,
):
    """
    Args:
        generate_features: Generate features for dynamic data.
        scaling: Scaling of dynamic and static data.
        use_static_features: Use static features.
        max_range: Maximum value in outcome.
        min_range: Minimum value in outcome.
        save_cache: Save recipe cache.
        load_cache: Load recipe cache.
    Returns:
        Preprocessed data.
    """
    super().__init__(
        generate_features=generate_features,
        scaling=scaling,
        use_static_features=use_static_features,
        save_cache=save_cache,
        load_cache=load_cache,
    )
    self.outcome_max = outcome_max
    self.outcome_min = outcome_min
    self.target_scaling = target_scaling_policy(target_scaling, outcome_min, outcome_max)

apply

apply(data: dict[str, dict[str, DataFrame]], vars: dict[str, Union[str, list[str]]]) -> dict[str, dict[str, DataFrame]]

Parameters:

Name Type Description Default
data dict[str, dict[str, DataFrame]]

Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.

required
vars dict[str, Union[str, list[str]]]

Variables for static, dynamic, outcome.

required

Returns: Preprocessed data.

Source code in icu_benchmarks/data/preprocessor.py
def apply(
    self,
    data: dict[str, dict[str, pd.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
) -> dict[str, dict[str, pd.DataFrame]]:
    """
    Args:
        data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.
        vars: Variables for static, dynamic, outcome.
    Returns:
        Preprocessed data.
    """
    self.target_transform = transform_outcomes(
        data, vars["LABEL"], self.target_scaling, self.outcome_min, self.outcome_max, self.target_transform
    )
    return super().apply(data, vars)

PandasImputationPreprocessor

PandasImputationPreprocessor(scaling: bool = True, use_static_features: bool = True, filter_missing_values: bool = True, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None)

Bases: Preprocessor

Preprocesses data for imputation.

Parameters:

Name Type Description Default
scaling bool

If the values in each column should be normalized. Defaults to True.

True
use_static_features bool

If static features should be included in the dataset. Defaults to True.

True

Methods:

Name Description
apply

Args:

Source code in icu_benchmarks/data/preprocessor.py
def __init__(
    self,
    scaling: bool = True,
    use_static_features: bool = True,
    filter_missing_values: bool = True,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
):
    super().__init__(
        generate_features=False,
        scaling=scaling,
        use_static_features=use_static_features,
        save_cache=save_cache,
        load_cache=load_cache,
    )
    self.scaling = scaling
    self.use_static_features = use_static_features
    self.filter_missing_values = filter_missing_values

apply

apply(data: dict[str, dict[str, DataFrame]], vars: dict[str, Union[str, list[str]]]) -> dict[str, dict[str, DataFrame]]

Parameters:

Name Type Description Default
data dict[str, dict[str, DataFrame]]

Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.

required
vars dict[str, Union[str, list[str]]]

Variables for static, dynamic, outcome.

required

Returns: Preprocessed data.

Source code in icu_benchmarks/data/preprocessor.py
def apply(
    self,
    data: dict[str, dict[str, pd.DataFrame]],
    vars: dict[str, Union[str, list[str]]],
) -> dict[str, dict[str, pd.DataFrame]]:
    """
    Args:
        data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome.
        vars: Variables for static, dynamic, outcome.
    Returns:
        Preprocessed data.
    """
    logging.info("Preprocessor static features.")
    data = {step: self._process_dynamic_data(data[step], vars) for step in data}

    dyn_rec = Recipe(
        data[DataSplit.train][DataSegment.dynamic],
        [],
        vars[DataSegment.dynamic],
        vars["GROUP"],
        vars["SEQUENCE"],
    )
    if self.scaling:
        dyn_rec.add_step(StepScale())
    data = apply_recipe_to_splits(dyn_rec, data, DataSegment.dynamic, self.save_cache, self.load_cache)

    if not (isinstance(vars["GROUP"], str) and isinstance(vars["SEQUENCE"], str)):
        raise TypeError(
            f'Expected keys "GROUP" and "SEQUENCE" to be of type str, got {type(vars["GROUP"])} and {type(vars["SEQUENCE"])} instead.'
        )
    selected_vars: list[str] = [str(item) for item in vars[DataSegment.dynamic]] + [
        vars["GROUP"],
        vars["SEQUENCE"],
    ]
    data[DataSplit.train][DataSegment.features] = data[DataSplit.train].pop(DataSegment.dynamic).loc[:, selected_vars]
    data[DataSplit.val][DataSegment.features] = data[DataSplit.val].pop(DataSegment.dynamic).loc[:, selected_vars]
    data[DataSplit.test][DataSegment.features] = data[DataSplit.test].pop(DataSegment.dynamic).loc[:, selected_vars]
    return data

apply_recipe_to_splits

apply_recipe_to_splits(recipe: Recipe, data: dict[str, dict[str, Union[DataFrame, DataFrame]]], type: str, save_cache: Optional[Union[str, Path]] = None, load_cache: Optional[Union[str, Path]] = None) -> dict[str, dict[str, Union[DataFrame, DataFrame]]]

Fits and transforms the training features, then transforms the validation and test features with the recipe. Works with both Polars and Pandas versions of recipys.

Parameters:

Name Type Description Default
load_cache Optional[Union[str, Path]]

Load recipe from cache, for e.g. transfer learning.

None
save_cache Optional[Union[str, Path]]

Save recipe to cache, for e.g. transfer learning.

None
recipe Recipe

Object containing info about the features and steps.

required
data dict[str, dict[str, Union[DataFrame, DataFrame]]]

Dict containing 'train', 'val', and 'test' and types of features per split.

required
type str

Whether to apply recipe to dynamic features, static features or outcomes.

required

Returns:

Type Description
dict[str, dict[str, Union[DataFrame, DataFrame]]]

Transformed features divided into 'train', 'val', and 'test'.

Source code in icu_benchmarks/data/preprocessor.py
def apply_recipe_to_splits(
    recipe: Recipe,
    data: dict[str, dict[str, Union[pd.DataFrame, pl.DataFrame]]],
    type: str,
    save_cache: Optional[Union[str, Path]] = None,
    load_cache: Optional[Union[str, Path]] = None,
) -> dict[str, dict[str, Union[pd.DataFrame, pl.DataFrame]]]:
    """
    Fits and transforms the training features, then transforms the validation and test features with the recipe.
     Works with both Polars and Pandas versions of recipys.

    Args:
        load_cache: Load recipe from cache, for e.g. transfer learning.
        save_cache: Save recipe to cache, for e.g. transfer learning.
        recipe: Object containing info about the features and steps.
        data: Dict containing 'train', 'val', and 'test' and types of features per split.
        type: Whether to apply recipe to dynamic features, static features or outcomes.

    Returns:
        Transformed features divided into 'train', 'val', and 'test'.
    """

    if isinstance(load_cache, (str, Path)):
        load_cache = Path(str(load_cache) + f"_{type}")
        # Load existing recipe
        recipe = restore_recipe(load_cache)
        data[DataSplit.train][type] = recipe.bake(data[DataSplit.train][type])
    elif isinstance(save_cache, (str, Path)):
        save_cache = Path(str(save_cache) + f"_{type}")
        # Save prepped recipe
        data[DataSplit.train][type] = recipe.prep()
        cache_recipe(recipe, save_cache)
    else:
        # No saving or loading of existing cache
        data[DataSplit.train][type] = recipe.prep()

    data[DataSplit.val][type] = recipe.bake(data[DataSplit.val][type])
    data[DataSplit.test][type] = recipe.bake(data[DataSplit.test][type])
    return data

cache_recipe

cache_recipe(recipe: Recipe, cache_file: Path) -> None

Cache recipe to make it available for e.g. transfer learning.

Source code in icu_benchmarks/data/preprocessor.py
def cache_recipe(recipe: Recipe, cache_file: Path) -> None:
    """Cache recipe to make it available for e.g. transfer learning."""
    recipe_cache = copy.deepcopy(recipe)
    recipe_cache.cache()
    if not (cache_file.parent).exists():
        (cache_file.parent).mkdir(parents=True, exist_ok=True)
    cache_file.touch()
    with open(cache_file, "wb") as f:
        pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL)
    logging.info(f"Cached recipe in {cache_file}.")

restore_recipe

restore_recipe(cache_file: Path) -> Recipe

Restore recipe from cache to use for e.g. transfer learning.

Source code in icu_benchmarks/data/preprocessor.py
def restore_recipe(cache_file: Path) -> Recipe:
    """Restore recipe from cache to use for e.g. transfer learning."""
    if cache_file.exists():
        with open(cache_file, "rb") as f:
            logging.info(f"Loading cached recipe from {cache_file}.")
            recipe = pickle.load(f)
            return recipe
    else:
        raise FileNotFoundError(f"Cache file {cache_file} not found.")

icu_benchmarks.data.loader

Classes:

Name Description
CommonPolarsDataset
PredictionPolarsDataset

Subclass of common dataset for prediction tasks.

CommonPandasDataset

Common dataset: subclass of Torch Dataset that represents the data to learn on.

PredictionPandasDataset

Subclass of common dataset for prediction tasks.

ImputationPandasDataset

Subclass of Common Dataset that contains data for imputation models.

ImputationPredictionDataset

Subclass of torch dataset that represents data with missingness for imputation.

Functions:

Name Description
check_class_labels

Balanced weights are indexed by class, so the labels must be 0..K-1.

CommonPolarsDataset

CommonPolarsDataset(data: dict, split: str = train, vars: dict[str, Union[str, list[str]]] | object = REQUIRED, grouping_segment: str = outcome, mps: bool = False, name: str = '')

Bases: Dataset

Methods:

Name Description
get_input_names

The columns of the deep learning input tensors (self[i][0]), in order.

Source code in icu_benchmarks/data/loader.py
def __init__(
    self,
    data: dict,
    split: str = DataSplit.train,
    vars: dict[str, Union[str, list[str]]] | object = gin.REQUIRED,
    grouping_segment: str = DataSegment.outcome,
    mps: bool = False,
    name: str = "",
):
    if not isinstance(vars, dict):
        raise ValueError(f"Expected vars to be of type dict, got {type(vars)} instead")
    self.split = split
    self.vars = vars
    self.grouping_df = data[split][grouping_segment]
    # Get the row indicators for the data to be able to match predicted labels
    if not isinstance(vars["SEQUENCE"], str):
        raise ValueError(f'Expected key "SEQUENCE" to be of type str, got {type(vars["SEQUENCE"])} instead')
    if "SEQUENCE" in self.vars and self.vars["SEQUENCE"] in data[split][DataSegment.features].columns:
        # We have a time series dataset
        self.row_indicators = data[split][DataSegment.features][self.vars["GROUP"], self.vars["SEQUENCE"]]

        self.row_indicators = self.row_indicators.with_columns(pl.col(self.vars["SEQUENCE"]))  # already in hours?
        self.features_df = data[split][DataSegment.features]
        self.features_df = self.features_df.sort([self.vars["GROUP"], self.vars["SEQUENCE"]])
        self.features_df = self.features_df.drop(self.vars["SEQUENCE"])
    else:
        # We have a static dataset
        logging.info("Using static dataset")
        self.row_indicators = data[split][DataSegment.features][self.vars["GROUP"]]
        self.features_df = data[split][DataSegment.features]

    # order columns: index, features (alphabetically), indicator (alphabetically)
    cols = self.features_df.columns
    m_index = [self.vars["GROUP"]]
    front = sorted([c for c in cols if not c.startswith("MissingIndicator_") and c not in m_index])
    back = sorted([c for c in cols if c.startswith("MissingIndicator_") and c not in m_index])
    self.features_df = self.features_df[m_index + front + back]

    # calculate basic info for the data
    self.num_stays = self.grouping_df[self.vars["GROUP"]].unique().shape[0]
    self.maxlen = self.features_df.group_by([self.vars["GROUP"]]).len().max().item(0, 1)
    self.mps = mps
    self.name = name

get_input_names

get_input_names() -> List[str]

The columns of the deep learning input tensors (self[i][0]), in order.

Unlike get_feature_names in prediction datasets, this keeps every feature column, including counter.

Source code in icu_benchmarks/data/loader.py
def get_input_names(self) -> List[str]:
    """The columns of the deep learning input tensors (``self[i][0]``), in order.

    Unlike get_feature_names in prediction datasets, this keeps every feature column, including ``counter``.
    """
    return [c for c in self.features_df.columns if c != self.vars["GROUP"]]

PredictionPolarsDataset

PredictionPolarsDataset(*args, ram_cache: bool = True, **kwargs)

Bases: CommonPolarsDataset

Subclass of common dataset for prediction tasks.

Parameters:

Name Type Description Default
ram_cache bool

Whether the complete dataset should be stored in ram. Defaults to True.

True

Methods:

Name Description
get_stay_ids

The stay ID of each item, in item order.

get_balance

Return the weight balance for the split of interest.

get_data_and_labels

Function to return all the data and labels aligned at once.

Source code in icu_benchmarks/data/loader.py
def __init__(self, *args, ram_cache: bool = True, **kwargs):
    super().__init__(*args, **kwargs)
    self.outcome_df = self.grouping_df

    # One-pass partition keyed by stay id. Avoids the O(N_stays) full-frame filter
    # that previously ran inside every __getitem__.
    GROUP = self.vars["GROUP"]
    LABEL = self.vars["LABEL"]
    label_partitions = self.outcome_df.partition_by(GROUP, maintain_order=True)
    feat_partitions = self.features_df.partition_by(GROUP, maintain_order=True)
    self._stay_order = [part[GROUP][0] for part in label_partitions]
    self._feat_arrays = {
        part[GROUP][0]: part.select(pl.exclude(GROUP)).to_numpy().astype(np.float32) for part in feat_partitions
    }
    for arr in self._feat_arrays.values():
        arr.setflags(write=False)

    self._label_arrays = {part[GROUP][0]: part[LABEL].to_numpy().astype(np.float32) for part in label_partitions}
    for arr in self._label_arrays.values():
        arr.setflags(write=False)

    self.ram_cache(ram_cache)

get_stay_ids

get_stay_ids() -> list

The stay ID of each item, in item order.

Source code in icu_benchmarks/data/loader.py
def get_stay_ids(self) -> list:
    """The stay ID of each item, in item order."""
    return list(self._stay_order)

get_balance

get_balance() -> list

Return the weight balance for the split of interest.

Returns:

Type Description
list

Weights for each label.

Source code in icu_benchmarks/data/loader.py
def get_balance(self) -> list:
    """Return the weight balance for the split of interest.

    Returns:
        Weights for each label.
    """
    labels = self.outcome_df[self.vars["LABEL"]].drop_nulls()
    if labels.dtype.is_float():
        labels = labels.drop_nans()
    # value_counts has no fixed order, so sort by label: weight i must belong to class i
    classes, counts = labels.value_counts(sort=False).sort(labels.name).get_columns()
    check_class_labels(classes.to_numpy())
    counts = counts.to_numpy()
    weights = list((1 / counts) * np.sum(counts) / counts.shape[0])
    return weights

get_data_and_labels

get_data_and_labels() -> tuple[ndarray, ndarray, ndarray]

Function to return all the data and labels aligned at once.

We use this function for the ML methods which don't require an iterator.

Returns:

Type Description
tuple[ndarray, ndarray, ndarray]

A Tuple containing data points and label for the split.

Source code in icu_benchmarks/data/loader.py
def get_data_and_labels(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Function to return all the data and labels aligned at once.

    We use this function for the ML methods which don't require an iterator.

    Returns:
        A Tuple containing data points and label for the split.
    """
    labels = self.outcome_df[self.vars["LABEL"]].to_numpy().astype(np.float32)
    rep = self.features_df
    group = self.vars["GROUP"]

    if len(labels) == self.num_stays:
        # One row per stay. polars group_by does not guarantee the group order, so align the rows to the
        # label order explicitly
        rep = self.outcome_df.select(group).join(rep.group_by(group).last(), on=group, how="left")
        # The hour index from the preprocessor is a feature for the hourly tasks only
        rep = rep.drop("counter", strict=False)
        row_indicators = self.outcome_df.select(group)
    else:
        row_indicators = self.row_indicators
    # Don't use group as model feature
    rep = rep.drop(group).to_numpy().astype(np.float32)
    logging.debug(f"rep shape: {rep.shape}")
    logging.debug(f"labels shape: {labels.shape}")
    valid = np.isfinite(labels.reshape(len(labels), -1)).all(axis=1)
    return rep[valid], labels[valid], row_indicators.to_numpy()[valid]

CommonPandasDataset

CommonPandasDataset(data: dict, split: str = train, vars: dict[str, Union[str, list[str]]] | object = REQUIRED, grouping_segment: str = outcome, mps: bool = False, name: str = '')

Bases: Dataset

Common dataset: subclass of Torch Dataset that represents the data to learn on.

Args: data: Dict of the different splits of the data. split: Either 'train','val' or 'test'. vars: Contains the names of columns in the data. grouping_segment: str, optional: The segment of the data contains the grouping column with only unique values. Defaults to Segment.outcome. Is used to calculate the number of stays in the data.

Source code in icu_benchmarks/data/loader.py
def __init__(
    self,
    data: dict,
    split: str = DataSplit.train,
    vars: dict[str, Union[str, list[str]]] | object = gin.REQUIRED,
    grouping_segment: str = DataSegment.outcome,
    mps: bool = False,
    name: str = "",
):
    warnings.warn(
        "CommonPandasDataset is deprecated. Use CommonPolarsDataset instead.",
        DeprecationWarning,
        stacklevel=2,
    )
    if not isinstance(vars, dict):
        raise ValueError(f"Expected vars to be of type dict, got {type(vars)} instead")
    self.split = split
    self.vars = vars
    self.grouping_df = data[split][grouping_segment].set_index(self.vars["GROUP"])
    features = data[split][DataSegment.features]
    hours = features[self.vars["SEQUENCE"]]
    if pd.api.types.is_timedelta64_dtype(hours):
        hours = hours / pd.Timedelta(hours=1)
    # Identifies the rows of the hourly tasks in the saved predictions, as in the polars dataset
    self.hours = hours.to_numpy()
    self.features_df = features.set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1)

    # calculate basic info for the data
    self.num_stays = self.grouping_df.index.unique().shape[0]
    self.maxlen = self.features_df.groupby([self.vars["GROUP"]]).size().max()
    self.mps = mps
    self.name = name

PredictionPandasDataset

PredictionPandasDataset(*args, ram_cache: bool = True, **kwargs)

Bases: CommonPandasDataset

Subclass of common dataset for prediction tasks.

Parameters:

Name Type Description Default
ram_cache bool

Whether the complete dataset should be stored in ram. Defaults to True.

True

Methods:

Name Description
get_balance

Return the weight balance for the split of interest.

get_data_and_labels

Function to return all the data and labels aligned at once.

Source code in icu_benchmarks/data/loader.py
def __init__(self, *args, ram_cache: bool = True, **kwargs):
    super().__init__(*args, grouping_segment=DataSegment.outcome, **kwargs)
    self.outcome_df = self.grouping_df
    self.ram_cache(ram_cache)

get_balance

get_balance() -> list

Return the weight balance for the split of interest.

Returns:

Type Description
list

Weights for each label.

Source code in icu_benchmarks/data/loader.py
def get_balance(self) -> list:
    """Return the weight balance for the split of interest.

    Returns:
        Weights for each label.
    """
    # Pandas sorts value_counts by frequency, so sort by label: weight i must belong to class i
    counts = self.outcome_df[self.vars["LABEL"]].value_counts().sort_index()
    check_class_labels(counts.index.to_numpy())
    return list((1 / counts) * np.sum(counts) / counts.shape[0])

get_data_and_labels

get_data_and_labels() -> tuple[ndarray, ndarray, ndarray]

Function to return all the data and labels aligned at once.

We use this function for the ML methods which don't require an iterator.

Returns:

Type Description
tuple[ndarray, ndarray, ndarray]

A Tuple containing data points and label for the split.

Source code in icu_benchmarks/data/loader.py
def get_data_and_labels(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Function to return all the data and labels aligned at once.

    We use this function for the ML methods which don't require an iterator.

    Returns:
        A Tuple containing data points and label for the split.
    """
    labels = self.outcome_df[self.vars["LABEL"]].to_numpy().astype(np.float32)
    rep = self.features_df
    if len(labels) == self.num_stays:
        rep = rep.groupby(level=self.vars["GROUP"], sort=False).last().reindex(self.outcome_df.index)
        row_indicators = self.outcome_df.index.to_numpy().reshape(-1, 1)
    else:
        row_indicators = np.column_stack((self.features_df.index.to_numpy(), self.hours))
    rep = rep.to_numpy().astype(np.float32)
    valid = np.isfinite(labels.reshape(len(labels), -1)).all(axis=1)
    return rep[valid], labels[valid], row_indicators[valid]

ImputationPandasDataset

ImputationPandasDataset(data: dict[str, DataFrame], split: str = train, vars: dict[str, Union[str, list[str]]] | object = REQUIRED, mask_proportion=0.3, mask_method='MCAR', mask_observation_proportion=0.3, ram_cache: bool = True, name: str = '')

Bases: CommonPandasDataset

Subclass of Common Dataset that contains data for imputation models.

Parameters:

Name Type Description Default
data Dict[str, DataFrame]

data to use

required
split str

split to apply. Defaults to Split.train.

train
vars Dict[str, str]

contains names of columns in the data. Defaults to gin.REQUIRED.

REQUIRED
mask_proportion float

proportion to artificially mask for amputation. Defaults to 0.3.

0.3
mask_method str

masking mechanism. Defaults to "MCAR".

'MCAR'
mask_observation_proportion float

poportion of the observed data to be masked. Defaults to 0.3.

0.3
ram_cache bool

if the dataset should be completely stored in ram and not generated on the fly during training. Defaults to True.

True
name str

name of the dataset, used in log messages. Defaults to "".

''
Source code in icu_benchmarks/data/loader.py
def __init__(
    self,
    data: dict[str, DataFrame],
    split: str = DataSplit.train,
    vars: dict[str, Union[str, list[str]]] | object = gin.REQUIRED,
    mask_proportion=0.3,
    mask_method="MCAR",
    mask_observation_proportion=0.3,
    ram_cache: bool = True,
    name: str = "",
):
    """
    Args:
        data (Dict[str, DataFrame]): data to use
        split (str, optional): split to apply. Defaults to Split.train.
        vars (Dict[str, str], optional): contains names of columns in the data. Defaults to gin.REQUIRED.
        mask_proportion (float, optional): proportion to artificially mask for amputation. Defaults to 0.3.
        mask_method (str, optional): masking mechanism. Defaults to "MCAR".
        mask_observation_proportion (float, optional): poportion of the observed data to be masked. Defaults to 0.3.
        ram_cache (bool, optional): if the dataset should be completely stored in ram and not generated on the fly during
            training. Defaults to True.
        name (str, optional): name of the dataset, used in log messages. Defaults to "".
    """
    if not isinstance(vars, dict):
        raise ValueError(f"Expected vars to be of type dict, got {type(vars)} instead")
    super().__init__(data, split, vars, grouping_segment=DataSegment.static, name=name)
    self.amputated_values, self.amputation_mask = ampute_data(
        self.features_df, mask_method, mask_proportion, mask_observation_proportion
    )
    self.amputation_mask = (self.amputation_mask + self.features_df.isna().values).bool()
    self.amputation_mask = DataFrame(self.amputation_mask, columns=np.array(self.vars[DataSegment.dynamic]))
    self.amputation_mask[self.vars["GROUP"]] = self.features_df.index
    self.amputation_mask = self.amputation_mask.set_index(self.vars["GROUP"])

    self.target_missingness_mask = self.features_df.isna()
    self.features_df.fillna(0, inplace=True)
    self.ram_cache(ram_cache)

ImputationPredictionDataset

ImputationPredictionDataset(data: DataFrame, grouping_column: Optional[str] = 'stay_id', select_columns: Optional[list[str]] = None, ram_cache: bool = True)

Bases: Dataset

Subclass of torch dataset that represents data with missingness for imputation.

Parameters:

Name Type Description Default
data DataFrame

dict of the different splits of the data

required
grouping_column str

column that is used for grouping. Defaults to "stay_id".

'stay_id'
select_columns List[str]

the columns to serve as input for the imputation model. Defaults to None.

None
ram_cache bool

whether the dataset should be stored in ram. Defaults to True.

True
Source code in icu_benchmarks/data/loader.py
def __init__(
    self,
    data: DataFrame,
    grouping_column: Optional[str] = "stay_id",
    select_columns: Optional[list[str]] = None,
    ram_cache: bool = True,
):
    self.dyn_df = data

    if select_columns is not None:
        self.dyn_df = self.dyn_df[list(select_columns) + [grouping_column]]

    if grouping_column is not None:
        self.dyn_df = self.dyn_df.set_index(grouping_column)
    else:
        self.dyn_df = data

    # calculate basic info for the data
    self.group_indices = self.dyn_df.index.unique()
    self.maxlen = self.dyn_df.groupby(grouping_column).size().max()

    self._cached_dataset = None
    if ram_cache:
        logging.info("Caching dataset in ram.")
        self._cached_dataset = [self[i] for i in range(len(self))]

check_class_labels

check_class_labels(classes: ndarray)

Balanced weights are indexed by class, so the labels must be 0..K-1.

Source code in icu_benchmarks/data/loader.py
def check_class_labels(classes: np.ndarray):
    """Balanced weights are indexed by class, so the labels must be 0..K-1."""
    if not np.array_equal(classes, np.arange(len(classes))):
        raise ValueError(f"Balanced class weights need labels 0..K-1, got {classes.tolist()}.")

icu_benchmarks.data.target_transform

Single-target affine normalization shared by preprocessing, ML and DL.

Classes:

Name Description
TargetTransform

z = y * scale + offset. Scalar arithmetic preserves NumPy/Torch shape and device.

Functions:

Name Description
target_scaling_policy

Resolve legacy bounds without silently accepting partial or conflicting settings.

transform_outcomes

Fit once before padding; reuse on all splits (or use a saved source transform).

TargetTransform dataclass

TargetTransform(label: str, policy: str, scale: float, offset: float, minimum: float, maximum: float, version: int = 1, metric_space: str = 'original_target_units')

z = y * scale + offset. Scalar arithmetic preserves NumPy/Torch shape and device.

target_scaling_policy

target_scaling_policy(policy, lower, upper)

Resolve legacy bounds without silently accepting partial or conflicting settings.

Source code in icu_benchmarks/data/target_transform.py
def target_scaling_policy(policy, lower, upper):
    """Resolve legacy bounds without silently accepting partial or conflicting settings."""
    has_bounds = lower is not None or upper is not None
    if policy is None:
        policy = "fixed" if has_bounds else "minmax"
        if has_bounds:
            warnings.warn(
                "Implicit fixed scaling from outcome bounds is deprecated; set target_scaling='fixed'.",
                FutureWarning,
                stacklevel=2,
            )
    if policy not in ("minmax", "fixed", "none"):
        raise ValueError(f"Unknown target scaling policy: {policy}")
    if policy == "fixed":
        if lower is None or upper is None or not np.isfinite([lower, upper]).all() or lower >= upper:
            raise ValueError("Fixed target scaling requires finite outcome_min < outcome_max.")
    elif has_bounds:
        raise ValueError("Outcome bounds are only valid with target_scaling='fixed'.")
    return policy

transform_outcomes

transform_outcomes(data, label, policy, lower, upper, target_transform=None)

Fit once before padding; reuse on all splits (or use a saved source transform).

Source code in icu_benchmarks/data/target_transform.py
def transform_outcomes(data, label, policy, lower, upper, target_transform=None):
    """Fit once before padding; reuse on all splits (or use a saved source transform)."""
    if isinstance(label, list) and len(label) == 1:
        label = label[0]
    if not isinstance(label, str):
        raise ValueError("Target scaling currently supports one selected regression label.")
    if target_transform is None:
        target_transform = TargetTransform.fit(
            data[DataSplit.train][DataSegment.outcome][label].to_numpy(), label, policy, lower, upper
        )
    if target_transform.label != label:
        raise ValueError(f"Target label {label!r} does not match saved label {target_transform.label!r}.")
    for split in data.values():
        outcome = split[DataSegment.outcome]
        if np.isinf(outcome[label].to_numpy()).any():
            raise ValueError("Infinite regression targets are not supported.")
        if isinstance(outcome, pl.DataFrame):
            split[DataSegment.outcome] = outcome.with_columns(target_transform.transform(pl.col(label)))
        else:
            split[DataSegment.outcome] = outcome.assign(**{label: target_transform.transform(outcome[label])})
    return target_transform

icu_benchmarks.data.pooling

Classes:

Name Description
PooledData

PooledData

PooledData(data_dir, vars, datasets, file_names, shuffle=True, stratify=None, runmode=classification, save_test=True)

Generate pooled data from existing datasets. Args: data_dir: Where to read the data from vars: Variables dictionary datasets: Which datasets to pool file_names: Which files to read from shuffle: Whether to shuffle data; classification pools are stratified, which needs shuffling stratify: Stratify data runmode: Which task runmode save_test: Save left over test data to test on without leakage

Methods:

Name Description
generate

Generate pooled data from existing datasets.

Source code in icu_benchmarks/data/pooling.py
def __init__(
    self,
    data_dir,
    vars,
    datasets,
    file_names,
    shuffle=True,
    stratify=None,
    runmode=RunMode.classification,
    save_test=True,
):
    """
    Generate pooled data from existing datasets.
    Args:
        data_dir: Where to read the data from
        vars: Variables dictionary
        datasets: Which datasets to pool
        file_names: Which files to read from
        shuffle: Whether to shuffle data; classification pools are stratified, which needs shuffling
        stratify: Stratify data
        runmode: Which task runmode
        save_test: Save left over test data to test on without leakage
    """
    self.data_dir = data_dir
    self.vars = vars
    self.datasets = datasets
    self.file_names = file_names
    self.shuffle = shuffle
    self.stratify = stratify
    self.runmode = runmode
    self.save_test = save_test

generate

generate(datasets, samples=10000, seed=42)

Generate pooled data from existing datasets. Args: datasets: Which datasets to pool samples: Amount of samples to pool seed: Random seed

Source code in icu_benchmarks/data/pooling.py
def generate(
    self,
    datasets,
    samples=10000,
    seed=42,
):
    """
    Generate pooled data from existing datasets.
    Args:
        datasets: Which datasets to pool
        samples: Amount of samples to pool
        seed: Random seed
    """
    data = {}
    for folder in self.data_dir.iterdir():
        if folder.is_dir():
            if folder.name in datasets:
                data[folder.name] = {
                    f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) for f in self.file_names
                }
    data = self._pool_datasets(
        datasets=data,
        samples=samples,
        vars=self.vars,
        shuffle=self.shuffle,
        seed=seed,
        runmode=self.runmode,
        data_dir=self.data_dir,
        save_test=self.save_test,
    )
    self._save_pooled_data(self.data_dir, data, datasets, self.file_names, samples=samples)

icu_benchmarks.data.constants