temperature-based-fertility-prediction-thesis/code/utils/dataset_creation.py

import os

import lmdb
import numpy as np
import pandas as pd
import torch
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import joblib

from utils.lmdb_utils import load_from_lmdb, get_lmdb_keys


def get_features(cycle: dict,
                 feature_config: dict) -> dict:
    """
    Computes the feature for a given cycle defined by the feature_config
    :param cycle: cycle data as dictionary
    :param feature_config: feature config as dictionary
    :return: dict with feature according feature config
    """
    features = {}
    feature_sets = feature_config["feature_sets"]
    for feature_set in feature_sets:
        for feature_def in feature_config[feature_set]:
            if feature_set not in features:
                features[feature_set] = {}

            # skip features that are marked as ignored
            if "ignored_features" in feature_config and feature_def["name"] in feature_config["ignored_features"]:
                continue

            feature_return = feature_def["fn"](cycle=cycle)
            if isinstance(feature_return, dict):
                if len(feature_return) > 1:
                    for feature_name, feature in feature_return.items():
                        features[feature_set][f"{feature_def['name']}_{feature_name}"] = feature
                else:
                    features[feature_set][feature_def["name"]] = list(feature_return.values())[0]
            else:
                features[feature_set][feature_def["name"]] = feature_return

        # check, if all features have same length
        if feature_set in features and len(features[feature_set]) > 0 and isinstance(
                list(features[feature_set].values())[0], np.ndarray):
            feature_lengths = [len(x) for x in features[feature_set].values()]
            if len(set(feature_lengths)) > 1:
                raise ValueError(f"Feature set {feature_set} has features of different lengths: {feature_lengths}")

    return features


def save_scalers(scalers: dict,
                 scaler_dir: str):
    if not os.path.exists(scaler_dir):
        os.makedirs(scaler_dir)
    for feature_type in scalers:
        for feature_name in scalers[feature_type]:
            joblib.dump(scalers[feature_type][feature_name],
                        f"{scaler_dir}/{feature_name}.pkl")


def load_scalers(scaler_dir: str) -> dict:
    scalers = dict()
    for scaler_file in os.listdir(scaler_dir):
        if scaler_file.endswith(".pkl"):
            feature_name = scaler_file.replace(".pkl", "")
            scalers[feature_name] = joblib.load(f"{scaler_dir}/{scaler_file}")
    return scalers


def get_scalers_for_model(model_configuration: dict):
    """
    Get the scalers for the features of a model configuration
    Args:
        model_configuration: configuration of the model

    Returns:
        scalers: scalers for the model

    """

    scaler_dir = os.path.join(model_configuration["feature_config"]["dataset_dir"], "scalers")
    scalers = load_scalers(scaler_dir)
    return scalers


def get_feature_values(feature_type: str | None,
                       feature_name: str,
                       env: lmdb.Environment,
                       keys: list = None) -> list:
    """
    Get the values of a feature from the lmdb dataset for all keys
    :param feature_type: type of feature, can be None, then the first feature with the given name will be used
    :param feature_name: feature name to extract
    :param env: lmdb environment
    :param keys: keys to extract the feature from, if None, all keys will be used / fetched from the database
    :return: list with feature values
    """
    if keys is None:
        keys = get_lmdb_keys(env)
    feature_values = []
    for key in keys:
        data = load_from_lmdb(env, key)
        if feature_type is None:
            for feature_type, feature_data in data.items():
                if feature_name in feature_data:
                    feature_values.append(feature_data[feature_name])
                    break
        else:
            for current_feature_name, feature_data in data[feature_type].items():
                # catch sub features that have been prefixed with the feature name
                if current_feature_name.startswith(feature_name):
                    feature_values.append(feature_data)
                    break
    return feature_values


def train_scalers(feature_type: str,
                  feature_name: str,
                  scaler_type,
                  sample,
                  env) -> dict:
    scalers = dict()
    all_features = sample[feature_type]
    individual_feature_names = list()
    for individual_feature_name in all_features:
        if individual_feature_name.startswith(feature_name) and not individual_feature_name.endswith("_scaled"):
            individual_feature_names.append(individual_feature_name)

    for individual_feature_name in individual_feature_names:
        feature_values = get_feature_values(feature_type, individual_feature_name, env)
        if len(feature_values) == 0:
            raise ValueError(f"No feature values found for feature {individual_feature_name}")
        if isinstance(feature_values[0], list) or isinstance(feature_values[0], np.ndarray):
            features_reshaped = np.concatenate(feature_values).reshape(-1, 1)
        else:
            features_reshaped = np.array(feature_values).reshape(-1, 1)
        del feature_values

        if scaler_type is not None:
            scaler = scaler_type()
            scaler.fit(features_reshaped)
        else:
            scaler = None

        scalers[individual_feature_name] = scaler
    return scalers


def scale_item(data: dict,
               scalers: dict):
    data_format = dict()
    for feature_set in data:
        if feature_set not in data_format:
            data_format[feature_set] = dict()
        for feature_name in data[feature_set]:
            if feature_name.endswith("_scaled"):
                continue
            data_format[feature_set][feature_name] = data[feature_set][feature_name]

    for feature_set in data_format:
        for feature_name in data_format[feature_set]:
            feature_values = data[feature_set][feature_name]
            if scalers[feature_name] is not None:
                if isinstance(feature_values, list) or isinstance(feature_values, np.ndarray):
                    scaled_feature_values = scalers[feature_name].transform(
                        feature_values.reshape(-1, 1)).flatten()
                else:
                    scaled_feature_values = scalers[feature_name].transform(
                        np.array(feature_values).reshape(-1, 1)).flatten()
            else:
                scaled_feature_values = feature_values
            data[feature_set][f"{feature_name}_scaled"] = scaled_feature_values

    return data


def inverse_scale_feature(input_feature: np.ndarray | int | float | torch.Tensor,
                          feature_names: np.ndarray | list | str,
                          scalers: dict) -> np.ndarray:
    """
    Inverse scales the input features, can handle single and multi feature input
    Args:
        input_feature: input feature as a numpy array
        feature_names: names of input features as reference for scalers
        scalers: dict of feature scalers

    Returns:
        scaled input features as numpy array
    """

    if isinstance(feature_names, str):
        feature_names = [feature_names]

    if isinstance(input_feature, np.ndarray) or isinstance(input_feature, torch.Tensor):
        input_dim = input_feature.shape[1] if len(input_feature.shape) > 1 else 1
    elif isinstance(input_feature, float) or isinstance(input_feature, int):
        # handle case, where input is scalar
        input_scaled = scalers[feature_names[0]].inverse_transform([[input_feature]])[0][0]
        return input_scaled
    elif isinstance(input_feature, list):
        input_feature = np.array(input_feature)
        input_dim = input_feature.shape[1] if len(input_feature.shape) > 1 else 1
    else:
        raise ValueError(f"Unsupported input type: {type(input_feature)}")

    # reshape, if input is one dimensional
    if input_dim == 1:
        input_feature = input_feature.reshape(-1, 1)

    input_scaled = input_feature.copy()
    for i in range(input_dim):
        input_scaled[:, i] = scalers[feature_names[i]].inverse_transform(np.array([input_scaled[:, i]]))

    return input_scaled


def combine_features(cycles: list[dict], feature_config: dict) -> dict:
    combined_features = dict()
    for feature_set in feature_config["feature_sets"]:
        combined_features[feature_set] = dict()
        if feature_set not in cycles[0]:
            continue
        features_in_set = cycles[0][feature_set].keys()
        for feature in features_in_set:
            feature_values = [cycles[i][feature_set][feature] for i in range(len(cycles))]
            if isinstance(feature_values[0], dict):
                for key in feature_values[0].keys():
                    feature_array = np.concatenate([feature_values[i][key] for i in range(len(feature_values))])
                    combined_features[feature_set][key] = feature_array
            else:
                feature_array = np.concatenate(feature_values)
                combined_features[feature_set][feature] = feature_array

    return combined_features