255 lines
17 KiB
TeX
255 lines
17 KiB
TeX
%! Author = alex
|
|
%! Date = 3/6/25
|
|
|
|
|
|
\section{Results}\label{sec:results}
|
|
|
|
We begin by comparing the overall performance of all trained models and baselines using four key evaluation metrics:
|
|
mean absolute error (MAE), mean squared error (MSE), and the coefficient of determination (\(R^2\)) for the regression target,
|
|
as well as the metrics for their respective sub-intervals.
|
|
Figure~\ref{fig:results_performance_overview_by_model_type} provides an overview of these metrics across all model types.
|
|
|
|
Overall, transformer-based models consistently outperformed LSTM variants and baseline methods across most evaluation criteria.
|
|
Among the baselines, [e.g., "the rule-based method"] showed the weakest performance,
|
|
while the [e.g., "windowed logistic regression"] performed competitively in certain contexts.
|
|
Differences across models were most pronounced in MSE and \(R^2\),
|
|
indicating that advanced architectures better captured higher-order dynamics and reduced large prediction errors.
|
|
|
|
The previous section detailed the design and implementation of our fertility prediction pipeline,
|
|
including data preprocessing, feature engineering, input encoding, and the development of several deep learning architectures.
|
|
We now present the results of our evaluation, focusing on the predictive accuracy of the proposed models across different temporal resolutions,
|
|
cycle types and use cases.
|
|
Model performance is assessed using both overall metrics and biologically targeted subintervals,
|
|
allowing for a nuanced comparison of approaches and their practical relevance to real-time fertility forecasting.
|
|
Additionally, model performance is compared to the three baseline models introduced.
|
|
|
|
% provide information about the training behaviour and statistic of the different models??
|
|
|
|
\subsection{Overall Model Performance Across Architectures}\label{subsec:overall_model_performance_across_architectures}
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=0.9\textwidth]{resources/figures/results/model_performance_overview}
|
|
\caption{Overview of the performances of all model types including the baselines on 4 selected performance metrics.}
|
|
\label{fig:results_performance_overview_by_model_type}
|
|
\end{figure}
|
|
|
|
\subsection{Fertility Probability Prediction Accuracy}\label{subsec:fertility_probability_precition_accuracy}
|
|
|
|
\begin{landscape}
|
|
\begin{table}[ht]
|
|
\centering
|
|
\caption{Model comparison for fertility prediction using MAE and MSE}
|
|
\begin{adjustbox}{max width=\linewidth}
|
|
\begin{tabular}{lllcccccc}
|
|
\toprule
|
|
\textbf{Model} & \textbf{Window} & \textbf{Daily} &
|
|
\textbf{MAE$_{fert}$} & \textbf{MAE$_{fert,during}$} & \textbf{MAE$_{fert,non}$} &
|
|
\textbf{MSE$_{fert}$} & \textbf{MSE$_{fert,during}$} & \textbf{MSE$_{fert,non}$} \\
|
|
\midrule
|
|
ModelA & 7 & Yes & 0.67 & 0.59 & 0.73 & 0.89 & 0.82 & 0.95 \\
|
|
ModelB & 14 & No & 0.65 & 0.58 & 0.71 & 0.87 & 0.80 & 0.93 \\
|
|
% More rows...
|
|
\bottomrule
|
|
\end{tabular}
|
|
\end{adjustbox}
|
|
\label{tab:fertility_comparison}
|
|
\end{table}
|
|
\end{landscape}
|
|
|
|
|
|
% show why I selected the individual input configs for model config training
|
|
% selected by best mse fertility, use 2nd best, as it provides basically the same performance, but more input data for more complex model configs
|
|
|
|
\subsubsection{Fertility Probability Prediction}\label{subsubsec:fertility_probability_prediction}
|
|
|
|
\subsubsection{Impact of Input Window Length}\label{subsubsec:fert_impact_of_historical_context}
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/lstm_fertility_results_by_input_length}
|
|
\caption{Performance of LSTM models with different input window lengths and fixes input resolutions for the fertility probability target}
|
|
\label{fig:results_performance_lstm_fertility_input_window_length}
|
|
\end{figure}
|
|
|
|
\begin{landscape}
|
|
\begin{table}
|
|
% \begin{tabular}{lrrrrrrr}
|
|
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
|
\toprule
|
|
\multirow{2}{*}{Input-Length in Days} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
|
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
|
& Fertility Overall & Fertile Days & Non-Fertile Days & Fertility Overall & Fertile Days & Non-Fertile Days \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{LSTM}} \\ \midrule
|
|
10 & 0.0399 & 0.0857 & 0.0212 & 0.0045 & 0.0105 & 0.0021 \\
|
|
20 & 0.0421 & 0.0994 & 0.0180 & 0.0052 & 0.0144 & \underline{\textbf{0.0013}} \\
|
|
40 & 0.0406 & 0.0938 & 0.0184 & 0.0049 & 0.0128 & 0.0016 \\
|
|
80 & 0.0420 & 0.1006 & \underline{\textbf{0.0176}} & 0.0053 & 0.0147 & 0.0014 \\
|
|
160 & \underline{0.0378} & \underline{\textbf{0.0834}} & 0.0192 & \underline{0.0043} & \underline{\textbf{0.0102}} & 0.0019 \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Transformer}} \\ \midrule
|
|
10 & 0.0443 & 0.0868 & 0.0273 & 0.0046 & 0.0110 & 0.0021 \\
|
|
20 & 0.0472 & 0.0949 & 0.0274 & 0.0051 & 0.0133 & 0.0018 \\
|
|
40 & 0.0413 & 0.0861 & 0.0233 & \underline{0.0044} & 0.0111 & 0.0017 \\
|
|
80 & 0.0437 & \underline{0.0859} & 0.0269 & 0.0045 & \underline{0.0108} & 0.0021 \\
|
|
160 & \underline{0.0411} & 0.0882 & \underline{0.0218} & 0.0045 & 0.0116 & \underline{0.0016} \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Convolution-LSTM}} \\ \midrule
|
|
10 & 0.0435 & \underline{0.0954} & 0.0224 & 0.0050 & 0.0133 & 0.0017 \\
|
|
20 & \underline{0.0414} & 0.0996 & \underline{0.0179} & 0.0050 & 0.0145 & \underline{\textbf{0.0013}} \\
|
|
40 & 0.0394 & 0.0911 & 0.0184 & \underline{0.0045} & \underline{0.0122} & 0.0014 \\
|
|
80 & 0.0481 & 0.1005 & 0.0278 & 0.0054 & 0.0146 & 0.0019 \\
|
|
160 & 0.0505 & 0.1067 & 0.0290 & 0.0060 & 0.0166 & 0.0020 \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Convolution-Transformer}} \\ \midrule
|
|
10 & 0.0459 & 0.0962 & 0.0256 & 0.0049 & 0.0136 & \underline{0.0015} \\
|
|
20 & 0.0423 & \underline{0.0848} & 0.0257 & 0.0045 & \underline{\textbf{0.0102}} & 0.0022 \\
|
|
40 & \underline{\textbf{0.0376}} & 0.0854 & \underline{0.0186} & \underline{\textbf{0.0041}} & 0.0108 & \underline{0.0015} \\
|
|
80 & 0.0414 & 0.0932 & 0.0210 & 0.0048 & 0.0128 & 0.0018 \\
|
|
160 & 0.0523 & 0.1018 & 0.0337 & 0.0059 & 0.0148 & 0.0026 \\
|
|
\bottomrule
|
|
% \end{tabular}
|
|
\end{tabularx}
|
|
\caption{Evaluation Metrics for the Fertility-Probability target for each model architecture across different input lenghts.
|
|
Underlined values indicate the best value per metric for a model and bold and underlined indicate global best values for a metric.}
|
|
\label{tab:fertility_results}
|
|
\end{table}
|
|
\end{landscape}
|
|
|
|
For the LSTM model, the error metrics show a small improvement with larger input window size.
|
|
Also, the MSE metric is numerically smaller than the MAE, which suggests, that the errors are rather small and tightly clustered with few large outliers.
|
|
For the MSE metric, the non-fertile phase seem to behave inversely proportional to the other both the fertile phase error and the overall error.
|
|
This is not the case for the MAE metric.
|
|
Overall, the improvement over a growing input window size is very small
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/transformer_fertility_results_by_input_length}
|
|
\caption{Performance of Transformer models with different input window lengths and fixes input resolutions for the fertility probability target}
|
|
\label{fig:results_performance_transformer_fertility_input_window_length}
|
|
\end{figure}
|
|
|
|
\subsubsection{Impact of Input Resolution}\label{subsubsec:fert_impact_of_input_resolution}
|
|
|
|
Figure~\ref{fig:results_performance_lstm_fertility_input_resolution} presents the results for different input resolutions used in the LSTM models,
|
|
alongside the corresponding mean squared error (MSE) metrics.
|
|
The plot shows the MSE values for three target categories: non-fertile days, overall MSE, and fertile days MSE\@.
|
|
|
|
Notably, the MSE for non-fertile days appears to behave inversely in relation to the other two metrics.
|
|
However, there is no consistent upward or downward trend when changing the input window length with a fixed input resolution.
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/transformer_fertility_results_by_input_resolution}
|
|
\caption{Performance of transformer models with different input window lengths and fixes input resolutions for the fertility probability target}
|
|
\label{fig:results_performance_transformer_fertility_input_resolution}
|
|
\end{figure}
|
|
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.00\textwidth]{resources/figures/results/lstm_fertility_results_by_input_resolution}
|
|
\caption{Performance of LSTM models with different input resolutions and fixes input window lengths for the fertility probability target}
|
|
\label{fig:results_performance_lstm_fertility_input_resolution}
|
|
\end{figure}
|
|
|
|
\subsubsection{Comparison with Baselines}\label{subsubsec:fert_comparison_with_baselines}
|
|
|
|
\subsection{Ovulation-Over Prediction}\label{subsubsec:ov_over_prediction}
|
|
\begin{landscape}
|
|
\begin{table}
|
|
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
|
\toprule
|
|
\multirow{2}{*}{Input-Length in Days} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
|
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
|
& OV-Over Overall & OV-Over Before OV & OV-Over After OV & OV-Over Overall & OV-Over Before OV & OV-Over After OV \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{LSTM}} \\
|
|
\midrule
|
|
10 & 0.1218 & 0.1044 & 0.1223 & \underline{0.0612} & 0.0289 & 0.0711 \\
|
|
20 & 0.1153 & \underline{\textbf{0.0745}} & 0.1312 & 0.0641 & \underline{\textbf{0.0212}} & 0.0822 \\
|
|
40 & 0.1066 & 0.0820 & 0.1128 & 0.0616 & 0.0281 & 0.0740 \\
|
|
80 & 0.1173 & 0.0842 & 0.1291 & 0.0647 & 0.0263 & 0.0801 \\
|
|
160 & \underline{0.1039} & 0.1139 & \underline{0.0973} & 0.0557 & 0.0462 & \underline{0.0580} \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Transformer}} \\
|
|
\midrule
|
|
10 & 0.1137 & 0.1006 & 0.1186 & 0.0618 & 0.0336 & 0.0740 \\
|
|
20 & 0.1204 & 0.0771 & 0.1366 & 0.0690 & \underline{0.0255} & 0.0864 \\
|
|
40 & \underline{\textbf{0.1017}} & 0.1409 & \underline{\textbf{0.0883}} & \underline{\textbf{0.0533}} & 0.0621 & \underline{\textbf{0.0520}} \\
|
|
80 & 0.1138 & \underline{0.0897} & 0.1234 & 0.0654 & 0.0356 & 0.0788 \\
|
|
160 & 0.1076 & 0.0959 & 0.1141 & 0.0606 & 0.0379 & 0.0714 \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Convolution-LSTM}} \\
|
|
\midrule
|
|
10 & 0.1847 & 0.1878 & 0.1840 & 0.0819 & 0.0581 & 0.0949 \\
|
|
20 & 0.1493 & 0.1274 & 0.1562 & \underline{0.0699} & \underline{0.0389} & \underline{0.0833} \\
|
|
40 & \underline{0.1455} & \underline{0.1089} & \underline{0.1561} & 0.0722 & 0.0358 & 0.0852 \\
|
|
80 & 0.2327 & 0.1796 & 0.2507 & 0.1168 & 0.0686 & 0.1327 \\
|
|
160 & 0.2518 & 0.2040 & 0.2757 & 0.1293 & 0.0878 & 0.1503 \\
|
|
\midrule
|
|
\multicolumn{7}{c}{\textbf{Convolution-Transformer}} \\
|
|
\midrule
|
|
10 & 0.1472 & 0.1053 & 0.1651 & 0.0768 & 0.0317 & 0.0966 \\
|
|
20 & 0.1530 & 0.1307 & 0.1637 & 0.0745 & 0.0443 & 0.0886 \\
|
|
40 & \underline{0.1448} & 0.1228 & \underline{0.1514} & \underline{0.0709} & 0.0435 & \underline{0.0820} \\
|
|
80 & 0.1685 & 0.1164 & 0.1889 & 0.0865 & 0.0345 & 0.1089 \\
|
|
160 & 0.2440 & \underline{0.1051} & 0.3117 & 0.1403 & \underline{0.0286} & 0.1946 \\
|
|
\bottomrule
|
|
\end{tabularx}
|
|
\caption{Evaluation Metrics for the Ovulation-Over target for each model architecture across different input lenghts.
|
|
Underlined values indicate the best value per metric for a model and bold and underlined indicate global best values for a metric.}
|
|
\label{tab:ov_over_results}
|
|
\end{table}
|
|
\end{landscape}
|
|
|
|
\subsubsection{Performance around Ovulation}\label{subsubsec:ov_over_performance_around_ovulation}
|
|
|
|
\subsubsection{Impact of Input Window Length}\label{subsubsec:ov_over_impact_of_historical_context}
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/lstm_ov_over_results_by_input_length}
|
|
\caption{Performance of LSTM models with different input window lengths and fixes input resolutions for the ovulation-over target}
|
|
\label{fig:results_performance_lstm_ov_over_input_window_length}
|
|
\end{figure}
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/transformer_ov_over_results_by_input_length}
|
|
\caption{Performance of Transformer models with different input window lengths and fixes input resolutions for the ovulation-over target}
|
|
\label{fig:results_performance_transformer_ov_over_input_window_length}
|
|
\end{figure}
|
|
|
|
\subsubsection{Impact of Input Resolution}\label{subsubsec:ov_over_impact_of_input_resolution}
|
|
|
|
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.00\textwidth]{resources/figures/results/lstm_ov_over_results_by_input_resolution}
|
|
\caption{Performance of LSTM models with different input resolutions and fixes input window lengths for the ovulation-over target}
|
|
\label{fig:results_performance_lstm_ov_over_input_resolution}
|
|
\end{figure}
|
|
\begin{figure}
|
|
\centering
|
|
\includegraphics[width=1.0\textwidth]{resources/figures/results/transformer_ov_over_results_by_input_resolution}
|
|
\caption{Performance of transformer models with different input window lengths and fixes input resolutions for the ovulation-over target}
|
|
\label{fig:results_performance_transformer_ov_over_input_resolution}
|
|
\end{figure}
|
|
|
|
\subsubsection{Comparison with Baselines}\label{subsubsec:ov_over_comparison_with_baselines}
|
|
|
|
\subsection{Stratified Analysis}\label{subsec:stratified_analysis}
|
|
|
|
\subsubsection{Regular vs Irregular Cycles}\label{subsubsec:regular_vs_irregular_cycles}
|
|
|
|
\subsubsection{Influence of User History Depth}\label{subsubsec:influence_of_past_user_data}
|
|
|
|
\subsection{Use-Case Evaluation Results}\label{subsec:use_case_evaluation_results}
|
|
|
|
\subsubsection{Contraception Use-Case Results}\label{subsubsec:use_case_contraception_results}
|
|
|
|
\subsubsection{Pregnancy Use-Case Results}\label{subsubsec:use_case_pregnancy_results}
|
|
|
|
\subsection{Summary of Key Findings}\label{subsec:summary_of_key_findings}
|