diff --git a/thesis/main.tex b/thesis/main.tex index f9ca317..e06552f 100644 --- a/thesis/main.tex +++ b/thesis/main.tex @@ -9,19 +9,27 @@ \usepackage{amsmath} \usepackage{array} \usepackage{amsfonts} -\usepackage[a4paper, margin=1in]{geometry} - -\usepackage{graphicx} -\graphicspath{{resources/figures/}} - -\usepackage{blindtext} -\usepackage[style=ieee, backend=biber]{biblatex} -\addbibresource{../main.bib} \usepackage{booktabs} \usepackage{tabularx} \usepackage{pdflscape} \usepackage{adjustbox} \usepackage{multirow} +\usepackage{blindtext} +\usepackage[a4paper, margin=1in]{geometry} + +\usepackage{graphicx} +\graphicspath{{resources/figures/}} + +\usepackage[style=ieee, backend=biber]{biblatex} +\addbibresource{../main.bib} + +\usepackage{hyperref} +\hypersetup{ + colorlinks = true, % Colours links instead of ugly boxes + urlcolor = blue, % Colour for external hyperlinks + linkcolor = blue, % Colour of internal links + citecolor = red % Colour of citations +} % Document \begin{document} diff --git a/thesis/resources/figures/results/lstm_fertility_results_by_input_length.png b/thesis/resources/figures/results/lstm_fertility_results_by_input_length.png index 04e394b..25f6ebb 100644 Binary files a/thesis/resources/figures/results/lstm_fertility_results_by_input_length.png and b/thesis/resources/figures/results/lstm_fertility_results_by_input_length.png differ diff --git a/thesis/resources/figures/results/lstm_fertility_results_by_input_resolution.png b/thesis/resources/figures/results/lstm_fertility_results_by_input_resolution.png index 54631c5..be9f185 100644 Binary files a/thesis/resources/figures/results/lstm_fertility_results_by_input_resolution.png and b/thesis/resources/figures/results/lstm_fertility_results_by_input_resolution.png differ diff --git a/thesis/resources/figures/results/lstm_ov_over_results_by_input_length.png b/thesis/resources/figures/results/lstm_ov_over_results_by_input_length.png index f725103..e6ff112 100644 Binary files a/thesis/resources/figures/results/lstm_ov_over_results_by_input_length.png and b/thesis/resources/figures/results/lstm_ov_over_results_by_input_length.png differ diff --git a/thesis/resources/figures/results/model_performance_overview.png b/thesis/resources/figures/results/model_performance_overview.png index 1042cd0..ebd54a4 100644 Binary files a/thesis/resources/figures/results/model_performance_overview.png and b/thesis/resources/figures/results/model_performance_overview.png differ diff --git a/thesis/sections/methodology.tex b/thesis/sections/methodology.tex index b252ec5..dd138e2 100644 --- a/thesis/sections/methodology.tex +++ b/thesis/sections/methodology.tex @@ -687,6 +687,22 @@ Tables~\ref{tab:fertility_mae_metrics} and~\ref{tab:ov_over_mae_metrics} summari \label{tab:ov_over_mae_metrics} \end{table} +\subsubsection{Best Model Configuration Selection}\label{subsubsec:methodology_best_model_config_selection} +Based on the results from the input window length, input resolution, +and model parameter space exploration, we selected a single best configuration for each model architecture to carry forward. +This step was necessary to keep the computational effort manageable, +as exhaustively testing all possible configurations for every subsequent metric would have been prohibitively expensive. + +Selection was based primarily on the \textbf{Fertility-Overall MSE} metric, +as it most directly reflects the main objective of this study: predicting fertility. +In cases where the difference between configurations was small, +we preferred the option that aligned with the general tendency of the model architecture. +For example, if an architecture consistently performed better with more input data or longer sequences, +but the Fertility-Overall MSE was only marginally better for a shorter window, we selected the longer window. + +This selection process was applied to all tested parameters—input window length, input resolution, and model complexity. +The resulting configurations were then used in the irregular cycles study and the use case evaluation. + \subsubsection{Regular and Irregular Cycles}\label{subsubsec:methodology_regular_and_irregular_cycles} A perfectly regular menstrual pattern does not require a sophisticated algorithm to predict ovulation— diff --git a/thesis/sections/results.tex b/thesis/sections/results.tex index 002833c..feb65a9 100644 --- a/thesis/sections/results.tex +++ b/thesis/sections/results.tex @@ -3,35 +3,51 @@ \section{Results}\label{sec:results} -% -%We begin by comparing the overall performance of all trained models and baselines using four key evaluation metrics: -%mean absolute error (MAE) and mean squared error (MSE), as well as the metrics for their respective sub-intervals. -% -%Overall, transformer-based models consistently outperformed LSTM variants and baseline methods across most evaluation criteria. -%Among the baselines, [e.g., "the rule-based method"] showed the weakest performance, -%while the [e.g., "windowed logistic regression"] performed competitively in certain contexts. -%Differences across models were most pronounced in MSE and \(R^2\), -%indicating that advanced architectures better captured higher-order dynamics and reduced large prediction errors. -% -%The previous section detailed the design and implementation of our fertility prediction pipeline, -%including data preprocessing, feature engineering, input encoding, and the development of several deep learning architectures. -%We now present the results of our evaluation, focusing on the predictive accuracy of the proposed models across different temporal resolutions, -%cycle types and use cases. -%Model performance is assessed using both overall metrics and biologically targeted subintervals, -%allowing for a nuanced comparison of approaches and their practical relevance to real-time fertility forecasting. -%Additionally, model performance is compared to the three baseline models introduced. -% -% provide information about the training behaviour and statistic of the different models?? + +We summarize the main findings from our modeling experiments, +beginning with overall model performance across architectures, +followed by a detailed analysis of the individual architectures' performances for irregular and regular cycles and the use cases +introduced in the last section. +These results will also be compared with the three baseline methods to evaluate their performance versus less sophisticated methods. + +We summarize the main findings from our modeling experiments, beginning with overall model performance across architectures. +We then provide a detailed analysis of model performance for irregular and regular cycles, as well as for +the pregnancy and contraception use-cases described earlier. +Additionally, these results will be compared to those of the three baseline methods to evaluate the benefit of more advanced modeling approaches. \subsection{Overall Model Performance Across Architectures}\label{subsec:overall_model_performance_across_architectures} +We evaluated multiple model architectures to compare their effectiveness in predicting the fertility-probability and ovulation-over targets. +Based on these results, we selected the best-performing configuration for each architecture for further analysis. -% show why I selected the individual input configs for model config training -% selected by best mse fertility, use 2nd best, as it provides basically the same performance, but more input data for more complex model configs +\subsubsection{Fertility-Probability Prediction}\label{subsec:fertility_probability_prediction} -\subsection{Fertility Probability Prediction}\label{subsec:fertility_probability_prediction} +This section examines model performance in predicting the probability of fertility, +focusing on the effects of input window length, input resolution, and key architecture parameters. + +\paragraph{Impact of Input Window Length:}\label{subsubsec:fert_impact_of_historical_context} +Table~\ref{tab:fertility_results_by_window_length} summarizes the effect of varying the input sequence length on model performance for the fertility-probability target. +Across all architectures, no single window length consistently outperformed others across all metrics. + +For the \textbf{LSTM}, the longest input (160~days) yielded the lowest overall MAE (0.0378) and the best +fertile-day performance (MAE~$=0.0834$, MSE~$=0.0102$), while shorter sequences tended to perform slightly worse +, particularly for fertile-day prediction. +Non-fertile day performance was best at 20~days (MSE~$=0.0013$). + +In the \textbf{Transformer}, the optimal MAE for overall fertility occurred at 160~days (0.0411), +but the lowest fertile-day error was achieved at 80~days (MAE~$=0.0859$). +The best non-fertile-day performance was seen with 160~days (MSE~$=0.0016$). + +For the \textbf{Convolution-LSTM}, the shortest windows generally underperformed, with the best overall MAE (0.0394) and MSE (0.0045) obtained at 40~days. +Fertile-day metrics were optimal at 40~days as well, while non-fertile-day performance peaked at 20~days (MSE~$=0.0013$). + +The \textbf{Convolution-Transformer} achieved the global best MAE (0.0376) and MSE (0.0041) for overall fertility at 40~days, +indicating that intermediate historical context was most effective for this architecture. +Fertile-day performance was strongest at 20~days (MSE~$=0.0102$), while non-fertile-day predictions benefited from shorter inputs (10 or 40~days). + +Overall, results suggest that the optimal input length is architecture-dependent, with intermediate windows (20--40~days) +frequently yielding competitive or best performance, while extremely long sequences (160~days) only benefited certain architectures such as the LSTM\@. -\subsubsection{Impact of Input Window Length}\label{subsubsec:fert_impact_of_historical_context} \begin{landscape} \begin{table} \small @@ -77,31 +93,23 @@ \end{table} \end{landscape} -Table~\ref{tab:fertility_results_by_window_length} summarises performance across metrics and input window lengths for the fertility-probability target. +\paragraph{Impact of Input Resolution:}\label{subsubsec:fert_impact_of_input_resolution} -For both the LSTM and Transformer, longer input windows generally yield better results. -The LSTM achieves the top scores in four of six metrics with the 160-day window, including the global best for MAE and MSE on both fertile and non-fertile days. -The only exception is non-fertile day errors, where shorter windows perform better. +Table~\ref{tab:fertility_results_by_window_resolution} shows the effect of varying the input resolution (values per day) +on model performance for the fertility-probability target. +Note, that the convolutional models are not included here, as they have their own learned input representation via convolution. +No single resolution consistently outperformed others across all metrics, and optimal settings varied by architecture. -The Transformer peaks at the 160-day window in three of six metrics but improves on non-fertile days with shorter inputs. -Across all window lengths, it is consistently outperformed by the LSTM\@. +For the \textbf{LSTM}, the lowest overall MAE (0.0402) was obtained at 48~values/day, while the best overall MSE (0.0046) occurred at 4~values/day. +Fertile-day performance was optimal at 288~values/day (MAE~$=0.0872$, MSE~$=0.0110$), and non-fertile-day metrics were best at +12~values/day (MAE~$=0.0180$, MSE~$=0.0013$), both of which represent the global best values for these categories. -Convolutional models perform best with medium-length windows (20--40 days). -The convolutional LSTM reaches its lowest errors for most metrics at 40 days, except for non-fertile MSE, where 20 days is optimal---also a global best. -The convolutional Transformer similarly favours 40 days for overall and non-fertile-day metrics, while fertile-day metrics perform best with 20-day inputs. -It achieves global best scores for 40-day MAE and MSE, and outperforms the convolutional LSTM in all but non-fertile day metrics. - -\subsubsection{Impact of Input Resolution}\label{subsubsec:fert_impact_of_input_resolution} -Shifting focus from temporal span to sampling density, Table~\ref{tab:fertility_results_by_window_resolution} reports performance for the fertility-probability target across varying input resolutions. -Convolutional models are omitted, as their convolution layers inherently perform learnable resampling. - -The LSTM outperforms the Transformer at all resolutions, with best-performing metrics scattered across the medium-to-high range (4--288 values/day) and no single optimum. -Fertile-day errors are lowest at 288 values/day, while non-fertile day errors peak at 12 values/day. -Overall MAE and MSE minima occur at 48 and 4 values/day, respectively. - -The Transformer shows a clearer trend towards higher-resolution inputs. -Its fertile-day metrics are best at 4 values/day, whereas non-fertile day, overall MAE, and overall MSE scores peak between 48 and 288 values/day. +In the \textbf{Transformer}, the lowest overall MAE (0.0447) occurred at 48~values/day, while the best overall MSE (0.0048) was shared between 48~and 288~values/day. +Fertile-day performance peaked at 4~values/day (MAE~$=0.0891$, MSE~$=0.0115$), +whereas non-fertile-day metrics were best at 48~values/day (MAE~$=0.0254$) and 72~values/day (MSE~$=0.0014$). +Overall, the results indicate that intermediate input resolutions (4--48~values/day) often yielded the best overall performance, +while extreme resolutions (1 or 288~values/day) only benefited specific metrics such as fertile-day prediction for the LSTM\@. \begin{landscape} \begin{table} @@ -142,11 +150,39 @@ Its fertile-day metrics are best at 4 values/day, whereas non-fertile day, overa \end{table} \end{landscape} -\subsubsection{Impact of Model Parameters}\label{subsubsec:fert_impaoct_of_model_parameters} +\paragraph{Impact of Model Parameters:}\label{subsubsec:fert_impaoct_of_model_parameters} + +Tables~\ref{tab:fertility_results_by_model_parameters_lstm}--\ref{tab:fertility_results_by_model_parameters_conv_transformer} report +the results of the model parameter search across all architectures. +Each table shows the effect of varying hidden layer size and number of LSTM layers (for recurrent models), +or embedding size, number of encoder layers, and attention heads (for Transformer-based models). + +For the \textbf{LSTM}, performance improved with increasing hidden layer size, +reaching the best overall values at 512 units with four layers (MAE~$=0.0399$, MSE~$=0.0047$). The lowest fertile-day errors were also +observed in this configuration, while non-fertile-day performance peaked at 256 units (MAE~$=0.0175$, MSE~$=0.0011$). + +In the \textbf{Transformer}, larger embeddings and deeper networks generally improved performance. +The best overall MAE (0.0395) was achieved with a 512-dimensional embedding, eight encoder layers, and eight attention heads. +The lowest fertile-day errors occurred with a 512-dimensional embedding and four layers (MAE~$=0.0831$, +MSE~$=0.0103$), whereas non-fertile-day performance was strongest at 512~×~8 (MAE~$=0.0159$, MSE~$=0.0011$). + +For the \textbf{Convolution-LSTM}, the best overall configuration was 256 hidden units with four layers, +yielding the lowest overall MSE (0.0042) and fertile-day MSE (0.0100). +Increasing to 512 units slightly reduced overall MAE (0.0380) and non-fertile-day MAE (0.0184). + +In the \textbf{Convolution-Transformer}, the optimal configuration used a 512-dimensional embedding with four encoder layers and four attention heads, +achieving the best overall MAE (0.0377) and non-fertile-day performance (MAE~$=0.0177$, MSE~$=0.0013$). +Fertile-day prediction was strongest with eight encoder layers (MAE~$=0.0842$, MSE~$=0.0107$). + +Taken together, these results indicate that larger model capacities generally improved performance across all architectures, +with the best configurations typically found at the higher end of the tested parameter ranges. + \begin{landscape} \begin{table} \scriptsize \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{LSTM}} \\ \toprule \multirow{2}{*}{Hidden Layer Size} & \multirow{2}{*}{\# LSTM Layers} & @@ -173,6 +209,8 @@ Its fertile-day metrics are best at 4 values/day, whereas non-fertile day, overa \begin{table} \scriptsize \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{9}{c}{\textbf{Transformer}} \\ \toprule \multirow{2}{*}{Size of Embedding} & \multirow{2}{*}{\# Encoder Layers} & @@ -198,22 +236,96 @@ Its fertile-day metrics are best at 4 values/day, whereas non-fertile day, overa \textbf{Bold} values represent the best value for each metric within a model.} \label{tab:fertility_results_by_model_parameters_transformer} \end{table} + + \begin{table} + \scriptsize + \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{Convolutional-LSTM}} \\ + \toprule + \multirow{2}{*}{Hidden Layer Size} & + \multirow{2}{*}{\# LSTM Layers} & + \multicolumn{3}{c}{MAE} & + \multicolumn{3}{c}{MSE} \\ + \cmidrule(lr){3-5} \cmidrule(lr){6-8} + & & Fert Overall & Fert Days & Non-Fert Days & Fert Overall & Fert Days & Non-Fert Days \\ + \midrule + 16 & 1 & 0.0487 & 0.1029 & 0.0264 & 0.0056 & 0.0154 & \textbf{0.0016} \\ + 32 & 1 & 0.0461 & 0.0956 & 0.0266 & 0.0050 & 0.0133 & 0.0017 \\ + 32 & 2 & 0.0448 & 0.0975 & 0.0232 & 0.0051 & 0.0139 & \textbf{0.0016} \\ + 64 & 2 & 0.0432 & 0.0921 & 0.0241 & 0.0048 & 0.0123 & 0.0019 \\ + 128 & 2 & 0.0400 & 0.0895 & 0.0204 & 0.0045 & 0.0119 & \textbf{0.0016} \\ + 128 & 4 & 0.0398 & 0.0871 & 0.0204 & 0.0044 & 0.0113 & \textbf{0.0016} \\ + 256 & 4 & 0.0389 & \textbf{0.0826} & 0.0209 & \textbf{0.0042} & \textbf{0.0100} & 0.0018 \\ + 512 & 4 & \textbf{0.0380} & 0.0869 & \textbf{0.0184} & 0.0044 & 0.0112 & 0.0017 \\ + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Fertility-Probability Target across Different Model Parameters for the convolutional LSTM model with + a fixed input window length of 40 days. + \textbf{Bold} values represent the best value for each metric within a model.} + \label{tab:fertility_results_by_model_parameters_conv_lstm} + \end{table} + \begin{table} + \scriptsize + \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{Convolutional-Transformer}} \\ + \toprule + \multirow{2}{*}{Size of Embedding} & + \multirow{2}{*}{\# Encoder Layers} & + \multirow{2}{*}{\# Attention Heads} & + \multicolumn{3}{c}{MAE} & + \multicolumn{3}{c}{MSE} \\ + \cmidrule(lr){4-6} \cmidrule(lr){7-9} + & & & Fert Overall & Fert Days & Non-Fert Days & Fert Overall & Fert Days & Non-Fert Days \\ + \midrule + 16 & 1 & 1 & 0.0574 & 0.1005 & 0.0427 & 0.0061 & 0.0146 & 0.0032 \\ + 32 & 1 & 1 & 0.0462 & 0.0963 & 0.0266 & 0.0051 & 0.0135 & 0.0018 \\ + 64 & 1 & 1 & 0.0546 & 0.1006 & 0.0379 & 0.0059 & 0.0145 & 0.0027 \\ + 64 & 2 & 2 & 0.0427 & 0.0903 & 0.0237 & 0.0046 & 0.0118 & 0.0018 \\ + 128 & 2 & 2 & 0.0406 & 0.0868 & 0.0225 & 0.0044 & 0.0113 & 0.0017 \\ + 128 & 4 & 4 & 0.0411 & 0.0886 & 0.0230 & 0.0044 & 0.0116 & 0.0017 \\ + 256 & 4 & 4 & 0.0420 & 0.0866 & 0.0240 & \textbf{0.0043} & 0.0113 & 0.0016 \\ + 512 & 4 & 4 & \textbf{0.0377} & 0.0878 & \textbf{0.0177} & \textbf{0.0043} & 0.0117 & \textbf{0.0013} \\ + 512 & 8 & 8 & 0.0399 & \textbf{0.0842} & 0.0224 & 0.0044 & \textbf{0.0107} & 0.0019 \\ + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Fertility-Probability Target across Different Model Parameters for the convolutional Transformer model with + a fixed input window length of 40 days. + \textbf{Bold} values represent the best value for each metric within a model.} + \label{tab:fertility_results_by_model_parameters_conv_transformer} + \end{table} \end{landscape} -\subsubsection{Comparison with Baselines}\label{subsubsec:fert_comparison_with_baselines} +\subsubsection{Ovulation-Over Prediction}\label{subsubsec:ov_over_prediction} -\subsection{Ovulation-Over Prediction}\label{subsubsec:ov_over_prediction} +This section examines model performance in predicting the indicator for a passed ovulation, with attention to the influence of input window length, +the input resolution and the individual model architecture parameters. -\subsubsection{Impact of Input Window Length}\label{subsubsec:ov_over_impact_of_historical_context} +\paragraph{Impact of Input Window Length:}\label{subsubsec:ov_over_impact_of_historical_context} -Table~\ref{tab:ov_over_results_by_window_length} presents performance across metrics and input window lengths for the ovulation-over target. +Table~\ref{tab:ov_over_results_by_window_length} reports the impact of input sequence length on performance for the ovulation-over target. +The optimal length varied across architectures and prediction phases. -The LSTM and Transformer both outperform their convolutional counterparts. -The LSTM benefits from longer windows (160 days) except for the before-ovulation metric, where 20 days is optimal---also the \textbf{global best} across models. -The Transformer performs best with mid-length windows (40 days) for most metrics, with exceptions in after-ovulation performance (MAE: 80 days, MSE: 20 days). +For the \textbf{LSTM}, the best overall MAE (0.1039) and MSE (0.0557) were obtained with 160~days, +which also gave the lowest after-ovulation errors (MAE~$=0.0973$, MSE~$=0.0580$). However, the best before-ovulation +performance occurred at 20~days (MAE~$=0.0745$, MSE~$=0.0212$), which were the global bests for this phase. -Among convolutional models, the convolutional LSTM peaks at mid-length windows, reaching lowest MAE at 40 days and lowest MSE at 20 days. -The convolutional Transformer also favours 40 days overall, but before-ovulation performance benefits from longer inputs (160 days). +In the \textbf{Transformer}, 40~days yielded the lowest overall MAE (0.1017) and MSE (0.0533), +as well as the lowest after-ovulation errors (MAE~$=0.0883$, MSE~$=0.0520$), all of which were global bests. +The best before-ovulation results were achieved at 80~days (MAE~$=0.0897$) and 20~days (MSE~$=0.0255$). + +For the \textbf{Convolution-LSTM}, the shortest effective length was 40~days, +which achieved the lowest overall MAE (0.1455) and before-ovulation MAE (0.1089). +The best overall MSE (0.0699) and before-ovulation MSE (0.0389) were observed at 20~days. +After-ovulation errors were smallest at 20~days (MSE~$=0.0833$) and 40~days (MAE~$=0.1561$). + +In the \textbf{Convolution-Transformer}, the best overall MAE (0.1448) and MSE (0.0709) occurred at 40~days, +which also minimized after-ovulation MAE (0.1514) and MSE (0.0820). +The best before-ovulation performance came from 160~days for MAE (0.1051) and 10~days for MSE (0.0317). + +Overall, intermediate input lengths (20--40~days) were often optimal, particularly for before-ovulation prediction, +while longer sequences (160~days) occasionally improved after-ovulation accuracy. \begin{landscape} \begin{table} @@ -265,14 +377,25 @@ The convolutional Transformer also favours 40 days overall, but before-ovulation \end{table} \end{landscape} -\subsubsection{Impact of Input Resolution}\label{subsubsec:ov_over_impact_of_input_resolution} +\paragraph{Impact of Input Resolution:}\label{subsubsec:ov_over_impact_of_input_resolution} -Table~\ref{tab:ov_over_results_by_resolution} shows results for the ovulation-over target across different resolutions. -Convolutional models are excluded, as their convolution layers inherently perform resampling. +Table~\ref{tab:ov_over_results_by_resolution} presents the effect of varying input resolution (values per day) +on performance for the ovulation-over target. +Note, that the convolutional models are not included here, as they have their own learned input representation via convolution. +The best-performing resolution differed across architectures and prediction phases. -The LSTM generally performs best at medium resolutions (12--48 values/day), except for the after-ovulation metric, where higher resolutions yield better results. -The Transformer prefers higher resolutions, with optimal performance for all but the before-ovulation metric at 72 or 288 values/day. -Before-ovulation performance peaks at a medium resolution of 12 values/day. +For the \textbf{LSTM}, the lowest overall MAE (0.1153) was achieved at 12~values/day, +which also produced the global best before-ovulation results (MAE~$=0.0745$, MSE~$=0.0212$). +The best overall MSE (0.0633) was observed at 24~values/day. +After-ovulation performance was strongest at 48~values/day (MAE~$=0.0973$, MSE~$=0.0550$), the latter being a global best. + +In the \textbf{Transformer}, the optimal overall MAE (0.1041) and MSE (0.0585) were both achieved at 72~values/day, which also yielded the +lowest after-ovulation MAE (0.0947), all of which were global bests. +The best before-ovulation MAE (0.0771) and MSE (0.0255) were found at 12~values/day. +The lowest after-ovulation MSE (0.0578) occurred at 288~values/day. + +Overall, intermediate input resolutions (12--72~values/day) tended to perform best for ovulation-over prediction, +with 12~values/day favouring before-ovulation performance and 48--72~values/day improving after-ovulation accuracy. \begin{landscape} \begin{table} @@ -313,11 +436,43 @@ Before-ovulation performance peaks at a medium resolution of 12 values/day. \end{table} \end{landscape} -\subsubsection{Impact of Model Parameters}\label{subsubsec:ov_over_impaoct_of_model_parameters} +\paragraph{Impact of Model Parameters:}\label{subsubsec:ov_over_impaoct_of_model_parameters} + +Tables~\ref{tab:ov_over_results_by_model_parameters_lstm}--\ref{tab:ov_over_results_by_model_parameters_conv_transformer} +show the results of the parameter exploration for the ovulation-over target. +Each table reports the effect of varying hidden layer size and number of LSTM layers (for recurrent models), +or embedding size, number of encoder layers, and number of attention heads (for Transformer-based models). + +For the \textbf{LSTM}, performance improved with increasing hidden layer size, +with the best overall MAE (0.1120) and MSE (0.0616) obtained at 512 units with four layers. +This configuration also yielded the lowest after-ovulation errors (MAE~$=0.0983$, MSE~$=0.0613$). +The lowest before- ovulation errors were observed at 128 units with two layers (MAE~$=0.0861$, MSE~$=0.0274$). + +In the \textbf{Transformer}, smaller configurations performed poorly, while larger ones markedly improved results. +The best overall MAE (0.1114) was achieved with a 128-dimensional embedding and four encoder layers, +whereas the best overall MSE (0.0543) occurred with a 512-dimensional embedding and eight encoder layers. +The lowest before-ovulation errors were found at 64 dimensions with two layers (MAE~$=0.0810$, MSE~$=0.0302$), +while after-ovulation performance was best at 512 dimensions with eight layers (MAE~$=0.0753$, MSE~$=0.0410$). + +For the \textbf{Convolution-LSTM}, the best overall configuration used 256 hidden units with four layers, +reaching the lowest overall MAE (0.1424) and MSE (0.0687). +This configuration also minimized after-ovulation errors (MAE~$=0.1369$, MSE~$=0.0715$). +Before-ovulation performance was strongest with 128 units and two layers (MAE~$=0.1138$, MSE~$=0.0357$). + +In the \textbf{Convolution-Transformer}, the best overall MAE (0.1495) and MSE (0.0703) +were achieved with a 512-dimensional embedding and four encoder layers. +This configuration also gave the lowest after-ovulation errors (MAE~$=0.1551$, MSE~$=0.0814$). +Before-ovulation errors were lowest at 512 dimensions with eight layers (MAE~$=0.1025$, MSE~$=0.0339$). + +Overall, the results show that larger configurations generally improved performance across architectures for the ovulation-over target, +with the best outcomes typically found at the higher-capacity settings. + \begin{landscape} \begin{table} \scriptsize \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{LSTM}} \\ \toprule \multirow{2}{*}{Hidden Layer Size} & \multirow{2}{*}{\# LSTM Layers} & @@ -344,6 +499,8 @@ Before-ovulation performance peaks at a medium resolution of 12 values/day. \begin{table} \scriptsize \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{9}{c}{\textbf{Transformer}} \\ \toprule \multirow{2}{*}{Size of Embedding} & \multirow{2}{*}{\# Encoder Layers} & @@ -369,15 +526,169 @@ Before-ovulation performance peaks at a medium resolution of 12 values/day. \textbf{Bold} values represent the best value for each metric within a model.} \label{tab:ov_over_results_by_model_parameters_transformer} \end{table} + + \begin{table} + \scriptsize + \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{Convolutional-LSTM}} \\ + \toprule + \multirow{2}{*}{Hidden Layer Size} & + \multirow{2}{*}{\# LSTM Layers} & + \multicolumn{3}{c}{MAE} & + \multicolumn{3}{c}{MSE} \\ + \cmidrule(lr){3-5} \cmidrule(lr){6-8} + & & Fert Overall & Fert Days & Non-Fert Days & Fert Overall & Fert Days & Non-Fert Days \\ + \midrule + 16 & 1 & 0.2920 & 0.2198 & 0.3344 & 0.1264 & 0.0602 & 0.1653 \\ + 32 & 1 & 0.2141 & 0.1598 & 0.2377 & 0.0912 & 0.0398 & 0.1149 \\ + 32 & 2 & 0.1990 & 0.1556 & 0.2135 & 0.0852 & 0.0419 & 0.1025 \\ + 64 & 2 & 0.1743 & 0.1345 & 0.1845 & 0.0789 & 0.0400 & 0.0923 \\ + 128 & 2 & 0.1524 & 0.1138 & 0.1633 & 0.0717 & \textbf{0.0357} & 0.0845 \\ + 128 & 4 & 0.1579 & 0.1185 & 0.1670 & 0.0777 & 0.0401 & 0.0894 \\ + 256 & 4 & \textbf{0.1424} & 0.1425 & \textbf{0.1369} & \textbf{0.0687} & 0.0546 & \textbf{0.0715} \\ + 512 & 4 & 0.1436 & \textbf{0.1166} & 0.1523 & 0.0699 & 0.0382 & 0.0820 \\ + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Ovulation-Over Target across Different Model Parameters for the convolutional LSTM model with + a fixed input window length of 40 days. + \textbf{Bold} values represent the best value for each metric within a model.} + \label{tab:ov_over_results_by_model_parameters_conv_lstm} + \end{table} + \begin{table} + \scriptsize + \begin{tabularx}{\linewidth}{l*{8}{X}} + \midrule + \multicolumn{8}{c}{\textbf{Convolutional-Transformer}} \\ + \toprule + \multirow{2}{*}{Size of Embedding} & + \multirow{2}{*}{\# Encoder Layers} & + \multirow{2}{*}{\# Attention Heads} & + \multicolumn{3}{c}{MAE} & + \multicolumn{3}{c}{MSE} \\ + \cmidrule(lr){4-6} \cmidrule(lr){7-9} + & & & Fert Overall & Fert Days & Non-Fert Days & Fert Overall & Fert Days & Non-Fert Days \\ + \midrule + 16 & 1 & 1 & 0.3759 & 0.3602 & 0.3855 & 0.1597 & 0.1498 & 0.1659 \\ + 32 & 1 & 1 & 0.1757 & 0.1119 & 0.2017 & 0.0916 & 0.0314 & 0.1167 \\ + 64 & 1 & 1 & 0.2771 & 0.2395 & 0.2978 & 0.1303 & 0.1133 & 0.1402 \\ + 64 & 2 & 2 & 0.1610 & 0.1149 & 0.1777 & 0.0816 & \textbf{0.0310} & 0.1021 \\ + 128 & 2 & 2 & 0.1501 & 0.0997 & 0.1725 & 0.0783 & 0.0316 & 0.0991 \\ + 128 & 4 & 4 & 0.1668 & 0.1472 & 0.1758 & 0.0801 & 0.0488 & 0.0953 \\ + 256 & 4 & 4 & 0.1529 & 0.1317 & 0.1597 & 0.0742 & 0.0435 & 0.0861 \\ + 512 & 4 & 4 & \textbf{0.1495} & 0.1298 & \textbf{0.1551} & \textbf{0.0703} & 0.0409 & \textbf{0.0814} \\ + 512 & 8 & 8 & 0.1576 & \textbf{0.1025} & 0.1848 & 0.0825 & 0.0339 & 0.1066 \\ + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Ovulation-Over Target across Different Model Parameters for the convolutional Transformer model with + a fixed input window length of 40 days. + \textbf{Bold} values represent the best value for each metric within a model.} + \label{tab:ov_over_results_by_model_parameters_conv_transformer} + \end{table} \end{landscape} -\subsubsection{Comparison with Baselines}\label{subsubsec:ov_over_comparison_with_baselines} +\subsubsection{Best Model Configuration Selection}\label{subsubsec:results_best_model_config_selection} + +Following the selection procedure described in Section~\ref{subsubsec:methodology_best_model_config_selection} +the best configuration for each model architecture was identified based on the Fertility-Overall MSE and, where applicable, +the general tendencies of the model. +Table~\ref{tab:best_configs_lstm} and~\ref{tab:best_configs_transformer} summarize the selected input window length, +input resolution, and model complexity for each architecture. +These configurations are used in all subsequent experiments, including the irregular cycles analysis and the use case evaluation. + +\begin{table}[htbp] + \centering + \scriptsize + \begin{tabularx}{\linewidth}{lXXXX} + \toprule + \textbf{Architecture} & + \textbf{Input Window Length} & + \textbf{Input Resolution} & + \textbf{Hidden Layer Size} & + \textbf{\# LSTM Layers} \\ + \midrule + LSTM & 160 & 12 & 512 & 4 \\ + Convolution-LSTM & 40 & 288 & 512 & 4 \\ + \bottomrule + \end{tabularx} + \caption{Selected configurations for LSTM-based architectures. Input Window Length is given in days and input resolution in values per day.} + \label{tab:best_configs_lstm} +\end{table} + +\begin{table}[htbp] + \centering + \scriptsize + \begin{tabularx}{\linewidth}{lXXXXX} + \toprule + \textbf{Architecture} & + \textbf{Input Window Length} & + \textbf{Input Resolution} & + \textbf{Embedding Size} & + \textbf{\# Encoder Layers} & + \textbf{\# Attention Heads} \\ + \midrule + Transformer & 160 & 12 & 512 & 4 & 4 \\ + Convolution-Transformer & 40 & 288 & 512 & 4 & 4 \\ + \bottomrule + \end{tabularx} + \caption{Selected configurations for Transformer-based architectures. Input Window Length is given in days and input resolution in values per day.} + \label{tab:best_configs_transformer} +\end{table} \subsection{Stratified Analysis}\label{subsec:stratified_analysis} -\subsubsection{Regular vs Irregular Cycles}\label{subsubsec:regular_vs_irregular_cycles} - \subsubsection{Influence of User History Depth}\label{subsubsec:influence_of_past_user_data} +% don't forget to also add baseline to tables + +\subsubsection{Regular vs Irregular Cycles}\label{subsubsec:regular_vs_irregular_cycles} +\begin{landscape} + \begin{table} + \small + \begin{tabularx}{\linewidth}{l*{6}{X}} + \toprule + \multirow{2}{*}{Model Architecture} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\ + \cmidrule(r){2-4} \cmidrule(r){5-7} + & Fertility Overall & Fertile Days & Non-Fertile Days & Fertility Overall & Fertile Days & Non-Fertile Days \\ + \midrule + \multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\ + \midrule + + \midrule + \multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\ + \midrule + + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Fertility-Probability Target across Different Model Architectures for the Regular and Irregular Cycle Groups. + \underline{Underlined} values represent the best value for each metric within a model. + \textbf{\underline{Bold + Underlined}} values represent the global best values across all models for a given metric.} + \label{tab:regular_vs_irregular_fertility_results} + \end{table} +\end{landscape} +\begin{landscape} + \begin{table} + \small + \begin{tabularx}{\linewidth}{l*{6}{X}} + \toprule + \multirow{2}{*}{Model} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\ + \cmidrule(r){2-4} \cmidrule(r){5-7} + & OV-Over Overall & OV-Over Before OV & OV-Over After OV & OV-Over Overall & OV-Over Before OV & OV-Over After OV \\ + \midrule + \multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\ + \midrule + + \midrule + \multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\ + \midrule + + \bottomrule + \end{tabularx} + \caption{Evaluation Metrics for the Ovulation-Over Target across Different Model Architectures for the Regular and Irregular Cycle Groups. + \underline{Underlined} values represent the best value for each metric within a model. + \textbf{\underline{Bold + Underlined}} values represent the global best values across all models for a given metric.} + \label{tab:regular_vs_irregular_ov_over_results} + \end{table} +\end{landscape} \subsection{Use-Case Evaluation Results}\label{subsec:use_case_evaluation_results}