finished results and started discussion
This commit is contained in:
Alex Blank
Binary file not shown.
|
After Width: | Height: | Size: 495 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 474 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 446 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 338 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 373 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 503 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 350 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 455 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 467 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 500 KiB |
@@ -416,3 +416,75 @@
|
||||
\label{tab:ov_over_results_by_model_parameters_conv_transformer}
|
||||
\end{table}
|
||||
\end{landscape}
|
||||
|
||||
\begin{landscape}
|
||||
\begin{table}
|
||||
\small
|
||||
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
||||
\toprule
|
||||
\multirow{2}{*}{Model} & \multicolumn{2}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
||||
& Fertility Overall & Fertile Days & Non-Fertile Days & Fertility Overall & Fertile Days & Non-Fertile Days \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\midrule
|
||||
LSTM & 0.026034 & 0.072578 & \textbf{0.007398} & 0.002563 & 0.007850 & \textbf{0.000415} \\
|
||||
Transformer & 0.025847 & \textbf{0.065979} & 0.009693 & \textbf{0.002376} & \textbf{0.006651} & 0.000626 \\
|
||||
Convolutional LSTM & 0.026750 & 0.070371 & 0.009774 & 0.002460 & 0.007313 & 0.000573 \\
|
||||
Convolutional Transformer & \textbf{0.025161} & 0.066934 & 0.008789 & 0.002424 & 0.006919 & 0.000655 \\
|
||||
Last-Cycle Baseline & 0.037961 & 0.085792 & 0.016133 & 0.006592 & 0.014177 & 0.003198 \\
|
||||
Population Mean Baseline & 0.080743 & 0.139663 & 0.052513 & 0.016657 & 0.027671 & 0.011292 \\
|
||||
User Mean Baseline & 0.032624 & 0.077341 & 0.012325 & 0.005195 & 0.011715 & 0.002312 \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\midrule
|
||||
LSTM & \textbf{0.036813} & 0.093108 & \textbf{0.014657} & 0.004490 & 0.013415 & \textbf{0.001076} \\
|
||||
Transformer & 0.039474 & \textbf{0.084206} & 0.022359 & 0.004122 & \textbf{0.010774} & 0.001783 \\
|
||||
Convolutional LSTM & 0.037182 & 0.085657 & 0.018791 & \textbf{0.004096} & 0.011188 & 0.001538 \\
|
||||
Convolutional Transformer & 0.038007 & 0.086143 & 0.019175 & 0.004281 & 0.011374 & 0.001639 \\
|
||||
Last-Cycle Baseline & 0.049164 & 0.124943 & 0.022958 & 0.009423 & 0.023650 & 0.004613 \\
|
||||
Population Mean Baseline & 0.051673 & 0.123722 & 0.027299 & 0.009795 & 0.022330 & 0.005751 \\
|
||||
User Mean Baseline & 0.054256 & 0.129195 & 0.028745 & 0.010548 & 0.023935 & 0.006159 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Evaluation Metrics for the fertility probability target across Different Model Architectures for the Regular and Irregular Cycle Groups.
|
||||
\textbf{Bold} values represent the best values across all models for a given metric.}
|
||||
\label{tab:regular_vs_irregular_fertility_results}
|
||||
\end{table}
|
||||
\end{landscape}
|
||||
|
||||
\begin{landscape}
|
||||
\begin{table}
|
||||
\small
|
||||
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
||||
\toprule
|
||||
\multirow{2}{*}{Model} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
||||
& OV-Over Overall & OV-Over Before OV & OV-Over After OV & OV-Over Overall & OV-Over Before OV & OV-Over After OV \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\midrule
|
||||
LSTM & \textbf{0.059506} & 0.087519 & \textbf{0.047856}&\textbf{0.028519}& 0.031695 & \textbf{0.026116} \\
|
||||
Transformer & 0.075639 & 0.083443 & 0.069310 & 0.032929 & \textbf{0.024774} & 0.033411 \\
|
||||
Convolutional LSTM & 0.076335 & 0.090550 & 0.068022 & 0.034224 & 0.028962 & 0.033929 \\
|
||||
Convolutional Transformer & 0.075421 & 0.111256 & 0.060485 & 0.033333 & 0.039045 & 0.029329 \\
|
||||
Last-Cycle Baseline & 0.087916 & 0.093465 & 0.074191 & 0.087916 & 0.093465 & 0.074191 \\
|
||||
Population Mean Baseline & 0.285887 & \textbf{0.008876} & 0.418879 & 0.285887 & 0.008876 & 0.418879 \\
|
||||
User Mean Baseline & 0.069245 & 0.069152 & 0.055724 & 0.069245 & 0.069152 & 0.055724 \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\midrule
|
||||
LSTM & \textbf{0.094527} & \textbf{0.063073} & \textbf{0.106232} & \textbf{0.051747} & \textbf{0.016777} & 0.070419 \\
|
||||
Transformer & 0.117531 & 0.096482 & 0.116790 & 0.055990 & 0.026159 & 0.067520 \\
|
||||
Convolutional LSTM & 0.108166 & 0.092732 & 0.110939 & 0.054523 & 0.030092 & \textbf{0.066643} \\
|
||||
Convolutional Transformer & 0.111817 & 0.094195 & 0.114122 & 0.056075 & 0.029638 & 0.067965 \\
|
||||
Last-Cycle Baseline & 0.222906 & 0.134945 & 0.258047 & 0.222906 & 0.134945 & 0.258047 \\
|
||||
Population Mean Baseline & 0.171368 & 0.130366 & 0.140363 & 0.171368 & 0.130366 & 0.140363 \\
|
||||
User Mean Baseline & 0.180702 & 0.088884 & 0.217144 & 0.180702 & 0.088884 & 0.217144 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Evaluation Metrics for the Ovulation-Over Target across Different Model Architectures for the Regular and Irregular Cycle Groups.
|
||||
\textbf{Bold} values represent the best values across all models for a given metric.}
|
||||
\label{tab:regular_vs_irregular_ov_over_results}
|
||||
\end{table}
|
||||
\end{landscape}
|
||||
+105
-13
@@ -5,19 +5,104 @@
|
||||
\section{Discussion}\label{sec:discussion}
|
||||
|
||||
\paragraph{Limitations.}
|
||||
This analysis reports aggregate test-set metrics without uncertainty quantification.
|
||||
Because we did not retain per-user predictions, we cannot compute user-level bootstrap confidence intervals or perform paired significance testing.
|
||||
As a result, apparent performance differences—especially small ones—may reflect sampling variability.
|
||||
Future re-evaluation that stores per-user predictions will enable user-level bootstrapping, calibration assessment, and formal comparisons.
|
||||
|
||||
%In this study, we investigated the performance of different machine learning architectures on the task of fertility prediction,
|
||||
%with the aim to find a model that performs well for natural family planning and natural contraception on regular and irregular cycles.
|
||||
%
|
||||
%
|
||||
%Based on an extensive real-world database and established model architectures for timeseries analysis,
|
||||
%we expect our models to outperform the rule-based baselines.
|
||||
%We think, that for regular cycles, the performance difference will be lower than irregular cycles,
|
||||
%as the baseline models have no way of adapting to irregularities.
|
||||
|
||||
The model selection study based on the input window length, input resolution and model complexity parameters has shown,
|
||||
longer input windows and higher complexity seem to be favourable for the purposes of this study.
|
||||
The convolutional models show better performance for medium length input windows, which might be a limitation
|
||||
of the model or its convolutional layers.
|
||||
The LSTM and Transformer models also work better with medium resolution input, which suggests a sweet-spot of information.
|
||||
Too small a resolution might omit important information, while the opposite might introduce too much noise,
|
||||
that the models in the current configuration cannot handle.
|
||||
It could be worth investigating how more complex models handle more complex input data,
|
||||
as this grid-search was not possible in this study due to time and resource limitations.
|
||||
|
||||
The study also showed, that the LSTM and Transformer base models outperform the convolutional models.
|
||||
However, this is only the case for longer input sequences.
|
||||
When isolating a 40-day input window, the convolutional models outperform the non-convolution models,
|
||||
which suggests, that there is an informational benefit in the convolution.
|
||||
However, the convolutional models seem to struggle with longer input windows,
|
||||
which clearly represent a benefit for the predictive tasks, as the normal LSTM and Transformer models perform better
|
||||
on longer input windows.
|
||||
Additional work might be necessary to investigate, whether the convolutional models need additional complexity
|
||||
or an optimized convolutional pre-processing to deal with longer input sequences.
|
||||
|
||||
|
||||
All prediction curve in this section have been generated with the best model selected in section~\ref{subsubsec:results_best_model_config_selection},
|
||||
i.e., the LSTM model.
|
||||
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=1.0\textwidth]{resources/figures/discussion/regular_cycle_fertility_prediction}
|
||||
\caption{
|
||||
Temperature rolling average and fertility-probability prediction for a user with a regular cycle pattern. (Values are scaled features)
|
||||
}
|
||||
\label{fig:discussion_regular_cycle_fertility_prediction}
|
||||
\end{figure}
|
||||
|
||||
Figure~\ref{fig:discussion_regular_cycle_fertility_prediction} shows the prediction curve for the fertility-probability target
|
||||
for a user with a regular cycle pattern.
|
||||
It is clearly visible, that the predictions improve with each cycle, until they almost exactly match the targets.
|
||||
Another noteworthy observation is the correlation between a clear temperature drop preceding ovulation and
|
||||
the fertility rising.
|
||||
This can be seen more prominently in Figure~\ref{fig:discussion_temperature_drop_fertility_prediction}.
|
||||
The model seems to identify that this drop must be an indication for a heightened fertility.
|
||||
However, this decrease in temperature is not visible for all cycles, and thus significantly impacts the prediction qualities of the model.
|
||||
Figure~\ref{fig:discussion_spike_in_temperature_drop} shows an anomaly that results in a temperature spike during
|
||||
the pre-ovulatory temperature dip.
|
||||
This confuses the model into ending the fertile phase earlier.
|
||||
|
||||
This suggests that the model may interpret the termination of the pre-ovulatory temperature decline as a
|
||||
signal that ovulation is imminent or has just occurred, thus marking the end of the fertile window.
|
||||
Another pattern that is visible for many users is a cycle with no clear temperature drop.
|
||||
Figure~\ref{fig:discussion_unclear_temperature_drop} shows such a curve.
|
||||
Here, potentially unrelated circumstances lead to a drop in temperature that the model incorrectly identifies as fertile days.
|
||||
The actual fertile days don't result in an equally visible temperature drop.
|
||||
This indicates, that a clear temperature drop is a precursor to the ovulation and comes with the fertile days,
|
||||
but its intensity varies between users and also between cycles of the same user.
|
||||
Such a pattern can turn out to be a useful predictor for ovulation / fertility, but the models we trained don't seem
|
||||
to be able to differentiate between ovulation-related and unrelated temperature drops.
|
||||
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=1.0\textwidth]{resources/figures/discussion/temperature_drop_fertility}
|
||||
\caption{
|
||||
Temperature rolling average and fertility-probability prediction with a clear correlation between the temperature drop
|
||||
pre-ovulation and the fertility. (Values are scaled features)
|
||||
}
|
||||
\label{fig:discussion_temperature_drop_fertility_prediction}
|
||||
\end{figure}
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=1.0\textwidth]{resources/figures/discussion/temperature_spike_in_temperature_dop}
|
||||
\caption{
|
||||
Temperature rolling average and fertility-probability prediction with a disruptive temperature spike during the pre-ovulatory phase,
|
||||
leading to an early end of the predicted fertile window.
|
||||
}
|
||||
\label{fig:discussion_spike_in_temperature_drop}
|
||||
\end{figure}
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=1.0\textwidth]{resources/figures/discussion/temperature_unclear_temperature_drop}
|
||||
\caption{
|
||||
Temperature rolling average and fertility-probability prediction with no clear temperature drop and a resulting
|
||||
incorrect fertility-probability prediction. (Values are scaled features)
|
||||
}
|
||||
\label{fig:discussion_unclear_temperature_drop}
|
||||
\end{figure}
|
||||
|
||||
The results of the irregular vs regular cycle study, as shown in~\ref{subsubsec:regular_vs_irregular_cycles},
|
||||
indicate, that the models seem to learn recurrent cycle pattern on a per-user basis.
|
||||
We conclude this based on the fact, that regular cycle performance is significantly better than the performance on the irregular cycle set.
|
||||
If there is a reliable indicator in the temperature data, or its ablations, then there would not be a large difference between the two subsets.
|
||||
The temperature patterns should show up in both regular and irregular cycle users.
|
||||
Thus, all models must learn or extract information beyond the actual temperature data, which they seem to rely on more than
|
||||
the temperature related features.
|
||||
This is likely information indicating some form of regularity, which the models then use to predict the targets for new cycles.
|
||||
It would be interesting to take a close look at how and in what intensity the models use certain features,
|
||||
and whether the performance changes upon omitting certain features.
|
||||
|
||||
|
||||
%
|
||||
%In general, we expect the transformer based model to outperform the LSTM basd models, as they have proven to be
|
||||
%a more effective for time-series analysis tasks especially for longer sequences.
|
||||
@@ -38,6 +123,8 @@ Future re-evaluation that stores per-user predictions will enable user-level boo
|
||||
% explain the need for further medical interpretation of the results of either model
|
||||
|
||||
|
||||
% regular vs irregular cycles: mean baselines perform well on regular cycles as expected, as they get closer to the regularities with every past cycle
|
||||
|
||||
% transformer can deal with more data and does also perform better for some cases
|
||||
% lstm outperforms
|
||||
|
||||
@@ -50,8 +137,13 @@ Future re-evaluation that stores per-user predictions will enable user-level boo
|
||||
%of a given cycle is already over for any given day, there a variety of external factors that should be taken into
|
||||
%consideration for a direct output to the user.
|
||||
|
||||
\subsection{Limitations}\label{subsec:limiations}
|
||||
This analysis reports aggregate test-set metrics without uncertainty quantification.
|
||||
Because we did not retain per-user predictions, we cannot compute user-level bootstrap confidence intervals or perform paired significance testing.
|
||||
As a result, apparent performance differences, especially small ones, may reflect sampling variability.
|
||||
Future re-evaluation that stores per-user predictions will enable user-level bootstrapping, calibration assessment, and formal comparisons.
|
||||
|
||||
\section{Future Work}\label{sec:future_work}
|
||||
\subsection{Future Work}\label{sec:future_work}
|
||||
There are several directions in which this study could be extended,
|
||||
most of which were omitted due to time and resource constraints but represent valuable areas for future exploration.
|
||||
|
||||
|
||||
@@ -725,7 +725,7 @@ For a user to be included in the analysis, they must have at least five complete
|
||||
Additionally, ovulation must occur no later than cycle day 150, as later values likely indicate measurement errors or
|
||||
biologically atypical cases that fall outside the scope of this study.
|
||||
|
||||
\subsubsection{Use Case Evaluation}
|
||||
\subsubsection{Use Case Evaluation}\label{subsubsec:use_case_evaluation}
|
||||
We further evaluate the two distinct use cases introduced in Section~\ref{subsubsec:practical_use_cases}.
|
||||
For this purpose, two specialized evaluation algorithms were developed,
|
||||
enabling comparability between models and providing interpretable performance metrics for each scenario.
|
||||
|
||||
+449
-124
@@ -12,28 +12,32 @@ Finally, we compare the best configurations to three baselines and present strat
|
||||
|
||||
\subsection{Overall Model Performance Across Architectures}\label{subsec:overall_model_performance_across_architectures}
|
||||
|
||||
We evaluated multiple model architectures to compare their effectiveness in predicting the fertility probability and ovulation-over targets.
|
||||
Based on these results, we selected the best-performing configuration for each architecture for further analysis.
|
||||
We systematically evaluated multiple model architectures to assess their performance in
|
||||
predicting fertility probability and ovulation-over targets.
|
||||
Each architecture was tested across a range of input window lengths, temporal resolutions, and model capacities.
|
||||
The best-performing configurations for each architecture were selected for downstream analyses.
|
||||
|
||||
All metrics below are point estimates without uncertainty bounds; differences between models should be interpreted descriptively.
|
||||
All metrics reported in this section are point estimates without confidence intervals; therefore, comparisons are descriptive.
|
||||
|
||||
\subsubsection{Fertility-Probability Prediction}\label{subsec:fertility_probability_prediction}
|
||||
|
||||
This section examines model performance in predicting the probability of fertility,
|
||||
focusing on the effects of input window length, input resolution, and key architecture parameters.
|
||||
We analyze model performance in predicting fertility probability,
|
||||
focusing on the influence of temporal context (window length), measurement resolution, and model capacity.
|
||||
|
||||
\paragraph{Impact of input window length.}
|
||||
Across architectures, the optimal context is model-dependent.
|
||||
Intermediate windows (20–40\,days) often minimize overall error, with the \textbf{Convolutional Transformer}
|
||||
achieving the best withing grid at 40\,days (MAE 0.0376; MSE 0.0041).
|
||||
The \textbf{LSTM} benefits from long context (160\,days: overall MAE 0.0378; MSE 0.0043) and ties the
|
||||
global-best fertile-day MSE (0.0102), whereas the lowest non-fertile-day MSE occurs at 20\,days (0.0013; LSTM/Convolutional LSTM).
|
||||
For the \textbf{Transformer}, overall MAE is lowest at 160\,days (0.0411), fertile-day error at 80\,days (MAE 0.0859; MSE 0.0108),
|
||||
and non-fertile-day MSE at 160\,days (0.0016).
|
||||
The \textbf{Convolutional LSTM} peaks at 40\,days (MAE 0.0394; MSE 0.0045), with non-fertile-day MSE best at 20\,days (0.0013).
|
||||
Table~\ref{tab:fertility_windows_compact_mse} shows the compact results for the different architectures,
|
||||
full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:fertility_results_by_window_length}.
|
||||
|
||||
Across all architectures, predictive performance is sensitive to the temporal context length, but the optimal window varies by model.
|
||||
We consider 10 days short, 20--40 medium and 80--160 long input windows.
|
||||
|
||||
The \textbf{LSTM} performs best with long context: at 160 days, it reaches the lowest fertile-day MSE (0.0102) and a competitive overall error(0.0043).
|
||||
The lowest non-fertile-day MSE (0.0013) is tied between LSTM and Convolutional LSTM, both at 20 days.
|
||||
The \textbf{Transformer} shows more varied behavior: its overall error is minimized at 40 days (0.0044),
|
||||
fertile-day MSE at 80 days (0.0108), and non-fertile-day MSE at 160 days (0.0016).
|
||||
For the \textbf{Convolutional LSTM}, the best results occur at 40 days overall (MSE 0.0045), with non-fertile-day MSE again best at 20 days.
|
||||
The \textbf{Convolutional Transformer} achieves the lowest overall MSE (0.0041) at 40 days.
|
||||
|
||||
A summary of best-performing window lengths per architecture is shown in Table~\ref{tab:fertility_windows_compact_mse};
|
||||
full results including MAE are provided in Appendix Table~\ref{tab:fertility_results_by_window_length}.
|
||||
|
||||
\begin{table}[t]
|
||||
\small
|
||||
@@ -65,21 +69,31 @@ full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:fertility_results_by_win
|
||||
& 10, 40 / 0.0015 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Fertility probability: best input window per architecture (MSE only) at a fixed input resolution of 12 values/day.
|
||||
\caption{Best-performing parameter settings per architecture for fertility probability prediction,
|
||||
highlighting model-specific trade-offs between capacity and error minimization.
|
||||
Bold entries are bests within a column; ties are shown with commas.
|
||||
Full grid (incl.\ MAE) in Appendix, Table~\ref{tab:fertility_results_by_window_length}.}
|
||||
\label{tab:fertility_windows_compact_mse}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Impact of input resolution.}
|
||||
We vary input resolution at a fixed 20\,day window and exclude convolutional architectures (they consume the full 288 values/day and learn downsampling internally).
|
||||
For \textbf{LSTM}, overall MAE is lowest at 48 values/day (0.0402), while overall MSE is globally best at 4/day (0.0046).
|
||||
Fertile-day errors are minimized at 288/day (MAE 0.0872; MSE 0.0110, best within grid), whereas non-fertile days are best at 12/day (MAE 0.0180; MSE 0.0013, both within grid).
|
||||
For the \textbf{Transformer}, overall MAE is lowest at 48/day (0.0447); overall MSE ties at 48 and 288/day (0.0048).
|
||||
Its fertile-day best is at 4/day (MAE 0.0891; MSE 0.0115), and non-fertile-day bests occur at 48/day (MAE 0.0254) and 72/day (MSE 0.0014).
|
||||
In summary, intermediate resolutions (4–48/day) tend to minimize overall error, while an extreme (288/day) benefits LSTM on fertile-day performance.
|
||||
Table~\ref{tab:fertility_resolution_compact_mse} shows the condensed results per architecture,
|
||||
full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:fertility_results_by_window_resolution}.
|
||||
To assess the impact of temporal resolution, we fix the input window at 20 days and evaluate LSTM and Transformer models across a range of input rates.
|
||||
We exclude convolutional models from this comparison, as they always receive the full-resolution input (288 values/day) and learn temporal compression internally.
|
||||
|
||||
We consider 1--8 values per day low resolution, 12--48 medium resolution and 72--288 high resolution.
|
||||
|
||||
For the \textbf{LSTM}, the lowest overall MSE (0.0046) occurs at 4 values/day,
|
||||
while fertile-day MSE is minimized at full resolution (288/day, MSE 0.0110).
|
||||
The best non-fertile-day MSE (0.0013) is achieved at 12/day.
|
||||
|
||||
The \textbf{Transformer} performs best overall at both 48 and 288/day (MSE 0.0048),
|
||||
with the best fertile-day MSE at 4/day (0.0115) and best non-fertile-day MSE at 72/day (0.0014).
|
||||
|
||||
Overall, low to medium input resolutions (4–48/day) often minimize overall error,
|
||||
though higher resolutions can be advantageous for detecting short-term fertile-day signals.
|
||||
Results are summarized in Table~\ref{tab:fertility_resolution_compact_mse}; full metrics are in Appendix Table~\ref{tab:fertility_results_by_window_resolution}.
|
||||
|
||||
|
||||
\begin{table}[t]
|
||||
\small
|
||||
\renewcommand{\arraystretch}{1.15}
|
||||
@@ -108,19 +122,25 @@ full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:fertility_results_by_win
|
||||
\end{table}
|
||||
|
||||
\paragraph{Impact of model parameters.}
|
||||
At fixed input settings (LSTM/Transformer: 160\,days, 12 values/day; Convolutional models: 40\,days),
|
||||
larger capacities generally improve performance.
|
||||
For \textbf{LSTM}, the best overall MSE is at \(512\times4\) (Hidden Layer Size, Number of LSTM Layers) (0.0047),
|
||||
with the lowest fertile-day MSE also at \(512\times4\) (0.0122); the best non-fertile-day MSE occurs at \(256\times4\) (\,0.0011\,).
|
||||
For the \textbf{Transformer}, overall MSE is minimized at \(512\times4\times4\) (Size of Embeddings, Number of Encoders, Number of Attention Heads)
|
||||
(0.0043), fertile-day MSE at the same setting (0.0103), and non-fertile-day MSE at \(512\times8\times8\) (\,0.0011\,).
|
||||
For the \textbf{Convolutional LSTM}, \(256\times4\) (Hidden Layer Size, Number of LSTM Layers) yields the lowest overall MSE (0.0042)
|
||||
and the global-best fertile-day MSE (0.0100), while several smaller settings tie for the best non-fertile-day MSE (0.0016).
|
||||
For the \textbf{Convolutional Transformer}, overall MSE ties at \(256\times4\times4\) and \(512\times4\times4\)
|
||||
(Size of Embeddings, Number of Encoders, Number of Attention Heads) (\,0.0043\,);
|
||||
fertile-day MSE is best at \(512\times8\times8\) (0.0107), and the non-fertile-day MSE is minimized at \(512\times4\times4\) (0.0013).
|
||||
Table~\ref{tab:fertility_params_compact_mse} shows the compact results for the parameter search per architecture,
|
||||
full parameter grids (incl.\ MAE) are in Appendix, Tables~\ref{tab:fertility_results_by_model_parameters_lstm}–\ref{tab:fertility_results_by_model_parameters_conv_transformer}.
|
||||
|
||||
We next evaluate how architectural complexity, i.e., hidden size, number of layers,
|
||||
and number of attention heads—influences performance under fixed input settings (160 days at 12/day for LSTM/Transformer, 40 days for convolutional models).
|
||||
Parameters are reported as (hidden size × number of layers) for LSTM models and (embedding size × number of encoders × attention heads) for Transformers.
|
||||
|
||||
For the \textbf{LSTM}, the best overall and fertile-day MSEs are both observed at a configuration with 512 hidden units and 4 layers.
|
||||
The lowest non-fertile-day MSE (0.0011) is reached at 256×4.
|
||||
|
||||
The \textbf{Transformer} performs best overall and on fertile days with 512-dimensional embeddings, 4 encoder layers,
|
||||
and 4 attention heads (0.0043 and 0.0103, respectively).
|
||||
Its lowest non-fertile-day MSE (0.0011) occurs at a larger configuration with 8 layers and 8 heads.
|
||||
|
||||
For the \textbf{Convolutional LSTM}, the best overall and fertile-day MSEs are reached with 256 hidden units and 4 layers,
|
||||
while several lower-capacity settings tie for the best non-fertile-day MSE (0.0016).
|
||||
The \textbf{Convolutional Transformer} performs best overall with either 256×4×4 or 512×4×4, and reaches its lowest non-fertile-day MSE at 512×4×4.
|
||||
|
||||
Detailed comparisons are shown in Table~\ref{tab:fertility_params_compact_mse},
|
||||
with full results in Appendix Tables~\ref{tab:fertility_results_by_model_parameters_lstm}–\ref{tab:fertility_results_by_model_parameters_conv_transformer}.
|
||||
|
||||
|
||||
\begin{table}[t]
|
||||
\scriptsize
|
||||
@@ -165,19 +185,19 @@ We evaluate prediction of a binary indicator denoting whether ovulation has alre
|
||||
We examine the influence of input window length, input resolution, and model capacity.
|
||||
|
||||
\paragraph{Impact of input window length.}
|
||||
At a fixed input resolution of 12 values/day, the optimal window length for OV-Over is architecture- and phase-dependent.
|
||||
The \textbf{Transformer} achieves the best overall and after-ovulation errors at 40\,days
|
||||
(MAE 0.1017; MSE 0.0533 overall; MAE 0.0883; MSE 0.0520 after-OV).
|
||||
The \textbf{LSTM} benefits from long context (160\,days: overall MAE 0.1039; MSE 0.0557; after-OV MSE 0.0580)
|
||||
but attains the best before-ovulation scores at 20\,days (MAE 0.0745; MSE 0.0212).
|
||||
For the \textbf{Convolutional LSTM}, short/intermediate windows work best (20–40\,days):
|
||||
overall MSE 0.0699 at 20\,days; before-OV MSE 0.0389 and after-OV MSE 0.0833 at 20\,days.
|
||||
The \textbf{Convolutional Transformer} peaks overall at 40\,days (MSE 0.0709; after-OV MSE 0.0820),
|
||||
with its best before-OV MSE at 160\,days (0.0286).
|
||||
In summary, \textit{intermediate} windows (20–40\,days) are often optimal—especially
|
||||
pre-ovulation—while \textit{long} windows (160\,days) can improve post-ovulation for LSTM (and Convolutional Transformer before-OV MSE).
|
||||
Table~\ref{tab:ovover_windows_compact_mse} shows the compact results per architecture,
|
||||
full parameter grids (incl.\ MAE) are in Appendix, Tables~\ref{tab:ov_over_results_by_window_length}.
|
||||
For predicting whether ovulation has occurred (OV-over),
|
||||
optimal window lengths again vary by architecture and target phase (pre- vs post-ovulation).
|
||||
|
||||
The \textbf{Transformer} yields the lowest overall (0.0533) and post-ovulation MSE (0.0520) at 40 days.
|
||||
The \textbf{LSTM} performs best before ovulation (MSE 0.0212 at 20 days), while its overall MSE improves with longer context (160 days).
|
||||
|
||||
The \textbf{Convolutional LSTM} favors short windows, with the best overall MSE (0.0699), before-OV (0.0389),
|
||||
and after-OV (0.0833) all occurring at 20 days.
|
||||
The \textbf{Convolutional Transformer} performs best overall at 40 days and best before ovulation at 160 days (MSE 0.0286).
|
||||
|
||||
These trends suggest that intermediate windows (20–40 days) often balance short- and long-term signal,
|
||||
while long windows (e.g., 160 days) help capture post-ovulation patterns.
|
||||
See Table~\ref{tab:ovover_windows_compact_mse} for a summary.
|
||||
|
||||
\begin{table}[t]
|
||||
\small
|
||||
@@ -215,16 +235,17 @@ full parameter grids (incl.\ MAE) are in Appendix, Tables~\ref{tab:ov_over_resul
|
||||
\end{table}
|
||||
|
||||
\paragraph{Impact of input resolution.}
|
||||
With the window fixed at 20\,days, optimal resolution for OV-Over is architecture- and phase-specific.
|
||||
For \textbf{LSTM}, overall MAE is lowest at 12/day (0.1153) and overall MSE at 24/day (0.0633);
|
||||
the \emph{before-ovulation} MSE reaches the best within grid at 12/day (0.0212) and
|
||||
the \emph{after-ovulation} MSE reaches the best within grid at 48/day (0.0550).
|
||||
For the \textbf{Transformer}, both overall MAE (0.1041) and overall MSE achieve bests within grid at 72/day (0.0585),
|
||||
with the best before-OV MSE at 12/day (0.0255) and the best after-OV MSE at 288/day (0.0578).
|
||||
Overall, intermediate resolutions (12–72/day) tend to minimize overall error; 12/day favors pre-ovulation,
|
||||
while 48–72/day improves post-ovulation, with 288/day giving a modest after-OV edge for the Transformer.
|
||||
Table~\ref{tab:ovover_resolution_compact_mse} shows the condensed results per architecture,
|
||||
full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:ov_over_results_by_resolution}.
|
||||
Fixing the input window to 20 days, we compare input resolutions for LSTM and Transformer models.
|
||||
|
||||
For the \textbf{LSTM}, the lowest overall MSE (0.0633) occurs at 24/day,
|
||||
with best before-OV and after-OV MSEs at 12/day (0.0212) and 48/day (0.0550), respectively.
|
||||
|
||||
The \textbf{Transformer} performs best overall at 72/day (0.0585),
|
||||
with lowest before-OV MSE at 12/day (0.0255) and after-OV MSE at 288/day (0.0578).
|
||||
This indicates that high resolution benefits post-ovulation prediction, while lower rates suffice for pre-ovulation.
|
||||
|
||||
Results are summarized in Table~\ref{tab:ovover_resolution_compact_mse};
|
||||
full resolution grids are in Appendix Table~\ref{tab:ov_over_results_by_resolution}.
|
||||
|
||||
\begin{table}[t]
|
||||
\small
|
||||
@@ -254,18 +275,19 @@ full grids (incl.\ MAE) are in Appendix, Table~\ref{tab:ov_over_results_by_resol
|
||||
\end{table}
|
||||
|
||||
\paragraph{Impact of model parameters.}
|
||||
At fixed input settings (LSTM/Transformer: 160 days with 12 values/day; Convolutional models: 40 days),
|
||||
larger capacities generally improve OV-Over performance.
|
||||
For \textbf{LSTM}, the best overall MSE is at \(512\times4\) (Hidden Layer Size, Number of LSTM Layers)
|
||||
(0.0616) with the global-best \emph{before-OV} MSE at \(128\times2\) (0.0274) and the best \emph{after-OV} MSE at \(512\times4\) (0.0613).
|
||||
For the \textbf{Transformer}, scaling to \(512\times8\times8\) (Size of Embeddings, Number of Encoders, Number of Attention Heads)
|
||||
yields the global-best overall MSE (0.0543) and \emph{after-OV} MSE (0.0410); the best \emph{before-OV} MSE is at \(256\times4\times4\) (0.0293).
|
||||
For the \textbf{Convolutional LSTM}, \(256\times4\) (Hidden Layer Size, Number of LSTM Layers) minimizes
|
||||
overall (0.0687) and \emph{after-OV} MSE (0.0715), while \(128\times2\) is best \emph{before-OV} (0.0357).
|
||||
For the \textbf{Convolutional Transformer}, \(512\times4\times4\) (Size of Embeddings, Number of Encoders, Number of Attention
|
||||
Heads) is best overall (0.0703) and \emph{after-OV} (0.0814), with \(512\times8\times8\) best \emph{before-OV} (0.0339).
|
||||
Table~\ref{tab:ovover_params_compact_mse} shows the compact results for the parameter search per architecture,
|
||||
full parameter grids (incl.\ MAE) are in Appendix, Tables~\ref{tab:ov_over_results_by_model_parameters_lstm}–\ref{tab:ov_over_results_by_model_parameters_conv_transformer}.
|
||||
Using fixed input settings (160 days at 12/day for LSTM/Transformer; 40 days for convolutional models), we evaluate model scaling.
|
||||
|
||||
The \textbf{Transformer} achieves the lowest overall (0.0543) and after-ovulation MSE (0.0410) at a large configuration (512×8×8).
|
||||
Before-OV MSE is lowest at 256×4×4 (0.0293).
|
||||
|
||||
For the \textbf{LSTM}, the best overall and after-OV performance is at 512×4,
|
||||
while the best before-OV MSE occurs at 128×2 (0.0274).
|
||||
The \textbf{Convolutional LSTM} performs best overall and after-OV at 256×4, with the best before-OV MSE at 128×2.
|
||||
|
||||
Finally, the \textbf{Convolutional Transformer} achieves its lowest overall and after-OV MSE at 512×4×4, and best before-OV MSE at 512×8×8.
|
||||
|
||||
Table~\ref{tab:ovover_params_compact_mse} summarizes these parameter-dependent results;
|
||||
full comparisons are included in Appendix Tables~\ref{tab:ov_over_results_by_model_parameters_lstm}–\ref{tab:ov_over_results_by_model_parameters_conv_transformer}.
|
||||
|
||||
\begin{table}[t]
|
||||
\scriptsize
|
||||
@@ -305,13 +327,14 @@ full parameter grids (incl.\ MAE) are in Appendix, Tables~\ref{tab:ov_over_resul
|
||||
|
||||
\subsubsection{Best Model Configuration Selection}\label{subsubsec:results_best_model_config_selection}
|
||||
|
||||
Following the selection procedure described in Section~\ref{subsubsec:methodology_best_model_config_selection}
|
||||
the best configuration for each model architecture was identified based on the fertility overall MSE and, where applicable,
|
||||
the general tendencies of the model.
|
||||
Table~\ref{tab:best_configs_lstm} and~\ref{tab:best_configs_transformer} summarize the selected input window length,
|
||||
input resolution, and model complexity for each architecture.
|
||||
These configurations are used in all subsequent experiments, including the irregular cycles analysis and the use case evaluation.
|
||||
Note, that the convolutional models always get the full resolution data (288 measurements per day).
|
||||
To identify the best configuration for each model architecture, we followed the procedure described in Section~\ref{subsubsec:methodology_best_model_config_selection}.
|
||||
Configurations were selected based on the overall fertility MSE on the validation set,
|
||||
while also considering the general performance trends of each model.
|
||||
|
||||
Tables~\ref{tab:best_configs_lstm} and~\ref{tab:best_configs_transformer} summarize the selected hyperparameters for each architecture,
|
||||
including input window length, temporal resolution, and model complexity.
|
||||
For convolutional models, the input resolution was always fixed at the full 288 measurements per day.
|
||||
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
@@ -328,7 +351,7 @@ Note, that the convolutional models always get the full resolution data (288 mea
|
||||
Convolutional LSTM & 40 & 288 & 512 & 4 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Selected configurations for LSTM-based architectures. Input Window Length is given in days and input resolution in values per day.}
|
||||
\caption{Selected configurations for LSTM-based architectures. Input window length is given in days; input resolution in samples per day.}
|
||||
\label{tab:best_configs_lstm}
|
||||
\end{table}
|
||||
|
||||
@@ -348,10 +371,15 @@ Note, that the convolutional models always get the full resolution data (288 mea
|
||||
Convolutional Transformer & 40 & 288 & 512 & 4 & 4 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Selected configurations for Transformer-based architectures. Input Window Length is given in days and input resolution in values per day.}
|
||||
\caption{Selected configurations for Transformer-based architectures. Input window length is given in days; input resolution in samples per day.}
|
||||
\label{tab:best_configs_transformer}
|
||||
\end{table}
|
||||
|
||||
Final evaluation of the selected configurations was performed on a held-out test set with no reuse of training or validation data.
|
||||
Table~\ref{tab:results_model_selection_metrics} reports the MSE values for both prediction targets:
|
||||
the fertility probability and the OV-over indicator.
|
||||
All trained architectures outperform the baseline models across all evaluation metrics.
|
||||
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
||||
@@ -369,24 +397,22 @@ Note, that the convolutional models always get the full resolution data (288 mea
|
||||
Population-Mean Baseline & 0.0127 & 0.0258 & 0.0074 & 0.2145 & 0.0400 & 0.3749 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Evaluation Metrics for the fertility probability and OV-over targets (MSE) for the best model selection and the baselines for comparison.
|
||||
\textbf{Bold} values represent the best values across all models for a given metric.}
|
||||
\caption{Evaluation metrics (MSE) for fertility probability and OV-over targets using the selected model configurations.
|
||||
Bold values indicate the best-performing architecture for each metric.}
|
||||
\label{tab:results_model_selection_metrics}
|
||||
\end{table}
|
||||
|
||||
Table~\ref{tab:results_model_selection_metrics} shows the performance of the best selected models for each architecture,
|
||||
as well as the baselines for comparison.
|
||||
The final numbers in Table~\ref{tab:results_model_selection_metrics} are on a held-out test set with no reuse.
|
||||
All trained model architectures outperform the baselines across all metrics.
|
||||
The LSTM achieves the global best MSE for both targets (\mbox{0.0036} fertility; \mbox{0.0437} OV-Over)
|
||||
and the best non-fertile-day MSE (\mbox{0.0008}).
|
||||
The Transformer attains the global best fertile-day MSE (\mbox{0.0089}).
|
||||
Relative to the User-Mean baseline, the best LSTM reduces fertility MSE by \mbox{44\%} and OV-Over MSE by \mbox{59\%}.
|
||||
The LSTM model achieves the best overall performance, with the lowest MSE for overall fertility (0.0036),
|
||||
non-fertile days (0.0008), and all post-ovulatory metrics.
|
||||
The Transformer model, however, yields the lowest MSE on fertile days (0.0089), slightly outperforming the LSTM in that subset.
|
||||
|
||||
Compared to the strongest baseline (User-Mean), the LSTM reduces the overall fertility MSE by 44\% and the post-ovulatory MSE by 59\%.
|
||||
The performance gap is even larger when compared to the population-mean and last-cycle baselines,
|
||||
confirming the advantage of personalized and temporally-aware modeling.
|
||||
|
||||
\subsection{Stratified Analysis}\label{subsec:stratified_analysis}
|
||||
|
||||
\subsubsection{Influence of User History Depth}\label{subsubsec:influence_of_past_user_data}
|
||||
% don't forget to also add baseline to tables
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=0.8\textwidth]{resources/figures/results/performance_on_different_historical_contexts}
|
||||
@@ -408,63 +434,362 @@ In contrast, the Population-Mean Baseline performs worse (i.e., MSE increases) a
|
||||
Across all models and baselines, the variability of results increases with greater history depth:
|
||||
results are more tightly clustered around the trend line with short histories,
|
||||
but show greater scatter for users with longer data records.
|
||||
This suggests that longer histories introduce additional heterogeneity, which may reflect increased user-specific variability.
|
||||
|
||||
\subsubsection{Regular vs Irregular Cycles}\label{subsubsec:regular_vs_irregular_cycles}
|
||||
\begin{landscape}
|
||||
\begin{table}
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\small
|
||||
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
||||
\begin{tabularx}{\linewidth}{l*{3}{X}}
|
||||
\toprule
|
||||
\multirow{2}{*}{Model Architecture} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
||||
& Fertility Overall & Fertile Days & Non-Fertile Days & Fertility Overall & Fertile Days & Non-Fertile Days \\
|
||||
\multirow{2}{*}{Model} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4}
|
||||
& Fertility Overall & Fertile Days & Non-Fertile Days \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\multicolumn{4}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\midrule
|
||||
|
||||
LSTM & 0.002563 & 0.007850 & \textbf{0.000415} \\
|
||||
Transformer & \textbf{0.002376} & \textbf{0.006651} & 0.000626 \\
|
||||
Convolutional LSTM & 0.002460 & 0.007313 & 0.000573 \\
|
||||
Convolutional Transformer & 0.002424 & 0.006919 & 0.000655 \\
|
||||
Last-Cycle Baseline & 0.006592 & 0.014177 & 0.003198 \\
|
||||
Population Mean Baseline & 0.016657 & 0.027671 & 0.011292 \\
|
||||
User Mean Baseline & 0.005195 & 0.011715 & 0.002312 \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\multicolumn{4}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\midrule
|
||||
|
||||
LSTM & 0.004490 & 0.013415 & \textbf{0.001076} \\
|
||||
Transformer & 0.004122 & \textbf{0.010774} & 0.001783 \\
|
||||
Convolutional LSTM & \textbf{0.004096} & 0.011188 & 0.001538 \\
|
||||
Convolutional Transformer & 0.004281 & 0.011374 & 0.001639 \\
|
||||
Last-Cycle Baseline & 0.009423 & 0.023650 & 0.004613 \\
|
||||
Population Mean Baseline & 0.009795 & 0.022330 & 0.005751 \\
|
||||
User Mean Baseline & 0.010548 & 0.023935 & 0.006159 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Evaluation Metrics for the fertility probability target across Different Model Architectures for the Regular and Irregular Cycle Groups.
|
||||
\underline{Underlined} values represent the best value for each metric within a model.
|
||||
\textbf{\underline{Bold + Underlined}} values represent the best values across all models for a given metric.}
|
||||
\label{tab:regular_vs_irregular_fertility_results}
|
||||
\caption{MSEs for fertility probability predictions across different model architectures for regular and irregular cycle groups. Bold values denote the best scores per column.}
|
||||
\label{tab:fertility_mse_regular_irregular}
|
||||
\end{table}
|
||||
\end{landscape}
|
||||
\begin{landscape}
|
||||
\begin{table}
|
||||
|
||||
\noindent
|
||||
Table~\ref{tab:fertility_mse_regular_irregular} shows the MSE metric results for all models on the regular and irregular
|
||||
cycle groups for the fertility-probability target.
|
||||
For this target, all learned models substantially outperform the baseline methods in both regular and irregular cycle groups.
|
||||
In the regular group, the Transformer achieves the lowest overall MSE and MSE on fertile days, while the LSTM excels on non-fertile days.
|
||||
Convolutional variants also perform competitively.
|
||||
Among baselines, the user-specific mean performs better than both the last-cycle and population mean, but still falls short of all learned models.
|
||||
|
||||
In the irregular group, model performance is uniformly lower (i.e., MSEs are higher).
|
||||
The Convolutional LSTM shows the lowest overall MSE, while the Transformer performs best on fertile days.
|
||||
The LSTM remains the most effective on non-fertile days.
|
||||
Baselines degrade more sharply in this group.
|
||||
|
||||
The full table with MSE and MAE for all models can be found in the appendix, Table~\ref{tab:regular_vs_irregular_fertility_results}.
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\small
|
||||
\begin{tabularx}{\linewidth}{l*{6}{X}}
|
||||
\begin{tabularx}{\linewidth}{l*{3}{X}}
|
||||
\toprule
|
||||
\multirow{2}{*}{Model} & \multicolumn{3}{c}{MAE} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4} \cmidrule(r){5-7}
|
||||
& OV-Over Overall & OV-Over Before OV & OV-Over After OV & OV-Over Overall & OV-Over Before OV & OV-Over After OV \\
|
||||
\multirow{2}{*}{Model} & \multicolumn{3}{c}{MSE} \\
|
||||
\cmidrule(r){2-4}
|
||||
& OV-Over Overall & OV-Over Before OV & OV-Over After OV \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\multicolumn{4}{c}{\textbf{Regular Cycle Group}} \\
|
||||
\midrule
|
||||
|
||||
LSTM & \textbf{0.028519} & 0.031695 & \textbf{0.026116} \\
|
||||
Transformer & 0.032929 & \textbf{0.024774} & 0.033411 \\
|
||||
Convolutional LSTM & 0.034224 & 0.028962 & 0.033929 \\
|
||||
Convolutional Transformer & 0.033333 & 0.039045 & 0.029329 \\
|
||||
Last-Cycle Baseline & 0.087916 & 0.093465 & 0.074191 \\
|
||||
Population Mean Baseline & 0.285887 & 0.008876 & 0.418879 \\
|
||||
User Mean Baseline & 0.069245 & 0.069152 & 0.055724 \\
|
||||
\midrule
|
||||
\multicolumn{7}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\multicolumn{4}{c}{\textbf{Irregular Cycle Group}} \\
|
||||
\midrule
|
||||
|
||||
LSTM & \textbf{0.051747} & \textbf{0.016777} & 0.070419 \\
|
||||
Transformer & 0.055990 & 0.026159 & 0.067520 \\
|
||||
Convolutional LSTM & 0.054523 & 0.030092 & \textbf{0.066643} \\
|
||||
Convolutional Transformer & 0.056075 & 0.029638 & 0.067965 \\
|
||||
Last-Cycle Baseline & 0.222906 & 0.134945 & 0.258047 \\
|
||||
Population Mean Baseline & 0.171368 & 0.130366 & 0.140363 \\
|
||||
User Mean Baseline & 0.180702 & 0.088884 & 0.217144 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Evaluation Metrics for the Ovulation-Over Target across Different Model Architectures for the Regular and Irregular Cycle Groups.
|
||||
\underline{Underlined} values represent the best value for each metric within a model.
|
||||
\textbf{\underline{Bold + Underlined}} values represent the best values across all models for a given metric.}
|
||||
\label{tab:regular_vs_irregular_ov_over_results}
|
||||
\caption{MSEs for ovulation-over target across different model architectures for regular and irregular cycle groups. Bold values denote the best scores per column.}
|
||||
\label{tab:ov_over_mse_regular_irregular}
|
||||
\end{table}
|
||||
\end{landscape}
|
||||
|
||||
\noindent
|
||||
Table~\ref{tab:ov_over_mse_regular_irregular} shows the MSE metric results for all models on the regular and irregular cycle groups for the OV-over target.
|
||||
For this target, performance patterns differ more distinctly across phases.
|
||||
In the regular group, the LSTM performs best overall and after ovulation, while the Transformer performs best before ovulation.
|
||||
Convolutional models perform slightly worse than their recurrent and transformer-based counterparts across all phases.
|
||||
The population mean baseline shows a spurious low MSE before ovulation, but this is not consistent across phases, suggesting it is not reliable.
|
||||
In general, baselines are notably weaker than learned models across all splits.
|
||||
|
||||
For irregular cycles, all model performances deteriorate relative to regular cycles.
|
||||
The LSTM remains the most robust, achieving the best MSEs both overall and before ovulation.
|
||||
Convolutional LSTM performs best after ovulation.
|
||||
Transformer-based models perform less consistently in this group, suggesting decreased robustness to cycle irregularity.
|
||||
As before, all learned models outperform the baselines by a wide margin.
|
||||
|
||||
The full table with MSE and MAE for all models can be found in the appendix, Table~\ref{tab:regular_vs_irregular_ov_over_results}.
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=0.8\textwidth]{resources/figures/results/model_results_on_regular_cycles}
|
||||
\caption{
|
||||
MSE performance trends across increasing historical data availability for fertility and ovulation-over targets on regular cycles.
|
||||
}
|
||||
\label{fig:results_performance_on_regular_cycles}
|
||||
\end{figure}
|
||||
|
||||
\begin{figure}[htbp]
|
||||
\centering
|
||||
\includegraphics[width=0.8\textwidth]{resources/figures/results/model_results_on_irregular_cycles}
|
||||
\caption{
|
||||
MSE performance trends across increasing historical data availability for fertility and ovulation-over targets on irregular cycles.
|
||||
}
|
||||
\label{fig:results_performance_on_irregular_cycles}
|
||||
\end{figure}
|
||||
|
||||
\paragraph{Evaluation on Historical Context Depth.}
|
||||
Figures~\ref{fig:results_performance_on_regular_cycles} and~\ref{fig:results_performance_on_irregular_cycles} visualize performance improvements with increasing numbers of past cycles.
|
||||
For regular cycles, all models except the population-mean baseline benefit from more historical data, showing consistent MSE reductions for both targets.
|
||||
Interestingly, the population-mean baseline performs worse as more cycles are added.
|
||||
The largest improvement can be seen for the last-cycle baseline and user-mean baseline models.
|
||||
|
||||
In contrast, irregular cycles exhibit more variability and less performance gain with more cycles available.
|
||||
All trained models improve on both targets with more context.
|
||||
The population-mean and user-mean baselines show improvement on the OV-over target.
|
||||
In contrast, their MSE increases with more historical context for the fertility-probability target.
|
||||
The last-cycle baseline shows marginal performance improvement for the OV-over target and the same increase for the fertility-probability
|
||||
target as the other two baselines.
|
||||
|
||||
All models show increased output variability as the number of available past cycles grows,
|
||||
as indicated by the wider deviations from the trend lines.
|
||||
|
||||
\subsection{Use-Case Evaluation Results}\label{subsec:use_case_evaluation_results}
|
||||
|
||||
We evaluated the use-case scenarios described in Section~\ref{subsubsec:practical_use_cases} using the algorithms
|
||||
in Section~\ref{subsubsec:use_case_evaluation} across multiple fertility thresholds.
|
||||
Each evaluation used a test set of 100 users (100 user-years) and was repeated for 200 iterations;
|
||||
we report means and 95\% confidence intervals (CIs).
|
||||
|
||||
\subsubsection{Contraception Use-Case Results}\label{subsubsec:use_case_contraception_results}
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Denials & Incorrect Denials \\
|
||||
\midrule
|
||||
Convolutional LSTM & 1154 (1149-1158) & 34.9 (34.1-35.7) & 1615 (1611-1620) & 4963 (4954-4971) \\
|
||||
Transformer & 3200 (3194-3207) & 4.4 (4.1-4.7) & 1817 (1812-1822) & 2717 (2710-2724) \\
|
||||
Convolutional Transformer & 1156 (1151-1160) & 34.3 (33.5-35.0) & 1613 (1608-1619) & 4974 (4965-4983) \\
|
||||
LSTM & 3206 (3199-3213) & 4.6 (4.3-4.9) & 1832 (1827-1838) & 2711 (2705-2717) \\
|
||||
Last-Cycle Baseline & 6066 (6057-6075) & 127.5 (125.9-129.1) & 918 (915-922) & 762 (759-766) \\
|
||||
Population-Mean Baseline & 5874 (5865-5883) & 153.7 (151.9-155.5) & 749 (746-752) & 1121 (1116-1125) \\
|
||||
User-Mean Baseline & 5933 (5924-5942) & 105.8 (104.4-107.2) & 1030 (1026-1034) & 776 (772-779) \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Contraception metrics at threshold \textbf{0.01} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.}
|
||||
\label{tab:results_contraception_use_case_0_01}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.01 (Table~\ref{tab:results_contraception_use_case_0_01}).}
|
||||
At the strictest threshold of 0.01, Transformer and LSTM models achieve the lowest pregnancy rates, only 4.4 to 4.6
|
||||
on average, despite allowing approximately 3,200 intercourse events.
|
||||
This equates to about 1.4 pregnancies per 1,000 intercourse events, a strong result for contraceptive reliability.
|
||||
|
||||
In contrast, convolutional variants (ConvLSTM and ConvTransformer) restrict intercourse events to around 1,150,
|
||||
leading to roughly 34–35 pregnancies, or about 30 per 1,000 events—far less efficient in terms of balance between access and protection.
|
||||
All baseline models perform substantially worse, with pregnancy counts exceeding 100 in all cases, confirming the value of personalized predictions.
|
||||
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Denials & Incorrect Denials \\
|
||||
\midrule
|
||||
Convolutional LSTM & 2509 (2502-2515) & 55.0 (54.0-56.0) & 1448 (1444-1452) & 3768 (3760-3775) \\
|
||||
Transformer & 3801 (3793-3810) & 8.2 (7.8-8.6) & 1768 (1763-1773) & 2162 (2157-2168) \\
|
||||
Convolutional Transformer & 2510 (2503-2516) & 55.6 (54.5-56.6) & 1448 (1443-1453) & 3776 (3769-3783) \\
|
||||
LSTM & 3801 (3794-3809) & 8.2 (7.8-8.7) & 1772 (1766-1777) & 2160 (2154-2165) \\
|
||||
Last-Cycle Baseline & 6106 (6096-6116) & 130.2 (128.7-131.7) & 888 (885-892) & 750 (746-753) \\
|
||||
Population-Mean Baseline & 5924 (5914-5933) & 156.1 (154.3-157.8) & 716 (713-720) & 1100 (1096-1104) \\
|
||||
User-Mean Baseline & 5988 (5978-5997) & 108.8 (107.3-110.3) & 998 (994-1002) & 764 (761-768) \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Contraception metrics at threshold \textbf{0.05} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.}
|
||||
\label{tab:results_contraception_use_case_0_05}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.05 (Table~\ref{tab:results_contraception_use_case_0_05}).}
|
||||
Increasing the threshold to 0.05 improves access: Transformer and LSTM models now permit approximately 3,800 intercourse
|
||||
events while still limiting pregnancies to around 8.2 (2.2 per 1,000).
|
||||
This represents a ~19\% increase in events over threshold 0.01, at the cost of a modest rise in pregnancies (+3.8 absolute, +86\% relative).
|
||||
|
||||
Convolutional models also allow more events (~2,510) but continue to produce significantly more pregnancies (~55),
|
||||
yielding a less favorable risk-benefit profile.
|
||||
Baselines remain underperforming.
|
||||
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Denials & Incorrect Denials \\
|
||||
\midrule
|
||||
Convolutional LSTM & 3596 (3588-3603) & 75.2 (74.0-76.4) & 1292 (1288-1297) & 2847 (2840-2853) \\
|
||||
Transformer & 4226 (4217-4234) & 13.3 (12.8-13.7) & 1692 (1687-1696) & 1822 (1817-1827) \\
|
||||
Convolutional Transformer & 3592 (3584-3599) & 73.8 (72.7-75.0) & 1296 (1292-1301) & 2846 (2840-2852) \\
|
||||
LSTM & 4222 (4214-4230) & 13.3 (12.8-13.8) & 1694 (1688-1699) & 1818 (1812-1823) \\
|
||||
Last-Cycle Baseline & 6145 (6135-6155) & 134.8 (133.3-136.3) & 865 (861-868) & 744 (740-747) \\
|
||||
Population-Mean Baseline & 5960 (5951-5969) & 160.3 (158.5-162.1) & 694 (691-697) & 1088 (1084-1093) \\
|
||||
User-Mean Baseline & 6014 (6005-6022) & 111.8 (110.3-113.3) & 969 (965-973) & 758 (755-762) \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Contraception metrics at threshold \textbf{0.10} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.}
|
||||
\label{tab:results_contraception_use_case_0_10}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.10 (Table~\ref{tab:results_contraception_use_case_0_10}).}
|
||||
A further increase to 0.10 raises Transformer/LSTM events to ~4,220, but also raises pregnancies to ~13.3 (3.1 per 1,000).
|
||||
This is a ~11\% gain in access compared to 0.05, but the pregnancy count increases by ~62\%.
|
||||
|
||||
Meanwhile, convolutional models permit ~3,590 events and result in 74–75 pregnancies, indicating a consistent trade-off in favor of sequential models.
|
||||
|
||||
\paragraph{Contraception—Recommended Threshold.}
|
||||
Threshold 0.05 achieves a favorable balance between access and effectiveness.
|
||||
Transformer and LSTM models perform best, allowing a relatively high number of intercourse events while keeping pregnancies low.
|
||||
This threshold offers the best compromise and is selected as the most promising setting for contraceptive use.
|
||||
|
||||
\subsubsection{Pregnancy Use-Case Results}\label{subsubsec:use_case_pregnancy_results}
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Deferrals & Incorrect Deferrals \\
|
||||
\midrule
|
||||
Convolutional LSTM & 4899 (4890-4908) & 216.9 (214.8-219.1) & 10038 & 1734 \\
|
||||
Transformer & 3685 (3678-3692) & 264.9 (262.7-267.0) & 17489 & 350 \\
|
||||
Convolutional Transformer & 4903 (4894-4912) & 214.3 (212.2-216.4) & 10038 & 1734 \\
|
||||
LSTM & 3688 (3680-3696) & 266.9 (264.7-269.1) & 17489 & 350 \\
|
||||
Last-Cycle Baseline & 1536 (1531-1541) & 138.7 (137.1-140.2) & 24593 & 4038 \\
|
||||
Population-Mean Baseline & 1704 (1699-1709) & 113.5 (112.1-114.9) & 22956 & 4841 \\
|
||||
User-Mean Baseline & 1652 (1647-1656) & 158.4 (156.8-160.0) & 24528 & 3523 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Pregnancy metrics at threshold \textbf{0.01} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.
|
||||
Under our simulation at a fixed threshold, correct/incorrect deferrals are deterministic; CIs are therefore omitted for these columns.}
|
||||
\label{tab:results_pregnancy_use_case_0_01}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.01 (Table~\ref{tab:results_pregnancy_use_case_0_01}).}
|
||||
In the conception use-case, Transformer and LSTM models generate the highest pregnancy counts (~265–267) at ~3,685 intercourse events (72 per 1,000).
|
||||
Convolutional models yield fewer pregnancies (~214–217) but allow ~4,900 intercourse events, resulting in ~44 pregnancies per 1,000.
|
||||
|
||||
Baselines underperform on both metrics, allowing fewer events and achieving lower pregnancy counts, suggesting they are
|
||||
overly conservative without yielding benefits in effectiveness.
|
||||
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Deferrals & Incorrect Deferrals \\
|
||||
\midrule
|
||||
Convolutional LSTM & 6174 (6164-6184) & 236.1 (234.1-238.2) & 4332 & 1076 \\
|
||||
Transformer & 4253 (4245-4261) & 270.9 (268.6-273.1) & 14825 & 190 \\
|
||||
Convolutional Transformer & 6173 (6164-6182) & 235.7 (233.6-237.9) & 4332 & 1076 \\
|
||||
LSTM & 4255 (4247-4262) & 273.0 (270.6-275.3) & 14825 & 190 \\
|
||||
Last-Cycle Baseline & 1581 (1576-1586) & 139.8 (138.2-141.3) & 24327 & 4091 \\
|
||||
Population-Mean Baseline & 1752 (1747-1757) & 117.1 (115.6-118.6) & 22655 & 4888 \\
|
||||
User-Mean Baseline & 1695 (1690-1701) & 158.3 (156.6-160.1) & 24271 & 3567 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Pregnancy metrics at threshold \textbf{0.05} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.}
|
||||
\label{tab:results_pregnancy_use_case_0_05}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.05 (Table~\ref{tab:results_pregnancy_use_case_0_05}).}
|
||||
At threshold 0.05, Transformer/LSTM models slightly increase intercourse access (~4,250 events) with pregnancies rising to ~271–273 (64 per 1,000).
|
||||
Notably, these models also achieve very low incorrect deferral counts (~190), indicating they rarely block opportunities for conception when they shouldn’t.
|
||||
|
||||
Convolutional models allow substantially more intercourse (~6,170) with lower pregnancy counts (~236),
|
||||
but at the cost of higher incorrect deferrals (~1,076).
|
||||
This suggests they are more permissive but less selective.
|
||||
|
||||
|
||||
\begin{table}
|
||||
\scriptsize
|
||||
\begin{tabularx}{\linewidth}{l*{5}{X}}
|
||||
\toprule
|
||||
Model & No. of Inter. Events & Pregnancies & Correct Deferrals & Incorrect Deferrals \\
|
||||
\midrule
|
||||
Convolutional LSTM & 3887 (3880-3895) & 198.0 (196.1-199.9) & 14544 & 2296 \\
|
||||
Transformer & 3291 (3284-3298) & 260.7 (258.6-262.7) & 19261 & 551 \\
|
||||
Convolutional Transformer & 3883 (3875-3891) & 198.4 (196.6-200.3) & 14544 & 2296 \\
|
||||
LSTM & 3296 (3289-3303) & 261.1 (259.1-263.2) & 19261 & 551 \\
|
||||
Last-Cycle Baseline & 1509 (1504-1513) & 135.2 (133.6-136.8) & 24778 & 4010 \\
|
||||
Population-Mean Baseline & 1668 (1663-1673) & 109.5 (108.1-110.9) & 23156 & 4811 \\
|
||||
User-Mean Baseline & 1619 (1614-1624) & 154.0 (152.4-155.6) & 24717 & 3501 \\
|
||||
\bottomrule
|
||||
\end{tabularx}
|
||||
\caption{Pregnancy metrics at threshold \textbf{0.10} for all models. Values are means over 200 iterations; 95\% CIs in parentheses.}
|
||||
\label{tab:results_pregnancy_use_case_0_10}
|
||||
\end{table}
|
||||
|
||||
\paragraph{Threshold 0.10 (Table~\ref{tab:results_pregnancy_use_case_0_10}).}
|
||||
At the highest threshold, Transformer/LSTM models see a drop in access (~3,290 events) and in pregnancies (~261),
|
||||
but with an increase in incorrect deferrals (~551).
|
||||
Convolutional models again show a more permissive profile (~3,880 events, ~198 pregnancies), but with higher rates of incorrect deferrals (~2,296).
|
||||
|
||||
This threshold leads to fewer pregnancies and more unnecessary blocks, especially for the Transformer/LSTM models.
|
||||
|
||||
\paragraph{Pregnancy—Recommended Threshold.}
|
||||
Threshold 0.05 strikes the best balance for conception as well.
|
||||
Transformer and LSTM models provide high pregnancy counts and good access with minimal incorrect deferrals.
|
||||
Compared to 0.10, it results in more successful conceptions with fewer missed opportunities, making it the optimal setting for this use-case.
|
||||
|
||||
\subsection{Summary of Key Findings}\label{subsec:summary_of_key_findings}
|
||||
%best model per target, typical window lengths (20–40 d often optimal; LSTM benefits from 160 d), best resolutions (4–48/day for fertility; 12–72/day for OV-Over), and %-gains over baselines.
|
||||
|
||||
\paragraph{Model Performance.}
|
||||
LSTM models consistently achieved the lowest overall mean squared errors (MSE) across both prediction targets,
|
||||
fertility probability and ovulation-over (OV-over), on the held-out test set.
|
||||
Transformer models slightly outperformed LSTMs on fertile-day predictions, suggesting higher sensitivity to short-term signals.
|
||||
Convolutional models (ConvLSTM and ConvTransformer) performed competitively in some configurations but generally
|
||||
exhibited higher error and less favorable trade-offs in practical scenarios.
|
||||
|
||||
\paragraph{Effect of Input Settings.}
|
||||
\textit{Input window length} significantly influenced performance.
|
||||
LSTMs and Transformers benefited from longer windows (e.g., 160\,days), especially for post-ovulation detection,
|
||||
while convolutional models favored shorter windows (e.g., 20--40\,days).
|
||||
\textit{Input resolution} showed that intermediate sampling rates (4--48\,values/day) often minimized overall error,
|
||||
though full resolution (288/day) was most useful for detecting fertile windows in LSTM models.
|
||||
|
||||
\paragraph{Model Scaling.}
|
||||
Larger configurations (e.g., 512 hidden units, 4--8 layers, 4--8 attention heads) generally improved performance across architectures,
|
||||
especially for Transformers in OV-over prediction.
|
||||
|
||||
\paragraph{Comparison to Baselines.}
|
||||
All trained models substantially outperformed baseline predictors (user mean, last-cycle, population mean) across metrics and targets.
|
||||
The LSTM model reduced fertility MSE by 44\% and post-ovulation MSE by 59\% compared to the strongest baseline.
|
||||
|
||||
\paragraph{Stratified Analyses.}
|
||||
\textit{User history depth:} More historical cycles consistently improved prediction accuracy, especially for LSTM models.
|
||||
However, output variability increased with longer histories.
|
||||
\textit{Cycle regularity:} All models performed better on regular cycles than irregular ones.
|
||||
LSTM remained the most robust across both groups.
|
||||
|
||||
\paragraph{Use-Case Scenarios.}
|
||||
In \textit{contraceptive settings}, Transformer and LSTM models maintained pregnancy rates below 2.2 per 1{,}000 intercourse events
|
||||
at a 0.05 threshold---demonstrating both high reliability and user access.
|
||||
In \textit{conception settings}, the same models yielded the highest pregnancy rates ($\sim$64 per 1{,}000 events)
|
||||
with minimal missed opportunities, again at the 0.05 threshold.
|
||||
Convolutional models allowed more events but were less selective,
|
||||
resulting in higher pregnancy rates in contraception and lower in conception scenarios.
|
||||
Baseline methods were consistently outperformed in both use-cases, often sacrificing either effectiveness or access.
|
||||
Reference in New Issue
Block a user