started tft part in background

main.bib

@@ -1404,3 +1404,104 @@ For additional {GBD} results and resources, visit the {GBD} 2019 Data Resources
 	langid = {english},
 	file = {PDF:/home/alex/Zotero/storage/Y73KE57K/Murray - 2005 - Diagnosis and treatment of ectopic pregnancy.pdf:application/pdf},
 }
+
+@collection{pfannstiel_entrepreneurship_2018,
+	location = {Wiesbaden},
+	title = {Entrepreneurship im Gesundheitswesen {II}},
+	rights = {http://www.springer.com/tdm},
+	isbn = {978-3-658-14780-8 978-3-658-14781-5},
+	url = {http://link.springer.com/10.1007/978-3-658-14781-5},
+	publisher = {Springer Fachmedien Wiesbaden},
+	editor = {Pfannstiel, Mario A. and Da-Cruz, Patrick and Rasche, Christoph},
+	urldate = {2025-03-17},
+	date = {2018},
+	langid = {german},
+	doi = {10.1007/978-3-658-14781-5},
+	file = {PDF:/home/alex/Zotero/storage/DTVM5NBP/Pfannstiel et al. - 2018 - Entrepreneurship im Gesundheitswesen II.pdf:application/pdf},
+}
+
+@online{noauthor_temporal_nodate,
+	title = {Temporal Fusion Transformer ({TFT}) — darts documentation},
+	url = {https://unit8co.github.io/darts/generated_api/darts.models.forecasting.tft_model.html},
+	urldate = {2025-03-19},
+	file = {Temporal Fusion Transformer (TFT) — darts documentation:/home/alex/Zotero/storage/5QNI6WSL/darts.models.forecasting.tft_model.html:text/html},
+}
+
+@software{sherar_mattsherartemporal_fusion_transform_2025,
+	title = {mattsherar/Temporal\_Fusion\_Transform},
+	url = {https://github.com/mattsherar/Temporal_Fusion_Transform},
+	abstract = {Pytorch Implementation of Google's {TFT}},
+	author = {Sherar, Matthew},
+	urldate = {2025-03-19},
+	date = {2025-03-07},
+	note = {original-date: 2020-01-11T17:54:01Z},
+}
+
+@software{noauthor_playtikaosstft-torch_2025,
+	title = {{PlaytikaOSS}/tft-torch},
+	rights = {{MIT}},
+	url = {https://github.com/PlaytikaOSS/tft-torch},
+	abstract = {A Python library that implements ״Temporal Fusion Transformers for Interpretable Multi-horizon Time Series Forecasting״},
+	publisher = {Playtika},
+	urldate = {2025-03-19},
+	date = {2025-01-27},
+	note = {original-date: 2021-11-28T07:08:32Z},
+}
+
+@online{noauthor_create_nodate,
+	title = {Create baseline model - {ValueError}: too many values to unpack (expected 2) · Issue \#230 · sktime/pytorch-forecasting},
+	url = {https://github.com/sktime/pytorch-forecasting/issues/230},
+	shorttitle = {Create baseline model - {ValueError}},
+	abstract = {{PyTorch}-Forecasting version: 0.7.1 {PyTorch} version: 1.7.1 Python version: 3.7 Operating System: {MAC} {OS} Big Sur: Version 11.1 Expected behavior I executed code actuals = torch.cat([y for x, (y, weig...},
+	titleaddon = {{GitHub}},
+	urldate = {2025-03-19},
+	langid = {english},
+}
+
+@misc{clevert_fast_2016,
+	title = {Fast and Accurate Deep Network Learning by Exponential Linear Units ({ELUs})},
+	url = {http://arxiv.org/abs/1511.07289},
+	doi = {10.48550/arXiv.1511.07289},
+	abstract = {We introduce the "exponential linear unit" ({ELU}) which speeds up learning in deep neural networks and leads to higher classification accuracies. Like rectified linear units ({ReLUs}), leaky {ReLUs} ({LReLUs}) and parametrized {ReLUs} ({PReLUs}), {ELUs} alleviate the vanishing gradient problem via the identity for positive values. However, {ELUs} have improved learning characteristics compared to the units with other activation functions. In contrast to {ReLUs}, {ELUs} have negative values which allows them to push mean unit activations closer to zero like batch normalization but with lower computational complexity. Mean shifts toward zero speed up learning by bringing the normal gradient closer to the unit natural gradient because of a reduced bias shift effect. While {LReLUs} and {PReLUs} have negative values, too, they do not ensure a noise-robust deactivation state. {ELUs} saturate to a negative value with smaller inputs and thereby decrease the forward propagated variation and information. Therefore, {ELUs} code the degree of presence of particular phenomena in the input, while they do not quantitatively model the degree of their absence. In experiments, {ELUs} lead not only to faster learning, but also to significantly better generalization performance than {ReLUs} and {LReLUs} on networks with more than 5 layers. On {CIFAR}-100 {ELUs} networks significantly outperform {ReLU} networks with batch normalization while batch normalization does not improve {ELU} networks. {ELU} networks are among the top 10 reported {CIFAR}-10 results and yield the best published result on {CIFAR}-100, without resorting to multi-view evaluation or model averaging. On {ImageNet}, {ELU} networks considerably speed up learning compared to a {ReLU} network with the same architecture, obtaining less than 10\% classification error for a single crop, single model network.},
+	number = {{arXiv}:1511.07289},
+	publisher = {{arXiv}},
+	author = {Clevert, Djork-Arné and Unterthiner, Thomas and Hochreiter, Sepp},
+	urldate = {2025-03-26},
+	date = {2016-02-22},
+	eprinttype = {arxiv},
+	eprint = {1511.07289 [cs]},
+	keywords = {Computer Science - Machine Learning},
+	file = {Full Text PDF:/home/alex/Zotero/storage/PU3ZGP4G/Clevert et al. - 2016 - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs).pdf:application/pdf;Snapshot:/home/alex/Zotero/storage/3MAW2IWE/1511.html:text/html},
+}
+
+@misc{ba_layer_2016,
+	title = {Layer Normalization},
+	url = {http://arxiv.org/abs/1607.06450},
+	doi = {10.48550/arXiv.1607.06450},
+	abstract = {Training state-of-the-art, deep neural networks is computationally expensive. One way to reduce the training time is to normalize the activities of the neurons. A recently introduced technique called batch normalization uses the distribution of the summed input to a neuron over a mini-batch of training cases to compute a mean and variance which are then used to normalize the summed input to that neuron on each training case. This significantly reduces the training time in feed-forward neural networks. However, the effect of batch normalization is dependent on the mini-batch size and it is not obvious how to apply it to recurrent neural networks. In this paper, we transpose batch normalization into layer normalization by computing the mean and variance used for normalization from all of the summed inputs to the neurons in a layer on a single training case. Like batch normalization, we also give each neuron its own adaptive bias and gain which are applied after the normalization but before the non-linearity. Unlike batch normalization, layer normalization performs exactly the same computation at training and test times. It is also straightforward to apply to recurrent neural networks by computing the normalization statistics separately at each time step. Layer normalization is very effective at stabilizing the hidden state dynamics in recurrent networks. Empirically, we show that layer normalization can substantially reduce the training time compared with previously published techniques.},
+	number = {{arXiv}:1607.06450},
+	publisher = {{arXiv}},
+	author = {Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E.},
+	urldate = {2025-03-26},
+	date = {2016-07-21},
+	eprinttype = {arxiv},
+	eprint = {1607.06450 [stat]},
+	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
+	file = {Full Text PDF:/home/alex/Zotero/storage/MJWRDPWE/Ba et al. - 2016 - Layer Normalization.pdf:application/pdf;Snapshot:/home/alex/Zotero/storage/F9WSU957/1607.html:text/html},
+}
+
+@misc{dauphin_language_2017,
+	title = {Language Modeling with Gated Convolutional Networks},
+	url = {http://arxiv.org/abs/1612.08083},
+	doi = {10.48550/arXiv.1612.08083},
+	abstract = {The pre-dominant approach to language modeling to date is based on recurrent neural networks. Their success on this task is often linked to their ability to capture unbounded context. In this paper we develop a finite context approach through stacked convolutions, which can be more efficient since they allow parallelization over sequential tokens. We propose a novel simplified gating mechanism that outperforms Oord et al (2016) and investigate the impact of key architectural decisions. The proposed approach achieves state-of-the-art on the {WikiText}-103 benchmark, even though it features long-term dependencies, as well as competitive results on the Google Billion Words benchmark. Our model reduces the latency to score a sentence by an order of magnitude compared to a recurrent baseline. To our knowledge, this is the first time a non-recurrent approach is competitive with strong recurrent models on these large scale language tasks.},
+	number = {{arXiv}:1612.08083},
+	publisher = {{arXiv}},
+	author = {Dauphin, Yann N. and Fan, Angela and Auli, Michael and Grangier, David},
+	urldate = {2025-03-26},
+	date = {2017-09-08},
+	eprinttype = {arxiv},
+	eprint = {1612.08083 [cs]},
+	keywords = {Computer Science - Computation and Language},
+	file = {Full Text PDF:/home/alex/Zotero/storage/4SBUNZ4A/Dauphin et al. - 2017 - Language Modeling with Gated Convolutional Networks.pdf:application/pdf;Snapshot:/home/alex/Zotero/storage/TQBL4EZ7/1612.html:text/html},
+}

thesis/main.tex

@@ -45,6 +45,8 @@
     \end{abstract}
     \pagebreak
 
+    \tableofcontents
+
     \include{sections/introduction}
 
     \include{sections/background}

thesis/sections/background.tex

@@ -1,10 +1,10 @@
 %! Author = alex
 %! Date = 3/7/25
 
-
 \section{Background}\label{sec:background}
 
 \subsection{Physiological Background}\label{subsec:physiological_background}
+
 \subsubsection{Menstrual Cycle}\label{subsec:menstrual_cycle}
 The menstrual cycle describes the physiological changes in the female body that prepare it for pregnancy.
 It is divided into two phases: the \textbf{follicular phase} and the \textbf{luteal phase}.
@@ -98,6 +98,7 @@ Advances in wearable technology have further enabled continuous and automated te
 improving accessibility and usability\cite{alexander_fertilitatsmonitoring_2014, luo_detection_2020, yu_tracking_2022}.
 
 \subsection{Technical Background}\label{subsec:technological_background}
+
 \subsubsection{Time Series Analysis}\label{subsubsec:time_series_analysis}
 Time series analysis is a fundamental tool for studying sequential data that evolves over time.
 Unlike other data types, time series data has an inherent temporal order, where each data point is associated
@@ -107,4 +108,60 @@ understanding the underlying mechanisms that lead to the observed data and predi
 historical information and potentially external factors\cite{cryer_time_2008}
 \\
 Time series analysis encompasses various methods, ranging from simple statistical models to complex deep learning architectures.
-Classical methods
+Classical methods
+
+\subsubsection{Transformer Models}\label{subsubsec:transformer_models}
+
+\subsubsection{Temporal-Fusion-Transformer Models}\label{subsubsec:temporal_fusion_transformer_models}
+The Temporal Fusion Transformer (TFT) is a deep learning model designed for time series forecasting.
+It's a novel approach at time series analysis and forecasting presented by Google in 2020\cite{lim_temporal_2020}.
+The main focus of this architecture is to both achieve high prediction accuracy while maintaining interpretability.
+It differs from a traditional implementation of a transformer architecture by incorporating the support for:
+\begin{itemize}
+    \item Multiple inputs (static, past and future variables)
+    \item Long sequences
+    \item Uncertainty
+\end{itemize}
+Figure~\ref{fig:background_tft_architecture} provides an overview of the TFT architecture.
+
+\paragraph{1. Gating Mechanisms}
+As the precise relationship between input variables is hard to anticipate in advance, the authors introduce a mechanisms,
+that allows the model to learn this relationship, giving it the ability to choose whether to apply a non-linear
+transformation to the inputs.
+Sometimes, the relationship between variables is linear, and applying a non-linear transformation would only add noise.
+They call this mechanism Gated Residual Network (GRN).
+GRNs take the primary input vector $a$ and an optional context vector $c$ and compute the output as follows:
+\begin{align}
+\text{GRN}_{\omega}(a, c) &= \text{LayerNorm}(a + \text{GLU}_{\omega}(\eta_1)) \label{eq:grn} \\
+\eta_1 &= W_{1,\omega} \eta_2 + b_{1,\omega} \label{eq:eta1} \\
+\eta_2 &= \text{ELU}(W_{2,\omega} a + W_{3,\omega} c + b_{2,\omega}) \label{eq:eta2}
+\end{align}
+
+ELU refers to the Exponential Linear Unit activation function, which is a variant of the ReLU function,
+that speeds up training and alleviates the vanishing gradient problem\cite{clevert_fast_2016}.
+LayerNorm refers to a more robust normalization technique compared to batch normalization, which is used to stabilize
+and speed up training in deep neural networks\cite{ba_layer_2016}.
+Here, $\omega$ denotes weight sharing.
+
+For large $W_{2,\omega}\ a + W_{3\omega}\   c + b_{2,\omega} \gg 0$ the output of the ELU is approximately $a$, acting
+as identity function.
+For negative values, it generates a constant output, resulting in a linear transformation of the input.
+Gating layers based on the Gated Linear Unit (GLU)\cite{dauphin_language_2017} provide the model with the ability to
+suppress any parts of the architecture not needed for the prediction in a dataset.
+
+
+\paragraph{2. Variable Selection Networks (VSNs)}
+Variable Selection Networks (VSNs) are a key component of the TFT architecture, allowing the model to both
+select input variables most relevant to the prediction and removing unnecessary, noisy inputs, that could negatively
+impact prediction performance.
+Each component, i.e., static, past, and future variables, use separate VSNs to select the most relevant features.
+Inputs to the VSN use entity embeddings for categorical features or a linear transformation for continuous features.
+This transforms the inputs into the ($d_{model}$)-dimensional space, where $d_{model}$ is the model's hidden dimension.
+
+
+\begin{figure}
+    \centering
+    \includegraphics[width=0.8\textwidth]{background_tft_architecture}
+    \caption{TFT architecture overview\cite{lim_temporal_2020}.}
+    \label{fig:background_tft_architecture}
+\end{figure}

thesis/sections/methodology.tex

@@ -36,12 +36,17 @@ pregnancies, which would make the prediction task ambiguous.
 
 %TODO: show that a model without marker context and one with them is trained, to show the impact of the markers
 
+%TODO: show that for initial testing and fine tuning the dataset was reduced to 10% of size to speed up early impressions of performance
+%TODO: show that initial feature set with only temperature as observable and only pregnancy "chance" as target
+% did not show promise on small 10% dataset, so the features got extended
 
 \subsection{Time-Series Modeling Approach}\label{subsec:time-series_modeling_approach}
 \subsubsection{Limitations of Traditional Time-Series Models}\label{subsubsec:limitations_of_traditional_time-series_models}
 \subsubsection{Temporal Fusion Transformer}\label{subsubsec:temporal_fusion_transformer}
 \subsubsection{Input \& Output Modeling}\label{subsubsec:input_output_modeling}
 
+% build in the probability curve for the pregnancy chance
+
 \subsection{Model Training}\label{subsec:model_training}
 \subsubsection{Training Setup}\label{subsubsec:training_setup}
 \subsubsection{Hyperparameter Tuning}\label{subsubsec:hyperparameter_tuning}