\documentclass[../main.tex]{subfiles}
\begin{document}

\subsection{Program 1}

\begin{lstlisting}
let x=0 in (1-x)
\end{lstlisting}

\begin{lstlisting}
(define ex1
  (a-program
   (let-exp 'x (const-exp 0) (diff-exp (const-exp 1) (var-exp 'x)))))
\end{lstlisting}

\begin{lstlisting}
Evaluating let x = 0 in (1 - x) in cc [ ]
Evaluating 0 in cc let x = [ ] in <(1 - x)>
Sending Number: 0 to cc let x = [ ] in <(1 - x)>
Evaluating (1 - x) in cc [ ]
Evaluating 1 in cc ([ ] - <x>)
Sending Number: 1 to cc ([ ] - <x>)
Evaluating x in cc (1 - [ ])
Sending Number: 0 to cc (1 - [ ])
Sending Number: 1 to cc [ ]
End of computation.
\end{lstlisting}

\subsection{Program 2}

\begin{lstlisting}
(15 - if zero?0 then 1 else 2)
\end{lstlisting}

\begin{lstlisting}
(define ex2
  (a-program
   (diff-exp (const-exp 15)
             (if-exp (zero?-exp (const-exp 0))
                     (const-exp 1)
                     (const-exp 2)))))
\end{lstlisting}

\begin{lstlisting}
Evaluating (15 - if zero? 0 then 1 else 2) in cc [ ]
Evaluating 15 in cc ([ ] - <if zero? 0 then 1 else 2>)
Sending Number: 15 to cc ([ ] - <if zero? 0 then 1 else 2>)
Evaluating if zero? 0 then 1 else 2 in cc (15 - [ ])
Evaluating zero? 0 in cc (15 - if [ ] then <1> else <2>)
Evaluating 0 in cc (15 - if zero?([ ]) then <1> else <2>)
Sending Number: 0 to cc (15 - if zero?([ ]) then <1> else <2>)
Sending Boolean: #t to cc (15 - if [ ] then <1> else <2>)
Evaluating 1 in cc (15 - [ ])
Sending Number: 1 to cc (15 - [ ])
Sending Number: 14 to cc [ ]
End of computation.
\end{lstlisting}

\subsection{Program 3}

\begin{lstlisting}
let double = proc (f) proc (x) (f(f x)) in ((double proc(x) (x-1)3)
\end{lstlisting}

\begin{lstlisting}
(define ex3
  (a-program
   (let-exp 'double
            (proc-exp 'f
                      (proc-exp 'x
                                (call-exp  (var-exp 'f)
                                           (call-exp (var-exp 'f)
                                                     (var-exp 'x)))))
            (call-exp (call-exp (var-exp 'double)
                                (proc-exp 'x
                                          (diff-exp (var-exp 'x)
                                                    (const-exp 1))))
                      (const-exp 3)))))
\end{lstlisting}

\begin{lstlisting}
Evaluating let double = proc(f)proc(x)(f (f x)) in ((double proc(x)(x - 1)) 3) in cc [ ]
Evaluating proc(f)proc(x)(f (f x)) in cc let double = [ ] in <((double proc(x)(x - 1)) 3)>
Sending Procedure: {proc}  to cc let double = [ ] in <((double proc(x)(x - 1)) 3)>
Evaluating ((double proc(x)(x - 1)) 3) in cc [ ]
Evaluating (double proc(x)(x - 1)) in cc ([ ] <3>)
Evaluating double in cc (([ ] <proc(x)(x - 1)>) <3>)
Sending Procedure: {proc}  to cc (([ ] <proc(x)(x - 1)>) <3>)
Evaluating proc(x)(x - 1) in cc (({proc} [ ] <3>)
Sending Procedure: {proc}  to cc (({proc} [ ] <3>)
Evaluating proc(x)(f (f x)) in cc ([ ] <3>)
Sending Procedure: {proc}  to cc ([ ] <3>)
Evaluating 3 in cc ({proc} [ ]
Sending Number: 3 to cc ({proc} [ ]
Evaluating (f (f x)) in cc [ ]
Evaluating f in cc ([ ] <(f x)>)
Sending Procedure: {proc}  to cc ([ ] <(f x)>)
Evaluating (f x) in cc ({proc} [ ]
Evaluating f in cc ({proc} ([ ] <x>)
Sending Procedure: {proc}  to cc ({proc} ([ ] <x>)
Evaluating x in cc ({proc} ({proc} [ ]
Sending Number: 3 to cc ({proc} ({proc} [ ]
Evaluating (x - 1) in cc ({proc} [ ]
Evaluating x in cc ({proc} ([ ] - <1>)
Sending Number: 3 to cc ({proc} ([ ] - <1>)
Evaluating 1 in cc ({proc} (3 - [ ])
Sending Number: 1 to cc ({proc} (3 - [ ])
Sending Number: 2 to cc ({proc} [ ]
Evaluating (x - 1) in cc [ ]
Evaluating x in cc ([ ] - <1>)
Sending Number: 2 to cc ([ ] - <1>)
Evaluating 1 in cc (2 - [ ])
Sending Number: 1 to cc (2 - [ ])
Sending Number: 1 to cc [ ]
End of computation.
#(struct:num-val 1)
\end{lstlisting}

\end{document}


\documentclass[12pt]{article}
\usepackage{bm}
\usepackage{graphicx}
\usepackage{gensymb}
\usepackage{float}
\usepackage{siunitx}
\usepackage{listings}
\usepackage{hyperref}
\usepackage[a4paper, total={6in, 8in}]{geometry}

\graphicspath{{./../images/}}

\begin{titlepage}

\title{\textbf{Optimum Loft Angle for Greatest Carry Distance}}
\author{\textbf{Group D}\\
		Alison McIntosh\\
		Emily Dark\\
		Henry Archer\\
		Kyle Stewart\\
		Stuart Ballantyne}
\date{}

\end{titlepage}

\begin{document}

\begin{titlepage}
\maketitle
\thispagestyle{empty}
\pagebreak
\end{titlepage}

\pagenumbering{arabic}

\section{Response}
The loft angle of the golf club which maximises the range of the golf ball trajectory is referred to as the optimum loft angle. This investigation consisted of two main parts: the impact of the club, and the flight of the golf ball. The golf ball considered is the Titleist Pro V1x, with characteristics outlined under (2.1) Assumptions.

\subsubsection{Atmospheric conditions at different golf courses}
\begin{table}[H]
\caption{Atmospheric conditions at each golf course}
\label{tab:aconditions}
\resizebox{\textwidth}{!}{%
\begin{tabular}{|l|l|l|l|l|}
\hline
Location                  & Temperature ($K$) & Humidity ($\%$) & Altitude ($m$) & Air density ($kg/m^3$)		\\ \hline
Renaissance, East Lothian &  287            & 85            & 84         & 1.128                              \\ \hline
La Paz, Bolivia           & 278             &60             & 3342         & 0.861                               \\ \hline
Sentosa, Singapore        & 307              &  75        & 33      & 1.154                               \\ \hline
\end{tabular}%
}
\end{table}

The tournament in Singapore took place in January 17th-20th 2019.The tournament in East Lothian is scheduled to take place in July, 8th-14th (2019) and the Bolivian tournament is scheduled for August (2019). The values for temperature and humidity are average values for the given time of year.

\subsection{Results}
The following graphs are for the conditions at Renasissance, East Lothian.

\begin{figure}[H]
\centering
\caption{Effect of loft angle on carry distance}
\includegraphics[scale=0.6]{results1128}
\end{figure}

\begin{figure}[H]
\centering
\caption{Loft angle vs carry distance to obtain optimum loft angle}
\includegraphics[scale=0.6]{results1128range}
\end{figure}

From the investigation, the recommended loft angle to maximise the carry distance based on a club speed of 51.4 $ms^-1$  for the Scottish Open at the Renaissance Club in East Lothian is \SI{13.4}{\degree} which produces a flight distance of \SI{186}{\metre}.

The following graphs are for the conditions at La Paz, Bolivia.

\begin{figure}[H]
\centering
\caption{Effect of loft angle on carry distance}
\includegraphics[scale=0.6]{results0861}
\end{figure}

\begin{figure}[H]
\centering
\caption{Loft angle vs carry distance to obtain optimum loft angle}
\includegraphics[scale=0.6]{results0861range}
\end{figure}

The recommended loft angle to maximise the flight distance based on a club speed of \SI{51.4}{\metre\per\second} for the La Paz golf club in Bolivia is \SI{16.2}{\degree} which produces a carry distance of \SI{218}{\metre}.

 The following graphs are for the conditions at Sentosa, Singapore.

\begin{figure}[H]
\centering
\caption{Effect of loft angle on carry distance}
\includegraphics[scale=0.6]{results1154}
\end{figure}

\begin{figure}[H]
\centering
\caption{Loft angle vs carry distance to obtain optimum loft angle}
\includegraphics[scale=0.6]{results1154range}
\end{figure}

The recommended loft angle to maximise the flight distance based on a club speed of \SI{51.4}{\metre\per\second} for the Sentosa, is \SI{13.2}{\degree} which produces a carry distance of \SI{183}{\metre}.

\begin{figure}[H]
\centering
\caption{Effect of club speed on carry distance and optimum loft angle, $\omega_{spin}=$ \SI{300}{\radian\per\second}, $v_i =$ \SI{50}{\metre\per\second}}
\includegraphics[scale=0.6]{effectofairdensity2}
\end{figure}

\section{Theory}

\subsection{Assumptions}
The following assumptions are made throughout the report and model:
\begin{description}
  \item[$\cdot$] golf course is level and has no effect on trajectory;
  \item[$\cdot$] height of the tee is negligible;
  \item[$\cdot$] gravitational field strength is constant (\SI{9.81}{\metre\per\second^{2}}) and does not flucuate with height;
  \item[$\cdot$] driver is roughly a flat plate and strikes the ball precisely at the center, with no draw or fade;
  \item[$\cdot$] the mass of the club head is significantly greater than that of the shaft, so we consider the shaft's influence to be negligible;
  \item[$\cdot$] the golf ball is a Titleist Pro V1x, with a mass of \SI{45.93}{\gram}, diameter of \SI{42.67}{\milli\metre}, a moment of inertia of \SI{0.009145}{\gram\cdot\metre^2}, and 352 circular dimples.
\end{description}

\subsection{Impact}
\subsubsection{Conservation of energy}
The collision between the club head and golf ball is inelastic, meaning the kinetic energy of the system is not conserved, but rather some of the energy is converted into different forms. One such form is elastic energy in the deformation of the golf ball. This has significant effect on the initial velocity of the golf ball, as the more the ball is deformed, the less kinetic energy the ball will have after the collision. The coefficient of restitution $e$ is the ratio of the final and initial velocities between the golf ball and club head after the collision and it is introduced in calculations. Further energy is lost as heat and sound.

\subsubsection{Conservation of momentum}
The law of conservation of momentum was applied and the velocities were resolved into components.
\begin{equation} \label{linearmomentum1}
M v_{cfn}+mv_{bfn}=Mv_{ci} \cos{\theta}
\end{equation}
\begin{equation} \label{linearmomentum2}
M v_{cfp}+mv_{bfp}=-Mv_{ci} \sin{\theta}
\end{equation}
$v_{bfn}$ and $v_{bfb}$ can be expressed in terms of $v_{ci}$, the initial club speed, as equations (\ref{clubspeed1}-\ref{clubspeed2}).

\begin{equation} \label{clubspeed1}
v_{bfn}=(1+e)v_{ci} \frac{\cos{\theta}}{1+\frac{m}{M}}
\end{equation}
\begin{equation} \label{clubspeed2}
v_{bfp}=-v_{ci} \frac{\sin{\theta}}{1+\frac{m}{M}+\frac{mr^2}{I}}
\end{equation}
These two vector components can be added to yield $v_{bo}$ and $\phi_{bo}$, the velocity and its direction for the ball on departure; equations (\ref{initialvelocity}-\ref{launchanglephi})\cite{Penner2001}
\begin{equation} \label{initialvelocity}
v_{bo}=v_{bf}=\sqrt{v_{bfn}^2+v_{bfp}^2}
\end{equation}
\begin{equation} \label{launchanglephi}
\phi_{bo}=\theta+\tan^{-1}{\frac{v_{bfp}}{v_{bfn}}}
\end{equation}

Lieberman and Johnson give values for $e$ decreasing from approximately $0.76$ for impact speeds of \SI{37}{\metre\per\second} to values of around $0.72$ for impact speeds of \SI{50}{\metre\per\second}. Applying a linear fit gives the empirical equation (\ref{restitution})\cite{Penner2001}.
\begin{equation} \label{restitution}
e=0.86-0.0029 v_{impact} \cos{\theta}
\end{equation}

\begin{figure}[H]
\centering
\caption{Loft angle and spin}
\includegraphics[scale=0.6]{loftangleandspin}
\end{figure}

\subsection{Flight conditions}
The main atmospheric factor affecting the golf ball’s flight is air density which is dependent on air pressure, temperature and humidity. When air density increases, there is increased resistance against the ball during its flight, thus the maximum carry distance is less. Air molecules have greater kinetic energy as the temperature of the air increases, causing them to occupy a larger volume. This results in a decrease in air density. When the air pressure is increased, air density also increases as more collisions between particles occur. The relative humidity of air is a measure of the water vapour relative to temperature and is the percentage of water vapour that could potentially be held in the air at that given temperature.  The air density of humid air is less than dry air. Relative humidity is given by equation (\ref{pressures}):
\begin{equation} \label{pressures}
\phi = \frac{P_w}{P_{w}^\prime} \times 100
\end{equation}
where $P_w$ is the pressure for water vapour and $P_{w}^\prime$ is the equilibrium vapour pressure, that is the maximum pressure it could be in that given temperature value. This value of $P_{w}^\prime$ can be obtained by Antoine's equation (\ref{antoine}) \cite{Roizard2014}:
\begin{equation} \label{antoine}
P_{w}^\prime = e^{\frac{A-B}{C+T}}
\end{equation}
where $T$ is the temperature in K and $A$, $B$, and $C$ are are component specific constants for the given medium. Through the use of Dalton's law for partial pressures, the molar fraction of elements which compose the atmosphere can be calculated using equation (\ref{daltons}):
\begin{equation} \label{daltons}
x_w = \frac{P_w}{P_T}
\end{equation}
where $x_w$ is the molar fraction of water vapour and $n_T$ is the total number of moles which is obtained using the ideal gas law (\ref{idealgaslaw}):
\begin{equation} \label{idealgaslaw}
P_T V = n_T R T
\end{equation}
where $P_T$ is the total pressure, which is found using the barometric formula (\ref{barometric}) \cite{BerberanSantos1997}:
\begin{equation} \label{barometric}
P_T = P_0 e^{\frac{-Mg}{R T_0} h}
\end{equation}
where $P_0$ and $T_0$ are, respectively, pressure and temperature at sea level (\SI{103125}{\pascal}, \SI{288.15}{\kelvin}), $g$ is the gravitational field strength, L is the temperature lapse rate (\SI{0.0065}{\kelvin\per\metre}). At higher altitudes, the air molecules can spread out further resulting in a decrease in air density.\\
The molar fraction of water vapour can be obtained through equation (\ref{molar})
\begin{equation} \label{molar}
x_w = \frac{n_w}{n_T}
\end{equation}
From here, the number of moles for each major component of the atmosphere - nitrogen, oxygen, argon, and water vapour - was obtained using equation (\ref{components}):
\begin{equation} \label{components}
n_T - n_w = n_O + n_N + n_{Ar}
\end{equation}
and considering the fraction of each component of air:
\begin{equation}
n_N = 0.7808(n_T - n_w)
\end{equation}
\begin{equation}
n_O = 0.20195(n_T - n_w)
\end{equation}
\begin{equation}
n_{Ar} = 0.0093(n_T - n_w)
\end{equation}
By the consideration of the molar masses, the density of air can be calculated by the equation (\ref{airdensity}):
\begin{equation} \label{airdensity}
\rho = \frac{((M_N P_{N}) + (M_O P_{O}) + (M_{Ar} P_{Ar}))}{RT}
\end{equation}
where $P$ is the partial pressure of an element, found by equation (\ref{ppressures}):
\begin{equation} \label{ppressures}
P = \frac{n_x RT}{V}
\end{equation}
where $n_x$ is the number of moles for a given element. Consequently, this allowed the air density to be found for each course based on environmental conditions. \cite{Legg2017}

\subsection{Flight}
\subsubsection{Simple golf ball}
A simple golf ball experiencing only weight may be modelled by the following system of differential equations;
\begin{equation} \label{simpleaccelx}
a_x=\frac{\partial v_x}{\partial t}=0
\end{equation}
\begin{equation} \label{simpleaccely}
a_y=\frac{\partial v_y}{\partial t}=-g
\end{equation}
where $g$ is the gravitional field strength.

Assuming the initial velocity is $v_0$ and the launch angle is $\theta$, equations (\ref{simpleaccelx}-\ref{simpleaccely}) can be solved to give equations (\ref{simplevelx}-\ref{simplevely}):
\begin{equation} \label{simplevelx}
v_x=v_0 \cos{\theta}
\end{equation}
\begin{equation} \label{simplevely}
v_y=v_0 \sin{\theta}-gt
\end{equation}
Integrating equations (\ref{simplevelx}-\ref{simplevely}) with respect to time yields the displacement as a function of time to give equations (\ref{simpleposx}-\ref{simpleposy}) \cite{Penner2001}:
\begin{equation} \label{simpleposx}
x=v_0 t \cos{\theta}
\end{equation}
\begin{equation} \label{simpleposy}
y=v_0 t \sin{\theta}-\frac{1}{2} g t^2
\end{equation}

\begin{figure}[H]
\centering
\caption{Golf ball trajectory, no drag or lift}
\includegraphics[scale=0.6]{simple}
\end{figure}

\begin{figure}[H]
\centering
\caption{Range as a function of loft angle, no drag or lift}
\includegraphics[scale=0.6]{simplerange}
\end{figure}

Figures (9-10) show the maximum range is when the loft angle at 45.0\degree, as predicted by the equations of projectile motion.
\subsubsection{Smooth golf ball experiencing drag}
The drag equation (\ref{drageqn}) is
\begin{equation} \label{drageqn}
\vec{F_{d}} = \frac{1}{2} A C_{d} \rho_{air} |\vec{v}| \vec{v}
\end{equation}
where $\rho_{air}$ is the density of air; $A$ is the reference area, which in the case of a smooth sphere of radius $r$, is the cross-sectional area $\pi r^2$; $C_d$ is the coefficient of drag, which is dependent on the Reynolds number; and $\vec{v}$ is the flow velocity relative to the golf ball. In this case, we assume the air is stationary and the golf ball is moving through the air with velocity $\vec{v}$.

Applying equation (\ref{drageqn}) to equations (\ref{simpleaccelx}-\ref{simpleaccely}), we get
\begin{equation} \label{dragaccelx}
\frac{\partial v_x}{\partial t}=-k |v_x| v_x
\end{equation}
\begin{equation} \label{dragaccely}
\frac{\partial v_y}{\partial t}=-g-k |v_y| v_y
\end{equation}
where $k=\frac{1}{2} A C_d \rho_{air}$. These equations already do not have a closed-form solution and require numerical methods.
\begin{figure}[H]
\centering
\caption{Golf ball trajectory when experiencing drag but no lift, $C_d$ = 0.5}
\includegraphics[scale=0.6]{dragcd}
\end{figure}

\begin{figure}[H]
\centering
\caption{Range as a function of loft angle, $C_d$ = 0.5}
\includegraphics[scale=0.6]{dragcdrange}
\end{figure}

Figure (12) shows that the maximum range is achieved at 40.2\degree, for an initial velocity of \SI{50}{\metre\per\second}. Additionally, the range is significantly decreased when drag was added to the model. The greatest range with drag is about \SI{114}{\metre} versus the previous range of \SI{255}{\metre} at 45.0\degree.
However, $C_d$ is not constant and depends on the Reynolds number, which is proportional to the velocity of the golf ball. The Reynolds number is given by equation (\ref{reynolds})\cite{Alam2011}:
\begin{equation} \label{reynolds}
R=\frac{2vr}{\nu}
\end{equation}
where $\nu$ is the kinematic viscosity of air.

Using data from \textit{Alam 2011}\cite{Alam2011}, Python was used to construct a quartic fit of drag coeffiecient as a function of Reynolds Number, as shown in Figure (13).\\
A quartic approximation is best as any greater degree would begin to oscillate too much at the edges (Runge's phenomenon). Likewise, a lesser degree would be too linear to give any meaningful relation.\\
One of the limitions of this approach is that there is no data above Reynolds number $112,000$ (\SI{38.39}{\meter\per\second}) or below $63,300$ (\SI{21.70}{\meter\per\second}), so the approximation does not hold for those conditions, and may in fact be drastically worse due to the chaotic behaviour of the polynomial at the aforementioned values. To overcome this, the drag coefficient was fixed at 0.8 for Reynolds numbers less than 53,000 and fixed at 0.37 for Reynolds numbers greater than 120,000.

\begin{figure}[H]
\centering
\caption{Drag coefficient as a function of Reynolds number with curve of best fit}
\includegraphics[scale=0.6]{reynolds}
\end{figure}

Applying the approxmation for the coefficient of drag, the optimum loft angle decreases down to about 34.5\degree, shown in Figures (14-15).
\begin{figure}[H]
\centering
\caption{Trajectory of golf ball with drag considering Reynolds number, $v_i=$ \SI{50}{\metre\per\second}}
\includegraphics[scale=0.6]{dragwithreynolds}
\end{figure}

\begin{figure}[H]
\centering
\caption{Range against loft angle for golf ball with drag considering Reynolds number, $v_i=$ \SI{50}{\metre\per\second}}
\includegraphics[scale=0.6]{dragwithreynoldsrange}
\end{figure}

\subsubsection{Smooth golf ball experiencing lift}
Lift on a golf ball is caused by the Magnus effect, which is dependent on the backspin of the ball. The equation is similar to the drag equation, however the direction of the force is perpendicular to both angular velocity and translational velocity.

\begin{equation} \label{lifteqn}
F_l = \frac{1}{2} \rho_{air} C_{l} |v|^2 (\hat{\omega} \times \hat{v})
\end{equation}

where $C_l$ is dependent on the spin parameter of the ball according to \textit{Bearman}\cite{Bearman1976}.
\begin{equation} \label{clifteqn}
C_l = -3.25 S^2 + 1.99 S
\end{equation}
The spin parameter is given by the ratio of the magnitude of tangential velocity to the magnitude of translational velocity.
\begin{equation} \label{spineqn}
S = \frac{r|\omega|}{|v|}
\end{equation}

\bibliography{bibliography}{}
\bibliographystyle{plain}

\pagebreak

\lstdefinestyle{PythonStyle}{
  language=Python,
  numbers=left,
  stepnumber=1,
  numbersep=10pt,
  tabsize=4,
  showspaces=false,
  showstringspaces=false
}

\textit{The model was written using Python version 3.8, on a Linux machine, although earlier versions (i.e. 3.5) should work. It requires the following Python packages: NumPy, SciPy, and Matplotlib. Use ./model3d.py -h for a list of parameters. It also supports 3d plots, though these are experimental. The source code is also hosted as a git repository at \url{https://github.com/s-ballantyne/gdp}.}
\begin{lstlisting}[language=Python, caption=Python model, style=PythonStyle, basicstyle=\tiny]
#!/usr/bin/env python3

import numpy as np
import argparse

from scipy.integrate import odeint as integrate
from matplotlib import pyplot as plot
from numpy.linalg import norm
from mpl_toolkits.mplot3d import Axes3D

parser = argparse.ArgumentParser()

# Ball parameters
constants = parser.add_argument_group("Constants")
constants.add_argument("-m", "--mass", default=0.04593, help="Mass of ball (kg)")
constants.add_argument("-r", "--radius", default=0.04267/2, help="Radius of ball (m)")

constants.add_argument("-g", "--gravity", type=float, default=9.81, help="For when we get a Mars base (m/s/s)")
constants.add_argument("-d", "--density", type=float, default=1.225, help="Density of air (kg m^-3)")
constants.add_argument("--viscosity", type=float, default=1.46e-5, help="Kinematic viscosity of air")

# Initial parameters
initialparams = parser.add_argument_group("Initial parameters")
#initialparams.add_argument("-vi", "--velocity", type=float, default=50, help="Initial velocity (m/s)")
initialparams.add_argument("-yi", "--height", type=float, default=0, help="Initial height (m)")

#initialparams.add_argument("-sp", "--spin", type=float, default=0, help="Spin (z)")
#initialparams.add_argument("-spy", "--spiny", type=float, default=0, help="Spin (y)")
#initialparams.add_argument("-spx", "--spinx", type=float, default=0, help="Spin (x)")

# Loft angle
parser.add_argument("-li", "--loftinitial", type=float, default=10, help="Loft angle (initial)")
parser.add_argument("-lf", "--loftfinal", type=float, default=35, help="Loft angle (final)")
parser.add_argument("-st", "--step", type=float, default=5, help="Loft angle (step)")

# Debugging
parser.add_argument("-v", "--verbose", action="store_true")

# Ball speed calculations
parser.add_argument("--clubmass", type=float, default=0.2, help="Mass of club head (kg)")
parser.add_argument("--vclub", type=float, default=51.4, help="Club speed (m/s)")
parser.add_argument("--inertia", type=float, default=9.145e-6, help="Inertia of golf ball")

# Parse args
args = parser.parse_args()

# Input validation
assert args.loftfinal > args.loftinitial, "Final loft angle must be gretaer than initial loft angle!"
assert args.step != 0, "Step must be non-zero!"
assert ((args.loftfinal - args.loftinitial) / args.step).is_integer(), "Step size must divide the change in loft angle!"

assert args.mass != 0, "Mass must be non-zero."
assert args.radius != 0, "Radius must be non-zero."
assert args.viscosity != 0, "Kinematic viscosity must be non-zero."
assert args.density != 0, "Density of air must be non-zero."

g = args.gravity
density = args.density

# Ball speed from club speed and loft angle
def ball_speed(theta):
	theta = np.radians(theta)
	e = 0.86 - 0.0029 * args.vclub * np.cos(theta)

	bfn = (1 + e) * args.vclub * np.cos(theta) / (1 + args.mass / args.clubmass)
	bfp = args.vclub * np.sin(theta) / (1 + args.mass / args.clubmass + (args.mass * args.radius**2 / args.inertia))
	return np.sqrt(bfn**2 + bfp**2)

# Spin
def ball_spin(theta):
	theta = np.radians(theta)
	bfp = args.vclub * np.sin(theta) / (1 + args.mass / args.clubmass + (args.mass * args.radius**2 / args.inertia))

	return args.mass * bfp * args.radius / args.inertia

# Coefficient of drag from Reynolds number, based on degree four polynomial.
def re_to_cd(re):
	# Clamp output value as it is only an approximation
	if re > 120000:
		return 0.370
	elif re < 53000:
		return 0.8

	# Array of coefficients
	coeffs = np.array([
			  9.46410458e-20, -3.80736984e-14,
			  5.72048806e-09, -3.81337408e-04,
			  9.92620188e+00
			])

	# Return value of polynomial approximation
	return np.polyval(coeffs, re)

# Linear velocity to Reynolds number (Re = velocity * diameter / k. viscosity)
def reynolds(velocity, radius):
	return 2 * radius * velocity / args.viscosity

# Linear velocity to drag coefficient
def sphere_cd(velocity, radius):
	cd = re_to_cd(reynolds(velocity, radius))
	return cd

# Drag equation
# F_d = 1/2 * air density * ref. area * coefficient * |velocity| * v
def drag(density, area, cd, velocity):
	return -0.5 * density * area * cd * norm(velocity) * velocity

# Lift equation
# F_l = 1/2 * air density * ref. area * coefficient * |v|^2 * (what x vhat)
def lift(density, area, cl, velocity, rvelocity):
	if cl == 0:
		return np.array([0, 0, 0])

	S = 0.5 * density * area * cl

	# Cross product of angular velocity and linear velocity, for direction of spin
	rxv = np.cross(rvelocity, velocity)
	rxv /= norm(rxv)

	# Magnitude of spin is considered in coefficient of lift
	return S * norm(velocity)**2 * rxv

# Simple golfball, no drag, no lift, smooth
class BasicGolfball:
	def __init__(self):
		# Properties
		self.mass = args.mass
		self.radius = args.radius

		# Coordinates
		self.x = 0
		self.y = args.height
		self.z = 0

		self.vx = 0
		self.vy = 0
		self.vz = 0

		# Rotational velocities
		self.rvx = 0
		self.rvy = 0
		self.rvz = 0

	# Reference area, for a sphere this is the cross-section.
	def area(self):
		return np.pi * self.radius**2

	# Set initial velocity
	def set_velocity(self, v, theta):
		self.vx = v * np.cos(np.radians(theta))
		self.vy = v * np.sin(np.radians(theta))

	# Set spin
	def set_spin(self, spin):
		self.rvx, self.rvy, self.rvz = spin

	# Get all coordinates
	def coords(self):
		return np.array([self.x, self.y, self.z, self.vx, self.vy, self.vz, self.rvx, self.rvy, self.rvz])

	# Set all coordinates [x, y, z, vx, vy, vz, rvx, rvy, rvz]
	def set_coords(self, coords):
		self.x, self.y, self.z, self.vx, self.vy, self.vz, self.rvx, self.rvy, self.rvz = coords

	# Returns numpy array of position coordinates
	def position(self):
		return np.array([self.x, self.y, self.z])

	# Returns numpy array of velocity at the current position
	def velocity(self):
		return np.array([self.vx, self.vy, self.vz])

	# Returns numpy array of acceleration at the current position
	def acceleration(self):
		return np.array([0, -g, 0])

	# Returns numpy array of rotational velocity (spin) at the current position
	def rvelocity(self):
		return np.array([self.rvx, self.rvy, self.rvz])

	# Returns numpy array of rotational acceleration at the current position
	def racceleration(self):
		return np.array([0, 0, 0])

	# Returns numpy array of differential eqns to be solved by odeint
	def differentials(self):
		d = np.zeros(9)

		d[0:3] = self.velocity()
		d[3:6] = self.acceleration()

		d[6:9] = self.racceleration()

		return d

	# (Internal) Updates coordinates and returns list of equations to solve (for odeint)
	def __eqns(self, t, coords):
		self.set_coords(coords)

		if args.verbose:
			print(t, self.velocity(), self.rvelocity(), self.acceleration(), self.racceleration())

		return self.differentials()

	# Solve for trajectory over given interval
	def solve(self, t0, t1, dt=0.01):
		interval = np.linspace(t0, t1, int((t1 - t0) / dt))
		res = integrate(self.__eqns, self.coords(), interval, tfirst=True)[:, :3]

		out = np.array([e for e in res if e[1] >= 0])
		return out

# Simple golf ball but with drag
class DragGolfball(BasicGolfball):
	def __init__(self):
		BasicGolfball.__init__(self)

	# Coefficient of drag from velocity & radius
	def cd(self):
		return sphere_cd(norm(self.velocity()), self.radius)

	def acceleration(self):
		fd = drag(density, self.area(), self.cd(), self.velocity())
		return BasicGolfball.acceleration(self) + fd / self.mass

# Golfball with lift and drag
class LiftGolfball(DragGolfball):
	def __init__(self):
		DragGolfball.__init__(self)

	# Returns spin factor
	def spinf(self):
		v = norm(self.velocity())
		w = self.radius * norm(self.rvelocity())
		return w / v

	# Returns coefficient of lift based on spin factor
	def cl(self):
		s = self.spinf()
		return -3.25 * s**2 + 1.99 * s

	def acceleration(self):
		fl = lift(density, self.area(), self.cl(), self.velocity(), self.rvelocity())
		return DragGolfball.acceleration(self) + fl / self.mass

	# Spin decreases by about 1% every second
	def racceleration(self):
		return -0.01 * self.rvelocity()

if __name__ == "__main__":
	# Initial conditions
	for density in [1.128, 0.861, 1.154]:
		plot.figure()
		for theta in np.arange(args.loftinitial, args.loftfinal, args.step):
			ball = LiftGolfball()
			ball.set_velocity(ball_speed(theta), theta)
			ball.set_spin([0, 0, ball_spin(theta)])

			res = ball.solve(0, 10)
			x, y, z = res.T

			plot.plot(x, y, label="Loft angle: " + format(theta, ".1f"))

		plot.legend()
		plot.grid(True)
		plot.xlabel("Distance (m)")
		plot.ylabel("Height (m)")
		plot.title("Ballistic trajectory for air density " + format(density, ".3f") + " kg/m^3")

		plot.figure()
		xdata = []
		ydata = []
		for theta in np.arange(5, 30, 0.5):
			ball = LiftGolfball()
			ball.set_velocity(ball_speed(theta), theta)
			ball.set_spin([0, 0, ball_spin(theta)])

			res = ball.solve(0, 10)
			x, y, z = res.T

			xdata.append(theta)
			ydata.append(x[-1])

		plot.plot(xdata, ydata, 'o', label="Air density: " + format(density, ".3f"))
		plot.legend()
		plot.grid(True)
		plot.xlabel("Loft angle (m)")
		plot.ylabel("Carry distance (m)")

	plot.show()
\end{lstlisting}

\end{document}

\grid


\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\def\Pr{\mathop{\rm Pr}}
\usepackage[landscape,margin=1cm]{geometry}
\usepackage[english]{babel}
\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=blue,
}

\usepackage{multicol}
% colour themes to come. KnitR?

%-------------------------

\title{Probability and Statistical Inference}
\author{John S Butler }
%\date{July 2019}
\input{cheatsheet-template.tex}

%--------------------------------------------------------------------------------
\begin{document}
\small
\begin{multicols}{3}

%\maketitle
%\thispagestyle{empty}
\scriptsize
%\tableofcontents

%\section{Data Type}
\section*{Probability and Statistical Inference}
\subsection*{Cheat Sheet}
%\href{https://sites.google.com/dit.ie/math4001
%}{Course} Cheat Sheet}

\subsubsection*{\href{johnsbutler.netlify.com}{John S Butler} (TU Dublin) }
\subsubsection*{\href{https://twitter.com/StatisticalInf1}{Course Twitter Account} }
\begin{textbox}{Data Type}
\begin{multicols}{2}
\begin{itemize}
  \item Categorical
  \item Interval
  \item Ordinal
  \item Ratio
\end{itemize}

\end{multicols}

\end{textbox}

\begin{textbox}{Measures of Location}
Different aspects of a distribution of data can be summarised by the measures of location:
\begin{enumerate}
    \item The First Moment: Mean, Mode or Median;
    \item  The Second Moment: Variance, Standard Deviation;
    \item  The Third Moment: Skewness.
\end{enumerate}

\begin{subbox}{subbox}{First Moment: Middle}
\begin{center}
\tiny
    \includegraphics[width=\textwidth]{Figures/Measures_of_Location/middle.png}
\end{center}

\end{subbox}

\begin{subbox}{subbox}{Second Moment: Spread}
\begin{center}
\tiny
    \includegraphics[width=\textwidth]{Figures/Measures_of_Location/range.png}
\end{center}

\end{subbox}
\end{textbox}
%% NEW TEXT BOX MEASURES OF LOCATION
\begin{textbox}{Measures of Location (cont.)}

\begin{subbox}{subbox}{Third Moment: Symmetry}
\begin{center}
\tiny
    \includegraphics[width=\textwidth]{Figures/Measures_of_Location/skew.png}

\end{center}
\end{subbox}
\end{textbox}
\begin{textbox}{Mathematical Probability }
\begin{subbox}{subbox}{Definitions}
\tiny
Define some event $A$ that can be the outcome of an experiment.\\
$\Pr(A)$ is the probability of a given event $A$ will happen.\\
Rules:
\begin{itemize}
    \item $\Pr(A)$ is between $0$ and $1$, $0\leq Pr(A) \leq 1$;
    \item $\Pr(A)=1$, means it will definitely happen;
    \item $\Pr(A)=0$, means it will definitely \textbf{not} happen;
    \item $\Pr(A)=0.05$, is arbitrarily considered unlikely.
\end{itemize}
\end{subbox}

\begin{subbox}{subbox}{Sample Space and Events}
\tiny
The \textbf{Sample Space}, $S$, of an experiment is the universal set of all possible outcomes for that experiment, defined so, no two outcomes can occur simultaneously. For example:
\begin{itemize}
    \item Throwing a die $S={1,2,3,4,5,6};$
    \item Tossing two coins $S={HH,TH,HT,TT}.$
\end{itemize}

An event, $A$, is a subset of the sample space $S$. For example:
\begin{itemize}
    \item Throwing a die $S={3,4,6};$
    \item Tossing two coins $S={TH,TT}.$
\end{itemize}
\end{subbox}

\begin{subbox}{subbox}{Axioms of Probabilities}
\tiny
For an event $A$ subset $S$ associated a number $Pr(A)$, the probability of $A$, which must have the following properties

\begin{itemize}
     \item $\Pr(A \bigcap B)=0;$ $\Pr(A\bigcup B)= \Pr(A)+\Pr(B) $;
     \item Probability of the Null Event $\Pr(\emptyset)=0$;
     \item The probability of the complement of $A,$ $\Pr(\bar{A})=1-\Pr(A)$;
     \item $\Pr(A \bigcup B)= \Pr(A)+\Pr(B)-\Pr(A\bigcap B)$.
\end{itemize}
\end{subbox}

\end{textbox}

\begin{textbox}{Conditional Probability}
 The Conditional Probability $\Pr(A|B)$ denotes the probability of the event $A$ occurring given that the event $B$ has occurred,
\[ \Pr(A|B)=\frac{\Pr(A\bigcap B)}{\Pr(B)}.\]
\begin{subbox}{subbox}{Example: The rain in Ireland}
\tiny
A normal probability would be what is the probability it is going to rain, $\Pr(\text{rain})$.\\
A conditional probability would, be what is the probability it is going to rain \textbf{given} that you are in Ireland, $\Pr(\text{rain}|\text{Ireland}),$
\[ \Pr(\text{rain}|\text{Ireland})=\frac{\Pr(\text{rain}\bigcap \text{Ireland})}{\Pr(\text{Ireland})},\]
where the probability of rain is $\Pr(\text{rain})=0.3$, the probability of being in Ireland is $\Pr(\text{Ireland})=0.4)$ and the probability of being in Ireland and it raining is $\Pr(\text{rain}\bigcap \text{Ireland})=0.2$,
\[ \Pr(\text{rain}|\text{Ireland})=\frac{0.2}{0.4}=0.5,\]
You could be interested in the probability that you are in Ireland \textbf{given} that it is raining,
\[ \Pr(\text{Ireland}|\text{rain})=\frac{\Pr(\text{rain}\bigcap \text{Ireland})}{\Pr(\text{rain})}=\frac{0.2}{0.3}=0.75.\]
\end{subbox}
\end{textbox}

\begin{textbox}{Bayes Theorem}
 Bayes Theorem states
\[ \Pr(A|B)=\frac{\Pr(B|A)P(A)}{\Pr(B)}.\]
\begin{subbox}{subbox}{Example: Diagnostic test}
\tiny
The probability that an individual has a rare disease is $\Pr(\text{Disease})=0.01$. The probability that a diagnostic test results in a positive (+) test \textit{given you have} the disease is $\Pr(+|\text{Disease})=0.95$. On the other hand, the probability that the diagnostic test results in a positive (+) test \textit{given you do not have} the disease is $\Pr(+|\text{No Disease})=0.1$.
This raises the important question if you are given a positive diagnosis, what is the probability you have the disease $\Pr(\text{Disease}|+)$?
From Bayes Theorem we have:
\[ \Pr(\text{Disease}|+)=\frac{\Pr(+|\text{Disease})\Pr(\text{Disease})}{\Pr(+)}\]
The probability of a positive test is,
\[\Pr(+)= \Pr(+|\text{Disease})\Pr(\text{Disease})+\Pr(+|\text{No Disease})\Pr(\text{No Disease}),\]
\[\Pr(+)= 0.1085.\]
\[ \Pr(\text{Disease}|+)=\frac{\Pr(+|\text{Disease})\Pr(\text{Disease})}{\Pr(+)}=\frac{0.95\times 0.01}{0.1085}=0.0875576.\]
This can also be done in a simple table format, by assume a population of 10,000
\begin{center}
 \begin{tabular}{||c |c c |c||}
 \hline
  Group & + Diagnosis & - Diagnosis & Total \\
 \hline
 Disease & 95 & 5 & 100 \\
 \hline
 No Disease & 990 & 8,910 & 9,900 \\
 \hline
 Total & 1,085 & 8,915 & 10,000 \\
 \hline

\end{tabular}
\end{center}
From the table we can calculate the same answer,
 $\Pr(\text{Disease}|+)=\frac{95}{1085}.$\end{subbox}
\end{textbox}
%%% DISCRETE DISTRIBUTION
\begin{textbox}{Discrete Distribution}
\begin{subbox}{subbox}{Probability Mass Functions}
\tiny
\[  \begin{tabular}{r|r|r|r|r|r}
			$i$&0&1&2&3&4\\
			\hline
			$x_i$&-1&0&1&2&3\\
			\hline
			$\Pr(x_i)$&0.3&0.1&0.3&0.1&0.2\\
			\end{tabular}
			\]
The expected value of the distribution is:
   	\[\mu=E[X]=\Sigma_{i} x_i p(x_i),\]
\[\Sigma_{i} x_i p(x_i)=-1\times 0.4+0\times 0.1+1\times 0.3+0.1\times 2+0.2\times 3=0.7,\]
The variance of the distribution is:
	\[Var[X]=\Sigma_{i} (x_i-\mu)^2 p(x_i)=\Sigma_{i} (x_i-0.7)^2 p(x_i).\]

\end{subbox}

\begin{subbox}{subbox}{Binomial Distribution}
\tiny
The formula for the Binomial distribution is:
\[\Pr(k)= \left( \begin{array}{c}
	n  \\
	k  \\
	\end{array} \right)p^kq^{n-k}, \ \ k=0,1,2,...n, \]
	\[E[k]=np,\ \ \ Var[k]=npq.\]
    \includegraphics[width=\textwidth]{Figures/Distributions/Binomial.png}
\end{subbox}

\begin{subbox}{subbox}{Geometric Distribution}
\tiny
The formula for the Geometric distribution is:
\[\Pr(k)=q^{(k-1)}p, \ \ k=1,2,... \]
	\[E[k]=\frac{1}{p}, \ \ \ Var[k]=\frac{q}{p^2}.\]
    \includegraphics[width=\textwidth]{Figures/Distributions/Geometric.png}
\end{subbox}
\end{textbox}
%%% DISCRETE DISTRIBUTION
\begin{textbox}{Discrete Distribution}

\begin{subbox}{subbox}{Poisson Distribution}
	\tiny
	The formula for the Poisson distribution is:
	\[\Pr(k)=\frac{\lambda^ke^{-\lambda}}{k!}, \ \ k=0,1,2,... \]
	\[E[k]=\lambda, \ \ \ Var[k]=\lambda. \]

    \includegraphics[width=\textwidth]{Figures/Distributions/Poisson.png}
\end{subbox}

\end{textbox}
\begin{textbox}{Continuous Distribution}
\begin{subbox}{subbox}{Normal Distribution}
    \includegraphics[width=\textwidth]{Figures/Distributions/Normal.png}
\end{subbox}
\begin{subbox}{subbox}{Confidence Intervals}
\begin{center}
\tiny
    \includegraphics[width=\textwidth]{Figures/Distributions/CI_Normal.png}
\end{center}
\end{subbox}

\end{textbox}
\begin{textbox}{Hypothesis Testing}
Five steps for Hypothesis testings
\begin{enumerate}
\item  State the Null Hypothesis $H_0$;
\item  State an Alternative Hypothesis $H_{ alpha}$;
\item  Calculate a Test Statistic (see below);
\item  Calculate a p-value and/or set a rejection region;
\item  State your conclusions.
\end{enumerate}
\end{textbox}
\begin{textbox}{z-test}
\begin{subbox}{subbox}{Continuous Data}
The test statistic is given by
\[Z=\frac{\bar{x}-\mu}{\frac{\sigma}{\sqrt{n}}} \sim N(0,1), \]
where $\bar{x}$ is the observed mean, $\mu$ is the historical mean, $\sigma$ is the standard deviation and $n$ is the number of observations.

\begin{subbox}{subbox}{Do supplements make you faster? }
\tiny
The effect of a food supplements on the response time in rats is of interest to a biologist. They have established that the normal response time of rats is $\mu=1.2$ seconds. The $n=100$ rats were given a new food supplements. The following summary statistics were recorded from the data $\bar{x}=1.05$ and $\sigma= 0.5$ seconds
\begin{enumerate}
    \item The rats in the study are the same as normal rats,  $H_0 : \bar{x}=1.2$.
\item The rats are different, $H_\alpha:\bar{x}\not= 1.2$.
\item Calculate a Test Statistic $Z=\frac{1.05-1.2}{\frac{0.5}{\sqrt{100}}}=-3 $
\item Reject the Null hypothesis $h_0$ if $Z<-1.96$ and $Z>1.96$
\item The data suggests that rats are faster with the new food.
\end{enumerate}

\end{subbox}
\end{subbox}
\begin{subbox}{subbox}{Proportional Data}
The test statistic is given by
\[z=\frac{\hat{p}-p}{\sqrt{\frac{pq}{n}}} \sim N(0,1). \]
where $\hat{p}$ is the observed proportion, $p$ is the historical proportion, $q$ is the complement $q=1-p$,  and $n$ is the number of observations.
\end{subbox}
\end{textbox}
\begin{textbox}{t-test}
\begin{subbox}{subbox}{paired t-test}

 The test statistic is given by  \[ t={\frac {{\bar {x}}-{\bar \mu_0}}{\frac{s}{\sqrt n}}} \sim t_{\alpha,df}\]
where $\bar{x}$ is the observed mean, $\mu_0$ is the null mean, $s$ is the standard deviation and $n$ is the number of observations.
\end{subbox}
\begin{subbox}{subbox}{unpaired t-test}
The test statistic is given by
\[ t={\frac {{\bar {x}}_{1}-{\bar {x}}_{2}}{s_{p}{\sqrt {\frac {1}{n_1}+\frac{1}{n_2}}}}} \sim t_{\alpha,df}\]
 where
$s_{p}={\sqrt{\frac {s_{x_{1}}^{2}+s_{x_{2}}^{2}}{2}}}$ is the pooled sample standard deviation, $\bar{x}_1$ and $\bar{x}_2$ are the sample means, $n_1$ and $n_2$ are the sample sizes.
\end{subbox}
\end{textbox}
\begin{textbox}{$\chi^2$ Independence test}
The test statistic to test if data are independent of group is given by:
   	\[\chi^2_{Ind}=\sum \frac{(O-E)^2}{E} \sim \chi^2_{(r-1)(c-1)}.\]
where $O$ is the observed data, $E$ is the expected data if independent, $r$ is the number of rows and $c$ is the number of columns.\\
\begin{subbox}{subbox}{Does ice-cream flavour matter?}
\tiny
An ice-cream company had 500 people sample one of three different ice-cream flavours and asked them to say whether they liked or disliked the ice-cream.
\begin{center}
\begin{tabular}{|l|r|r|r|}
\hline
&Vanilla&Chocolate& Strawberry\\
\hline
Liked&130&170&100\\
\hline
Disliked&20&30&50\\
\hline
\end{tabular}
\end{center}
The $\chi^2_{Ind}$ independence test could be used to determine if the enjoyment of the ice-cream depends on the flavour.\\
\end{subbox}
\end{textbox}
\begin{textbox}{$\chi^2$ Goodness of Fit}
The test statistic to test if data come from a specific distribution is given by:
\[\chi^2_{GoF}=\sum \frac{(O-E)^2}{E} \sim\chi^2_{k-1},\]
where $O$ is the observed data, $E$ is the expected data from a chosen distribution and $k$ is the number of observation bins.
\begin{subbox}{subbox}{Does it fit?}
\tiny
The $\chi^2_{GoF}$ can test if the observed distribution of the height of Dutch people (grey) fits the expected distribution of heights (dark grey).

\includegraphics[width=\textwidth]{Figures/Distributions/GoF.png}
\end{subbox}
\end{textbox}

\begin{textbox}{Linear Regression}

A linear regression is used to model a linear relationship of the dependent variable $y$ and the regressors $x_1$, $x_2$, ...
	\[  y=\beta_0+\beta_1 x_{1} +\beta_2 x_{2}+..., \]
	where $\beta_0$, $\beta_1$ are the slopes of the regressors.

\begin{subbox}{subbox}{Height Prediction}
\tiny
A simple linear regression (correlation) is used to predict the height of 744 children $y$ using the height of their parent $x$,
\[  y=\beta_0+\beta_1 x.\]
The plot below shows the fit of the model:
\begin{center}
\includegraphics[width=\textwidth]{Figures/Regression/Linear_Regression.png}
\end{center}
The parents' height $x$ explained $12.7\%$ of the childrens' height $y$.
\end{subbox}
\end{textbox}
\begin{textbox}{Logistic Regression}
A logistic regression (or logit model) is used to model the probability of a binary events such as win/lose.
The general formula for the Logistic regression is
\[  p_i=\frac{e^{\eta}}{1+e^{\eta}},\]
where
\[\eta=\beta_0+\beta_1 x_{1} +\beta_2 x_{2}+... \]
and $\beta$ is the slope corresponding to the predictor variable $x$.
\begin{subbox}{subbox}{Sexton Conversion Rate}
\tiny
Data from $1000$ conversions kicks by Johnny Sexton was acquired; the  distance (m) from the goal-line and if the kick was a  miss $0$  or a conversion $1$. The data  was fit to a logistic regression. The model was
\[  p=\frac{e^{\eta}}{1+e^{\eta}},\]
where
\[\eta=\beta_0+\beta_1 \text{Distance} \]
and $p$ is the probability of a conversion.
The plot below shows the fit of the model:
\begin{center}
\includegraphics[width=\textwidth]{Figures/Regression/Sexton.png}
\end{center}
The model predicts that at the half-way line (50m) Sexton has a  $0.375$ probability of conversion.

\end{subbox}
\end{textbox}
\begin{textbox}{Bibliography}
\begin{enumerate}
\item
Devore \& Peck  - Statistics: The exploration and analysis of data (2011)
\item James, G., Witten, D., Hastie, T., \& Tibshirani, R. (2013). An introduction to statistical learning (Vol. 112, p. 18). New York: springer.
\href{https://www.statlearning.com}{book website}
\item Poldrack R. Statistical Thinking in the 21st Century 2020 \href{https://statsthinking21.github.io/statsthinking21-core-site/index.html}{website}
\item
Gareth, J, et al. - An introduction to statistical learning. Vol. 112. New York: Springer, 2013.
\item
Fry, H. - Hello World: How to be Human in the Age of the Machine, Doubleday, 2018
\item Butler, J. S., \href{https://github.com/john-s-butler-dit/Probability_and_Statistical_Inference}{Course GitHub Repository}

\end{enumerate}

\end{textbox}

\end{multicols}

\end{document}


\documentclass{article}

\usepackage{amsmath}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{xspace}

\begin{document}
\title{TBD}

\author{TBD}

\maketitle

\begin{abstract}
This document is a model and instructions for
\end{abstract}

\section{Introduction}
Welcome to the

\section{Ease of Use}

\subsection{Maintaining the Integrity of the Specifications}
The `article' class is used to format your paper and style the text. All margins, column widths, line spaces, and text fonts are prescribed.

\section{Styling Guide}

\subsection{Abbreviations and Acronyms}
Define abbreviations and acronyms the first time they are used in the text,
even after they have been defined in the abstract.

\subsection{Equations}
\begin{equation}
1+1=2
\end{equation}

Taylor series in a text would be $1+1=2$.

\subsection{Lists}
Bullet style list.

Number style list.

\subsection{Figures and Tables}
\paragraph{Positioning Figures and Tables} Figure captions should be below the figures; table heads should appear above the tables. Insert figures and tables after they are cited in the text. Use the abbreviation .

\begin{table}
\begin{center}
%\begin{tabular}
%
%\end{tabular}
\end{center}
\end{table}

\begin{figure}

\end{figure}

\subsection{Algorithms}
%\begin{algorithmic}
%
%\end{algorithmic}

\subsection{Source codes}
\begin{verbatim}

\end{verbatim}

\subsection{References}

\begin{thebibliography}{00}
\bibitem{Eason} G. Eason, B. Noble, and I. N. Sneddon, ``On certain integrals of Lipschitz-Hankel type involving products of Bessel functions,'' Phil. Trans. Roy. Soc. London, vol. A247, pp. 529--551, April 1955.
\bibitem{Maxwell} J. Clerk Maxwell, A Treatise on Electricity and Magnetism, 3rd ed., vol. 2. Oxford: Clarendon, 1892, pp.68--73.
\bibitem{Jacobs} I. S. Jacobs and C. P. Bean, ``Fine particles, thin films and exchange anisotropy,'' in Magnetism, vol. III, G. T. Rado and H. Suhl, Eds. New York: Academic, 1963, pp. 271--350
\end{thebibliography}

\end{document}


\newpage
\appendix

\section{KUKA youBot Design Specification}\label{app:kinematics}

\begin{figure}[h]
  \centering
  \includegraphics[width=\textwidth]{figures/p2/youBot_specs.pdf}
  \caption{Informal design specification for the KUKA youBot manipulator arm, which has five
    revolute joints and a two-finger gripper. The angle of joint $i$ is given by $\theta_i$, and the
    direction in which the angle is measured is shown with a purple arrow. Each link in the arm has a
    frame rigidly attached to it, and the $x$-, $y$-, and $z$-axes of the frame are shown in red, green,
    and blue, respectively (the one unshown axis for each frame points either out of or into the page).
    Note that $z_i$ is the axis of rotation for joint $i + 1$, \textbf{not} joint $i$. The frames are
    attached in this manner so that when joint $i$ is actuated, link $i$ and its attached frame $\{i\}$
    move. Finally, note that since axes $z_3$ and $z_4$ intersect, the origin of frame $\{4\}$ can be
    placed at any point along $z_4$. We have depicted it at the wrist for the sake of the visual, but in
  practice we would place it at the intersection of $z_3$ and $z_4$ to simplify the math.}
\end{figure}

\newpage

\section{NumPy Overview}\label{app:numpy}

\begin{description}
  \item[Importing NumPy] First, make sure to import NumPy: \texttt{import numpy as np}
  \item[Constructing an array] A NumPy array can be any number of dimensions. To create a
    $n$-dimensional array, you can use \texttt{np.array()}. For example, \texttt{a\_1D =
      np.array([1.,2.,3.])} creates a 1D array of floats, and \texttt{a\_2D =
    np.array([[2,2],[0,1]])} creates a 2D array of ints where the first row is [2, 2] and the second
    row is [0, 1].
  \item[Special arrays] NumPy includes some useful functions to create special arrays. One such
    function is \texttt{np.zeros(shape)}, which returns a $n$-dimensional array of the specified
    shape (a tuple specifying each of the $n$ dimensions). For instance, \texttt{np.zeros((3,2))}
    returns a $3 \times 2$ array of 0s. Another useful function is \texttt{np.identity(n)}, which
    returns a $n \times n$ identity matrix.
  \item[Constructing a matrix] A NumPy matrix is just a special 2D NumPy array. Use
    \texttt{np.matrix()} to create a matrix. For example, \texttt{m = np.matrix([[2,2],[0,1]])}
    creates a matrix with the same values as \texttt{a\_2D} from above. However, while they may be
    very similar and you can do the same things with both, the matrix and array types are not
    exactly the same. Besides the fact that a matrix can only be 2D, an important difference is in
    matrix multiplication, as described next.
  \item[Matrix multiplication] To perform matrix multiplication with the matrix type, you can simply
    use the * operator. For example, we can multiply the matrix \texttt{m} we defined above by doing
    \texttt{m}*\texttt{m}. If you're working with 2D matrices, the matrix type is convenient since
    matrix multiplication is very intuitive. \textbf{Be careful:} In this case, the * operator
    performs matrix multiplication like you might expect. However, if you use $n$-dimensional arrays
    instead of matrices, the * operator will multiply the arrays element-wise. You can do normal
    matrix multiplication with arrays by using NumPy's \texttt{dot()} function. For example,
    \texttt{a\_2D.dot(a\_2D)}, or alternatively \texttt{np.dot(a\_2D,a\_2D)}, will return the same
    matrix product as \texttt{m}*\texttt{m}, and these are NOT equal to
    \texttt{a\_2D}*\texttt{a\_2D}.
  \item[Inverse and transpose] Getting the transpose of a matrix/array with NumPy is easy. You just
    need to use \texttt{.T} (e.g. \texttt{m.T} or \texttt{a\_2D.T}). You can get the inverse of a
    square matrix/2D array using \texttt{np.linalg.inv()}. For example: \texttt{np.linalg.inv(m)}.
  \item[Indexing] You can access specific parts of an array/matrix by indexing particular cells like
    you would expect (e.g. \texttt{m[1,0]} would return 0). You can also use slicing to get subsets
    of a matrix/array. For example, \texttt{m[:,1]} returns the second column of \texttt{m}, where
    the colon selects all of the rows and the 1 specifies the column we want.
\end{description}

For a more in-depth NumPy guide, you can read the quickstart tutorial
\href{https://docs.scipy.org/doc/numpy-dev/user/quickstart.html}{here}.


\documentclass[12pt]{amsart}
% The srcltx package helps with inverse dvi search
%\usepackage{srcltx}

% This package has an easy way to set the margins
\usepackage[paper=letterpaper]{geometry}
\geometry{top=1in,left=1in,right=1in,bottom=1in,headheight=14pt}

% These packages include nice commands from AMS-LaTeX
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{amssymb,amsmath,amsthm}
\usepackage{bm}
\usepackage{booktabs}
\usepackage{cancel}
\usepackage{caption}
\usepackage{changepage}
\usepackage{color}
\usepackage{comment}
\usepackage[shortlabels]{enumitem}
\usepackage[mathscr]{euscript}
\usepackage{float}
\usepackage{graphicx}
\usepackage{hyperref}
%\usepackage[linktoc=none]{hyperref}
\usepackage{listings}
\usepackage{lscape}
\usepackage{multimedia}
\usepackage{setspace}
\usepackage{subfig}
\usepackage{tabularx}
\usepackage{textcomp}
\usepackage{tikz}
\usetikzlibrary{patterns}
\usetikzlibrary{scopes}
\usetikzlibrary{decorations.pathmorphing}
\usetikzlibrary{shapes,backgrounds, arrows}
\usepackage{xcolor}

%This package helps make nice headers
\usepackage{fancyhdr}
\pagestyle{fancy}
\lhead{\textsf{Note}}
\chead{\textsf{Jared McBride}}
\rhead{\textsf{\today}}
\renewcommand{\headrulewidth}{0pt}

% Make the space between lines slightly more
% generous than normal single spacing, but compensate
% so that the spacing between rows of matrices still
% looks normal.  Note that 1.1=1/.9090909...
\renewcommand{\baselinestretch}{1.1}
\renewcommand{\arraystretch}{1.1}

\renewcommand{\and}{\qquad\text{and}\qquad}
\newcommand{\ds}{\displaystyle}

\newcommand\Fontvi{\fontsize{8}{7.2}\selectfont}
\newcommand{\vs}{\vspace{2 cm}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\p}{\mathbb{P}}
\newcommand{\T}{\mathbb{T}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\F}{\mathscr{F}}
\newcommand{\scrE}{\mathscr{E}}
\newcommand{\spa}{\text{span}}
\newcommand{\rref}{\mathrm{rref}}

\newcommand{\X}{\mathfrak{X}}
\newcommand{\ssX}{\bm{\textsf{X}}}
\newcommand{\ssx}{\bm{\textsf{x}}}
\newcommand{\fS}{\mathfrak{S}}
\newcommand{\fC}{\mathfrak{C}}

\renewcommand{\b}[1]{{\color{blue} #1}}
\usepackage[T1]{fontenc}
\usepackage{beramono}
\usepackage{listings}

\renewcommand{\and}{\qquad\text{and}\qquad }

\renewcommand{\P}{\mathbb{P}}
\newcommand{\cov}{\mathrm{cov}}
\newcommand{\var}{\mathrm{var}}
\newcommand{\abox}[1]
{
	\begin{center}
		\fbox{$\displaystyle #1$}
	\end{center}
}
%%
%% Julia definition (c) 2014 Jubobs
%%
\lstdefinelanguage{Julia}%
{morekeywords={abstract,break,case,catch,const,continue,do,else,elseif,end,export,false,for,function,immutable,import,importall,if,in,macro,module,otherwise,quote,return,switch,true,try,type,typealias,using,while},%
	sensitive=true,%
	alsoother={$},%
	morecomment=[l]\#,%
	morecomment=[n]{\#=}{=\#},%
	morestring=[s]{"}{"},%
	morestring=[m]{'}{'},%
}[keywords,comments,strings]%

\lstset{%
	language         = Julia,
	basicstyle       = \ttfamily,
	keywordstyle     = \bfseries\color{blue},
	stringstyle      = \color{magenta},
	commentstyle     = \color{olive},
	showstringspaces = false,
}

\newtheorem{theorem}{Theorem}

\title{Numerical Spectral Factorization by Kalman Recursion}
\author{Jared A. McBride}
\date{\today}

%\doublespacing

\begin{document}

\begin{abstract}
	In this note I describe a method presented Kailath et al. in \cite{sayed2001} and \cite{kailath2000} for finding a spectral factor of the $z$-spectrum of a wide-sense stationary stochastic process under certain conditions. The method employs a version of Kalman filtering. As such I begin with a summary of relevant results from Kalman filtering. With those results in hand we present the method for finding spectral factors and conclude with examples.
\end{abstract}

\maketitle
\tableofcontents

\section{Introduction}

Given a wide-sense stationary (WSS) stochastic process $X = \big(X_n, n \in \Z\big)$ it's $z$-spectrum $S_X(z)$ is defined by\footnote{
	The superscript $*$ denotes the conjugate transpose operator for matrices and just the complex conjugate for real or complex numbers. The superscripts $-*$ together is an abbreviation for the conjugate of the multiplicative inverse, i.e. $$z^{-*} = \frac{1}{z^*}.$$
	}
$$S_X(z) = \sum_{n=-\infty}^\infty R_X[n]z^{-n},$$
where $R_X[n] = \E X_nX_0^*$ This function is positive definite on the unit circle since $R_X[-n] = R_X^*[n]$.
Spectral factorization refers to a particular factorization in which
$$S_X(z) = L(z)L^*(z^{-*})$$
where $L(z)$ is \emph{minimum phase}, meaning both $L(z)$ and $L^{-1}(z) = 1/L(z)$ are analytic on and outside the unit circle. The function $L$ will be referred to as a spectral factor of $S_X$. This factorization is not unique since for any spectral factor of $S_X$ right-multiplication by a unitary matrix produces another spectral factor of $S_X$.

In this note I describe a method presented Kailath et al. in \cite{sayed2001} and \cite[p.~336]{kailath2000} for finding a spectral factor under certain conditions. The method employs a version of Kalman filtering, more specifically the Kalman recursion for finding the innovations process for process with a finite state-space model. The innovations for such a process will be defined below. Section \ref{sec: Kalman} provides a summary of relevant results from Kalman filtering, together with some intuition relevant to present context. With those results in hand, Section \ref{sec: Factor} presents the method for finding spectral factors. Then, in Section \ref{sec: Example} is an example.

\section{Kalman Filtering}
\label{sec: Kalman}

The purpose of this section is to provide information from Kalman filtering theory that will that is pertinent to the development of the factorization algorithm to be discussed. To that end, the \textit{summum bonum} of the section is the computation of a so-called modeling filter for a given process. The Associated whitening filter also bears significance in this work and will be briefly treated.

Suppose we have a (discrete-time) stochastic process\footnote{
	In this section $X$ need not be WSS.
}
$X = \big(X_n, n \in \Z\big)$ with a finite state-space representation of the following form:
\begin{subequations}
\label{equ: ss}
\begin{align}
\label{equ: ss State}\theta_{i+1} &= F_i\theta_i + G_iu_i \\
\label{equ: ss Obs}X_i &= H_i\theta_i + v_i
\end{align}
\end{subequations}
where $F_i \in C^{m\times m},$ $G_i \in C^{m\times p},$ and $H_i \in C^{d\times m}$ are known matrices, and $u=\{u_i\}$, $v=\{v_i\}$, and $\theta_0$ are random variables with the following property\footnote{
	Through out the note $$\delta_{ij} = \begin{cases} 1 &\text{if }i = j\\ 0 &\text{if }i \ne j.\end{cases}$$
	}
\begin{equation}
\E\begin{pmatrix} \theta_0 \\u_i \\ v_i \end{pmatrix}
\begin{pmatrix} \theta_0 \\u_j \\ v_j\\1 \end{pmatrix}^*
=\begin{pmatrix}
\Pi_0 & 0                & 0              & 0 \\
0     & Q_i\delta_{ij}   & S_i\delta_{ij} & 0 \\
0     & S^*_i\delta_{ij} & R_i\delta_{ij} & 0
\end{pmatrix}
\label{equ: cond}
\end{equation}

In the context here, we think of $X_i$ as a given series of\textit{ observations}, they are directly measurable, and $\theta_i$ is the \textit{state} of the process. Both of these series are sequences of random variables, along with $u$ and $v$. The matrices $F_i$, $G_i$, $H_i$, $\Pi_0$, $Q_i$, $S_i$ and $R_i$ are deterministic for all $i$.

The Kalman filter allows us to find the linear least squares estimate of the state $\theta_i$ given all preceding observations $X_j$ for $j = 0,1,2,\dots, i$.
This quantity I will denote by $\hat{\theta}_{i|i-1}$. Since in this note I will only be concerned with conditioning on the $X_i$'s the subscript $\cdot_{|i}$ will denote conditioning on all $X_j$ for $j \le i$.
So with this convention when I write $\hat{X}_{i|i-1}$ I mean the linear least squares estimate of $X_i$ given $X_j$ for $j < i$.

Let $\mathbb{X}_d$ be the module of $\C^d$-valued random variables, over the ring of square ($d\times d$) complex matrices, together with the following inner product
$$\langle x,y \rangle = \E xy^*\qquad \text{for all }x,y\in \mathbb{X}_d.$$ Much of Hilbert space theory will apply to what we do here. I wanted to make the space explicit to facilitate the discussion in the sequel. It is helpful to have in mind some Hilbert space theory in the discussion to follow since associated with least square estimates is an orthogonality principle, and with it a geometric interpretation of conditioning on random variables in a Hilbert space, i.e. conditioning is a projection on the space spanned by the variable by which we are conditioning.

We seek to project $\theta_{i+1}\in \mathbb{X}_m$ onto the space $\mathcal{L}_i = \text{span} \{X_0,X_1,\dots,X_{i}\}\in \mathbb{X}_d$. A strait forward way to obtain this projection is to compute an orthogonal basis for $\mathcal{L}_i$, $\{e_i\}$.  Then we can simply write
\begin{equation}
\hat{\theta}_{i+1|i} = \sum_{j=0}^i \langle \theta_{i+1}, e_j \rangle R^{-1}_{e,j} e_j\label{equ: projection}
\end{equation}
where $R_{e,i} = \langle e_i, e_i \rangle$. To that end, let us build this orthogonal set in a way similar to the Grahm-Schmidt process, but without normalizing. Put
$$e_i = X_i - \hat{X}_{i|i-1}\quad \text{for }i>0\qquad\text{and}\qquad e_0 = X_0$$
The $\{e_i\}$ have the property $\mathcal{L}_i = \text{span}\{e_j,~j\le i\}$ for any $i\ge 0$. This stochastic process we will refer to as the \textit{innovations} associated with $X$. Observe that the innovations are an orthogonal set, having $\langle e_i,e_j \rangle = I\delta_{ij}$. There are other orthogonality relations I would generally like to highlight. First noting that $\theta_i \in \text{span}\{\theta_0, u_j,\; j<i\}$, then by use of (\ref{equ: ss Obs}) and the definition of $e_i$ one can show the following.
$$\langle \theta_i, u_j \rangle = 0 \and \langle \theta_i, v_j \rangle = 0 \qquad\text{for}\qquad i\le j;$$
$$\langle X_i, u_j \rangle = 0 \and \langle X_i, v_j \rangle = 0 \qquad\text{for}\qquad i < j;$$
$$\langle X_i, u_j \rangle = S^*_i \and \langle X_i, v_j \rangle = R_i \qquad\text{for}\qquad i = j;$$
$$\langle e_i, u_j \rangle = 0 \and \langle e_i, v_j \rangle = 0 \qquad\text{for}\qquad i < j;$$
$$\langle e_i, u_j \rangle = S^*_i \and \langle e_i, v_j \rangle = R_i \qquad\text{for}\qquad i = j.$$

\subsection{Kalman filter recursions for the innovations}

Kalman developed a recursion to cheaply find the innovations process, which we will derive here.
I like the way Kialath put it, ``It turns out that the recusive construction of the innovations combines nicely with the recursive evolution of the state variables to give a recursion for the innovations...''\cite[p.~312]{kailath2000}.

We seek a recursion for $\{e_i\}$ but start with finding a recursion for $\hat{\theta}_{i|i-1}$ using (\ref{equ: projection}),
\begin{align*}
\hat{\theta}_{i+1|i} &= \sum_{j=0}^i \langle \theta_{i+1}, e_j \rangle R^{-1}_{e,j} e_j\\
&= \left(\sum_{j=0}^{i-1} \langle \theta_{i+1}, e_j \rangle R^{-1}_{e,j} e_j\right) + \langle \theta_{i+1}, e_i \rangle R^{-1}_{e,i} e_i\\
& = \hat{\theta}_{i+1|i-1} + \langle \theta_{i+1}, e_i \rangle R^{-1}_{e,i} e_i\\
\end{align*}
The first term of the last line undermines the recursion in $\hat{\theta}_{j|j-1}$; however, we can remedy this by projecting equation (\ref{equ: ss State}) onto $\mathcal{L}_{i-1}$ to get
$$\hat{\theta}_{i+1|i-1} = F_i\hat{\theta}_{i|i-1} +G_i\hat{u}_{i|i-1} = F_i\hat{\theta}_{i|i-1} + 0$$
which provides
$$\hat{\theta}_{i+1|i} = F_i\hat{\theta}_{i|i-1} + \langle \theta_{i+1}, e_i \rangle R^{-1}_{e,i} e_i$$
The next thing to do is to make sure we can compute $e_i$ from the $\hat{\theta}_{i|i-1}$'s. If we project equation (\ref{equ: ss Obs}) onto $\mathcal{L}_{i-1}$ then
\begin{equation}
\hat{X}_{i|i-1} = H_i\hat{\theta}_{i|i-1} + \hat{v}_{i|i-1} = H_i\hat{\theta}_{i|i-1} + 0.\label{equ: X hat}
\end{equation}
So, that
$$e_i = X_i - \hat{X}_{i|i-1} = X_i - H_i\hat{\theta}_{i|i-1}.$$
From this point I will use the abbreviated notation $\hat{\theta}_i = \hat{\theta}_{i|i-1}$.

Putting this together, we get a recursion for the innovations
\begin{subequations}
\begin{align}
\label{equ: theta hat} \hat{\theta}_{i+1} &= F_i\hat{\theta}_{i} + K_ie_i, \quad \hat{\theta}_0 = 0\\
\label{equ: innov} e_i &= X_i - H_i \hat{\theta}_{i}.
\end{align}
\end{subequations}
where $K_i = \langle \theta_{i+1}, e_i \rangle R^{-1}_{e,i}$.

Here then is the recursion we sought, which recursively generates the innovations associated with the process $X$. For our interests, and for many others, it would be useful if the $K_i$'s and the $R_{e,i}$'s could be computed off-line, that is, independent of the observations. As it turns out this is possible and I will demonstrate how this may be accomplished.

Let $\tilde{\theta}_i = \theta_i - \hat{\theta}_{i}$, we get orthogonality between $\tilde{\theta}_{i}$ and $e_i$. With which, we can express the recursion of the covariance of the error, $\tilde{\theta}_{i}$ rather simply. Let $P_{i} = \langle \tilde{\theta}_{i},\tilde{\theta}_{i} \rangle$, then combining equations (\ref{equ: ss State}) and (\ref{equ: theta hat}) and taking the covariance of both sides gives the the discrete Lyapunov recursion,
\begin{equation}
P_{i+1} = F_iP_{i}F_i^* + G_iQ_iG_i^* - K_iR_{e,i}K^*_i,\qquad i\ge 0
\label{equ: DL}
\end{equation}
which we initialize by $$P_{0} = \langle \hat{\theta}_{0}, \hat{\theta}_{0} \rangle = \langle \theta_0, \theta_0 \rangle = \Pi_0.$$

It is easy to see by subtracting (\ref{equ: X hat}) from (\ref{equ: ss Obs}) that $e_i = H_i\tilde{\theta}_{i} - v_i$, and this is enough to put $R_{e,i}$ in terms of $P_i$.
Because $\hat{\theta}_{i}$ is orthogonal to $v_i$, we get
$$R_{e,i} = H_iP_iH^*_i + R_i.$$
Now to put $K_i$ in terms of only $P_i$ and $R_{e,i}$, we start with $K_i = \langle \theta_{i+1}, e_i \rangle R_{e,i}^{-1}$ and observe that by (\ref{equ: ss Obs}) $\langle X_{i+1}, e_i \rangle = F_i\langle X_{i}, e_i \rangle  + G_i\langle u_i, e_i \rangle$. Reflecting on both inner products in term provides
\begin{align*}
\langle X_{i}, e_i \rangle &= \langle X_{i}, \tilde{X}_i \rangle H^*_i + \langle X_{i}, v_i \rangle \\
&= \langle \hat{X}_{i} + \tilde{X}_i, \tilde{X}_i \rangle H^*_i + 0 \\
&= P_i H^*_i
\end{align*}
and $\langle u_{i}, e_i \rangle = S_i$, as we have seen. Substituting, we able to find that
$$K_i = (F_iP_iH_i^* + G_iS_i) R_{e,i}^{-1},$$
as desired.

We have just have just found a recursion for the $K_i$ and $R_{e,i}$, which are necessary in the computation of $e_i$, which is independent of observations. It also provides a recursion for the error covariance of the state estimate (which is useful in its own right).  The discussion in this section can be summarized into a single theorem
\begin{theorem}
	\label{thm: innov}
	Consider the standard statespace model
	$$\left\{\begin{array}{ r l}
	\theta_{i+1} &= F_i\theta_i + G_iu_i \\
	X_i &= H_i\theta_i + v_i
	\end{array}\right. $$
	for $i\ge 0$, and satisfying (\ref{equ: cond}). The innovations process of $X$ can be recursively computed using the equations
	\begin{subequations}
		\label{equ: innovations}
		\begin{align}
	e_i &= X_i - H_i\hat{\theta}_i, \qquad i \ge 0,\\
	\hat{\theta}_{i+1} &= F_i\hat{\theta}_i + K_{p,i}e_i,\qquad \hat{\theta}_0 = 0,
	\end{align}
	\end{subequations}
	where \begin{subequations}
		\label{equ: Kal}
		\begin{align}
		\label{equ: Kal Pi}  P_{i+1} &= F_iP_{i}F_i^* + G_iQ_iG_i^* - K_iR_{e,i}K^*_i,\qquad P_0 = \Pi_0 \\
		\label{equ: Kal Rei} R_{e,i} &= H_iP_iH^*_i + R_i \\
		\label{equ: Kal Ki}  K_i &= (F_iP_iH_i^* + G_iS_i) R_{e,i}^{-1}
		\end{align}
	\end{subequations}
	from (\ref{equ: cond}). Here, $P_i = \langle \tilde{\theta}_i,\tilde{\theta}_i \rangle$ where $\tilde{\theta}_i = \theta_i - \hat{\theta}_i$.
\end{theorem}

Observe that the discrete Lyanov recursion (\ref{equ: DL}) can now be written as a discrete Riccati recursion,
$$ P_{i+1} = F_iP_{i}F_i^* + G_iQ_iG_i^* - (F_iP_iH_i^* + G_iS_i)(H_iP_iH^*_i + R_i)^{-1}(H_iP_iF_i^* + S^*_iG^*_i).$$

\subsection{The Stationary Case and the Modeling Filter}

So far in this section all we have assumed about $X$ is that it is a discrete-time stochastic process that has a finite state-space representation defined by (\ref{equ: ss} - \ref{equ: cond}). But now it is time to assume conditions native to the problem at hand, which will be delineated shortly. These assumptions are afforded by the stationary of the processes I work with.  The plan is to realize stationarity and adjust the innovations process so as to produce both stationary innovations and state estimators.

Here are the further assumptions I impose on the process $X$,
\begin{itemize}
	\item the state-space representation for $X$ is time-invariant, meaning, all the parameters are constant in time. So,
	$$F_i = F,\quad G_i=G, \quad H_i = H,\quad Q_i=Q, \quad S_i=S \quad\text{and}\quad R_i = R\qquad \text{for }i\ge 0$$
	additionally I impose that $F$ be stable (meaning its eigenvalues lie strict in the unit cirlce) and $R$ be strictly positive-definite, $R>0$, also
	\item $X$ is in  steady state, meaning $\Pi_i = \Pi$ for $i\ge 0$ where is $\Pi$ is the positive-semi-definite solution to the Lyaponov equation.
	$$\Pi = F\Pi F^* + GQG^*.$$
	 Since $F$ is stable this has a unique solution.
\end{itemize}
A consequence of these assumptions in that $X$ is wide-sense stationary.
Under these conditions we now apply Theorem \ref{thm: innov} and find that the innovations sequence associated with $X$ has the following state-space representation
\begin{subequations}
	\label{equ: innovations stationary}
	\begin{align}
	e_i &= X_i - H\hat{\theta}_i, \qquad i \ge 0,\\
	\hat{\theta}_{i+1} &= F\hat{\theta}_{i} + K_{p,i}e_i,\qquad \hat{\theta}_0 = 0,
	\end{align}
\end{subequations}
	where now
	\begin{subequations}
	\label{equ: Kal stationary}
	\begin{align}
	\label{equ: Kal stationary Pi}  P_{i+1} &= FP_{i}F^* + GQG^* - K_iR_{e,i}K^*_i,\qquad P_0 = \Pi \\
	\label{equ: Kal stationary Rei} R_{e,i} &= HP_iH^* + R \\
	\label{equ: Kal stationary Ki}  K_i &= (FP_iH^* + GS) R_{e,i}^{-1}
	\end{align}
\end{subequations}

Re rearranging (\ref{equ: innovations stationary}) slightly we can write
\begin{subequations}
	\label{equ: model}
	\begin{align}
	\hat{\theta}_{i+1} &= F\hat{\theta}_{i} + K_{p,i}e_i,\qquad \hat{\theta}_0 = 0,\\
	X_i &= H\hat{\theta}_i + e_i, \qquad i \ge 0.
	\end{align}
\end{subequations}
Which provides an alternative way of modeling the process $X$, this time driven by the white noise sequence $e_i$. We recover $X_i$ exactly if the innovation sequence computed by (\ref{equ: innovations stationary}). We would like however to be able to feed in any white noise input and recover a stationary process with the same $z$-spectrum as $X$. For that job (\ref{equ: innovations stationary}) is not a good candidate since it represents a time-variant system. This begs the question is there a time invariant model filter that produces stationary outputs. It turns out that the system parameters $P_i$, $R_{e,i}$, and $K_i$ all converge as $i\rightarrow \infty$. Which suggests a time invariant system that when fed in a stationary input will certainly produce a stationary output.

The convergence of $P_i$ has been well studied \cite[sec.~14.5]{kailath2000} and there are many conditions one can check to ensure that the limit of the recursion converges to the stabilizing solution of the appropriate Racatti equation necessary to provide the causal and casually invertible modeling filter. Since this theory is more problem dependent I will defer this matter to when we have a specific problem.

So, assume that
$$\lim_{i\rightarrow \infty} P_i = P$$
So that $P$ in the unique solution to
$$P = FPF^* + GQG^* - KR_eK^*$$
where $R_e = HPH+R$ and $K = (FPH^* + GS)R^{-1}_e$, and that $P$ is such that $F_p = F - KH$ is stable. And let us consider the system $L: w = (w_i,\; i\ge 0) \mapsto Y = (Y_i,\; i\ge 0)$ given by
\begin{subequations}
	\label{equ: whitening}
	\begin{align}
	\eta_{i+1} &= F\eta_{i} + Kw_i,\\
	Y_i &= H\eta_i+ w_i, \qquad i \ge 0.
	\end{align}
\end{subequations}
where $ \langle e_i,e_j \rangle = R_e\delta_{ij}$, and $\eta_0$ is chosen such that $\langle \eta_0, \eta_0 \rangle = \Sigma$ where $$\Sigma =F\Sigma F^* +KR_eK^*$$.

\section{Spectral Factorization by Kalman Filtering}
\label{sec: Factor}

In this section I explain how the above results from Kalman filtering may be employed in finding an approximate spectral factor of a given power spectrum ($z$-spectrum).

Given a $z$-spectrum $S_X(z)$ of a WSS discrete-time process $X_n\in \C^d$ for $n>-\infty$, by definition, $$S_X(z) = \sum_{n=-\infty}^{\infty} R_X[n]z^{-n},$$
where $$R_X[n] = \cov(X_{k+n},X_{k}) = \E X_{k+n}X_{k}^* = \langle X_{k+n},X_{k}\rangle$$ is the covariance. Now, if the decay of the covariance is sufficiently fast it is reasonable to truncate $S_X(z)$ to a Laurent polynomial
$$\tilde{S}_X(z) = \sum_{n=-m}^{m} R_X[n]z^{-n}.$$
However, this isn't the always the best way to approximate the spectrum. It is usual to the apply some windowing function to the autocovariances. The Parzen windowing function $\Lambda_n$, given below, for some $m>0$ is an example.
$$\Lambda_n = \begin{cases}
1-6\left(\frac{n}{m}\right)^2+6\left(\frac{|n|}{m}\right)^3, & |n|\le m/2 \\
2\left(1-\frac{|n|}{m}\right)^3, & m/2 < |n| \le m \\
0, & |n| > m
\end{cases}.$$
The Fourier transform $\widehat{\lambda}(\omega)$ of the Parzen windowing function is
$$\widehat{\Lambda}(\omega) = \frac{3}{4}m\left(\frac{\sin \pi \omega m/2}{\pi \omega m/2}\right)^4,\qquad -\infty \le \omega \le \infty$$
Let $R_\text{win}[n] = \Lambda_nR_X[n]$ and
$$S_\text{win}(z) = \sum_{n=-m}^{m} R_\text{win}[n]z^{-n}.$$
The function $S_\text{win}(z)$ is generally a better approximation of $S_{X}(z)$ than $\tilde{S}_{X}(z).$

Now that we have approximated the $z$-spectrum we seek to factor by a Laurent polynomial, we can construct a process $Y_n$ with a finite state-space model whose $z$-spectrum is exactly $S_\text{win}(z)$, that is $S_{Y}(z) = S_\text{win}(z)$ \cite[p.~488]{sayed2001}. We then take the stat-space model and using Kalman, compute the innovations recursion. It is constructed using the following model:
$$\left\{\begin{array}{ r l}
\theta_{i+1} &= F\theta_i + Gv_i \\
\tilde{Y}_i &= H\theta_i + u_i
\end{array}\right.$$
Where
$$
F = \begin{pmatrix}
0 &   &        &        &   \\
I & 0 &        &        &   \\
& I & 0      &        &   \\
&   & \ddots & \ddots &   \\
&   &        & I      & 0
\end{pmatrix}\in \C^{md\times md}\and
H = \begin{pmatrix}
0 & \dots  & 0       & I
\end{pmatrix}\in \C^{d\times md},
$$
$$\E \begin{pmatrix} v_i & u_i \end{pmatrix}\begin{pmatrix} v_j^* \\ u_j^* \end{pmatrix} =
\begin{pmatrix} R\delta_{ij} & S\delta_{ij} \\ S^*\delta_{ij} & Q\delta_{ij} \end{pmatrix},$$
and the remaining quantities need only satisfy the following relations
$$\Pi = F\Pi F^* + GQG^*$$
$$GS = N - F\Pi H^*$$
$$R = R_\text{win}[0] - H\Pi H^*$$
where
$$\Pi = \cov(X_i,X_i) = \E X_iX_i^* \;\;\Big(\!\in \C^{m\times m}\Big) \and
N = \begin{pmatrix} R_\text{win}[m] \\ R_\text{win}[m-1] \\ \vdots \\ R_\text{win}[1] \end{pmatrix}$$
Now, provided of all the above can be satisfied it can be show that $Y_i$ is stationary and its autocovariance is
\begin{align*}
	R_{Y}[i] &= \begin{cases}
		HF^{i-1}N & i > 0 \\
		H\Pi H^* + R & i = 0 \\
		NF^{*(-i-1)}H^* & i < 0
	\end{cases} \\
	&=\begin{cases}
		0 & i > m \\
		R_\text{win}[i]& m \ge i \ge -m \\
		0 & i < -m
	\end{cases}
\end{align*}
So, that $\tilde{S}_Y(z) = S_{\tilde{Y}}(z)$ as desired.

So, we have a time-invariant, stationary state-space model that approximates the original process in the sense that the $z$-spectra are close. With this finite, linear state-space model we can use the Kalman filter to produce an innovations model that will correspond to a casual and causally invertible model filter for the process $\tilde{Y}_n$ and therefore the inverse constitute a whitening filter. The result is well know and we choose the following representation \cite[p.~335]{kailath2000}.

$$\left\{\begin{array}{ r l}
\theta_{i+1} &= F\theta_i + K_ie_i,\qquad \theta_0 = 0\\
\tilde{Y}_i &= H\theta_i + e_i
\end{array}\right.$$
where $\E e_ie_j^* =  R_{e,1}\delta_{ij}$,
$$K_i = (N - F\Sigma_i H^*)R_{e,i}^{-1}, \qquad R_{e,i} = R_Y[0] - H\Sigma_i H^*,$$
and
\begin{equation} \Sigma_{i+1} = F\Sigma_i F^* + K_iR_{e,i}K_i^*. \label{eqn: Sigmai}\end{equation}
Equation (\ref{eqn: Sigmai}), plugging in $R_{e,i}$ and $K_i$, can be written as a discrete-time algebraic Riccati recursion as follows
\begin{equation}
\Sigma_{i+1} = F\Sigma_i F^* + (N - F\Sigma_i H^*)( R_Y[0] - H\Sigma_i H^*)^{-1}(N^* - H\Sigma_i F^*).
\label{eqn: DARR}
\end{equation}

Now, we would like to take the timeseries to be in steady-state so that we have a time invariant representation. the question is then does $K_i$, $R_{e,i}$, and $\Sigma_i$ converge? It can be show that under certain assumptions they do \cite[ch.~14]{kailath2000}. And we get the following

\begin{equation}
\left\{\begin{array}{ r l}
\theta_{i+1} &= F\theta_i + Ke_i,\qquad \theta_0 = 0\\
\tilde{Y}_i &= H\theta_i + e_i
\end{array}\right.
\label{sys: Inno}
\end{equation}
where $\E e_ie_j^* =  R_{e,1}\delta_{ij}$,
$$K = (N - F\Sigma H^*)R_{e}^{-1}, \qquad R_{e} = R_Y[0] - H\Sigma H^*,$$
and
\begin{equation} \Sigma = F\Sigma F^* + KR_{e}K^*. \label{eqn: Sigma} \end{equation}
Equation (\ref{eqn: Sigma}) can be writen as a discrete-time algebraic Riccati equation (DARE) as follows
\begin{equation}
\Sigma = F\Sigma F^* + (N - F\Sigma H^*)( R_Y[0] - H\Sigma H^*)^{-1}(N^* - H\Sigma F^*).
\label{eqn: DARE}
\end{equation}
Observe that from System (\ref{sys: Inno}) it can be shown that $$\tilde{Y}_n = \sum_{i=1}^{m} HF^{i-1}Ke_{n-i} + e_n = \sum_{i = 0}^{m} K^{\#}_ie_{n-i}$$
where $$K^{\#}_i = \begin{cases}
K_{m-i+1} & 1\le i \le m\\
I & i = 0\end{cases}$$
The notation views $K \in C^{d\times md}$ as a column vector with matrix entries, $(d\times d)$-blocks.
And so defines an $MA(m)$ (model) process.

\section{An Example}
\label{sec: Example}

\appendix
\section{CKMS}

CKMS, named for Chandrasekhar, Kailath, Morf, and Sidhu who in various ways and times contributed to its developement, is a efficient algorithm for computing the Kalman filter recursion for innovations. We will begin by observing who the Kalman filter may be used to find the spectral factors of a rational power spectrum (and approximate that of a nonrational power spectrum) and then from there give details about the CKMS recurssion.

\bibliographystyle{unsrt}
\bibliography{../Notes}
\end{document}