%%% lecture 05 %%%
\documentclass{beamer}
\usepackage[utf8]{inputenc}
\usepackage{algorithm2e, amsmath, amssymb, amsfonts, graphicx}
% allow section.equation numbering
\numberwithin{equation}{section}
% use boadilla theme
\usetheme{Boadilla}
% remove navigation symbols
\usenavigationsymbolstemplate{}
% get numbered figure captions
\setbeamertemplate{caption}[numbered]
% changes itemize to circle + other things
\useoutertheme{split}
\useinnertheme{circles}

% command for the title string. change for each lecture
\newcommand{\lecturetitle}{Probabilistic generative models}
% allow automatic alert-highlighted references and hyperlinks
\newcommand{\aref}[1]{\alert{\ref{#1}}}
\newcommand{\ahref}[2]{\href{#1}{\alert{#2}}}
% title page stuff. brackets content displayed in footer bar
\title[\lecturetitle]{\lecturetitle}
% metadata. content in brackets is displayed in footer bar
\author[Derek Huang (BAC Advanced Team)]{Derek Huang}
\institute{BAC Advanced Team}
\date{October 1, 2021}

% change "ball" bullet to numbered bullet and section title for section
\setbeamertemplate{section in toc}{\inserttocsectionnumber.~\inserttocsection}
% change ball to gray square (copied from stackoverflow; \par needed for break)
\setbeamertemplate{subsection in toc}{
    \hspace{1.2em}{\color{gray}\rule[0.3ex]{3pt}{3pt}}~\inserttocsubsection\par
}
% use default enumeration scheme
\setbeamertemplate{enumerate items}[default]
% required line that fixes the problem of \mathbf, \bf not working in beamer
% for later (post-2019) TeX Live installations. see the issue on GitHub:
% https://github.com/josephwright/beamer/issues/630
\DeclareFontShape{OT1}{cmss}{b}{n}{<->ssub * cmss/bx/n}{}

\begin{document}

% title slide
\begin{frame}
    \titlepage
    \centering
    % relative path may need to be updated depending on .tex file location
    \includegraphics[scale = 0.1]{../bac_logo1.png}
\end{frame}

% table of contents slide
\begin{frame}{Overview}
    \tableofcontents
\end{frame}

% section
\section{Generative modeling}

\begin{frame}{Motivation}
    \begin{itemize}
        \item
        Let $ X : \Omega \rightarrow \mathbb{R}^d $ be the input variable,
        $ Y : \Omega \rightarrow \mathbb{R} $ be the response variable. Let
        $ \mathcal{D} \triangleq \{(\mathbf{x}_1, y_1), \ldots
        (\mathbf{x}_N, y_N)\} $ be the training data, each $ (\mathbf{x}_k,
        y_k) $ sampled independently from $ X, Y $.

        \item
        Assume a parametric model for $ X, Y $ joint or $ Y \mid X $
        conditional distribution, i.e. using a distribution parametrized by
        $ \theta \in \Theta $, $ \Theta \subseteq \mathbb{R}^q $.

        \item
        We showed that the joint likelihood $ p(\mathcal{D} \mid \theta) $
        is such that
        \begin{equation*}
            p(\mathcal{D} \mid \theta) =
            \prod_{k = 1}^Np(y_k, \mathbf{x}_k \mid \theta) \propto
            \prod_{k = 1}^Np(y_k \mid \mathbf{x}_n, \theta)
        \end{equation*}

        \item
        In linear regression, we directly modeled
        $ p(y \mid \mathbf{x}, \theta) $, positing
        $ Y \mid X \sim \mathcal{N}(\mathbf{w}^\top X + b,
        \sigma^2) $, for $ \mathbf{w} \in \mathbb{R}^d $,
        $ b \in \mathbb{R} $, $ \sigma \in (0, \infty) $.

        \item
        What's a way to model $ p(y, \mathbf{x} \mid \theta) $, the joint
        $ X, Y $ likelihood?
    \end{itemize}
\end{frame}

\subsection{Conditioning on class}

\begin{frame}{Conditioning on class}
    \begin{itemize}
        \item
        We can apply conditional Bayes' rule to $ p(\mathcal{D} \mid \theta) $
        to see that
        \begin{equation*}
            p(\mathcal{D} \mid \theta) =
            \prod_{k = 1}^Np(y_k, \mathbf{x}_k \mid \theta) =
            \prod_{k = 1}^Np(\mathbf{x}_k \mid y_k, \theta)p(y_k \mid \theta)
        \end{equation*}

        \item
        Not too helpful for regression, where usually
        $ Y(\Omega) = \mathbb{R} $, but useful for classification, where
        $ Y(\Omega) = \mathcal{C} $, $ \mathcal{C} $ some finite set of
        classes.

        \item
        $ p(\mathbf{x} \mid y, \theta) $ is a \textit{class-conditional
        likelihood}, giving the likelihood of $ \mathbf{x} $ given $ \theta $
        and knowledge that the pair $ (\mathbf{x}, y) $ belongs to class
        $ y \in \mathcal{C} $.

        \item
        $ p(y \mid \theta) $ is the \textit{class prior}, essentially
        $ \mathbb{P}\{Y = C \mid \theta\} $, $ C \in \mathcal{C} $.

        \item
        With $ p(\mathbf{x} \mid y, \theta) $ and $ p(y \mid \theta) $
        we can model $ p(\mathcal{D} \mid \theta) $ directly and even generate
        new $ (\mathbf{x}, y) $ examples, hence the moniker
        \textit{generative modeling}.
    \end{itemize}
\end{frame}

\begin{frame}{Conditioning on class}
    \begin{itemize}
        \item
        Generative modeling typically limited to classification, so from now on
        assume $ Y(\Omega) = \mathcal{C} $, $ \mathcal{C} \triangleq \{C_1,
        \ldots C_K\} $, $ K $ the number of classes.

        \item
        We usually model $ Y \mid \theta $ with the categorical distribution,
        i.e.
        \begin{equation} \label{cat_dist_like}
            p(y \mid \theta) \triangleq
            \sum_{k = 1}^Kp_k\mathbb{I}_{\{C_k\}}(y)
        \end{equation}
        Here $ K \in \mathbb{N} $, $ p_1, \ldots p_K \in (0, 1) $,
        $ \sum_{k = 1}^Kp_k = 1 $. Note $ p(C_k \mid \theta) \triangleq p_k $.

        \item
        The \textit{class priors} $ p_1, \ldots p_K $ are typically estimated
        through maximum likelihood, i.e. $ \hat{p}_j \triangleq \frac{1}{N}
        \sum_{k = 1}^N\mathbb{I}_{\{C_j\}}(y_k) $ is the estimate for $ p_j $.

        \item
        The major difference between generative models is how to model
        $ p(\mathbf{x} \mid y, \theta) $, which depends on assumptions and on
        the type of input.
    \end{itemize}
\end{frame}

\subsection{Bernoulli naive Bayes}

\begin{frame}{Bernoulli naive Bayes}
    \begin{itemize}
        \item
        Suppose $ X(\Omega) = \{0, 1\}^d $, where for $ i \in
        \{1, \ldots d\} $, $ j \in \{1, \ldots K\} $,
        $ X_i \mid Y \sim \operatorname{Bernoulli}(p_{Y, i}) $,
         $ X_1 \mid Y, \ldots X_d \mid Y $ mutually independent\footnote{
            Class-conditional input feature independence is the
            \textit{naive Bayes} assumption \cite{bishop_ml}.
        }, $ p_{C_1, 1}, \ldots p_{C_K, d} \in (0, 1) $. Writing the class
        explicitly, we have
        \begin{equation} \label{bern_nb_like}
            p(\mathbf{x} \mid C_j, \theta) =
            \prod_{i = 1}^dp(x_i \mid C_j, \theta) \triangleq
            \prod_{i = 1}^dp_{C_j, i}^{x_i}(1 - p_{C_j, i})^{1 - x_i}
        \end{equation}

        \item
        Again, the $ p_{C_j, i} $ parameters are typically\footnote{
            Not exactly true. Usually some smoothing is applied to prevent
            zero probabilities.
        } estimated by
        maximum likelihood, i.e. the estimate $ \hat{p}_{C_j, i} $ is such that
        \begin{equation} \label{bern_nb_param_mle}
            \hat{p}_{C_j, i} \triangleq \frac{N_{C_j, i}}{N_{C_j}} \triangleq
            \frac{
                |\{(\mathbf{x}, y) \in \mathcal{D} : x_i = 1, y = C_j\}|
            }{
                |\{(\mathbf{x}, y) \in \mathcal{D} : y = C_j\}|
            }
        \end{equation}
    \end{itemize}
\end{frame}

\subsection{Gaussian naive Bayes}

\begin{frame}{Gaussian naive Bayes}
    \begin{itemize}
        \item
        Suppose $ X(\Omega) = \mathbb{R}^d $, where for
        $ i \in \{1, \ldots d\} $, $ j \in \{1, \ldots K\} $,
        $ X_i \mid Y \sim \mathcal{N}(\mu_{Y, i}, \sigma_{Y, i}^2) $,
        $ X_1 \mid Y, \ldots X_d \mid Y $ mutually independent,
        $ \mu_{C_1, 1}, \ldots \mu_{C_K, d} \in \mathbb{R} $,
        $ \sigma_{C_1, 1}, \ldots \sigma_{C_K, d} \in (0, \infty) $. Then,
        \begin{equation} \label{normal_nb_like}
            p(\mathbf{x} \mid C_j, \hat{\theta}) \triangleq
            \prod_{i = 1}^d\frac{1}{\sqrt{2\pi}\sigma_{C_j, i}}e^{
                -\frac{1}{2}\sigma_{C_j, i}^{-2}(x_i - \mu_{C_j, i})^2
            }
        \end{equation}

        \item
        Maximum likelihood estimates for $ \mu_{C_j, i}, \sigma_{C_j, i}^2 $
        are
        \begin{equation} \label{normal_nb_param_mle}
            \begin{split}
            \hat{\mu}_{C_j, i} & \triangleq \frac{1}{|\mathcal{D}_{C_j}|}
                \sum_{(\mathbf{x}, y) \in \mathcal{D}_{C_j}}x_i \\
            \sigma_{C_j, i}^2 & \triangleq \frac{1}{|\mathcal{D}_{C_j}|}
                \sum_{(\mathbf{x}, y) \in \mathcal{D}_{C_j}}
                (x_i - \hat{\mu}_{C_j, i})^2
            \end{split}
        \end{equation}
        Here $ \mathcal{D}_{C_j} \triangleq \{(\mathbf{x}, y) \in
        \mathcal{D} : y = C_j\} $, the set of class $ C_j $ examples.
    \end{itemize}
\end{frame}

\section{Discriminant analysis}

\subsection{Linear discriminant analysis}

\begin{frame}{Linear discriminant analysis}
    \begin{itemize}
        \item
        Dropping the naive Bayes assumption, let
        $ X \mid Y \sim \mathcal{N}(\mu_Y, \mathbf{\Sigma}) $, $ \mu_{C_1},
        \ldots \mu_{C_K} \in \mathbb{R}^d $, $ \mathbf{\Sigma} \succ \mathbf{0}
        \in \mathbb{R}^{d \times d} $.
        Then, $ \forall j \in \{1, \ldots K\} $,
        \begin{equation} \label{lda_like}
            p(\mathbf{x} \mid C_j, \hat{\theta}) \triangleq \frac{1}{
                (2\pi)^d|\mathbf{\Sigma}|^{1 / 2}
            }e^{
                -\frac{1}{2}(\mathbf{x} - \mu_{C_j})^\top
                \mathbf{\Sigma}^{-1}(\mathbf{x} - \mu_{C_j})
            }
        \end{equation}

        \item
        Maximum likelihood estimates for $ \mu_{C_1}, \ldots \mu_{C_K},
        \mathbf{\Sigma} $ are
        \begin{equation}
            \begin{split}
            \hat{\mu}_{C_j} & \triangleq \frac{1}{|\mathcal{D}_{C_j}|}
                \sum_{(\mathbf{x}, y) \in \mathcal{D}_{C_j}}\mathbf{x} \\
            \hat{\mathbf{\Sigma}} & \triangleq
                \frac{1}{N}
                \sum_{k = 1}^N
                (\mathbf{x}_k - \hat{\mu}_{y_k})
                (\mathbf{x}_k - \hat{\mu}_{y_k})^\top
            \end{split}
        \end{equation}

        \item
        Note the shared covariance matrix $ \hat{\mathbf{\Sigma}} $ and that
        $ \hat{\mu}_{y_k} = \hat{\mu}_{C_j} $ if $ y_k = C_j $.
    \end{itemize}
\end{frame}

\subsection{Quadratic discriminant analysis}

\begin{frame}{Quadratic discriminant analysis}
    \begin{itemize}
        \item
        Suppose we let $ X \mid Y \sim \mathcal{N}(\mu_Y, \mathbf{\Sigma}_Y) $,
        $ \mu_{C_1}, \ldots \mu_{C_K} \in \mathbb{R}^d $,
        $ \mathbf{\Sigma}_{C_1}, \ldots \mathbf{\Sigma}_{C_K} \succ \mathbf{0}
        \in \mathbb{R}^{d \times d} $. Then, $ \forall j \in \{1, \ldots K\} $,
        \begin{equation} \label{qda_like}
            p(\mathbf{x} \mid C_j, \hat{\theta}) \triangleq \frac{1}{
                (2\pi)^d|\mathbf{\Sigma}_{C_j}|^{1 / 2}
            }e^{
                -\frac{1}{2}(\mathbf{x} - \mu_{C_j})^\top
                \mathbf{\Sigma}_{C_j}^{-1}(\mathbf{x} - \mu_{C_j})
            }
        \end{equation}

        \item
        Maximum likelihood estimates for $ \mu_{C_1}, \ldots \mu_{C_K},
        \mathbf{\Sigma}_{C_1}, \ldots \mathbf{\Sigma}_{C_K} $ are
        \begin{equation}
            \begin{split}
            \hat{\mu}_{C_j} & \triangleq \frac{1}{|\mathcal{D}_{C_j}|}
                \sum_{(\mathbf{x}, y) \in \mathcal{D}_{C_j}}\mathbf{x} \\
            \hat{\mathbf{\Sigma}}_{C_j} & \triangleq
                \frac{1}{|\mathcal{D}_{C_j}|}
                \sum_{(\mathbf{x}, y) \in \mathcal{D}_{C_j}}
                (\mathbf{x} - \hat{\mu}_{C_j})
                (\mathbf{x} - \hat{\mu}_{C_j})^\top
            \end{split}
        \end{equation}

        \item
        Note each class $ C_j $ has its own covariance matrix
        $ \mathbf{\Sigma}_{C_j} $.
    \end{itemize}
\end{frame}

\subsection{Decision boundaries}

\begin{frame}{Decision boundaries}
    \begin{itemize}
        \item
        For any fitted probabilistic classifier with estimated parameters
        $ \hat{\theta} $, given an unlabeled point $ \mathbf{x} \in
        \mathbb{R}^d $, the natural classification rule is
        \begin{equation} \label{class_rule}
            \arg\max_{C_j \in \mathcal{C}}p(C_j \mid \mathbf{x}, \hat{\theta})
        \end{equation}

        \item
        Note that for class $ C_j $, $ j \in \{1, \ldots K\} $, we have
        \begin{equation}
            p(C_j \mid \mathbf{x}, \hat{\theta}) =
            \frac{p(C_j, \mathbf{x} \mid \hat{\theta})}{
                p(\mathbf{x} \mid \hat{\theta})
            } = \frac{
                p(\mathbf{x} \mid C_j, \hat{\theta})p(C_j \mid \hat{\theta})
            }{
                \sum_{j = 1}^Kp(\mathbf{x} \mid C_j, \hat{\theta})
                p(C_j \mid \hat{\theta})
            }
        \end{equation}
        We use definition of conditional probability, Bayes' rule, and law of
        total probability since $ \mathcal{D}_{C_1} \cap \ldots
        \mathcal{D}_{C_K} = \emptyset $. Note
        $ p(\mathbf{x} \mid \hat{\theta}) = p(\mathbf{x}) $.

        \item
        For compactness of notation, let
        $ p_{C_j \mid \hat{\theta}} \triangleq p(C_j \mid \hat{\theta}) $.
    \end{itemize}
\end{frame}

\begin{frame}{Decision boundaries}
    \begin{itemize}
        \item
        $ p(\mathbf{x}) $ doesn't change with $ C_j $. Given a fitted
        generative model with parameters $ \hat{\theta} $, we can rewrite
        (\aref{class_rule}) as\footnote{
            $ \arg\max $ invariant to positive scaling and monotone
            increasing transformations \cite{bv_convex_opt}.
        }
        \begin{equation*}
            \arg\max_{C_j \in \mathcal{C}}\left\{
                \log p(\mathbf{x} \mid C_j, \hat{\theta}) +
                \log p_{C_j \mid \hat{\theta}}
            \right\}
        \end{equation*}

        \item
        Let $ p(\mathbf{x} \mid C_j, \hat{\theta}) $ be as defined in
        (\aref{qda_like}). Take two classes $ C_a, C_b \in \mathcal{C} $ and
        note the \textit{decision boundary} $ \mathcal{B}_{C_a, C_b} $ between
        them is
        \begin{equation} \label{decision_boundary}
            \begin{split}
            \mathcal{B}_{C_a, C_b} & \triangleq \big\{
                \mathbf{x} \in \mathbb{R}^d :
                p(C_a \mid \mathbf{x}, \hat{\theta}) =
                p(C_b \mid \mathbf{x}, \hat{\theta})
            \big\} \\ & =
                \left\{
                    \mathbf{x} \in \mathbb{R}^d : \log\frac{
                        p(\mathbf{x} \mid C_a, \hat{\theta})
                    }{p(\mathbf{x} \mid C_b, \hat{\theta})} + \log\frac{
                        p_{C_a \mid \hat{\theta}}
                    }{p_{C_b \mid \hat{\theta}}} = 0
                \right\}
            \end{split}
        \end{equation}
        We use properties of logarithms and the fact that the
        $ p(\mathbf{x} \mid \hat{\theta}) $ terms in the denominator of
        $ p(C_a \mid \mathbf{x}, \hat{\theta}), p(C_b \mid \mathbf{x},
        \hat{\theta}) $ cancel to see this.
    \end{itemize}

    % spacing for footnote
    \medskip
\end{frame}

\begin{frame}{Decision boundaries}
    \begin{itemize}
        \item
        After some tedious arithmetic\footnote{
            Try verifying the form of the decision boundary as an exercise.
        }, under QDA (\aref{decision_boundary})
        becomes
        \begin{equation} \label{qda_decision_boundary}
            \mathcal{B}_{C_a, C_b} = \left\{
                \mathbf{x} \in \mathbb{R}^d : \frac{1}{2}
                \mathbf{x}^\top\hat{\mathbf{Q}}_{C_a, C_b}\mathbf{x} -
                \hat{\mathbf{a}}_{C_a, C_b}^\top\mathbf{x} +
                \hat{b}_{C_a, C_b} = 0
            \right\}
        \end{equation}
        Here $ \hat{\mathbf{Q}}_{C_a, C_b} \in \mathbb{R}^{d \times d},
        \hat{\mathbf{a}}_{C_a, C_b} \in \mathbf{R}^d, \hat{b}_{C_a, C_b} \in
        \mathbb{R} $ are such that
        \begin{equation*}% \label{qda_decision_constants}
            \begin{split}
            \hat{\mathbf{Q}}_{C_a, C_b} & \triangleq
                \hat{\mathbf{\Sigma}}_{C_a}^{-1} -
                \hat{\mathbf{\Sigma}}_{C_b}^{-1} \\
            \hat{\mathbf{a}}_{C_a, C_b} & \triangleq
                \hat{\mathbf{\Sigma}}_{C_a}^{-1}\hat{\mu}_{C_a} -
                \hat{\mathbf{\Sigma}}_{C_b}^{-1}\hat{\mu}_{C_b} \\
            \hat{b}_{C_a, C_b} & \triangleq
                \frac{1}{2}\left(
                    \hat{\mu}_{C_a}^\top
                    \hat{\mathbf{\Sigma}}_{C_a}^{-1}\hat{\mu}_{C_a} -
                    \hat{\mu}_{C_b}^\top
                    \hat{\mathbf{\Sigma}}_{C_b}^{-1}\hat{\mu}_{C_b} +
                    \log\frac{
                        |\hat{\mathbf{\Sigma}}_{C_a}|
                    }{|\hat{\mathbf{\Sigma}}_{C_b}|}
                \right) - \log\frac{p_{C_a \mid \hat{\theta}}}{
                    p_{C_b \mid \hat{\theta}}
                }
            \end{split}
        \end{equation*}

        % remove some excess space
        \vspace{-5 pt}

        \item
        $ \mathcal{B}_{C_a, C_b} $ under QDA is the solution set of a
        quadratic equation\footnote{
            $ \mathcal{B}_{C_a, C_b} $ in this form is apparently known as a
            \textit{quadric hypersurface}.
        }.
    \end{itemize}

    % correct spacing for footnote
    \medskip
\end{frame}

\begin{frame}{Decision boundaries}
    \begin{itemize}
        \item
        If we let $ p(\mathbf{x} \mid C_j, \hat{\theta}) $ be defined as in
        (\aref{lda_like}), then $ \hat{\mathbf{\Sigma}}_{C_a} =
        \hat{\mathbf{\Sigma}}_{C_b} = \hat{\mathbf{\Sigma}} $, so
        \begin{equation} \label{lda_decision_boundary}
            \mathcal{B}_{C_a, C_b} = \big\{
                \mathbf{x} \in \mathbb{R}^d :
                    \hat{\mathbf{a}}_{C_a, C_b}^\top\mathbf{x} =
                    \hat{b}_{C_a, C_b}
            \big\}
        \end{equation}
        Note $ \hat{\mathbf{Q}}_{C_a, C_b} $ simplifies to $ \mathbf{0} $ and
        that $ \hat{\mathbf{a}}_{C_a, C_b}, \hat{b}_{C_a, C_b} $ are such that
        \begin{equation*}
            \begin{split}
            \hat{\mathbf{a}}_{C_a, C_b} & = \hat{\mathbf{\Sigma}}^{-1}
                \big(\hat{\mu}_{C_a} - \hat{\mu}_{C_b}\big) \\
            \hat{b}_{C_a, C_b} & = \frac{1}{2}
                \big(\hat{\mu}_{C_a} + \hat{\mu}_{C_b}\big)^\top
                \hat{\mathbf{\Sigma}}^{-1}
                \big(\hat{\mu}_{C_a} - \hat{\mu}_{C_b}\big)
            - \log\frac{
                p_{C_a \mid \hat{\theta}}
            }{p_{C_b \mid \hat{\theta}}}
            \end{split}
        \end{equation*}

        \item
        $ \mathcal{B}_{C_a, C_b} $ under LDA is the solution set of a linear
        equation (hyperplane).

        \item
        Assuming the $ K $ classes share the same covariance matrix
        $ \hat{\mathbf{\Sigma}} $ leads to \alert{linear} decision boundaries
        while assuming different covariance matrices
        $ \hat{\mathbf{\Sigma}}_{C_1}, \ldots \hat{\mathbf{\Sigma}}_{C_K} $
        per class leads to \alert{quadratic} decision boundaries.
    \end{itemize}
\end{frame}

%\section{Smoothing and shrinkage}
%
%\subsection{Probability smoothing}
%
%\begin{frame}{Probability smoothing}
%    \begin{itemize}
%        \item
%        Recall the Bernoulli naive Bayes model, class-conditional likelihood
%        defined in (\aref{bern_nb_like}). (\aref{bern_nb_param_mle}) gives
%        maximum likelihood estimates
%        $ \hat{p}_{C_j, i} $ for the probability parameters $ p_{C_j, i} $,
%        $ i \in \{1, \ldots d\} $, $ j \in \{1, \ldots K\} $.
%
%        \item
%        Suppose we estimate $ p_{C_j, i} $ using MAP. It's natural to
%        choose $ p_{C_j, i} \sim \operatorname{Beta}(\alpha, \beta) $ since
%        $ p_{C_j, i} \in (0, 1) $. The prior $ p(\theta) $ is such that
%        \begin{equation} \label{bern_nb_prior}
%            p(\theta) \propto \prod_{j = 1}^K\prod_{i = 1}^d
%                p_{C_j, i}^{\alpha - 1}(1 - p_{C_j, i})^{\beta - 1}
%        \end{equation}
%        Here $ \alpha, \beta \in (0, \infty) $. Then, the posterior
%        $ p(\theta \mid \mathcal{D} $ is such that
%        \begin{equation} \label{bern_nb_post}
%            p(\theta \mid \mathcal{D}) \propto
%                \prod_{(\mathbf{x}, y) \in \mathcal{D}}
%                \prod_{i = 1}^d p_{y, i}^{x_i}(1 - p_{y, i})^{1 - x_i}
%                \prod_{j = 1}^K p_{C_j, i}^{\alpha - 1}
%                (1 - p_{C_j, i})^{\beta - 1}
%        \end{equation}
%    \end{itemize}
%\end{frame}
%
%\begin{frame}{Probability smoothing}
%    \begin{itemize}
%        \item
%        Recalling $ N_{C_j, i}, N_{C_j} $ definitions in
%        (\aref{bern_nb_param_mle}), (\aref{bern_nb_post}) simplifies so
%        we have
%        \begin{equation*}
%            p(\theta \mid \mathcal{D}) \propto \prod_{i = 1}^d\prod_{j = 1}^K
%                p_{C_j, i}^{N_{C_j, i} + \alpha - 1}
%                (1 - p_{C_j, i})^{N_{C_j} - N_{C_j, i} + \beta - 1}
%        \end{equation*}
%        The MAP estimate $ \hat{p}_{C_j, i, \alpha, \beta} $ for each\footnote{
%            Each $ p_{C_j, i} $ independent, so
%            $ \hat{p}_{C_j, i, \alpha, \beta} = \arg\max_{p \in (0, 1)}
%                p^{N_{C_j, i} + \alpha - 1}
%                (1 - p)^{N_{C_j} - N_{C_j, i} + \beta - 1} $.
%        }
%        $ p_{C_j, k} $ is such that
%        \begin{equation} \label{bern_nb_param_map}
%            \hat{p}_{C_j, i, \alpha, \beta} \triangleq \frac{
%                N_{C_j, i} + \alpha - 1
%            }{N_{C_j} + \alpha + \beta - 2}
%        \end{equation}
%
%        \item
%        Commonly, for some $ \eta \in [0, \infty) $, one chooses
%        $ \alpha = \beta = 1 + \eta $, so the $ \eta $-parametrized MAP estimate
%        $ \hat{p}_{C_j, i, \eta} $ is
%        \begin{equation} \label{bern_nb_param_map_eta}
%            \hat{p}_{C_j, i, \eta} \triangleq \frac{N_{C_j, i} + \eta}{
%                N_{C_j} + 2\eta
%            }
%        \end{equation}
%%        \item
%%        Note $ |\{(\mathbf{x}, y) \in \mathcal{D} : x_i = 1, y = C_j\}| = 0
%%        \Rightarrow \hat{p}_{C_j, i} = 0 $.
%%
%%        \item
%%        If $ \mathbf{v} = \mathbf{x} + \mathbf{e}_i $ for some
%%        $ (\mathbf{x}, y) \in \mathcal{D} $, $ y = C_j $, the model predicts
%%        $ p(C_j \mid \mathbf{v}, \hat{\theta}) = 0 $ despite $ \mathbf{v} $,
%%        $ \mathbf{x} $ differing only at the $ i $th element\footnote{
%%            Smoothing is used in the
%%            \href{%
%%                https://scikit-learn.org/stable/modules/naive\_bayes.html\#%
%%                multinomial-naive-bayes%
%%            }{\alert{scikit-learn naive Bayes implementations}}.
%%        }.
%    \end{itemize}
%
%    % spacing for footnote
%    \medskip
%
%\end{frame}
%
%\subsection{Covariance matrix shrinkage}
%
%\begin{frame}{Covariance matrix shrinkage}
%    \begin{itemize}
%        \item
%        If there are many correlated predictors and/or $ d $ much larger than
%        $ n $, (tibshirani p439, sklearn image)
%    \end{itemize}
%\end{frame}

% BibTeX slide for references. should use either acm or ieeetr style
\begin{frame}{References}
    \bibliographystyle{acm}
    % relative path may need to be updated depending on .tex file location
    \bibliography{../master_bib}
\end{frame}

\end{document}


\documentclass[a4paper]{article}

%% Language and font encodings
\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage[T1]{fontenc}
\usepackage[linesnumbered,lined,boxed,commentsnumbered]{algorithm2e}

%% Sets page size and margins
\usepackage[a4paper,top=3cm,bottom=2cm,left=3cm,right=3cm,marginparwidth=1.75cm]{geometry}

%% Useful packages
\usepackage{amsmath}
\usepackage{float}
\usepackage{graphicx}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage[colorlinks=true, allcolors=blue]{hyperref}
\usepackage{subfig}
\usepackage{float}
\usepackage{amsmath}
\usepackage{caption}
\usepackage{adjustbox}
\usepackage{enumitem}
\usepackage{hyperref}

% Make lists more compact
\setitemize{noitemsep, topsep=0pt, parsep=0pt, partopsep=0pt}

\title{Computer Vision: Incisor Segmentation}
\author{Thierry Deruyttere (r0660485) \& Armin Halilovic (r0679689)}

\begin{document}
\maketitle

\section{Introduction}
This report describes our approach for automatic model based segmentation of incisor teeth in dental radiographs.
The approach mainly relies on ``Active Shape Modeling'', which is briefly explained in section~\ref{sect:asm}.
Then, in section~\ref{sect:preprocessing} we discuss how radiographs are preprocessed before model building and incisor detection is done.
Afterwards, we discuss two initialization and search methods we implemented and used in section~\ref{sect:init-estimate}.
Section~\ref{sect:evaluation} describes the formal performance evaluation procedure used for the results of the incisor segmentation process.
Experiments and results are be presented in~\ref{sect:experiments}.
We also did some experiments with neural networks, which are discussed in section~\ref{sect:nns}.
Finally, section~\ref{sect:conclusion} provides a conclusion of the work done.

\section{Active Shape Modeling (ASM)}
\label{sect:asm}
The idea behind ASM is to create a statistical model for a certain object which can be used to detect said object in unseen examples.
The approach is explained in this section.
To create such an active shape model, training data is needed.
The training data should consist of images accompanied with landmark annotations.
These landmark annotations consist of $n$ points $({x_i, y_i})$ and together all these points indicate the object that is to be recognized.
The full landmark can be represented as a vector
\begin{equation}
\textnormal{\textbf{l}} = (x_1, y_1, x_2, y_2, \ldots, x_n, y_n)^T
\end{equation}
A problem with landmark annotations across different images, is that they are not necessarily all at the same position nor do they have the same scale or rotation.
This complicates the learning of the shape of the object we wish to detect.
A solution for this is discussed below.

\subsection{Generalized Procrustes Analysis}
To train a model using landmarks with different positions, scales, and rotations, the landmarks are first projected into the same normalized coordinate system.
This is done by applying a ``Generalized Procrustes Analysis''.
The algorithm that does this can be seen in Algorithm \ref{algo:procr}.

\begin{algorithm}[!tbp]
\SetKwData{Left}{left}\SetKwData{This}{this}\SetKwData{Up}{up}
\SetKwFunction{Union}{Union}\SetKwFunction{FindCompress}{FindCompress}
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}
\Input{A list of landmarks}
\Output{A list of normalized landmarks}
\BlankLine

\For{$l$ in landmarks}{
	$l$ $\leftarrow$ translate $l$ such that its origin is (0,0) by subtracting the mean\;
    $l$ $\leftarrow$ rescale $l$ such that its scale is $1$\;
}
$ref$ $\leftarrow$ choose random reference landmark from landmarks\;
$converged \leftarrow false$\;
\While{not converged}{
	\For{$l$ in landmarks}{
    	$l$ $\leftarrow$ rotate $l$ towards $ref$\;
	}
$\bar{l}$ $\leftarrow$ calculate the mean landmark for all the landmarks\;
$d$ $\leftarrow$ distance between $\bar{l}$ and $ref$\;
$ref$ $\leftarrow$ $\bar{l}$\;
$converged \leftarrow d < threshold$\;
}
\Return{landmarks}
 \caption{Procrustes analysis}
 \label{algo:procr}
\end{algorithm}

\subsection{Principal Component Analysis (PCA)}
\label{subsect:pca}
Once all the landmarks have been projected into a normalized coordinate system, PCA is applied on them.
Applying PCA results in two matrices containing the eigenvectors and eigenvalues.
To create the statistical model, the top-$k$ eigenvectors are collected into a matrix $P$ and the mean landmark $\bar{l}$ is saved.
Using these eigenvectors and the mean landmark, a landmark $l$ can later be reconstructed with the following equations.
\begin{equation}
l \approx \bar{l} + Pb
\end{equation}
\begin{equation}
b = P^T (l - \bar{l})
\end{equation}
The vector $b$ in the equations above holds coefficients which are called the shape parameters.

\subsection{Matching the model on a new image}
When presented with a new image, we can apply the statistical model by first choosing an initial location where to put the model.
How we derive this initial location is explained in section \ref{sect:init-estimate}.
Once this is done, the search procedure looks at a region around each landmark point in this initial estimate.
For each of these regions around a landmark points, a better points is chosen to replace the current landmark point.
How these regions are viewed for each landmark point is described in the subsection below.
Once better landmark points have been chosen, the model can be matched to those points.
This is done by an iterative algorithm that updates the pose parameters scale, translation and rotation, and the shape parameters $b$~\cite{Cootes1992AnIT}.

\subsection{Grey Level Models}
\label{subsect:grey-level-models}
When we fit a model on a new unseen image, we would like that this model can detect the object we look for as well as possible.
%When we fit a model on a new image, we would like to detect the object we look for as well as possible.
To help this process, we use a training set to build a model that guides the search procedure.
For each of the landmarks in the training set, we build a grey level profile for each point $({x_i, y_i})$ of the landmark.
This is done is by sampling multiple points along the normal line for the point $({x_i, y_i})$ of the landmark.
For each of these sampled points, the grey level of the pixel they lie in is retrieved.
All of the grey values of these points are then stored in a vector \textbf{$g_i$}.
To reduce the effects of changes in global intensity, the derivative of each grey level profile is calculated~\cite{cootes1996statistical}.
The derivative of point $k$ in a grey level profile $j$ is calculated as
\begin{equation}
g'_{jk} = g_{jk+1} - g_{jk-1}
\end{equation}
With $g'_j$ the derived profile.
This derived profile is then normalized using the formula
\begin{equation}
\frac{g'_j}{\sum_k |g'_jk|}
\end{equation}
This normalized profile will be denoted as $g_j$ from here on.
After these normalized grey level profiles have been calculated for all the landmark points in all of the the different landmarks, the mean profile $\bar{g}_j$ is calculated for each landmark point $j$ of the landmarks, along with the covariance matrix $S_j$.
This is done under the assumption that the distribution of the grey level profiles follow a multivariate Gaussian distribution~\cite{Cootes1992AnIT}.
By calculating this covariance matrix and mean profile, we create a statistical model for each point.
The fit of a new profile is calculated by the ``Mahalanobis distance'' defined as
\begin{equation}
f(g) = (g-\bar{g_j})^TS_j(g-\bar{g_j})
\end{equation}
Finding the best new location for a point in the model is thus equivalent to minimizing the above distance metric.
To calculate a new profile $g$ for an unseen image, the model is first placed on an initialization position.
Normal lines are then calculated for each point of the model, and grey level profiles are calculated at different points on each of these lines.

\section{Preprocessing}
\label{sect:preprocessing}
In order to improve the speed and efficiency of the segmentation process, images are preprocessed before they are used for model building and detection of teeth.

\subsection{Region of Interest Cropping}
First, to improve the speed of the whole process, images are cropped so that only the most relevant region remains.
This reduces the total preprocessing work that has to be done for a radiograph.
To determine to which region to crop, we looked at the radiographs that came with this project.
All of the radiographs looked very similar in a broad sense; the centers of the mouths always lied in a certain region of the image, and the scale was consistent across all images.
Because of this, we decided for the region of interest (ROI) to be a rectangle around the center of each image.
%Since all images were similar, we have determined the exact location of the ROI experimentally.
The values were chosen so that the ROI for each of the images would contain each of the eight incisor teeth along with a small buffer around the teeth.
The values we set for the ROI rectangle were
\begin{align}
(x_{start}, y_{start}) &= (x_{mid} - 375, y_{mid} - 450) \\
(x_{end}, y_{end})     &= (x_{mid} + 375, y_{mid} + 700)
\end{align}
This results in a region of 750 by 1150 pixels.
A drawback of this hardcoded approach is that a new radiograph which is shifted significantly will cause the segmentation process to not work correctly, since (a part of) the incisor teeth will then be cut off.

\subsection{Filters}
The goal of using filters is to make teeth more visible in contrast to the rest of the image.
This should aid in the model building and tooth detection steps.
Making the teeth more visible than the mouth tissues also has a positive impact on the grey level models.
If the teeth are more visible than the rest, there will be a large difference in the grey levels at the edges of teeth.
The filters we eventually used are bilateral and contrast limited adaptive histogram equalization (CLAHE) filters.
These are applied on each image twice, each in alternating turns, starting with the CLAHE filter.
\bigskip

The CLAHE filter is part of the family of adaptive histogram equalization (AHE) algorithms.
These algorithms are different than the normal histogram equalization algorithms as they compute histograms for different parts of an image instead of computing one histogram for the full image.
These histograms are then used to redistribute the lightness values over the whole image.
By doing this, we increase the local contrast and enhance edges in each region of the image.
However, these AHE algorithms tend to amplify noise in regions where the histogram is highly concentrated~\cite{AHE}.
To avoid this we use the CLAHE filter which limits this amplification process.
CLAHE does this is by clipping the histogram at a predefined threshold.
The part of the histogram that is above this predefined threshold will then be evenly redistributed over the histogram.
The bilateral filter is then used to denoise the image by changing the intensity of each pixel by a weighted average of the intensity of the pixels around it.
The effect of applying these filters can be seen in Figure~\ref{fig:preprocessing} (a), (b) and (c).
% Ge kunt ni "we have tried many different filters" zeggen en dan maar praten over 2 he
We have also considered the use of adaptive threshold filters and Sobel filters to increase the contrast between the teeth and the rest of the images or to make their edges much clearer, but did not manage evaluate their effects in the time allocated to this project.

\subsection{Gaussian Pyramid}
\label{subsect:gaussian-pyramid}
For the implementation of a multi resolution search algorithm, a Gaussian pyramid of images is created.
This is a pyramid of a predetermined number of levels.
The pyramid starts at level 0, which contains the original image.
Each higher level then contains a downsampled image of the level before it using a Gaussian average.
Each pixel at a higher level contains a local average of a pixel neighborhood on the level below it in the pyramid.
Figure~\ref{fig:preprocessing} (d) shows a radiograph after the preprocessing step.
\begin{figure}[H]
    \centering
        \subfloat[The cropped image of radiograph ``01.tif''.]{{\includegraphics[width=5cm]{orig-1}}}
    \qquad
    \subfloat[The image after applying CLAHE and bilateral filter once.]{{\includegraphics[width=5cm]{preprocessed-1}}}
    \qquad
    \subfloat[The image after applying CLAHE and bilateral filter twice.]{{\includegraphics[width=5cm]{radiograph-01-preprocessed-level-0.png}}}
    \qquad
    \subfloat[The image after applying CLAHE and bilateral filter twice and moving three levels higher in the Gaussian pyramid.]{{\includegraphics[width=5cm]{radiograph-01-preprocessed-level-3.png} }}
    \caption{The effects of applying the different filters.}
  \label{fig:preprocessing}
\end{figure}

\section{Fitting a model to an image}
\label{sect:init-estimate}
This section describes the two approaches that were used for the initial positioning of a tooth model and the following search procedure to fit the model to an image.
The first approach (subsection~\ref{subsect:upper-lower-incisors}) makes use of an ASM for the upper four incisor teeth and another one for the lower four incisor teeth.
When those models have converged, the models to detect individual teeth are placed on the centers of the respective teeth in the converged initialization model.
The second method (subsection~\ref{subsect:multi-res}) uses a multi resolution ASM for both the initialization and complete incisor detection.
Both of these methods rely on finding the center of the mouth for the initialization step, which is explained first.

\subsection{Locating the center of a mouth}
\label{subsect:center of mouth}
In order to find the $y$ center of a mouth, a dynamic programming approach was implemented to find a line that can split the two jaws of the mouth.
Since the area between two jaws is one of the darkest regions on the preprocessed radiographs, we opted to find a line that travels through this dark area from left to right.
Such a line, which we named the ``jaw split line'', is discovered using the Viterbi algorithm.
To speed up the search we decided to only search for this ``jaw split line'' in a region around the vertical center of each ROI.
The region is described by the following variables:
\begin{align}
y_{search\_min} &= y_{roi\_max} / 2 - 300\\
y_{search\_max} &= y_{roi\_max} / 2 + 200
\end{align}
The Viterbi search starts on the leftmost $x$ position of each ROI with each of the $y$ values in the region initialized to the intensity of the pixels in the ROI.
The algorithm searches for the darkest path one $x$ position at a time.
For each next $x$ position, it loops over each $y$ position and looks at the best (i.e. lowest intensity) previous $y$ position that can reach the current $y$ position.
The algorithm has been set to move forward in the path by 10 pixels at each iteration, and to look in a window of 10 previous $y$ positions to find the next best $y$ position.
These settings are used to create a speed up for the algorithm.
Figure~\ref{fig:jawSplitLine} shows two jaw split lines.
The left one is the split line in a relatively simple case; the black area between the jaws follows a fairly straight path.
However, in the right one the black area is not straight at all, but the Viterbi approach still manages to find a good jaw split line in this case.

\begin{figure}[!tbp]
    \centering
    \subfloat[]{{\includegraphics[width=5cm]{jawSplitLine-easy.png}}}
    \qquad
    \subfloat[]{{\includegraphics[width=5cm]{jawSplitLine-harder.png} }}
    \caption{Left, a simple jaw split line. Right, a more complicated one that follows the curve of the mouth.}
  \label{fig:jawSplitLine}
\end{figure}
\bigskip

The $x$ center of each mouth is simply approximated by the middle $x$ position of the image.
This means that the $x$ centers will not be accurate for regions of interest in which the mouth is shifted significantly to the left or to the right.
However, we noticed that most of the radiographs are nicely centered horizontally, which makes the middle $x$ position a good approximation.
The vertical center of each mouth is approximated by using the mean $y$ value of the jaw split line.
Note that in contrast to the $x$ center, simply approximating the $y$ center by the middle y position would be much less accurate.
This can be seen in figure~\ref{fig:jawSplitLine}, the mean $y$ position would be well above the $y$ center of each mouth.
The use of the jaw split line allows us to more accurately estimate the $y$ center, even if this $y$ center is different across images.

\subsection{Model for the upper and lower incisors}
\label{subsect:upper-lower-incisors}
The first initialization approach makes use of two statistical models.
One for the upper four incisors and one for the lower four incisors.
To initialize these two models, we first locate the center of the mouth by using the method described above.
Once the center has been determined, we determine the origin of these models as follows:
\begin{itemize}
\item For the upper incisor model, we initialize the center of this model at a distance $d_u$ above the mouth center.
This distance $d_u$ is defined as the half of the mean length of the upper incisor teeth in the training data.
\item For the lower incisor model, we initialize the center of this model at a distance $d_l$ below the mouth center.
This distance $d_l$ is defined as the half of the mean length of the lower incisor teeth in the training data.
\end{itemize}
Once these models have been placed we apply the ASM algorithm until convergence.
The location of these converged models will then be the start location of the individual tooth models.
Each individual tooth model will then be placed at the center of this respective tooth in the converged model as its starting point.
%The individual tooth model that is bein will thus be placed on the respective center of that tooth in this converged model.

\subsection{Multi Resolution Active Shape Model}
\label{subsect:multi-res}
Aside from the initialization model described above, a multi resolution search approach was also implemented as described in~\cite{Cootes1992AnIT}.
In this case an ASM is trained for all eight incisor teeth at once.
Landmarks thus consist of 320 landmark points ($40\ points \times 8\ teeth$).
This approach makes use of a Gaussian pyramid as described in subsection~\ref{subsect:gaussian-pyramid}.
Such a pyramid is built for each train and test image.
\bigskip

The multi resolution active shape model is built as follows.
First, grey level models (subsection~\ref{subsect:grey-level-models}) are trained for each landmark point at each resolution level in the pyramid.
Procrustes analysis is then performed to normalize the landmarks, and to find the mean landmark, scale, and rotation.
To finish building the model, PCA is performed on the normalized landmarks (subsection~\ref{subsect:pca}) and the resulting eigenvalues and eigenvectors are stored.
The mean landmark and the eigenvectors are used in the search procedure to reconstruct landmarks.
The eigenvalues are used to impose limits on the shapes of the landmarks that are reconstructed.

\subsubsection{Search Procedure}
For the detection of incisor teeth, the search procedure is started at the highest resolution level (i.e. lowest resolution image) in the Gaussian pyramid.
A low resolution version of the mean landmark is placed at the center of the mouth (found by the approach described in subsection~\ref{subsect:center of mouth}) and is then iteratively improved (as described in section~\ref{sect:asm}) until convergence or the maximum amount of iterations is reached.
The grey level model trained for the current resolution level is used in each iteration to find better landmark points.
Convergence is reached when the shape distance between two successive landmarks is less than 3.
After convergence, the landmark is scaled up (all points multiplied by 2) and moved one resolution level lower.
This is repeated until convergence is reached at the lowest level in the pyramid.
The resulting landmark can then be used for segmentation of teeth.
\bigskip

An example of the multi resolution search approach can be seen in figure~\ref{fig:multi-resolution-search-example}.
The radiograph used for the figure has no landmarks and thus was not used in the training of the model used for incisor detection.
Figure~\ref{fig:multi-resolution-search-example} (a) shows the start of the search procedure, with the white polygons representing the mean landmark placed on the detected center of the mouth.
As described above, the procedure is started at the highest resolution level in the Gaussian pyramid.
Figure~\ref{fig:multi-resolution-search-example} (b) shows the same landmark after it has converged on the highest resolution level.
Figure~\ref{fig:multi-resolution-search-example} (c) shows the same landmark after convergence on the original radiograph image.

\begin{figure}[!tbp]
  \centering
  \begin{adjustbox}{center}
    \subfloat[]{{\includegraphics[width=5.5cm]{multi-resolution-24-init} }}
    \subfloat[]{{\includegraphics[width=5.5cm]{multi-resolution-24-init-converged} }}
    \subfloat[]{{\includegraphics[width=5.5cm]{multi-resolution-24-final-converged} }}
  \end{adjustbox}
  \caption{The multi resolution ASM search approach applied to the radiograph with filename ``24.tif''.}
  \label{fig:multi-resolution-search-example}
\end{figure}
\bigskip

The multi resolution search approach simplifies the initialization procedure.
At the highest level of the Gaussian pyramid, small movements correspond to large movements at the lowest level.
The result of this is that small movements of a landmark on the lowest resolution image can place it on a good initial position for the higher resolution image on the pyramid level below it. In the higher resolution image, the location of the landmark is then refined more and passed further to higher resolution images.
Furthermore, this means that multi resolution search is able to shift landmarks both horizontally and vertically by a lot so that the models can still move to correct positions when the approximated center of the mouth lies far from the actual center.

\subsubsection{PCA}
In this approach, landmarks consist of many more points than when modeling teeth individually.
This posed a problem for the eigenvectors that were obtained after PCA was performed on the landmarks.
Reconstructions of these landmarks were much less accurate and were more crinkly than smooth.
This effect can be seen in figure~\ref{fig:pca} (a).
\bigskip

An approach used to remedy this was to split each landmark up into eight landmarks and perform PCA for each tooth.
Reconstructions were then made separately for each tooth and grouped back together into landmarks that contained all eight teeth.
While this did cause the reconstructions to be much more smooth, it also caused teeth to overlap often, since they were reconstructed separately.
A benefit of performing PCA on all teeth together is that the reconstructed landmarks then force the teeth to not overlap.
\bigskip

Another approach to remedy this problem, the one that we ended up using, is to mirror all radiographs and their landmarks in order to double the training data.
After performing PCA on the set of original and mirrored radiographs, reconstructions were much smoother, while still being able to adapt to new shapes found in improvement iterations, and still keeping teeth from overlapping. The result of this is visible in figure~\ref{fig:pca} (b).

\begin{figure}[!tbp]
  \centering
  \begin{adjustbox}{center}
    \subfloat[]{{\includegraphics[width=8cm]{pca-crinkly} }}
    \subfloat[]{{\includegraphics[width=8cm]{pca-pretty} }}
  \end{adjustbox}
  \caption{Reconstructions after PCA is performed on landmarks that consist of all eight teeth. Left, a reconstruction using the learned eigenvectors using all 14 radiographs, right, using all 14 radiographs and their mirrored versions.}
  \label{fig:pca}
\end{figure}

\newpage
\subsubsection{Parameters}
The following parameters were set for the multi resolution search experiments:
\begin{itemize}
\item resolution levels = 5
\item maximum number of iterations allowed at each level = 20
\item number pixels on either side of point for grey level model = 7
\item number of sample points on either side of point for improvement search = 3
\item PCA components retained = 25.
\end{itemize}
These parameters were chosen based on subjective evaluation of performance. A better way to choose parameters would be by using a parameter optimization method such as grid search or Bayesian optimization, but in the limited time for this project we did not do this.

\section{Evaluation}
\label{sect:evaluation}
To evaluate the performance of our approaches formally, the landmarks produced by search procedures are used to create segments of teeth.
These segments are then compared to \textit{ground truth} segments which we received with the assignment.
For the actual performance metrics, we have decided to use accuracy, precision, and recall.
These performance metrics are defined in terms of true positives (TP), false positives (FP), true negatives (TN), and false negatives (FN).
These terms are defined as follows in this project.
\begin{itemize}
\item \textbf{True Positive:} The model correctly predicts that a pixel is part of the incisors.
\item \textbf{False Positive:} The model incorrectly predicts that a pixel is part of the incisors.
\item \textbf{True Negative:} The model correctly predicts that a pixel is not part of the incisors.
\item \textbf{False Negative:} The model incorrectly predicts that a pixel is not part of the incisors.
\end{itemize}

The performance metrics are then defined as follows.
\begin{align}
accuracy &= \frac{TP + TN}{TP + TN + FP + FN} \\
precision &= \frac{TP}{TP + FP} \\
recall &= \frac{TP}{TP + FN}
\end{align}
A drawback of using accuracy in this context is that it will always be high.
This is caused by the fact that most of the pixels in the binary segmentation images are black, which causes TN to be significantly higher than the other counts.
Since accuracy has TN both in the numerator and denominator, it will always be high.
Precision and recall avoid this effect by not using TN in their definitions.

\section{Experiments}
\label{sect:experiments}
For the experiments, we report the accuracy, precision, and recall results of leave one out cross-validation runs.
The results are presented in table \ref{tab:experiments-results}. The averages of the performance metrics are given at the bottom of the table.
Some segmentation overlays can be found in the Appendix (images \ref{fig:results-14-tif} and \ref{fig:results-08-tif}).
All segmentation overlays can be found in the ``overlays'' directory submitted with this report.

\begin{table}[!tpb]
\centering
\begin{tabular}{|c|c|c|c|c|c|c|}
\cline{1-7}
Radiograph & \multicolumn{3}{c|}{Initialization model} & \multicolumn{3}{c|}{Multi resolution model} \\ \cline{1-7}
  & Accuracy & Precision & Recall & Accuracy & Precision & Recall \\ \cline{1-7}
01 & 99.37\% & 86.14\% & 90.94\%  & 98.64\%   & 68.99\% & 88.59\% \\ \cline{1-7}
02 & 99.36\% & 91.75\% & 83.49\%  & 98.56\%   & 69.83\% & 81.08\% \\ \cline{1-7}
03 & 98.49\% & 83.51\% & 62.60\%  & 98.25\%   & 71.60\% & 70.26\% \\ \cline{1-7}
04 & 99.19\% & 90.42\% & 76.18\%  & 98.68\%   & 71.06\% & 80.92\% \\ \cline{1-7}
05 & 99.07\% & 86.83\% & 76.44\%  & 97.83\%   & 57.99\% & 63.42\% \\ \cline{1-7}
06 & 98.97\% & 93.78\% & 65.73\%  & 96.95\%   & 45.90\% & 83.97\% \\ \cline{1-7}
07 & 99.26\% & 93.87\% & 75.60\%  & 98.20\%   & 61.76\% & 75.70\% \\ \cline{1-7}
08 & 98.42\% & 54.02\% & 74.42\%  & 98.12\%   & 48.16\% & 77.10\% \\ \cline{1-7}
09 & 98.70\% & 73.29\% & 68.29\%  & 98.21\%   & 58.25\% & 77.94\% \\ \cline{1-7}
10 & 98.65\% & 78.63\% & 69.15\%  & 98.27\%   & 65.29\% & 77.03\% \\ \cline{1-7}
11 & 98.58\% & 90.86\% & 60.79\%  & 97.93\%   & 69.30\% & 60.96\% \\ \cline{1-7}
12 & 99.14\% & 68.75\% & 89.59\%  & 97.73\%   & 41.17\% & 82.44\% \\ \cline{1-7}
13 & 99.01\% & 86.27\% & 73.28\%  & 98.73\%   & 71.55\% & 83.99\% \\ \cline{1-7}
14 & 98.66\% & 94.61\% & 63.51\%  & 98.61\%   & 82.71\% & 74.16\% \\ \cline{1-7}
\textbf{Average} & \textbf{98.92\%} & \textbf{83.77\%} & \textbf{73.57\%} & \textbf{98.19\%} & \textbf{63.11\%} & \textbf{76.97\%}\\
\hline
\end{tabular}
\caption{The results of the experiments for search approaches}
\label{tab:experiments-results}
\end{table}

\section{Neural Networks}
\label{sect:nns}
Although Neural networks were not part of the assignment, we still decided to test them out even with the very limited data that was given.
The last couple of years, convolutional neural networks (CNN) have successfully been used in image recognition tasks.
For this reason, we have decided to test two different CNN architectures.
The first CNN is the YOLO\cite{yolov3} network which can draw bounding boxes around the objects of interest.
The second CNN is the Mask-RCNN\cite{MASK-RCNN} network which creates masks to annotate the found objects.
Both networks have been trained on the fourteen training images that came with this project and have been initialized with pretrained weights (YOLO has pretrained darknet weights and Mask-RCNN has pretrained imagenet weights).
To train each of these networks we manually labeled the two incisor groups present on each of these fourteen images.
When we tested the YOLO network, we didn't get a single bounding box back, not even on training data.
With the Mask-RCNN network, however, we got the results presented in \ref{fig:mask-rcnn-up-down}.
A question we asked ourselves is what would happen if we trained a network to recognize each of the incisor teeth individually.
The result of this network can be seen in the Appendix in figure \ref{fig:mask-rcnn-all}.
We can see that even with the very limited data we sometimes still get some pretty good results, certainly for the Mask-RCNN trained to recognize the two groups.

\section{Conclusion}
\label{sect:conclusion}
In this document, we explained two approaches we implemented to detect and segment incisor teeth in radiographs.
Both approaches were based on active shape models.
When using active shape models, the initial position of the model is very important.
This is why we have developed the two approaches.
One is based on the usage of two models for the upper and lower incisors, the other on a multi resolution active shape model for all eight teeth.
From the results presented in section~\ref{sect:experiments}, we can conclude that the former model performs better than the latter one.
We also tried using neural networks which gave interesting results, even with the limited amount of training data we had.
%A significant difference between using the neural networks and the ASM approaches is the amount of time it took to get the processes running.
%It took less than two hours to get a neural network that could slightly recognize the incisor teeth, while it took us more than a week to get good results using active shape models.

\section*{Appendix}
\begin{figure}[H]
  	\centering
    \begin{adjustbox}{center}
        \subfloat[The segmentation overlay for ``14.tif'' with the first initialization method.]{{\includegraphics[width=8cm]{init-model-1-14} } }
        \subfloat[The segmentation overlay for ``14.tif'' with the second initialization method.]{{\includegraphics[width=8cm]{final_14_multi_resolution} } }
    \end{adjustbox}
    \caption{Comparing the two initialization methods for ``14.tif''.}
    \label{fig:results-14-tif}
\end{figure}

\begin{figure}[H]
  	\centering
    \begin{adjustbox}{center}
        \subfloat[The segmentation overlay for ``08.tif'' with the first initialization method.]{{\includegraphics[width=8cm]{init-model-1-8} } }
        \subfloat[The segmentation overlay for ``08.tif'' with the second initialization method.]{{\includegraphics[width=8cm]{final_08_multi_resolution} } }
    \end{adjustbox}
    \caption{Comparing the two initialization methods for ``08.tif''.}
    \label{fig:results-08-tif}
\end{figure}

\begin{figure}[H]
    \centering
        \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``15.tif''.]{{\includegraphics[width=5cm]{up-down-tif15}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``18.tif''.]{{\includegraphics[width=5cm]{up-down-tif18}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``26.tif''.]{{\includegraphics[width=5cm]{up-down-tif26}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``28.tif''.]{{\includegraphics[width=5cm]{up-down-tif28} }}
    \caption{Results of using Mask-RCNN\cite{MASK-RCNN} trained on recognizing the two incisor groups.}
  \label{fig:mask-rcnn-up-down}
\end{figure}

\begin{figure}
    \centering
        \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``15.tif''.]{{\includegraphics[width=5cm]{all-tif15}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``18.tif''.]{{\includegraphics[width=5cm]{all-tif18}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``26.tif''.]{{\includegraphics[width=5cm]{all-tif26}}}
    \qquad
    \subfloat[Results of the Mask-RCNN\cite{MASK-RCNN} network on ``28.tif''.]{{\includegraphics[width=5cm]{all-tif28} }}
    \caption{Results of using Mask-RCNN\cite{MASK-RCNN} trained on recognizing each of the incisor teeth.}
  \label{fig:mask-rcnn-all}
\end{figure}
\newpage
\bibliographystyle{plain}
\bibliography{bibliography}

\end{document}


\documentclass[a4paper,12pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[margin=2cm]{geometry}
\usepackage{enumitem}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{mathtools}
\usepackage{cleveref}
\usepackage{xspace}

\setlength{\parindent}{0pt}
\setlength{\parskip}{10pt}
\setlist[itemize]{itemsep=0pt,topsep=0pt}

\newtheorem{lemma}{Lemma}
\newtheorem{theorem}{Theorem}

\DeclarePairedDelimiter{\floor}{\lfloor}{\rfloor}

\newcommand{\tcapture}{\texttt{capture}\xspace}
\newcommand{\taccept}{\texttt{accept}\xspace}
\newcommand{\tyes}{\texttt{yes}\xspace}
\newcommand{\tno}{\texttt{no}\xspace}
\newcommand{\tleader}{\texttt{leader}\xspace}
\newcommand{\tid}{\texttt{id}\xspace}
\newcommand{\tsize}{\texttt{size}\xspace}
\newcommand{\tactive}{\texttt{active}\xspace}
\newcommand{\tlevel}{\texttt{level}\xspace}
\newcommand{\towner}{\texttt{owner}\xspace}
\newcommand{\tqueue}{\texttt{queue}\xspace}
\newcommand{\tcontender}{\texttt{contender}\xspace}
\newcommand{\tnone}{\texttt{none}\xspace}
\newcommand{\ttrue}{\texttt{true}\xspace}
\newcommand{\tfalse}{\texttt{false}\xspace}
\newcommand{\midentifier}{\mathit{identifier}}
\newcommand{\mlevel}{\mathit{level}}
\newcommand{\mpotentiallevel}{\mathit{potential \ level}}

\begin{document}

\section*{Humblet's clique leader election algorithm}

Our goal is to elect a leader in a complete graph with $n$ nodes and asynchronous first-in-first-out communication.
We assume that every node has a unique identifier, and that all identifiers are totally ordered.
Initially, no node needs any knowledge about the identities of the nodes at the other ends of its incident edges,
and these edges may be arbitrarily ordered.

The simplest solution to this problem is to let every node send its identifier to all its neighbors,
and then let every node select the neighbor with the largest identifier.
This approach requires $O(n^2)$ messages.
Here we describe an algorithm developed by Pierre Humblet in 1984,
which solves this problem and only requires $O(n \lg n)$ messages.

\subsection*{Algorithm description}

The idea is that every node tries to ``capture'' its consecutive neighbors and expand its ``territory'',
until it is either defeated by another node or its territory is large enough so that it can announce itself as a leader.

Every node is in one of three states: \emph{active}, \emph{inactive} or \emph{captured}.
Every node remembers its \emph{level}, i.e., the number of nodes it has already captured.
Initially, all nodes are active.
An active node tries to capture its consecutive neighbors
by sending a \tcapture message, waiting for an \taccept response, increasing its level, and repeating this for the next node.
A capture may fail, and in this case no \taccept response is sent and the node waiting for it may not proceed.
The \tcapture message contains the sender's identifier and current level.

When an active or inactive node receives a \tcapture message,
it compares the sender's pair $(\mlevel, \midentifier)$ with its own
(they must be different, because the identifiers are unique),
and if the sender's pair is greater, it responds with an \taccept message and becomes captured.
Otherwise, no response is sent and the sender may not proceed.
A node may be captured multiple times and it remembers its \emph{owner}, i.e., the last node which captured it.

When an already captured node receives a \tcapture message,
it forwards this message to its owner (which is not necessarily still active)
and waits for a response with the result of comparison between the sender and the owner.
During this time, any other incoming \tcapture messages are queued and only processed after the awaited response,
which may potentially result in a change of the owner.
If the sender has won the comparison,
the captured node changes its owner to the sender and responds to it with an \taccept message.
In this case, if the previous owner was active, then it becomes inactive,
which means that it no longer tries to capture next nodes, but is not captured yet.
Otherwise, if the sender has lost the comparison,
the captured node keeps its owner and sends no response to the sender, which then may not proceed.
The queueing of \tcapture messages prevents a situation in which a node is about to change its owner,
but the \tyes message has not yet arrived, and the node erroneously forwards a message to its previous owner.

When an active node reaches a level such that its \emph{territory}, i.e., the node itself and all its captured nodes,
is a majority of all nodes (which means that $\mlevel + 1 > n / 2$), then this node announces itself as a leader and finishes.
All other nodes then immediately finish after receiving this announcement.

\subsection*{Implementation details}

Every node keeps the following variables:
\begin{itemize}
    \item \tid \ -- the node's identifier
    \item \tsize \ -- the size of the graph
    \item \tactive \ -- whether the node is active; initially \ttrue
    \item \tlevel \ -- the node's level; initially $0$
    \item \towner \ -- the index (neighbor number) of the node's owner;
        if \tnone, then this node is not yet captured
    \item \tqueue \ -- the queue for \tcapture messages
    \item \tcontender \ -- the index of the node which sent a \tcapture message when this node was already captured;
        only set for the time of waiting for a response from the owner;
        the \tqueue may only be processed when this variable is \tnone
\end{itemize}

A node is active if and only if \texttt{active \&\& owner == none},
inactive if and only if \texttt{!active \&\& owner == none},
and captured if and only if \texttt{!active \&\& owner != none}.

Every node starts by sending a \tcapture message to its first neighbor.
Then it enters a loop in which it processes the incoming messages.

After receiving a \tcapture message, the node checks if it is forwarded from a captured node and enqueues this message.
The message is forwarded if and only if the index of the sender is less than the node's current level,
because a captured node only forwards a message to its owner, and a non-forwarded message must come from a non-captured node.

After receiving an \taccept message, the node increases its level,
and if it is active, checks whether to capture next node or announce itself as a leader and finish.

When receiving a \tyes or \tno message, the node must have been already captured,
must have received a \tcapture message from \tcontender, and this must be a response from the owner.
If the response is \tyes, the node changes its owner to \tcontender,
sends \taccept message to it, and sets the \tcontender variable to \tnone.
If the response is \tno, it ignores the \tcontender and only sets this variable to \tnone.
After this, the processing of the \tqueue may continue.

After receiving a \tleader message, the node saves the leader's identifier and finishes.

After processing the incoming message, and while the \tcontender variable is \tnone, the node processes the \tqueue.

If the dequeued \tcapture message is forwarded, the node responds with \tyes or \tno
depending on the result of the comparison with the original sender.
If the sender has won, the node sets \tactive to \tfalse, which either makes it inactive or keeps it captured.

If the dequeued \tcapture message is not forwarded and the node is not captured, then the node compares itself with the sender.
If the sender has won, then the node becomes captured and responds with \taccept message to the sender.
Otherwise, the node ignores the sender.

If the dequeued \tcapture message is not forwarded and the node is captured,
then the node forwards the message to its owner and sets the \tcontender variable to the sender.
This variable can then only be unset after receiving a \tyes or \tno response from the owner,
and during this time the \tqueue may not be processed.

\subsection*{Correctness proof}

A node is a \emph{winner} when it has won its last capture attempt
and a relevant \tyes or \taccept message is currently in transit.

A node's \emph{potential level}
is its level when it is not a winner,
or its level plus one when it is a winner.

\begin{lemma}\label{lost-capture}
    After a node loses its capture attempt,
    neither its level nor its potential level can increase.
\end{lemma}
\begin{proof}
    The only way for the node's level to increase is by receiving an \taccept message,
    which can only happen after winning a capture attempt.
    The only way for the node's potential level to increase
    is by increasing its level or by winning a capture attempt.
\end{proof}

\begin{lemma}\label{defeated}
    After a node with level $l$ stops being active,
    its potential level can only become at most $l + 1$.
\end{lemma}
\begin{proof}
    The only way for the node's level to increase is by receiving an \taccept message,
    which in turn requires a \tcapture message to be sent.
    When the node becomes a winner, its potential level becomes $l + 1$,
    and it will eventually receive an \taccept message,
    which will make both its level and potential level equal $l + 1$.
    It will not however send a next \tcapture message,
    so neither its level nor its potential level will increase.
\end{proof}

A node is a \emph{candidate},
when it is active and has not lost its last capture attempt,
so either it is a winner or the \tcapture message is still in transit.

\begin{lemma}\label{no-candidate-again}
    Once a node stops being a candidate, it can not become a candidate again.
\end{lemma}
\begin{proof}
    A node can not become active again, and an ignored \tcapture message
    prevents the node from sending any other \tcapture messages.
\end{proof}

\begin{lemma}\label{always-one-candidate}
    At any moment at least one node is a candidate.
\end{lemma}
\begin{proof}
    Since all identifiers are unique,
    at any moment there must be a node $x$ with the largest $(\mpotentiallevel, \midentifier)$ pair.
    Suppose that this node is not a candidate.
    Then either it has stopped being active or it has lost its last capture attempt.
    If it has lost its last capture attempt,
    then at that moment some node $y$ must have had larger $(\mlevel, \midentifier)$ pair.
    By \cref{lost-capture}, the $x$'s $(\mpotentiallevel, \midentifier)$ pair
    could not have increased afterwards, so $y$'s pair must still be larger,
    but $x$'s pair was assumed to be currently the largest (contradiction).
    If it has stopped being active, then it must have been defeated by some node $y$.
    Let $i_x$ and $i_y$ be the identifiers of $x$ and $y$ respectively.
    Let $l_x$ and $l_y$ be the levels of $x$ and $y$ respectively, at the moment when $y$ defeated $x$.
    Let $p_x$ and $p_y$ be the current potential levels of $x$ and $y$ respectively.
    Since $y$ has won the capture attempt, there must be $(l_y, i_y) > (l_x, i_x)$ and $p_y \geq l_y + 1$.
    By \cref{defeated}, there must be $p_x \leq l_x + 1$.
    Thus, $(p_y, i_y) \geq (l_y + 1, i_y) > (l_x + 1, i_x) \geq (p_x, i_x)$,
    but $x$'s pair was assumed to be currently the largest (contradiction).
    Therefore, the node $x$ must be a candidate.
\end{proof}

\begin{lemma}\label{min-one-leader}
    Some node will announce itself as a leader.
\end{lemma}
\begin{proof}
    By \cref{no-candidate-again} and \cref{always-one-candidate},
    some node will be a candidate for the whole run time of the algorithm.
    This node will eventually reach a level high enough to announce itself as a leader.
\end{proof}

\begin{lemma}\label{disjoint-territories}
    If a node $x_1$ reaches level $l$ at moment $t_1$,
    and another node $x_2$ reaches the same level $l$ at moment $t_2$,
    then $x_1$'s territory from moment $t_1$ is disjoint with $x_2$'s territory from moment $t_2$.
\end{lemma}
\begin{proof}
    Assume, without loss of generality, that $t_1$ is before $t_2$.
    Suppose that some node belongs both to $x_1$'s territory at moment $t_1$ and to $x_2$'s territory at moment $t_2$.
    Then it must have been captured by $x_2$ in the time between $t_1$ and $t_2$.
    At that moment $x_1$ has already reached level $l$, but $x_2$ has not yet reached this level.
    Therefore, $x_2$ must have lost this capture attempt (contradiction).
\end{proof}

\begin{lemma}\label{max-for-level}
    At most $n / (l + 1)$ nodes can reach level $l$.
\end{lemma}
\begin{proof}
    Consider the set of all nodes which reached level $l$.
    Every one of them had territory of size $l + 1$ at the moment of reaching this level.
    By \cref{disjoint-territories}, all these territories must be mutually disjoint,
    so this set can have at most $n / (l + 1)$ elements.
\end{proof}

\begin{lemma}\label{max-one-leader}
    At most one node can announce itself as a leader.
\end{lemma}
\begin{proof}
    A node can only announce itself as a leader when its level $l$ is greater than $n / 2 - 1$.
    Then, $n / (l + 1) < n / (n / 2) = 2$, so by \cref{max-for-level}, at most one node can reach this level.
\end{proof}

\begin{theorem}\label{one-leader}
    Exactly one node will announce itself as a leader.
\end{theorem}
\begin{proof}
    Follows from \cref{min-one-leader} and \cref{max-one-leader}.
\end{proof}

\subsection*{Complexity analysis}

\begin{lemma}\label{capture-messages}
    Every \tcapture message sent from an active node generates at most three other messages.
\end{lemma}
\begin{proof}
    A \tcapture message may be forwarded to the receiver's owner,
    the owner does not forward this message any further and responds with \tyes or \tno message,
    and then an \taccept response may be sent.
\end{proof}

\begin{theorem}\label{messages}
    At most $4 n H_{\floor{n / 2}} + n - 1$ messages are sent,
    where $H_i$ is the $i$-th harmonic number.
\end{theorem}
\begin{proof}
    Group all \tcapture messages from active nodes by the sender's level.
    An active node may only sent one \tcapture message per level.
    The maximum level at which an active node may send a \tcapture message is $\floor{n / 2} - 1$.
    By \cref{max-for-level} and \cref{capture-messages},
    the total number of messages generated by capture attempts is at most
    $$\sum_{l=0}^{\floor{n / 2} - 1} 4 n / (l + 1) = 4 n \sum_{l=1}^{\floor{n / 2}} 1 / l = 4 n H_{\floor{n / 2}},$$
    and the final leader announcement generates exactly $n - 1$ messages.
\end{proof}

\subsection*{References}

P. Humblet, ``Selecting a leader in a clique in $O(N \log N)$ messages,''
The 23rd IEEE Conference on Decision and Control. IEEE, Dec. 1984. doi: 10.1109/cdc.1984.272191.

N. Santoro, ``Design and Analysis of Distributed Algorithms,''
Wiley Series on Parallel and Distributed Computing. John Wiley \& Sons, Inc., Nov. 22, 2006. doi: 10.1002/0470072644.

\end{document}


% This file was created (at least in part) by the script ParseMdtoLatex by Louis du Plessis
% (Available from https://github.com/taming-the-beast)

\documentclass[11pt]{article}
\input{preamble}

% Add your bibtex library here
\addbibresource{master-refs}

%%%%%%%%%%%%%%%%%%%%
% Do NOT edit this %
%%%%%%%%%%%%%%%%%%%%
\begin{document}
\renewcommand{\headrulewidth}{0.5pt}
\headsep = 20pt
\lhead{ }
\rhead{\textsc {BEAST v2 Tutorial}}
\thispagestyle{plain}

%%%%%%%%%%%%%%%%%%
% Tutorial title %
%%%%%%%%%%%%%%%%%%
\begin{center}

	% Enter the name of your tutorial here
	\textbf{\LARGE Structured coalescent}\\\vspace{2mm}

	% Enter a short description of your tutorial here
	\textbf{\textcolor{mycol}{\Large Population structure using MultiTypeTree}}\\

	\vspace{4mm}

	% Enter the names of all the authors here
	{\Large {\em Nicola F. Müller and Tim Vaughan}}
\end{center}

%%%%%%%%%%%%%%%%%
% Tutorial body %
%%%%%%%%%%%%%%%%%

\section{Background}\label{background}

Population dynamics can influence the shape of a tree. Another thing
that has strong influence on the shape of the tree is structure in a
population. This is the case as soon as sequences do not mix well,
i.e.~they cluster together. One cause of this clustering is due to
geography. Samples may not have been taken from the same geographic
region, leading to clustering of samples from the same region. This
clustering of samples can bias the estimation of parameters. The
extension of the classic coalescent to the structured coalescent by
allowing for migration between regions is trying to circumvent this by
allowing individual regions to have distinct coalescent rates and by
allowing migration between those regions.

\section{Programs used in this
Exercise}\label{programs-used-in-this-exercise}

\subsubsection{BEAST - Bayesian Evolutionary Analysis Sampling
Trees}\label{beast---bayesian-evolutionary-analysis-sampling-trees}

BEAST version 2.4.2 \citep{Bouckaert2014}.

\subsubsection{BEAUti - Bayesian Evolutionary Analysis
Utility}\label{beauti---bayesian-evolutionary-analysis-utility}

GUI to create \lstinline!*.xml! for BEAST2.

\subsubsection{Tracer}\label{tracer}

Tracer (\url{http://tree.bio.ed.ac.uk/software/tracer}) is used to
summarize the posterior estimates of the various parameters sampled by
the Markov Chain. It includes code to analyze estimates of past
population dynamics from \lstinline!*.log! files and \lstinline!*.trees!
files. It is mainly used for visual inspection and assessment of
convergence of MCMC runs. It helps to quickly view median estimate and
95\% credible intervals (which approximate the 95\% highest posterior
density intervals) of the parameters, and calculates the effective
sample sizes (ESS) of parameters. It also helps to visualize potential
parameter correlations.

\subsubsection{TreeAnnotator}\label{treeannotator}

TreeAnnotator is a program that comes with BEAST2. It allows to
summarize the analysis of sampled trees.

\section{Practical: MultiTypeTree}\label{practical-multitypetree}

In this tutorial we will estimate migration rates, effective population
sizes and locations of internal nodes using the structured coalescent
implemented in BEAST2, MultiTypeTree \citep{Vaughan2014}.

The aim is to:

\begin{itemize}

\item
  Learn how to infer structure from trees with sampling location
\item
  Get to know how to choose the set-up of such an analysis
\item
  Get to know the advantages and disadvantages of working with
  structured trees
\end{itemize}

\subsection{The Data}\label{the-data}

The dataset consists of 60 Influenza A/H3N2 sequences sampled in Hong
Kong and in New Zealand between 2001 and 2005. South-East Asia has been
hypothesized to be a global source location of seasonal Influenza, while
more temperate regions such as New Zealand are assumed to be global
sinks \citep{Rambaut2008,Russell2008,Lemey2014}, meaning that Influenza
strains are more likely to migrate from the tropic to the temperate
regions then vice versa. We want to see if we can infer this source-sink
dynamic from sequence data using the structured coalescent.

\subsection{Creating the Analysis File with
BEAUti}\label{creating-the-analysis-file-with-beauti}

We will use BEAUti to generate the input XML for BEAST2 from the
sequence alignment.

\subsubsection{Install BEAST 2 Plug-Ins}\label{install-beast-2-plug-ins}

The packages MultiTypeTree is not implemented in the core of BEAST, but
has to be installed (Figure \protect\hyperlink{fig:install_mtt}{1}).

\begin{figure}
    \centering
    \includegraphics[width=0.500000\textwidth]{figures/install_mtt.png}
    \caption{Install MultiTypeTree.}
    \label{fig:install_mtt}
\end{figure}

To be able to make \lstinline!.xml!'s for MultiTypeTree, we have to load
the MultiTypeTree template \lstinline!File > Template > MultiTypeTree!.
This template allows to specify additional things, such as sampling
location, which one can not specify using the standard interface, as
well as parameters such as the migration rates. After setting the
template, we can load the alignment of the H3N2 data
\lstinline!File > Add Alignment!. Since the sequences were sampled
through time, we have to specify the sampling dates. These are included
in the sequence names. To set the sampling dates, go to \textbf{Tip
Dates}, guess them by splitting after the ``\_" and then choose the last
group. There are two different ways in how BEAST can interpret sampling
dates. They are labeled as \textbf{Since some time in the past} and
\textbf{Before the present}. The easiest way to check if you have used
the correct one is by checking \lstinline!Height!. If the setup is
correct, the sequences sampled the most recently (i.e.~2005.66) should
have a Height of 0 while all other tips should be larger then 0 (Figure
\protect\hyperlink{fig:sampling_dates}{2}).

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/dates.png}
    \caption{Sampling dates.}
    \label{fig:sampling_dates}
\end{figure}

The main contrast in the setup to previous analyses is that we include
additional information about the sampling location of sequences.
Sequences were taken from patients in Hong Kong and New Zealand. We can
specify these sampling locations by going to \textbf{Tip Locations} in
BEAUti and guessing the locations. Use here the second group after
splitting the names on the character ``\_``. After guessing the tip
locations, the column \textbf{Location} should contain the entries Hong
Kong and New Zealand (Figure
\protect\hyperlink{fig:sampling_locations}{3}).

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/locations.png}
    \caption{Sampling locations.}
    \label{fig:sampling_locations}
\end{figure}

For this analysis, we will be using the HKY model. The HKY model infers
different rates for transversion and transition. Transition being the
change within purines (\textbf{A} and \textbf{G}) and pyrimidines
(\textbf{T} and \textbf{C}) and transversion being the change among
those groups.

We also want to allow for heterogeneity between sites, which we can do
by setting the \textbf{Gamma Category Count} to a value greater than 0
(normally between 4 and 6) and ticking the \textbf{estimate} box for the
shape parameter (Figure \protect\hyperlink{fig:hky}{4}).

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/choose_hky.png}
    \caption{Setup of the site model.}
    \label{fig:hky}
\end{figure}

To speed up convergence, we leave the branch model on the Strict Clock
model and set a different value for the clock rate (default is 1). A
value of 0.005 substitutions * site$^{-1}$ * year$^{-1}$ is closer to
the truth.

Since we have more than one deme (Hong Kong and New Zealand), we can
estimate the effective population size of those two demes separately.
Additionally, these demes are connected, so we can (or need to) allow
for migration between them. By default, the migration rates and
population sizes are not estimated. To change this, we have to go to the
Priors setting. There, we have to check the two \textbf{estimate} boxes
for the population sizes and the migration rates.

After checking those two boxes, there will be two new fields appearing,
where we can set the priors for the population sizes and the migration
rates. Since we know the time scale of our data (few years), we can
choose a proper prior for the migration rates, like an exponential prior
distribution with mean 1.\\
Figure \protect\hyperlink{fig:est_migrates}{5} shows the final setup for
the priors.

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/estimate_migration.png}
    \caption{Check the boxes that say estimate for the population size and the migration rates and change it to an exponential prior with mean 1.}
    \label{fig:est_migrates}
\end{figure}

The rest of the settings we can leave as they are.

After saving, we get an \lstinline!*.xml!, which we can use in BEAST2.
The run will take a bit of time. If the MultiTypeTree run consumes too
much CPU power, you can just close it and then use the ``pre-cooked''
\lstinline!*.log! and \lstinline!*.trees! files later instead.

\subsubsection{Analysis of the MultiTypeTree
run}\label{analysis-of-the-multitypetree-run}

Load the \lstinline!*.log! files into tracer. First, we should check
that the run has converged by looking at the ESS values. If all ESS
values are above 200, we should be on the safe side.

Next, we can have a look at the estimates of the effective population
sizes (Figure \protect\hyperlink{fig:estimated_peff}{6}).

\begin{figure}
    \centering
    \includegraphics[width=0.600000\textwidth]{figures/effectivePsize_tracer.png}
    \caption{Estimated effective population sizes.}
    \label{fig:estimated_peff}
\end{figure}

Hong Kong ($ \sim $7 Mio Inhabitants) is inferred to have
the larger effective population size than New Zealand
($ \sim $ 4.5 Mio Inhabitants). Keep in mind though that the
effective population size is not only dependent on the population size
itself, but also on e.g.~transmission rates or contact rates.
Differences in the real population size are therefore not necessarily
reflected in the effective population size. However, they can still act
as a sanity test. Next, we want to have a look at the inferred migration
rates (Figure \protect\hyperlink{fig:etimated_mig}{7}).

\begin{figure}
    \centering
    \includegraphics[width=0.600000\textwidth]{figures/migrationrates_tracer.png}
    \caption{Estimated migration rates.}
    \label{fig:etimated_mig}
\end{figure}

 \clearpage

As we stated at the beginning, it is assumed that South-East Asia works
as a global source of influenza, while e.g.~Oceania acts as a sink.

\begin{framed}
Do the inferred migration rates agree with this hypothesis?
\end{framed}

After having looked at the inferred parameters, we can look at the
inferred trees. We can either look at all the sampled trees individually
or can use a summary of all tree. This can be done by using the program
TreeAnnotator:

\begin{framed}
We need to specify at least 4 settings there. First, the burn-in
percentage, then the input tree file, the output tree file for which we
have to specify a file name and the \lstinline!Node Heights! (set it to
\lstinline!Mean Heights!)
\end{framed}

MultiTypeTree logs 3 different tree files:

\begin{itemize}

\item
  The first one is the \lstinline!*.map.trees! file. It contains the
  maximum posterior tree for the analysis up to a sample of the MCMC.
  This tree is the tree with the highest posterior probability that has
  been visited so far.
\item
  The second one is the \lstinline!*.trees! file. It contains the logged
  trees with so called single child node. In a normal tree, all nodes
  have two (or sometimes more) children. The nodes there are coalescent
  events. The single child nodes on the other hand are migration events
  (in this case a migration event between Hong Kong and New Zealand).
  MultiTypeTree infers the timing of those migration events on a branch
  and logs them in the \lstinline!*.trees! file.
\item
  The third tree file logged is the \lstinline!*.typedNode.trees!. This
  file does not use single child nodes. Instead, every coalescent event
  (here always a node with two daughter lineages) has a location (or
  state or color or type) where it was inferred to take place (see
  Figure \protect\hyperlink{fig:single_child}{8})
\end{itemize}

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/single_child.png}
    \caption{An example of a tree where the migration events are logged as single child nodes (left) and of the same tree where only the location of a coalescent event is logged (right).}
    \label{fig:single_child}
\end{figure}

To summarize all the trees, we will need the
\lstinline!*.typedNode.trees! files, since TreeAnnotator cannot handle
single child nodes. We will also need to specify the \textbf{Burnin
percentage}, which we can guess from looking at the traces of the
parameter estimates in Tracer (10\% should be more than enough). Next,
we also need to specify where the output will be saved and under what
name. After that, we can run the analysis.

When TreeAnnotator is finished, we can visualize the summarized
MultiTypeTree run with FigTree.

\begin{framed}
Open the program and go to \lstinline!File > Open!, and open the output
tree file from TreeAnnotator.

To color the tree, go to \textbf{Appearance} and change \textbf{Colour
by} and \textbf{Width by}.

To get the coloring by inferred location, one has to set it to
\textbf{type}. The expressions \textbf{type, state, location and color}
are often used interchangeably. The color gives the estimated (most
likely) location of a node (red for Hong Kong and blue for New Zealand),
the width gives the certainty of a color. The more certain the estimate
is, the wider the branch above (towards the root of) the node. You can
also change the \textbf{Line Weight} to better see difference in the
width of branches.
\end{framed}

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/figtree_color.png}
    \caption{Color tree according to the inferred location.}
    \label{fig:example}
\end{figure}

Next, we would like to know how certain we are about the node heights.

\begin{framed}
This can be visualized by going to \textbf{Node Bars} and there
\textbf{Display} the 95\% HPD of node heights, which gives the 95\%
credibility interval of node heights (the mean is indicated by the shown
node height).
\end{framed}

\begin{figure}
    \centering
    \includegraphics[max width=\textwidth, max height=0.9\textheight]{figures/figtree_nodeheights.png}
    \caption{Set the 95\% HPD for the node heights.}
    \label{fig:example}
\end{figure}

\subsubsection{Some considerations for using the structured Coalescent
(or any structured
method)}\label{some-considerations-for-using-the-structured-coalescent-or-any-structured-method}

\begin{itemize}

\item
  Inferring structure on a tree is hard and requires a lot of
  assumptions, e.g.~that the migration rates don't change over time or
  populations are constant.
\item
  The number of migration events on a tree might be very low, to infer a
  rate from such a low number of events can be very hard. In general, it
  is easier to infer rates within a region (here the effective
  population size) than it is to infer rates between them, as there are
  simply more events within than between regions.
\item
  Despite considering structure in a tree, there might still be states
  or locations etc. that were not sampled. Even if a node is inferred to
  be in a location with high certainty, the results could look
  completely different if samples from other locations would be
  considered as well.
\end{itemize}

\section{Useful Links}\label{useful-links}

\begin{itemize}

\item
  \href{http://www.beast2.org/book.html}{\emph{Bayesian Evolutionary
  Analysis with BEAST 2}} \citep{BEAST2book2014}
\item
  BEAST 2 website and documentation: \url{http://www.beast2.org/}
\item
  BEAST 1 website and documentation: \url{http://beast.bio.ed.ac.uk}
\item
  Join the BEAST user discussion:
  \url{http://groups.google.com/group/beast-users}
\end{itemize}

\clearpage

The content of this tutorial is based on the
\href{https://github.com/CompEvol/MultiTypeTree/wiki/Beginner's-Tutorial}{MultiTypeTree}
tutorial by Tim Vaughan. \clearpage

%%%%%%%%%%%%%%%%%%%%%%%
% Tutorial disclaimer %
%%%%%%%%%%%%%%%%%%%%%%%
% Please do not change the license
% Add the author names and relevant links
% Add any other aknowledgments here
\href{http://creativecommons.org/licenses/by/4.0/}{\includegraphics[scale=0.8]{figures/ccby.pdf}} This tutorial was written by Nicola F. Müller and Tim Vaughan for \href{https://taming-the-beast.github.io}{Taming the BEAST} and is licensed under a \href{http://creativecommons.org/licenses/by/4.0/}{Creative Commons Attribution 4.0 International License}.

%%%%%%%%%%%%%%%%%%%%
% Do NOT edit this %
%%%%%%%%%%%%%%%%%%%%
Version dated: \today

\newpage

%%%%%%%%%%%%%%%%
%  REFERENCES  %
%%%%%%%%%%%%%%%%

\printbibliography[heading=relevref]

\end{document}