% SIAM Article Template
\documentclass[review]{siamart0216}

% Information that is shared between the article and the supplement
% (title and author information, macros, packages, etc.) goes into
% ex_shared.tex. If there is no supplement, this file can be included
% directly.

% SIAM Shared Information Template
% This is information that is shared between the main document and any
% supplement. If no supplement is required, then this information can
% be included directly in the main document.

% Packages and macros go here
\usepackage{lipsum}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage{epstopdf}
\usepackage{algorithmic}
\ifpdf
  \DeclareGraphicsExtensions{.eps,.pdf,.png,.jpg}
\else
  \DeclareGraphicsExtensions{.eps}
\fi

% Declare title and authors, without \thanks
\newcommand{\TheTitle}{An Example Article}
\newcommand{\TheAuthors}{D. Doe, P. T. Frank, and J. E. Smith}

% Sets running headers as well as PDF title and authors
\headers{\TheTitle}{\TheAuthors}

% Title. If the supplement option is on, then "Supplementary Material"
% is automatically inserted before the title.
\title{{\TheTitle}\thanks{This work was funded by the Fog Research Institute under contract no.~FRI-454.}}

% Authors: full names plus addresses.
\author{
  Dianne Doe\thanks{Imagination Corp., Chicago, IL
    (\email{ddoe@imag.com}, \url{http://www.imag.com/\string~ddoe/}).}
  \and
  Paul T. Frank\thanks{Department of Applied Mathematics, Fictional
    University, Boise, ID (\email{ptfrank@fictional.edu},
    \email{jesmith@fictional.edu}).}
  \and
  Jane E. Smith\footnotemark[3]
}

\usepackage{amsopn}
\DeclareMathOperator{\diag}{diag}

%%% Local Variables:
%%% mode:latex
%%% TeX-master: "ex_article"
%%% End:

% Optional PDF information
\ifpdf
\hypersetup{
  pdftitle={\TheTitle},
  pdfauthor={\TheAuthors}
}
\fi

% The next statement enables references to information in the
% supplement. See the xr-hyperref package for details.

%\externaldocument{ex_supplement}

% FundRef data to be entered by SIAM
%<funding-group>
%<award-group>
%<funding-source>
%<named-content content-type="funder-name">
%</named-content>
%<named-content content-type="funder-identifier">
%</named-content>
%</funding-source>
%<award-id> </award-id>
%</award-group>
%</funding-group>

\begin{document}

\maketitle

% REQUIRED
\begin{abstract}
  This is an example SIAM \LaTeX\ article. This can be used as a
  template for new articles.  Abstracts must be able to stand alone
  and so cannot contain citations to the paper's references,
  equations, etc.  An abstract must consist of a single paragraph and
  be concise. Because of online formatting, abstracts must appear as
  plain as possible. Any equations should be inline.
\end{abstract}

% REQUIRED
\begin{keywords}
  example, \LaTeX
\end{keywords}

% REQUIRED
\begin{AMS}
  68Q25, 68R10, 68U05
\end{AMS}

\section{Introduction}
The introduction introduces the context and summarizes the
manuscript. It is importantly to clearly state the contributions of
this piece of work. The next two paragraphs are text filler,
generated by the \texttt{lipsum} package.

\lipsum[2-3]

% The outline is not required, but we show an example here.
The paper is organized as follows. Our main results are in
\cref{sec:main}, our new algorithm is in \cref{sec:alg}, experimental
results are in \cref{sec:experiments}, and the conclusions follow in
\cref{sec:conclusions}.

\section{Main results}
\label{sec:main}

We interleave text filler with some example theorems and theorem-like
items.

\lipsum[4]

Here we state our main result as \cref{thm:bigthm}; the proof is
deferred to \cref{sec:proof}.

\begin{theorem}[$LDL^T$ Factorization \cite{GoVa13}]\label{thm:bigthm}
  If $A \in \mathbb{R}^{n \times n}$ is symmetric and the principal
  submatrix $A(1:k,1:k)$ is nonsingular for $k=1:n-1$, then there
  exists a unit lower triangular matrix $L$ and a diagonal matrix
  \begin{displaymath}
    D = \diag(d_1,\dots,d_n)
  \end{displaymath}
  such that $A=LDL^T$. The factorization is unique.
\end{theorem}

\lipsum[6]

\begin{theorem}[Mean Value Theorem]\label{thm:mvt}
  Suppose $f$ is a function that is continuous on the closed interval
  $[a,b]$.  and differentiable on the open interval $(a,b)$.
  Then there exists a number $c$ such that $a < c < b$ and
  \begin{displaymath}
    f'(c) = \frac{f(b)-f(a)}{b-a}.
  \end{displaymath}
  In other words,
  \begin{displaymath}
    f(b)-f(a) = f'(c)(b-a).
  \end{displaymath}
\end{theorem}

Observe that \cref{thm:bigthm,thm:mvt,cor:a} correctly mix references
to multiple labels.

\begin{corollary}\label{cor:a}
  Let $f(x)$ be continuous and differentiable everywhere. If $f(x)$
  has at least two roots, then $f'(x)$ must have at least one root.
\end{corollary}
\begin{proof}
  Let $a$ and $b$ be two distinct roots of $f$.
  By \cref{thm:mvt}, there exists a number $c$ such that
  \begin{displaymath}
    f'(c) = \frac{f(b)-f(a)}{b-a} = \frac{0-0}{b-a} = 0.
  \end{displaymath}
\end{proof}

Note that it may require two \LaTeX\ compilations for the proof marks
to show.

Display matrices can be rendered using environments from \texttt{amsmath}:
\begin{equation}\label{eq:matrices}
S=\begin{bmatrix}1&0\\0&0\end{bmatrix}
\quad\text{and}\quad
C=\begin{pmatrix}1&1&0\\1&1&0\\0&0&0\end{pmatrix}.
\end{equation}
Equation \cref{eq:matrices} shows some example matrices.

We calculate the Fr\'{e}chet derivative of $F$ as follows:
\begin{subequations}
\begin{align}
  F'(U,V)(H,K)
  &= \langle R(U,V),H\Sigma V^{T} + U\Sigma K^{T} -
  P(H\Sigma V^{T} + U\Sigma K^{T})\rangle \label{eq:aa} \\
  &= \langle R(U,V),H\Sigma V^{T} + U\Sigma K^{T}\rangle
  \nonumber \\
  &= \langle R(U,V)V\Sigma^{T},H\rangle +
  \langle \Sigma^{T}U^{T}R(U,V),K^{T}\rangle. \label{eq:bb}
\end{align}
\end{subequations}
\Cref{eq:aa} is the first line, and \cref{eq:bb} is the last line.

\section{Algorithm}
\label{sec:alg}

\lipsum[40]

Our analysis leads to the algorithm in \cref{alg:buildtree}.

\begin{algorithm}
\caption{Build tree}
\label{alg:buildtree}
\begin{algorithmic}
\STATE{Define $P:=T:=\{ \{1\},\ldots,\{d\}$\}}
\WHILE{$\#P > 1$}
\STATE{Choose $C^\prime\in\mathcal{C}_p(P)$ with $C^\prime := \operatorname{argmin}_{C\in\mathcal{C}_p(P)} \varrho(C)$}
\STATE{Find an optimal partition tree $T_{C^\prime}$ }
\STATE{Update $P := (P{\setminus} C^\prime) \cup \{ \bigcup_{t\in C^\prime} t \}$}
\STATE{Update $T := T \cup \{ \bigcup_{t\in\tau} t : \tau\in T_{C^\prime}{\setminus} \mathcal{L}(T_{C^\prime})\}$}
\ENDWHILE
\RETURN $T$
\end{algorithmic}
\end{algorithm}

\lipsum[41]

\section{Experimental results}
\label{sec:experiments}

\lipsum[50]

\Cref{fig:testfig} shows some example results. Additional results are
available in the supplement in \cref{tab:foo}.

\begin{figure}[htbp]
  \centering
  \label{fig:a}\includegraphics{lexample_fig1}
  \caption{Example figure using external image files.}
  \label{fig:testfig}
\end{figure}

\lipsum[51]

\section{Discussion of \texorpdfstring{{\boldmath$Z=X \cup Y$}}{Z = X union Y}}

\lipsum[76]

\section{Conclusions}
\label{sec:conclusions}

Some conclusions here.

\appendix
\section{An example appendix}
\lipsum[71]

\section*{Acknowledgments}
We would like to acknowledge the assistance of volunteers in putting
together this example manuscript and supplement.

\bibliographystyle{siamplain}
\bibliography{references}
\end{document}


\documentclass[a4paper,12pt]{article}

% Font
\usepackage[T1]{fontenc}
\usepackage{gentium}

% Math packages
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{bm}

% Define symbol shortcuts
\newcommand{\cc}{\mathcal{C}}
\newcommand{\dd}{\mathcal{D}}
\newcommand{\hh}{\mathcal{H}}
\newcommand{\xx}{{\bm x}}
\newcommand{\yy}{{\bm y}}

% Math environment
\newtheorem*{thm}{Theorem}

% Better list management:
% - vertical spacing in lists
% - items in lists start with dash not bullet point.
\usepackage{enumitem}
\setlist{label=\textemdash,
  itemsep=0pt, topsep=3pt, partopsep=0pt}

% Include graphics
\usepackage{graphicx}
\usepackage{subcaption}

% Page format
\usepackage[top=2cm,left=2cm,right=2cm,bottom=2cm]{geometry}

\begin{document}
%%% HEADER
\raisebox{0.6in}[0in]{\makebox[\textwidth][r]{\it Unproofed version }}
\vspace{-0.7in}
\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{array,multirow,makecell}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage{float}
\setlength{\hoffset}{-18pt}
\setlength{\oddsidemargin}{1pt} % Marge gauche sur pages impaires
\setlength{\evensidemargin}{1pt} % Marge gauche sur pages paires
\setlength{\marginparwidth}{54pt} % Largeur de note dans la marge
\setlength{\textwidth}{481pt} % Largeur de la zone de texte (17cm)
\setlength{\voffset}{-18pt} % Bon pour DOS
\setlength{\marginparsep}{7pt} % Séparation de la marge
\setlength{\topmargin}{0pt} % Pas de marge en haut
\setlength{\headheight}{0pt} % Haut de page
\setlength{\headsep}{0pt} % Entre le haut de page et le texte
\setlength{\footskip}{3pt} % Bas de page + séparation
\setlength{\textheight}{800pt} % Hauteur de la zone de texte (25cm)
\usepackage{subcaption}
\usepackage{caption}
\pagenumbering{Roman}

\begin{center}
\bf\large MA2823: Foundations of Machine Learning \\
Chapter 3: Model evaluation and selection
\end{center}

\noindent
Lecturer: Chlo\'e-Agathe Azencott
\hfill
Scribe: Manon REVEL et Nathan VERMEERSCH

\noindent
\rule{\textwidth}{1pt}

\medskip

%%% NOTES START HERE
\section{Supervised Learning - Reminder}

From a training set $\mathcal{D}=\{x^{\text{i}},y^{\text{i}} \}$ ($x^{\text{i}}$ are the inputs objects and $y^{\text{i}}$ the expected outputs), one wants to deduce a function. The output of the function may be a regression ($y^{\text{i}}$ is a real number)  that models the behavior of a phenomenon or a classification ($y^{\text{i}}$ is 0 or 1) that predicts class label of the input objects. To achieve this, the model has to be generalized from the presented data to unseen situations in a "reasonable" way. Many models may fit with a set of data, however their complexity and accuracy are variable.

The point of this section is to find a way to evaluate and to select models.
\begin{figure}[!h]
\centering
\includegraphics[width=0.3\textwidth]{1}
\caption{\label{fig:1}Generalization error relatively to the model complexity}
\end{figure}

Even if, intuitively, one could have thought that increasing the model complexity would approach more accurately a data set, Figure 1 shows that a complex model that fits with a training set  will be unable to describe a new  set. Overfitting occurs: we model noise rather than intended inputs (Figure 2c). Inversely, a set described with too a simple model will be unable to model a phenomenon and underfitting occurs (Figure 2a).

\begin{figure}[!h]
    \centering
    \begin{subfigure}[b]{0.3\textwidth}
        \centering
        \includegraphics[width=\textwidth]{2A}
        \caption{$Underfitting$}
        %\label{fig:y equals x}
    \end{subfigure}
    \begin{subfigure}[b]{0.3\textwidth}
        \centering
        \includegraphics[width=0.4\textwidth]{2B}
        \caption{$Accurate model$}
        %\label{fig:three sin x}
    \end{subfigure}
    \begin{subfigure}[b]{0.3\textwidth}
        \centering
        \includegraphics[width=\textwidth]{2C}
        \caption{$Overfitting$}
        %\label{fig:five over x}
    \end{subfigure}
    \caption{Three simple graphs}
    \label{fig:three graphs}
\end{figure}

Therefore, models have to be evaluated in order to find the one that fits the best (Figure 2b).

Contrary to models previously studied in other fields like partial differential equations or mechanics, machine learning problems are ill-posed problems, that is to say they are no well-posed problems. Well-posed problems have the following properties: a unique solution exists, and it depends continuously on the initial condition. In machine learning, data gives information about the solution, but is not sufficient to lead one to a unique solution.
As a result, one has to find some criteria to choose the best model. In the case of supervised learning, that means making assumptions about $H$, the hypothesis space. The inductive bias is the set of assumptions that the learner makes to generalize beyond the training data. The model selection focuses on choosing the right inductive bias. We can evaluate this choice thanks to the empirical errors $E$ that says how our label defers from our prediction.

\textbf{Transitory question :} Empirical errors allows us to elaborate how good a label is on a training set. But what if we take a new set ?

\newpage
\section{Model Evaluation}

\textbf{Validation sets }

One should not be satisfied with tests on the training set. Indeed, they can be misleading, as the graph in Figure1 shows.
For instance, a very complex model might seem to work for the training set, but could yet be totally inefficient for other pieces of data. That is why a validation set, other than the training set, is needed.

Once we have chosen parameters, to tune them, we separate the data sets.
At first sight, it seems logical to simply divide the available data in two parts: one is the training set, the other the validation set. However, data should be used in the most efficient way in machine learning, as data can be scarce or difficult to obtain. Thus, there exists several methods to make the most of available data. We divide the data set in training set (used to see if the parameters fit) and a validation test \todo{Important concept}  (used to tune the parameters). We can also create a test set (used to assess the performance).
To compare optimal and suboptimal parameters, we have all interests to divide it more.
NB: Model selection pick the best model, model assessment estimates its prediction error on new data.

In this chapter, model selection methods will be used and two processes of sample re-use will be explained: cross-validation and bootstrap.

\textbf{Transitory question :} How much data should go in each of the training, validation and test sets ?
\subsection{Cross-Validation}
The Cross-Validation  \todo{Important concept} is a method based on the re-use of the sample. We cut the training set in k separate folds. We train on (k-1) folds and validate on one.

\begin{figure}[!h]
\centering
\includegraphics[width=0.3\textwidth]{a}
\caption{\label{fig:1}Cross-Validation with $k=4$}
\end{figure}

The cross-validation estimate of the prediction error is:
\[
CV(f) = \sum_{i=1}^n L(y_i,f_{k(i)}(x^{\text{i}})
\]

It estimates the expected prediction error.
We do then, k evaluations, with k outcomes and k empirical errors.  Notwithstanding, one should pay attention to the value of $k$ when using cross-validation. For instance, if one chooses $k=N$ where $N$ is the number of items in the set of available data, the following problems can be encountered: potential high variance and burdensome computation. As a result, values of $k=5$ or $k=10$ are usually chosen.

\begin{itemize}
\item \textbf{Positive points}

    Low bias estimator : we have a unbiased estimator of the expected error
\item \textbf{Negative points }

    High variance : we have overlap between the k evaluations, so they are correlated, so dependent. The variance is high because of the sum of the co-variances.

    The computation can become burdensome
\end{itemize}

With a small k, one trains on a little fraction of the original training data, so the variance is high and it gives an inaccurate model. Generally we take then k=5 or k=10 to have at least 80percent of the original training set in the newly defined one.

\newpage
\subsection{Bootstrap}

This method consists in building $B$ data sets by picking randomly with replacement from the available data. Typically, $B=100$.
Bootstrap \todo{Important concept} also asks several questions that one should be careful to: leave-one-out bootstrap error (the probability that a particular item belongs to a particular data set is only 63,2\%, so a data set can be restrictive), and the size of each data set.

\subsection{Confusion Matrix}

\begin{center}
\begin{tabular}{ c  c  c  c }
  & & True & Class \\
  &  & -1 & +1 \\
 Predicted & -1 & \textbf{T}rue \textbf{N}egative & \textbf{F}alse \textbf{N}egative \\
 Class & +1 & \textbf{F}alse \textbf{P}ositive & \textbf{T}rue \textbf{P}ositive
\end{tabular}
\end{center}

False positives can also be named type I errors or false alarms.

False negatives can also be called type II errors or misses.

We can then define a set of tools thanks to which we evaluate a model. Indeed, the proportion of positives that are correctly identified (sensitivity or recall), the proportion of negatives wrongly said positive (a wrong alarm or specificity) and inversely may become relevant rates to consider according to the environment of one's studies : in a clinical settings, we have better to have a very high sensitivity otherwise that means that we have failed to diagnostic patients.

\begin{figure}[!h]
	\centering
    \begin{subfigure}[b]{0.3\textwidth}
        \centering
        \includegraphics[width=1.3\textwidth]{aa}
        %\label{fig:y equals x}
    \end{subfigure}
    \begin{subfigure}[b]{0.3\textwidth}
        \centering
        \includegraphics[width=1.3\textwidth]{aaa}
        %\label{fig:three sin x}
    \end{subfigure}
    \caption{Tools for the evaluation of a model}
    \label{fig:three graphs}
\end{figure}

\subsection{Receiver-Operator Characteristic (ROC curve)}

The curves are a graphic mean to know if a model is optimal and to discard suboptimal models.
The ROC curve represents the true positive rate in ordinate and the false positive rate in abscissa.

\[TPR = \frac{TP}{TP + FN}\]
\[FPR = \frac{FP}{FP + TN}\]

      A good model should have a curve localized in the high left corner. We should maximize the area under the curve.
\begin{figure}[!h]
\centering
\includegraphics[width=0.5\textwidth]{4.jpg}
\caption{\label{fig:1}Cross-Validation with k=4}
\end{figure}

\subsection{Precision-Recall (PR curve)}

The Precision-Recall curve is a graph with the recall (the sensitivity) in ordinate and the precision (or positive predicated value) in abscissa.
\[TPR = Recall = \frac{TP}{TP + FN}\]
\[Precision = Positive Predicted Value = \frac{FP}{FP + TN}\]

An algorithm with a high precision is an algorithm that points out more relevant results than irrelevant.
An algorithm with a high recall is an algorithm that points the most relevant results.
In clinical settings, a high recall is appreciated.

\vspace{10pt}
\section{Model Selection}
\subsection{Penalizing model complexity}
As indicated by his name, a penalizing model complexity tool \todo{Important concept} penalize a model that is too much complex with an added contribution in the empirical error. We define then an augmented error E' :
\[E' = empirical error E + model complexity   \lambda \]
\begin{enumerate}
\item Mallow's Cp:
It compares the precision and bias of models. It is defined with a criteria to assess the fit of regression models with different numbers of parameters are being compared.
\item Akaike's Information Criterion (AIC)
\item Bayesian Information Criterion (BIC)
\end{enumerate}
\subsection{Minimum description length}

Based on the Shannon's information theory, the MDL method will choose the model that compress the data the best.
We calculate the sum of the average code length to transmit the difference between model prediction and true outputs and the average code length to transmit $\theta$ for each model. This is the code length needed to transmit the outputs Y. We then choose the model with the smallest length.
\subsection{Structural risk minimization}

Finally, the structural risk minimization separates the models according to their VC dimension. It fits a nested sequence of the models (their VC dimension increases). The best model is the one with the lower bound on test error.

\vspace{10pt}
We have seen some tools for measures of performance for classifiers and regressors.\todo{Important concept}
\vspace{60pt}

Bibliography :
%http://cazencott.info/index.php/pages/MA-2823-Foundations-of-Machine-Learning-%28Fall-2016%29
Computer Aided Diagnosis of Lung Ground Glass Opacity Nodules and Large Lung Cancers in CT.

\end{document}


\documentclass[numbers=enddot,12pt,final,onecolumn,notitlepage]{scrartcl}%
\usepackage[headsepline,footsepline,manualmark]{scrlayer-scrpage}
\usepackage[all,cmtip]{xy}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{framed}
\usepackage{amsmath}
\usepackage{comment}
\usepackage{color}
\usepackage{hyperref}
\usepackage[sc]{mathpazo}
\usepackage[T1]{fontenc}
\usepackage{amsthm}
%TCIDATA{OutputFilter=latex2.dll}
%TCIDATA{Version=5.50.0.2960}
%TCIDATA{LastRevised=Friday, May 31, 2019 21:43:46}
%TCIDATA{SuppressPackageManagement}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}
%TCIDATA{<META NAME="SaveForMode" CONTENT="1">}
%TCIDATA{BibliographyScheme=Manual}
%BeginMSIPreambleData
\providecommand{\U}[1]{\protect\rule{.1in}{.1in}}
%EndMSIPreambleData
\theoremstyle{definition}
\newtheorem{theo}{Theorem}[section]
\newenvironment{theorem}[1][]
{\begin{theo}[#1]\begin{leftbar}}
{\end{leftbar}\end{theo}}
\newtheorem{lem}[theo]{Lemma}
\newenvironment{lemma}[1][]
{\begin{lem}[#1]\begin{leftbar}}
{\end{leftbar}\end{lem}}
\newtheorem{prop}[theo]{Proposition}
\newenvironment{proposition}[1][]
{\begin{prop}[#1]\begin{leftbar}}
{\end{leftbar}\end{prop}}
\newtheorem{defi}[theo]{Definition}
\newenvironment{definition}[1][]
{\begin{defi}[#1]\begin{leftbar}}
{\end{leftbar}\end{defi}}
\newtheorem{remk}[theo]{Remark}
\newenvironment{remark}[1][]
{\begin{remk}[#1]\begin{leftbar}}
{\end{leftbar}\end{remk}}
\newtheorem{coro}[theo]{Corollary}
\newenvironment{corollary}[1][]
{\begin{coro}[#1]\begin{leftbar}}
{\end{leftbar}\end{coro}}
\newtheorem{conv}[theo]{Convention}
\newenvironment{condition}[1][]
{\begin{conv}[#1]\begin{leftbar}}
{\end{leftbar}\end{conv}}
\newtheorem{quest}[theo]{Question}
\newenvironment{algorithm}[1][]
{\begin{quest}[#1]\begin{leftbar}}
{\end{leftbar}\end{quest}}
\newtheorem{warn}[theo]{Warning}
\newenvironment{conclusion}[1][]
{\begin{warn}[#1]\begin{leftbar}}
{\end{leftbar}\end{warn}}
\newtheorem{conj}[theo]{Conjecture}
\newenvironment{conjecture}[1][]
{\begin{conj}[#1]\begin{leftbar}}
{\end{leftbar}\end{conj}}
\newtheorem{exmp}[theo]{Example}
\newenvironment{example}[1][]
{\begin{exmp}[#1]\begin{leftbar}}
{\end{leftbar}\end{exmp}}
\newenvironment{statement}{\begin{quote}}{\end{quote}}
\newenvironment{fineprint}{\begin{small}}{\end{small}}
\iffalse
\newenvironment{proof}[1][Proof]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\fi
\newenvironment{verlong}{}{}
\newenvironment{vershort}{}{}
\newenvironment{noncompile}{}{}
\excludecomment{verlong}
\includecomment{vershort}
\excludecomment{noncompile}
\newcommand{\kk}{\mathbf{k}}
\newcommand{\id}{\operatorname{id}}
\newcommand{\ev}{\operatorname{ev}}
\newcommand{\Comp}{\operatorname{Comp}}
\newcommand{\bk}{\mathbf{k}}
\newcommand{\Nplus}{\mathbb{N}_{+}}
\newcommand{\NN}{\mathbb{N}}
\let\sumnonlimits\sum
\let\prodnonlimits\prod
\renewcommand{\sum}{\sumnonlimits\limits}
\renewcommand{\prod}{\prodnonlimits\limits}
\setlength\textheight{22.5cm}
\setlength\textwidth{15cm}
\ihead{Math 4281, Spring 2019, Darij Grinberg: Syllabus}
\ohead{\today}
\begin{document}

\begin{center}
Math 4281, Spring 2019: \textbf{Introduction to Modern Algebra}

-- \textit{Syllabus} --

Darij Grinberg, VinH 203B, \texttt{dgrinber@umn.edu}

last update: \today

\end{center}

\bigskip

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

\textbf{WARNING!} You are reading the syllabus of a class that lies in the
past. If you're looking for the current iteration of Math 4281, you are in the
wrong place.

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

\bigskip

\section{Time \& Place}%

\begin{tabular}
[c]{|ll|}\hline
\textbf{Lectures:} & \textbf{Section 001:} MWF 9:05--9:55,
\href{http://campusmaps.umn.edu/vincent-hall}{Vincent Hall} 211.\\
& \textbf{Section 002:} MWF 11:15--12:05,
\href{http://campusmaps.umn.edu/vincent-hall}{Vincent Hall} 211.\\\hline
\textbf{Office hours:} & Monday 14:35--15:35.\\
& Tuesday 10:00--11:00.\\
& Tuesday 12:00--13:00.\\
& Friday 14:30--15:00 (half an hour).\\\hline
\end{tabular}

\begin{noncompile}
\vspace{0.1cm}Office hours are usually in
\href{http://campusmaps.umn.edu/lind-hall}{Lind Hall 4th floor (center of the
hall)}. If I am not there, try
\href{http://campusmaps.umn.edu/vincent-hall}{Vincent Hall 203B}. Otherwise,
by appointment (email).
\end{noncompile}

\bigskip

This class has \href{http://www.cip.ifi.lmu.de/~grinberg/t/19s/}{a website:
http://www.cip.ifi.lmu.de/\symbol{126}grinberg/t/19s/} and two Canvas boards:

\qquad\href{https://canvas.umn.edu/courses/99199}{Section 001:
https://canvas.umn.edu/courses/99199}.

\qquad\href{https://canvas.umn.edu/courses/99188}{Section 002:
https://canvas.umn.edu/courses/99188}.

\bigskip

\textbf{Homework} will usually be due on \textbf{Wednesday} (sometimes weekly,
sometimes every other week) \textbf{at the beginning of class}. You can also
submit your homework electronically via Canvas (see above) \textbf{provided
that the problem set is submitted as 1 single PDF file}. See \textquotedblleft
Grading\textquotedblright\ and \textquotedblleft Coursework\textquotedblright%
\ below for details.

\bigskip

\textbf{Midterms} (3 in total) are like homework, but they count for more, and
collaboration is not allowed (see below for details). There will be \textbf{no
final exam}.

\section{Requirements}

This is a pure mathematics class and relies heavily on proofs. You have to
feel at home reading and writing mathematical proofs. You can catch up on this from:

\begin{itemize}
\item {}[LeLeMe] Eric Lehman, F. Thomson Leighton, Albert R. Meyer,
\textit{Mathematics for Computer Science}, \newline%
\url{https://courses.csail.mit.edu/6.042/spring18/mcs.pdf} . (You should know
the material from Chapters 1--5, minus the CS parts.)

\item {}[Hammack] Richard Hammack, \textit{Book of Proof},\newline\url{http://www.people.vcu.edu/~rhammack/BookOfProof/}

\item {}[Day] Martin V. Day, \textit{An Introduction to Proofs and the
Mathematical Vernacular},\newline%
\url{https://www.math.vt.edu/people/day/ProofsBook/IPaMV.pdf} .
\end{itemize}

You should also know some linear algebra: Gaussian elimination, linear
independence, vector spaces, dimension, rank of matrices. The more you know,
the better. One of the best texts to learn linear algebra from is:

\begin{itemize}
\item {}[Hefferon] Jim Hefferon, \textit{Linear Algebra}, 2017,\newline\url{http://joshua.smcvt.edu/linearalgebra/}
\end{itemize}

\noindent if you have the time for it (it is long and thorough). If not, make
sure you know at least the topics named above. We won't start using linear
algebra until a few weeks into this class.

\section{Texts}

\subsection{required:}

\begin{noncompile}
I will write lecture notes accompanying the class:%
\[
\text{\url{http://www.cip.ifi.lmu.de/~grinberg/t/19s/notes.pdf}}%
\]
(just started; they'll grow). I am not sure how detailed they will be; I might
also supplement them with other freely available sources. Everything
examinable will be in these notes (and homework).
\end{noncompile}

There is no required textbook if you attend class. I won't follow a text
directly anyway. I will post lecture notes on the
\href{http://www.cip.ifi.lmu.de/~grinberg/t/19s/}{class website}. These will
not be as polished as a book, but will contain everything needed to do
homeworks and midterms.

\begin{noncompile}
class notes (= scans of what I'm projecting on the screen in class)
\end{noncompile}

\subsection{recommended:}

Here are some good texts I know:\footnote{The bracketed names (such as
\textquotedblleft\lbrack Goodman]\textquotedblright) are the abbreviations by
which I will refer to the texts in class.}

\begin{itemize}
\item {}[Goodman]
\href{http://homepage.divms.uiowa.edu/~goodman/algebrabook.dir/algebrabook.html}{Frederick
M. Goodman, \textit{Algebra: Abstract and Concrete}, edition 2.6}.\newline
This is a long book going beyond what we will do in this class. We won't
follow it directly either, but it has a lot of the material we care about.

\item {}[Siksek]
\href{http://homepages.warwick.ac.uk/staff/S.Siksek/teaching/aa/aanotes.pdf}{Samir
Siksek, \textit{Introduction to Abstract Algebra}, 2015}.\newline This is
probably the most amusing (though occasionally overdoing the goofiness) book
on abstract algebra I've ever seen. It is also really good at motivating definitions.

\item {}[Pinter]
\href{https://www.amazon.com/Book-Abstract-Algebra-Second-Mathematics/dp/0486474178}{Charles
C. Pinter, \textit{A book of abstract algebra}, 2nd edition, Dover
2010}.\newline The link goes to an ebook version that doesn't look very good,
but \href{http://store.doverpublications.com/0486474178.html}{you can get the
original for less than \$20}.

\item {}[Armstrong]
\href{http://www.math.miami.edu/~armstrong/561fa18.php}{Drew Armstrong,
\textit{Abstract Algebra I}, 2018}.\newline(Click on
\href{http://www.math.miami.edu/~armstrong/561fa18/561fa18notes.pdf}{\textquotedblleft
Course Notes\textquotedblright} for the main text.) This is new and I haven't
had much experience with it, but Armstrong is a good expositor. Note that this
only goes until group theory.
\end{itemize}

On specific topics:

\begin{itemize}
\item {}[Conrad-Gauss]
\href{http://www.math.uconn.edu/~kconrad/blurbs/ugradnumthy/Zinotes.pdf}{Keith
Conrad, \textit{The Gaussian integers}}.\newline This is a topic we'll spend
some time with early on in this course. And Conrad is a really good writer;
check out \href{http://www.math.uconn.edu/~kconrad/blurbs/}{his expository
papers} for various other pieces of algebra.

\item {}[Strickland]
\href{https://neil-strickland.staff.shef.ac.uk/courses/MAS201/MAS201.pdf}{Neil
Strickland, \textit{Linear mathematics for applications}}.\newline Would you
have guessed by the title that these lecture notes have one of the most
rigorous treatments of determinants I've seen in the linear algebra literature?

\item {}[Gallier-RSA]
\href{https://www.cis.upenn.edu/~jean/RSA-primality-testing.pdf}{Jean Gallier,
Jocelyn Quaintance, \textit{Notes on Primality Testing And Public Key
Cryptography, Part 1}}.\newline This contains most of the elementary number
theory we'll be going through (modular arithmetic, RSA, groups), plus a lot
that we won't (primality checking).
\end{itemize}

\begin{noncompile}
You don't have to buy any of these, but I assume any of them would look nice
on your shelf. (You can download [Bogart] and [GKP] from the URLs above, and
you might be able to get [Aigner] for free from the UMN network. Of course,
there are ways to get the other two as well if you know your way around the
internet\footnote{Search for \textquotedblleft reddit
textbooks\textquotedblright, for example.}.)
\end{noncompile}

\begin{fineprint}
For the sake of completeness, here are links to some old editions of this
class: \href{http://www-users.math.umn.edu/~kim00657/teaching/18f4281/}{Fall
2018}, \href{http://mchmutov.net/teaching/4281teaching.html}{Spring 2018},
\href{http://www-users.math.umn.edu/~musiker/4281/}{Fall 2017},
\href{http://www-users.math.umn.edu/~musiker/4281/}{Spring 2017}. Note that
they were structured differently from mine and used different texts.
\end{fineprint}

\section{Contact}

All material regarding the course (including homework) can be found on
\[
\text{\url{http://www.cip.ifi.lmu.de/~grinberg/t/19s/}}
\]

The best way to reach me is by email to \texttt{dgrinber@umn.edu} .

After I leave UMN (in Summer), the best way to reach me will be by email to
\texttt{darijgrinberg@gmail.com} , and the class materials will migrate to
\url{http://www.cip.ifi.lmu.de/~grinberg/t/19s/} .

\section{Topics (tentative)}

This is still far from finished and decided.

Topics marked with an * \textbf{may} be excluded. Topics marked with an **
probably \textbf{will} be excluded.

\begin{enumerate}
\item Introduction and motivating questions.

\begin{enumerate}
\item $n=x^{2}+y^{2}$.

\item Algebraic vs. transcendental numbers; sums of algebraics.

\item \href{https://en.wikipedia.org/wiki/Shamir%27s_Secret_Sharing}{Shamir's
secret sharing scheme} and the need for finite fields.

\item * Angle trisection, cube duplication, circle squaring.

\item * Solving degree-$3$ equations; higher orders.
\end{enumerate}

\item Elementary number theory.

\begin{enumerate}
\item Divisibility.

\item Congruences mod $n$.

\item Division with remainder.

\item gcd, Bezout, Euclidean algorithm.

\item Primes and prime factorization.

\item Modular inverses \& cancellation mod $n$.

\item Fermat's little theorem.

\item Euler's $\phi$-function and its properties.

\item Chinese Remainder Theorem.
\end{enumerate}

\item $\mathbb{Z}/n$.

\begin{enumerate}
\item Equivalence relations.

\item Equivalence classes and quotient sets.

\item $\mathbb{Z}/n$.

\item RSA and applications of the Chinese Remainder Theorem.

\item Primitive roots (no proofs).
\end{enumerate}

\item Complex numbers and Gaussian integers.

\begin{enumerate}
\item Complex numbers (the very basics).

\item Gaussian integers (follow Keith Conrad).

\item $n=x^{2}+y^{2}$ (follow Keith Conrad).

\item Finish with discussion of other quadratic and higher-order number rings.
\end{enumerate}

\item Rings and fields I.

\begin{enumerate}
\item Define rings, commutative rings and fields.

\item Examples (but no structure or homomorphisms). Include $\mathbb{Z}/p$,
$\mathbb{Z}\left[  i\right]  $, dual numbers, etc. Also $\mathbb{Z}\left[
\sqrt[3]{2}\right]  $, but note that just adding $\sqrt[3]{2}$ does not work.

\item General properties (e.g., finite sums).
\end{enumerate}

\item Linear algebra.

\begin{enumerate}
\item Modules and vector spaces.

\item Linear algebra (crash course, following Hefferon? Charlier? Artin?
whoever does it most slickly).

\item Direct sums/products of vector spaces.

\item Quotient vector spaces.

\item Linear algebra over $\mathbb{Z}/p$.

\item $\operatorname*{XOR}$ as vector addition over $\mathbb{Z}/2$.

\item Solve \href{https://en.wikipedia.org/wiki/Lights_Out_(game)}{lights-out}
using $\mathbb{Z}/2$-linear algebra.

\item What works and what fails over the base ring $\mathbb{Z}$ ? (Leave some
harder stuff unproven.)

\item * $\mathbb{Z}/26$ and affine ciphers as $2\times2$-matrices (follow Conrad).

\item The Smith normal form over $\mathbb{Z}$, proven by merging Gaussian
elimination with the Euclidean algorithm.
\end{enumerate}

\item Permutations and determinants.

\begin{enumerate}
\item Permutation basics (follow [detnotes]).

\item Lengths and signs (follow [detnotes]).

\item Cycles and transpositions (follow [detnotes]).

\item Cycle decompositions (follow Loeh?).

\item Determinants.

\item Properties of determinants.

\item $\det\left(  AB\right)  $.

\item Adjugate matrices.
\end{enumerate}

\item Groups.

\begin{enumerate}
\item Definition of a group.

\item Subgroups.

\item Homomorphisms.

\item Direct products.
\end{enumerate}

\item Group actions. (Include necklace counting.)

\item Quotient groups and homomorphism theorems.

\item Rings.

\begin{enumerate}
\item Recalling the definition.

\item Subrings.

\item Ring homomorphisms.

\item Ideals and quotient rings.
\end{enumerate}

\item Polynomials and their rings.

\begin{enumerate}
\item Define formal power series.

\item Define polynomials.

\item Polynomials over a field as vector space.

\item Division with remainder modulo monic (or invertible-LT) polynomial.

\item Use as generating functions.

\item Lucas's theorem reproved.

\item Companion matrices.

\item Cayley-Hamilton theorem.

\item Application: sums/products of algebraic integers.
\end{enumerate}

\item Fields.

\item Field extensions and angle trisection.

\item Finite fields. (Prove existence by counting irreducible polys.)

\item * Properties of rings (PID, UFD, etc.).

\item * Symmetric polynomials and the Fundamental Theorem.

\item * Galois theory introduction. (Apply to deg-$3$ maybe.)

\item * Modules and their basic applications. (Apply to algebraic integers again?)
\end{enumerate}

\section{Schedule (tentative)}

\begin{noncompile}
This is my \textbf{best guess} at this point (and \textquotedblleft
best\textquotedblright\ does not mean \textquotedblleft good\textquotedblright%
). Even the midterm dates are \textbf{not set in stone}. Topics inside
parentheses are the most likely to be sacrificed if I run out of time.

Asterisks (*) mean days when I'll be away; class will be substituted.
\end{noncompile}

\textbf{The due dates with question marks are not set in stone.}%

\[%
\begin{tabular}
[c]{||c||c||c||}\hline\hline
\textbf{week} & \textbf{material} & \textbf{due}\\\hline\hline
Jan 23, 25 &  & \\\hline\hline
Jan 28, 30, 1 &  & \\\hline\hline
Feb 4, 6, 8 &  & hw1\\\hline\hline
Feb 11, 13, 15 &  & hw2\\\hline\hline
Feb 18, 20, 22 &  & hw3\\\hline\hline
Feb 25, 27, 1 &  & \\\hline\hline
Mar 4, 6, 8 &  & MT1\\\hline\hline
Mar 11, 13, 15 &  & hw4\\\hline\hline
\textit{Mar 18--24} & \textit{break} & \\\hline\hline
Mar 25, 27, 29 &  & \\\hline\hline
Apr 1, 3, 5 &  & hw5 (Mon)\\\hline\hline
Apr 8, 10, 12 &  & MT2\\\hline\hline
Apr 15, 17, 19 &  & hw6?\\\hline\hline
Apr 22, 24, 26 &  & \\\hline\hline
Apr 29, 1, 3 &  & MT3\\\hline\hline
May 5 &  & \\\hline\hline
\end{tabular}
\ \ \ \ \ \ \ \ \ \ \ \
\]

\bigskip

\section{Grading}

The grade will be computed based on three take-home \textbf{midterms}
(totalling to \textbf{60\%} of the final grade, each giving 20\% of the final
grade) and somewhere between 5 and 10 \textbf{homework sets} (totalling to
\textbf{40\%} of the final grade, but the lowest score will be dropped).

Points will be deducted if your proofs are ambiguously worded or otherwise
hard to understand. Writing readable arguments is part of mathematics; you can
learn this from the references in the \textquotedblleft
Requirements\textquotedblright\ section above and you can practice it on
\href{https://math.stackexchange.com/}{math.stackexchange}.

\section{Coursework}

Collaboration on homework is allowed, as long as:

\begin{itemize}
\item you \textbf{write} up the solutions autonomously and in your own words
(in particular, this means that you have to \textbf{understand} them), and

\item you \textbf{list the names of your collaborators} (there will be no
penalties for collaboration, so you don't lose anything doing this!).
\end{itemize}

On the midterms, you have to \textbf{work alone} (you can \textbf{read}
whatever you want, but you must \textbf{not contact} anyone about the midterm
problems\footnote{It is OK to contact \textbf{me} with questions.}; in
particular, you must \textbf{not ask} them on the internet).

Homework and midterms should be submitted either in person during class, or
via Canvas.

\textbf{If you handwrite your solutions:}

\begin{itemize}
\item Make sure that your writing is legible.

\item If you submit your solutions by email, make sure that your submission is
\textbf{1 single PDF file} for a given homework set (not many 1-page JPGs!).
Double-check that your scans are readable and aren't missing any relevant text
near the margins.
\end{itemize}

\textbf{If you type up your solutions:}

\begin{itemize}
\item Again, make sure that your submission is \textbf{1 single PDF file} for
a given homework set.

\item Double-check that your text doesn't go over the margins (something that
often happens when using LaTeX). If something is not on the page, we cannot
grade it...
\end{itemize}

Calculators and computer algebra systems may be used, but are not necessary
(and you are responsible for any resulting errors).

\begin{noncompile}
For emails, I suggest using \textquotedblleft\lbrack Math 4281] Homework set
\#$n$ submission\textquotedblright\ ($n$ = the number of the problem set) as
the subject line.
\end{noncompile}

\textbf{Late} homework or late midterms are \textbf{not accepted} in any
situation; if you are not finished, submit whatever you have before the
deadline. If you want to update your submission, you can do so (before the
deadline!) by sending me an email that includes the whole updated submission
(not just the parts you want changed).

See also the following university policies:

\begin{itemize}
\item
\href{https://policy.umn.edu/education/gradingtranscripts}{https://policy.umn.edu/education/gradingtranscripts}%

\item
\href{https://policy.umn.edu/research/academicmisconduct}{https://policy.umn.edu/research/academicmisconduct}%

\end{itemize}

\begin{noncompile}
-----

Things I wanted to get to:

(a) angle trisection impossible, $\leftarrow$ This should take 1 extra hour
(once ring extensions and gcds of polynomials are done), but is probably not
worth the hassle.

(b) sum\&prod of algebraic numbers/integers, $\leftarrow$ This requires either
Cayley-Hamilton + modules, or symmetric polynomials. I'd love to get here but
I didn't.

(c) finite fields GF(q), $\leftarrow$ Gregg proved the existence, which is
really nice (use necklace enumeration to prove the existence of irreducible
polynomials of each degree; then use adjunction via polynomial quotients). But
I didn't get here. Uniqueness is prolly not worth it.

(d) the Lagrange interpolation cryptoscheme (Shamir) $\leftarrow$
\textbf{done}.

(e) Solving systems of linear equations over $\mathbb{Z}$ (a simple version of
Smith normal form). (elementary, Euclidean alg). $\leftarrow$ I sketched the
rough idea of this without going into details.

(f) Gaussian integers and writing $n$ as $x^{2}+y^{2}$ (this is in KConrad
except for counting), $\leftarrow$ \textbf{done}.

(g) Fundamental theorem of algebra, proven using Sylow (may be too advanced).
$\leftarrow$ Not tried. I think it isn't worth it, since algebra doesn't use it.

(h) quaternions (what exactly?). $\leftarrow$ I've done the very basics. Not
sure whether there are any good applications to go into short of the
four-squares theorem (which would probably take too long.)

(i) Solving deg-3 and deg-4 equations. (Gregg: \textquotedblleft solving
degree 3 polys and once when I taught it, I got a little further in Galois
theory and contrasted deg 4 polys with Galois group V\_4 vs C\_4 (but did not
discuss solving degree 4 in general)\textquotedblright) $\leftarrow$ not done,
and Vic confirms my suspicion that doing this is fairly orthogonal to most of
abstract algebra.

(j) Insolvability of deg-5 equation, probably with bare hints to its proof.
$\leftarrow$ not even tried; not worth it.
\end{noncompile}

\end{document}


\documentclass[12pt]{article}
\usepackage{amsmath,cclicenses}
\usepackage{harvard} % this is a downloaded package for harvard style of referencing (a .sty file)
\title{Tutorial on Bibliography}
\author{Author\\email\\ \byncsa}
\date{\today}
\bibliographystyle{ifac} % The style of bibliography to be used
% {plain} : plain bibliography
% {unsrt} : The references are ordered according to the sequence that appears first
% {alpha} : Style used by computer scientists
% *when using the harvard style* use {ifac} a downloaded .bst file

\begin{document}
\maketitle
\newpage
\section{Aryabatta's Identity for Control Design}

Polynomial equations of the form
\begin{align*}
X(z)+D(z) + Y(z)N(z) = C(z)
\end{align*}
arise frequently in control system design. In the above equation, $D(z)$, $N(z)$ and $C(z)$ are known polynomials and $X(z)$ and $Y(z)$ are unknowns, to be determined. This equation is known as Diophatine equation \cite{vk79,tk80} and Aryabatta's identity \cite{mv85}. A solution technique to this identity is presented in
\cite{cp82}.  Matlab and Scilab implementations of this solution are
available on the web \cite{kmm1-07}.

% cite as noun *works only for harvard style* of referencing, where the name of reference appears without the brackets
The textbook by \citeasnoun{kmm07}
illustrates several control design
examples using Aryabhatta's identity.  The approach followed in this
book is explained in \cite{ms04,km06}.  In addition to handling
control design problems in conventional domains, this approach will
be useful also for naturally discrete time problems that arise in
computing systems, see for example, \cite{mmr03,mrbm04,vs06}.

\bibliography{ref} % This tells LaTeX the file name in which the references can be found
\end{document}

% Procedure for adding references
% 1. Make the source as above
% 2. In command line, go the folder containing the *.tex and **.bib file (where *,** are to be replaced with your respective filenames)
% replace * with your filename in the following commands
% 3. run `pdflatex *'
% 4. run `bibtex *'
% 5. run `pdflatex *' twice


\subsection{Parameter Input by User}
At some point you might want to make a macro to ask the user to input numerical
values or file names. We now learn how to do that, by first examining the
following code. Run the code first.
\lstinputlisting{code/code05.ijm}
Running this macro,  a dialogue window pops-up.

\begin{figure}[htbp]
\begin{center}
\includegraphics[scale=0.6]{fig/getNumberDialog.png}
\caption{getNumber Dialog} \label{fig_getNUmber}
\end{center}
\end{figure}

The function \textit{getNumber} consists of two parameters (programmers call such parameters "arguments" so we use this word in the reminder of this textbook).
\begin{indentCom}
\textbf{getNumber}(message string, default number)
\end{indentCom}
The first argument is a string wrapped by double quotes (see code 5, line 3
and 4). This string will appear in the dialog window such as shown above.
Default number will appear in the input field in the dialog window,
and the user is expected to modifies this default number.
When OK button is clicked, the number given by the user will be returned to
the macro and then substituted to a variable. In the above case, this could be
either \textit{a} or \textit{b}.

To ask a user for providing a string in dialog, following is an example.
\lstinputlisting[morekeywords={*, getString}]{code/code06.ijm}

The function \textbf{getString} also has two arguments,
and only the difference is that the user input will be treated as a string. \\

\begin{indentexercise}{1}
Run the code 6 and input 1 for a and 2 for b. What happened? Explain the reason.
\end{indentexercise}


% !TeX program = pdfLaTeX
\documentclass[12pt]{article}
\usepackage{amsmath}
\usepackage{graphicx,psfrag,epsf}
\usepackage{enumerate}
\usepackage{natbib}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{longtable}
\usepackage{booktabs}

\usepackage{url} % not crucial - just used below for the URL

%\pdfminorversion=4
% NOTE: To produce blinded version, replace "0" with "1" below.
\newcommand{\blind}{0}

% DON'T change margins - should be 1 inch all around.
\addtolength{\oddsidemargin}{-.5in}%
\addtolength{\evensidemargin}{-.5in}%
\addtolength{\textwidth}{1in}%
\addtolength{\textheight}{1.3in}%
\addtolength{\topmargin}{-.8in}%

\begin{document}

\def\spacingset#1{\renewcommand{\baselinestretch}%
{#1}\small\normalsize} \spacingset{1}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\if0\blind
{
  \title{\bf Fundamental Kernel Density Estimation for Computation}

  \author{
        Michael Liou \thanks{All code is available at www.github.com/lioumens/stat771\_final} \\
    Department of Statistics, University of Wisconsin, Madison\\
      }
  \maketitle
} \fi

\if1\blind
{
  \bigskip
  \bigskip
  \bigskip
  \begin{center}
    {\LARGE\bf Fundamental Kernel Density Estimation for Computation}
  \end{center}
  \medskip
} \fi

\bigskip
\begin{abstract}
We cover fundamental principles of the Kernel Density Estimate (KDE) in
a computational setting. Influence of kernel choice and bandwidth tuning
on the density estimates are illustrated. Asymptotic bounds and decision
theoretic frameworks are presented for the KDE. Computational complexity
and efficiency of the Fast Fourier Transform (FFT) procedure in KDE
calculation are shown, and bandwidth choices are also shown to nearly
attain the asymptotic lower bound computationally. Multivariate
extensions of the univariate first principles are briefly mentioned.
\end{abstract}

\noindent%
{\it Keywords:} nonparametric, statistics, fast fourier transform, curse of dimensionality
\vfill

\newpage
\spacingset{1.45} % DON'T change the spacing!

\newcommand{\expect}[1]{\mathbf{E}\left[ #1 \right]}
\newcommand{\variance}[1]{\mathrm{Var}\left( #1 \right)}
\newcommand{\covariance}[1]{\mathrm{Cov}\left( #1 \right)}
\newcommand{\bigo}[1]{\mathcal{O}\left( #1 \right)}

\def\tightlist{}

\section{Introduction}\label{introduction}

Density estimation is a useful, nonparametric way to estimate the
underlying probability density function. This is common in any scenario
in which we are trying to determine a pattern of the underlying random
process from an observed dataset. Density estimation can provide insight
to the pattern of the underlying distribution, especially when the
underlying distribution is complicated. Here, we explore some of the
fundamental computational results that have shaped the univariate
density estimation. We then extend some of these principles into a
multidimensional setting and look at how current research areas have
addressed sparsity issues and parameter tuning in higher dimensions. We
illustrate these results with a mix of mature code bases and self-coded
crude approximations.

\section{Theoretical Background}\label{theoretical-background}

\subsection{Estimators}\label{estimators}

The kernel density estimator with bandwidth \(h > 0\) and kernel \(K\)
is defined as

\[
\begin{aligned}
\hat{f_n}(x) = \frac{1}{n} \sum_{i=1}^{n} \frac{1}{h} K\left(\frac{x-X_i}{h}\right)
\end{aligned}
\]

There are two parameters to specify for this estimator, the kernel \(K\)
and the bandwidth \(h\). It is well known from theory and practice that
the kernel does not have a significant on the final density estimate. On
the other hand, the bandwidth will have a large effect on the overall
fit of the estimate. The bandwidth determines the scaling parameter of
the kernel. Visually, the bandwidth determines how ``squiggly'' the
estimate looks. The KDE estimate at one particular point is the sum of
the Gaussian kernel at every point of evaluation.

\begin{figure}[h]
\label{fig:bandwidth}
\caption{Comparison of different bandwidths}
\includegraphics[scale=.31]{bandwidth_fig}
\end{figure}

\begin{figure}[h]
\caption{Comparison of different kernels}
\includegraphics[scale=.4]{fig2}
\centering
\end{figure}

The difficulty with kernel density estimation is the varying regions of
high density and low density sample regions, in which low bandwidth more
appropriate for low density regions and high bandwidth is more
appropriate for high density regions. Thus, many other estimators that
build off of this basic idea have been proposed
\citep{silverman2018density, MACK19791, silverman1982estimation, mack1979multivariate}.

\begin{longtable}[]{@{}lll@{}}
\caption{Non-Exhaustive List of Density Estimators}\tabularnewline
\toprule
\begin{minipage}[b]{0.20\columnwidth}\raggedright\strut
Estimator\strut
\end{minipage} & \begin{minipage}[b]{0.32\columnwidth}\raggedright\strut
Formula\strut
\end{minipage} & \begin{minipage}[b]{0.38\columnwidth}\raggedright\strut
Defined Terms\strut
\end{minipage}\tabularnewline
\midrule
\endfirsthead
\toprule
\begin{minipage}[b]{0.20\columnwidth}\raggedright\strut
Estimator\strut
\end{minipage} & \begin{minipage}[b]{0.32\columnwidth}\raggedright\strut
Formula\strut
\end{minipage} & \begin{minipage}[b]{0.38\columnwidth}\raggedright\strut
Defined Terms\strut
\end{minipage}\tabularnewline
\midrule
\endhead
\begin{minipage}[t]{0.20\columnwidth}\raggedright\strut
Adaptive Smooth (Location)\strut
\end{minipage} & \begin{minipage}[t]{0.32\columnwidth}\raggedright\strut
\(\frac{1}{nh(x)} \sum^{n}_{i=1}{K \left( \frac{x-X_i}{h(x)}\right)}\)\strut
\end{minipage} & \begin{minipage}[t]{0.38\columnwidth}\raggedright\strut
\(h(x)\) is a function of the location \(x\)\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.20\columnwidth}\raggedright\strut
Adaptive Smooth (Sample)\strut
\end{minipage} & \begin{minipage}[t]{0.32\columnwidth}\raggedright\strut
\(\frac{1}{n} \sum^{n}_{i=1}{\frac{1}{h_i}K \left( \frac{x-X_i}{h_i}\right)}\)\strut
\end{minipage} & \begin{minipage}[t]{0.38\columnwidth}\raggedright\strut
\(h_i\) is a function of the \(i\)th sample point\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.20\columnwidth}\raggedright\strut
Nearest Neighbor\strut
\end{minipage} & \begin{minipage}[t]{0.32\columnwidth}\raggedright\strut
\(\frac{1}{nd_k(x)} \sum^{n}_{i=1}{K \left( \frac{x-X_i}{d_k(x)}\right)}\)\strut
\end{minipage} & \begin{minipage}[t]{0.38\columnwidth}\raggedright\strut
\(d_k(x)\) is the Euclidean distance between \(x\) and the increasing
\(k\)th nearest neighbor sequence\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.20\columnwidth}\raggedright\strut
Orthogonal Series\strut
\end{minipage} & \begin{minipage}[t]{0.32\columnwidth}\raggedright\strut
\(\sum^{n}_{i=1}\theta_i\phi_i(x)\)\strut
\end{minipage} & \begin{minipage}[t]{0.38\columnwidth}\raggedright\strut
\(\phi_i\) is an orthogonal basis,
\(\theta = \expect{ \frac{1}{n} \sum_i\phi_i(x)}\)\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.20\columnwidth}\raggedright\strut
Max Penalized Likelihood\strut
\end{minipage} & \begin{minipage}[t]{0.32\columnwidth}\raggedright\strut
\(\arg \max_{f} \frac{1}{n} \sum^{n}_{i=1}\log f(X_i)- \frac{1}{2}\lambda\left[\log f, \log f\right]\)\strut
\end{minipage} & \begin{minipage}[t]{0.38\columnwidth}\raggedright\strut
\(\left[f,f\right] = \int D(f)D(f)\), \(D(f)\) is linear differential
operator\strut
\end{minipage}\tabularnewline
\bottomrule
\end{longtable}

\pagebreak

\subsection{Decision theory}\label{decision-theory}

In order to evaluate an estimator in a decision theoretic framework, a
commonly defined loss function is the Mean Integrated Squared Error
(MISE), which averages the estimator's performance over all samples that
could be observed. We can show that, under some regularity conditions,
there is a bias-variance trade-off with the MISE.

\begin{equation} \label{mise}
\begin{split}
\mathrm{MISE}(\hat{f}) &= \expect{\int_{-\infty}^{\infty}(\hat{f}(x) - f(x))^2 \, dx} \\
&= \int_{-\infty}^{\infty}\expect{(\hat{f}(x) - f(x))^2} \, dx \\
&= \int_{-\infty}^{\infty} \variance{\hat{f}(x)} + \left(\mathrm{bias}(\hat{f}(x))\right)^2 \, dx
\end{split}
\end{equation}

To calculate the bias term for the kernel density estimator, we
substitute a Taylor expansion and integrate.

\[
\begin{aligned}
\expect{\hat{f}(x)} &= f(x) + \frac{1}{2}h^2\sigma^2_Kf''(x) + o(h^2) \\
\left[\expect{\hat{f}(x)} - f(x)\right]^2 &= \frac{1}{4}h^4\sigma^4_K\left(f''(x)\right)^2 + o(h^4) \\
\int \left[\expect{\hat{f}(x)} - f(x)\right]^2 \, dx &= \frac{1}{4}h^4\sigma^4_K\left(\int f''(x)^2 \, dx\right) + o(h^4)
\end{aligned}
\] To calculate the variance term, we do a Taylor expansion about \(x\),
and change of variable \(t = \frac{x-X_i}{h}\).

\[
\begin{aligned}
\variance{\hat{f}(x)} &= \frac{1}{nh}\int K(t)^2f(x-ht)\, dt - \frac{1}{n}\left[\expect{ \frac{1}{h} K\left( \frac{x-X_i}{h}\right)}\right]^2 \\
&= \frac{1}{nh}\int K(t)^2[f(x) + o(1)] \, dt - \frac{1}{n}\left[f(x) + o(1)\right]^2\\
&= \frac{1}{nh}f(x) \left(\int K(t)^2\, dt\right) + o\left( \frac{1}{nh}\right) \\
\int \variance{\hat{f}(x)} \, dx &= \frac{1}{nh}\left(\int K(t)^2\, dt\right) + o\left( \frac{1}{nh}\right)
\end{aligned}
\] Hence, if we put these together back into MISE, we can evaluate our
density estimate as a function of the bandwidth parameter and the data.
The terms without the order approximation is know as the Asymptotic MISE
(AMISE) \citep{wasserman2006all}.

\[
\begin{aligned}
MISE(h) &= \frac{1}{nh}\left(\int K(t)^2\, dt\right) + \frac{1}{4}h^4\sigma^4_K\left(\int f''(x)^2 \, dx\right) + o \left( \frac{1}{nh} + h^4\right)
\end{aligned}
\] We often estimate this term with cross-validation, with an estimator
off by a true constant term.

\begin{equation}
\label{est_risk}
\hat J(h) = \int \left(\hat{f_n}(x)\right)^2 \, dx - \frac{2}{n} \sum^{n}_{i=1}{ \hat{f}_{(-i)}(X_i)}
\end{equation}

\subsection{Bandwidth Estimation}\label{bandwidth-estimation}

Since good bandwidth choice has shown to be highly influential on the
quality of the density estimate, many rules have been developed to
choose a good bandwidth. Some of the more common methods are shown
briefly below

\begin{longtable}[]{@{}ll@{}}
\toprule
\begin{minipage}[b]{0.37\columnwidth}\raggedright\strut
Name of Rule\strut
\end{minipage} & \begin{minipage}[b]{0.57\columnwidth}\raggedright\strut
Estimation Procedure\strut
\end{minipage}\tabularnewline
\midrule
\endhead
\begin{minipage}[t]{0.37\columnwidth}\raggedright\strut
Silverman's Rule of Thumb\strut
\end{minipage} & \begin{minipage}[t]{0.57\columnwidth}\raggedright\strut
\(h_n = \left( \frac{4 \hat{\sigma}^5}{3n}\right)^{1/5}\) where
\(\hat\sigma = \min\left(s, \frac{IQR}{1.34}\right)\)\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.37\columnwidth}\raggedright\strut
Sheather-Jones\strut
\end{minipage} & \begin{minipage}[t]{0.57\columnwidth}\raggedright\strut
See \citep{SJ}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.37\columnwidth}\raggedright\strut
Cross-Validation\strut
\end{minipage} & \begin{minipage}[t]{0.57\columnwidth}\raggedright\strut
See \citep{scott_terrell}\strut
\end{minipage}\tabularnewline
\bottomrule
\end{longtable}

\begin{figure}[h]
\label{fig:bw_selection}
\caption{Comparison of Bandwidth Selection Rules}
\includegraphics[scale=.5]{bw_selection}
\centering
\end{figure}

\subsection{Curse of Dimensionality}\label{curse-of-dimensionality}

Density estimation in practice is generally used with lower dimensional
data. Consider a sample
\(\mathbf{x_1},\mathbf{x_2}, \ldots, \mathbf{x_n}\) from a
\(d\)-dimensional density function \(f\). As the sample space increases
in dimension, the pairwise distance between each of the points
increases, and our space will be increasingly sparse, making it more
difficult to estimate the underlying density function. We would thus
require an exponentially increasing amount of data corresponding to the
dimension to achieve a similar MSE lower bound. This is known as the
\emph{curse of dimensionality} \citep{scott2015multivariate}. We focus
on the kernel density estimator. In higher dimensions, our general
kernel density estimator is defined by

\[
\begin{aligned}
\hat{f}_n(\mathbf{x}) = \frac{1}{n} \sum^{n}_{i=1}{|\mathbf{H}|^{-1/2}K\left(\mathbf{H}^{-1/2}(\mathbf{x} - \mathbf{X_i})\right)}
\end{aligned}
\]

For higher dimensions, it has been shown that the optimal MISE for an
estimator runs on the order of \(\bigo{n^{-4/(d+4)}}\). Based on this
lower bound for MSE, we see that as \(n\rightarrow \infty\) our
estimator \(\hat{f}(x)\) converges in probability to \(f(x)\). The
bandwidth selection also becomes more difficult because instead of a
single parameter, we must now estimate a matrix of bandwidths.

\section{Computation}\label{computation}

\subsection{Using the FFT}\label{using-the-fft}

In situations the density estimate needs to be calculated on a grid of
points, such as plotting, calculating the kernel density estimator
directly is inefficient, requiring \(\bigo{n^22^g}\) time to run on a
dataset with \(n\) data points and \(g\) grid points. If we note that
the kernel density estimate is simply a convolution of the empirical
density function \(g(x) = \frac{1}{n} \sum_{i=1}^{n}\delta(x-x_i)\)
(where \(\delta\) is the generalized Dirac delta function) and the
symmetric kernel \(K_h(x)\) with bandwidth parameter \(h\), we can
utilize the Fast Fourier Transform to reduce the computational
complexity to \(\bigo{ng\log n}\)
\citep{wand1994fast, lohne2017computational}. This calculation, due to
rounding and underflow, the estimate of the density may lead to small
negative values in some cases. We can simply smooth these estimates by
setting them to zeros \citep{silverman1982algorithm}. This is the method
(slightly simplified) used by the function \texttt{density()} in
\texttt{R}. The number of grid points we discretize the space will also
have a significant effect on the algorithm speed. In particular, the
direct calculation will grow exponentially with the grid we discretize
over.

\begin{algorithm}[H]
\caption{Univariate KDE with Fast Fourier Transform}
\begin{algorithmic}
\Function{KDE\_FFT}{$X$, Bandwidth}
  \State mesh $\gets [\min X - \mathrm{sd}(X), \max X + \mathrm{sd}(X)]$
  \State meshcount $\gets \mathrm{hist}(X, \mathrm{mesh})$
  \State meshkernel $\gets \mathrm{dKern}(\mathrm{mesh})$ \Comment{density values of kernel over mesh}
  \State density $\gets \mathrm{FFT}^{-1}(\mathrm{FFT}(\mathrm{meshcount}) * \mathrm{FFT}(\mathrm{meshkernel}))$
  \State density $\gets$ SmoothZeros(density)
  \State \Return density
\EndFunction
\end{algorithmic}
\end{algorithm}

\begin{figure}[h]
\caption{Old Faithful Data (N=272)}
\includegraphics[scale=.5]{density_comparison}
\centering
\label{fig:plot1}
\end{figure}

The routines \texttt{brute\_kde()} and \texttt{fft\_kde()} were coded
myself from scratch, and we compare computational speeds to
\texttt{density()} provided by \texttt{R} in the univariate case on the
eruption times of the Old Faithful Geyser Data in Yellowstone National
Park \citep{azzalini1990look}. For comparability, we calculate the
discretization over a grid of \(512\) points which is default for the
\texttt{density()} function. From Figure \ref{fig:plot1} we can see that
all the algorithms seem to match up fairly well on our Old Faithful
Eruption data. We can attribute the slight deviations to rounding errors
and having used a different discrete grid. From Table 3 we see that
there is over \(1000x\) speed-up from using a brute force approach to
using the FFT. Our hand coded algorithm is also slightly faster than the
standard \texttt{density()} function. This is likely because we do not
do error checking or argument processing in our algorithms, such as
weights or kernel changing.

\begin{longtable}[]{@{}llll@{}}
\caption{Computational Time comparison (avg of 10 runs)}\tabularnewline
\toprule
Algorithm & \texttt{brute\_kde} & \texttt{fft\_kde} & \texttt{density}
from R\tabularnewline
\midrule
\endfirsthead
\toprule
Algorithm & \texttt{brute\_kde} & \texttt{fft\_kde} & \texttt{density}
from R\tabularnewline
\midrule
\endhead
Time(s) & 0.3667 & 0.000464 & 0.00073\tabularnewline
\bottomrule
\end{longtable}

\begin{figure}[h]
\label{fig:gs}
\caption{Comparison of Algorithms dependency on Gridsize}
\includegraphics[scale=.3]{gridsize_time}
\centering
\end{figure}

\subsection{Accuracy with Data}\label{accuracy-with-data}

In the above theoretical work, we saw that in the univariate case, the
best estimator with optimal bandwidth selection would have a convergence
rate of \(\bigo{n^{-4/5}}\) as a function of the number of data points
\(n\). We illustrate the result by calculating the cross-validated risk
estimator as shown in Eq. \ref{est_risk} with a near-optimal bandwidth
selector, Silverman's Rule of Thumb.

\begin{figure}[h]
\centering
\label{fig:kde_data}
\caption{The KDE estimator with Silverman's Rule of Thumb converges closer and closer to the ``true" distribution with more data. Faded densities are estimates with less data.}
\includegraphics[scale=.5]{kde_data}
\end{figure}

\begin{figure}[h]
\centering
\label{fig:lb}
\caption{Estimated risk of estimator with increasing data nears the theoretical lower bound of $n^{-4/5}$}
\includegraphics[scale=.5]{lb_silverman}
\end{figure}

\subsection{Higher Dimensions}\label{higher-dimensions}

Again, we would like to use the FFT in higher dimensions because of the
massive computational speedup for calculating the KDE. Here we present a
two dimensional example based on a UNICEF dataset on life expectancy and
child mortality. The problem of using a definitive FFT-based algorithm
for unconstrained bandwidth matrices is an open question, though some
significant progress was made recently \citep{fft_unconstrained}. The
technical details are omitted, but the main result is an observation
that the generalization presented in \citep{wand1994kernel}, has two
orderings, the \emph{zero-padding} and \emph{wrap-arround ordering}. The
wrap-around ordering will only support kernels that are oriented with
the axes, and thus, reshaping the FFT input matrix to use only zero
padded procedures will remove some symmetry aliasing. This order was
causing some irregularity in the density estimate as shown in Figure
\ref{fig:unconstrainedH}. They also suggest a scaling parameter of the
mesh size based on the largest eigenvalue of the bandwidth matrix
\(\mathbf{H}\). This is an interesting result, but doesn't guarantee
that weaknesses will not be found with the zero-padded ordered matrix.
This result has already been integrated into the package \texttt{ks} in
\texttt{R}.

\begin{figure}[h]
\centering
\caption{Reproduction of Figure 1 of (Gramacki \& Gramacki 2017) showing weakness of KernSmooth FFT implementation. Note (a) is different from (b) and (c)}
\label{fig:unconstrainedH}
\includegraphics[scale=.33]{fig1_unconstrained}
\end{figure}

\section{Conclusions}\label{conclusions}

Density estimation has seen many useful applications for understanding
of underlying data structure or as a first step in downstream analysis.
We've shown some asymptotic results that have led to a wide array of
other estimators, in particular the kernel density estimate. The quality
of the KDE is highly subject to bandwidth changes, but less so to the
kernel choice. The fundamental computational results in one dimension of
using the FFT was explored, and we can see at least a 1000-fold increase
in computational speed depending on the chosen gridsize. We note the
work of extending these results to the multivariate case has matured
significantly since the 1990s, which has reduced the computational
complexity and increased the accuracy to the point they have been
integrated into common software packages \citep{chen2017tutorial}. In
future years, we are excited to see maturity in other density estimation
techniques such as orthogonal series estimators, splines, or tree-based
density estimation.

\bibliographystyle{agsm}
\bibliography{bibliography.bib}

\end{document}


%% LyX 2.2.1 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass{article}
\usepackage[T1]{fontenc}
\usepackage[utf8x]{inputenc}
\usepackage[a4paper]{geometry}
\geometry{verbose,tmargin=2cm}
\usepackage{color}
\usepackage{verbatim}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[unicode=true,
 bookmarks=false,
 breaklinks=false,pdfborder={0 0 1},backref=section,colorlinks=true]
 {hyperref}
\hypersetup{
 allcolors=blue}
\usepackage{breakurl}

\makeatletter
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.

%% Language and font encodings
\usepackage[english]{babel}

%% Sets page size and margins

\linespread{1.2}

%% Useful packages
\usepackage{blkarray}
\usepackage{bm}
\usepackage{graphicx}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage{indentfirst}

\DeclareMathOperator*{\argmax}{argmax}

\title{Linear Optimization of filtering for AUI}
\author{Yihao Du}

\makeatother

\begin{document}
\maketitle

\section{Problem definition}

In this problem, the goal is to develop an efficient algorithm for
optimize filter setting for the retrieval set associated with maximum
set-score.

The scenario of global element set $E$ includes $m$ unique elements.
Each of the elements in this global set is associated with its score
$S$, which is used for further estimation of set-score. We intend
to develop a algorithm ($f$) of optimize filter setting for the optimal
retrieval set ($L$).
\begin{equation}
\begin{aligned} & \underset{f}{\text{maximize}} &  & \emph{L(f(E))}\end{aligned}
\end{equation}

It is easy to observe two significant parts in this problem: How to
efficiently filter unnecessary elements ($f$)? How to evaluate different
sets ($L$)?

\section{Set Evaluation Metrics}

In information retrieval, a Retrieval Set (RS) is usually evaluated
using three main metrics: precision, recall and F-score, which are
defined as follows:

\begin{equation}
\emph{\ensuremath{P(RS)}}=\dfrac{{\displaystyle \sum_{i\in RS}S(i)}}{|RS|}=\dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{\sum_{i=0}^{m}I(i)}
\end{equation}

\begin{equation}
\emph{\ensuremath{R(RS)}}=\dfrac{{\displaystyle \sum_{i\in RS}S(i)}}{|RE|}=\dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{{\displaystyle \sum_{i=0}^{m}S(i)}}=\dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{C}
\end{equation}

\begin{equation}
\emph{\ensuremath{F(RS)}}=\dfrac{2\times Precision\times Recall}{Precision+Recall}=\dfrac{2\times{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{{\displaystyle \sum_{i=0}^{m}I(i)}+{\displaystyle \sum_{i=0}^{m}S(i)}}=\dfrac{2\times{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{{\displaystyle \sum_{i=0}^{m}I(i)}+C}
\end{equation}

\noindent where, $S(i)$ is a binary score indication function defined
as:

\begin{equation}
S(i)=\begin{cases}
1, & \text{if element \ensuremath{i} is relevant}\\
0, & \text{otherwise}
\end{cases}
\end{equation}
Similarly, $I(i)$ is a binary variable that indicates whether an
element $i$ is selected in retrieval set, and is formally defined
as:

\begin{equation}
I(i)=\begin{cases}
1, & \text{iff \ensuremath{i\in RS}}\\
0, & \text{otherwise}
\end{cases}
\end{equation}
Finally, $\left|RE\right|$ is the size of the relevant element set.
The denominator in \emph{Recall} is score sum of global element set
$E$, which only depends on score of data. Therefore, it is a constant
$C$ in our problem.

In practice, a probabilistic score is used to approximate $S(i)$
because of unavailability of real label. Therefore, this relaxation
of the score $S(i)$ into a continuous value results in \emph{expected}
values for the metrics above.

\section{Search Algorithm Validation - Metrics Optimization}

In order to validate search algorithm, we are able to compare its
result with the one obtained by metrics optimization. The objective
function in this optimization problem is formulated based on different
metrics (Precision, Recall and F-score). The constraints are defined
based on different fields (time, keyword and position).

\subsection{Objective functions}

\subsubsection{Precision}

The optimization problem to maximize precision is formulated as follows:

\begin{equation}
\begin{array}{ll}
\underset{I}{\text{maximize}} & \dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{{\displaystyle \sum_{i=0}^{m}I(i)}}\\
s.t & I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

Note this is a Mixed-Integer Linear Fractional Programming (MILFP)
problem, so Charnes-Cooper transformation is used to transform the
problem into a Mixed-Integer Nonlinear Programming (MINLP) as follows:

\begin{equation}
\begin{array}{ll}
\underset{I}{\text{maximize}} & {\displaystyle \sum_{i=0}^{m}S(i)\cdot u\cdot I(i)}\\
s.t & {\displaystyle \sum_{i=0}^{m}u\cdot I(i)}=1\\
 & u>0,I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

\noindent where $u$ is defined as follows:

\begin{equation}
u=\dfrac{1}{{\displaystyle \sum_{i=0}^{m}I(i)}}
\end{equation}

Next, Glover's linearization scheme is used to convert the MINLP into
an equivalent Mixed-Integer Linear Programming (MILP) problem that
can be directly optimized. Hence, a new variables $y(i)=u\cdot I(i)$
is introduced again and using this, an equivalent MILP is given:

\begin{equation}
\begin{array}{ll}
\underset{y}{\text{maximize}} & {\displaystyle \sum_{i=0}^{m}S(i)\cdot y(i)}\\
s.t & {\displaystyle \sum_{i=0}^{m}y(i)}=1\\
 & y(i)\leqslant u,\forall i\in RS\\
 & y(i)\leqslant M\cdot I(i),\forall i\in RS\\
 & y(i)\geqslant u-M\cdot(1-I(i)),\forall i\in RS\\
 & u>0,y(i)\geq0,I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

\subsubsection{Recall}

Maximization of Recall can be formulated as the following optimization
problem:

\begin{equation}
\begin{array}{ll}
\underset{I}{\text{maximize}} & \dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{C}\\
s.t & I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

This problem is reformulated with Glover's linearization as follows:

\begin{equation}
\begin{array}{ll}
\underset{y}{\text{maximize}} & {\displaystyle \sum_{i=0}^{m}S(i)\cdot y(i)}\\
s.t & y(i)\leqslant\dfrac{1}{C},\forall i\in RS\\
 & y(i)\leqslant M\cdot I(i),\forall i\in RS\\
 & y(i)\geqslant u-M\cdot(1-I(i)),\forall i\in RS\\
 & y(i)\geq0,I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

\subsubsection{F-score}

Maximization of F-score can be formulated as the following optimization
problem:

\begin{equation}
\begin{array}{ll}
\underset{I}{\text{maximize}} & \dfrac{{\displaystyle \sum_{i=0}^{m}S(i)\cdot I(i)}}{{\displaystyle \sum_{i=0}^{m}I(i)}+C}\\
s.t & I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

—

Note this is a Mixed-Integer Linear Fractional Programming (MILFP)
problem, so Charnes-Cooper transformation is used to transform the
problem into a Mixed-Integer Nonlinear Programming (MINLP) as follows:

\begin{equation}
\begin{array}{ll}
\underset{I}{\text{maximize}} & {\displaystyle \sum_{i=0}^{m}S(i)\cdot u\cdot I(i)}\\
s.t & {\displaystyle \sum_{i=0}^{m}u\cdot I(i)}\leq1\\
 & u>0,I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

\noindent where $u$ is defined as follows:

\begin{equation}
u=\dfrac{1}{{\displaystyle \sum_{i=0}^{m}I(i)}+C}
\end{equation}

—

With Charnes-Cooper transformation and Glover's linearization method,
this optimization problem can be transformed into the following problem.

\begin{equation}
\begin{array}{ll}
\underset{y}{\text{maximize}} & \sum_{i=0}^{m}S(i)\cdot y(i)\\
s.t & \sum_{i=0}^{m}y(i)+uC=1\\
 & y(i)\leqslant u,\forall i\in RS\\
 & y(i)\leqslant M\cdot I(i),\forall i\in RS\\
 & y(i)\geqslant u-M\cdot(1-I(i)),\forall i\in RS\\
 & u>0,y(i)\geq0,I(i)\in\{0,1\},\forall i\in RS
\end{array}
\end{equation}

\subsection{Constraints}

\subsubsection{Time}

Time filter setting is a two - element tuple $(ts,te)$, which indicates
start ($ts$) and end ($te$) of time-line. For element $i$ with
its time $t(i)$, the time constraint for $I(i)$ can be formulated
as follows:

\begin{comment}
\%\textbackslash{}begin\{equation\}

\%(ts \textbackslash{}leqslant t(i)) \textbackslash{}land (t(i) \textbackslash{}leqslant
te) \textbackslash{}implies (I(i) = 1)

\%\textbackslash{}end\{equation\}
\end{comment}

\begin{equation}
I(i)=\begin{cases}
1, & \text{if \ensuremath{(ts\leqslant t(i))\land(t(i)\leqslant te)}}\\
0, & \text{otherwise}
\end{cases}
\end{equation}

\subsubsection{Keyword}

Keyword filter setting is a set of unigram ($K$), which filters the
element does not have these words. For element $i$ with its whole
unigram set $W(i)$, the keyword constraint for $I(i)$ can be formulated
as follows:

\begin{equation}
I(i)=\begin{cases}
1, & \text{if \ensuremath{K\cap W(i)\neq\emptyset}}\\
0, & \text{otherwise}
\end{cases}
\end{equation}

\textcolor{red}{The implementation method is still unclear in keyword
field.}

\subsubsection{Position}

Position filter setting is a four - element tuple (xmin, ymin, xmax,
ymax), which filters the element out of a bounding box created based
on tuple. For element $i$ with its position ($x(i),y(i)$), the position
constraint for $I(i)$ can be formulated as follows:

\begin{equation}
I(i)=\begin{cases}
1, & \text{if \ensuremath{(xmin\leqslant x(i))\land(x(i)\leqslant xmin)\land(ymin\leqslant y(i))\land(y(i)\leqslant ymax)}}\\
0, & \text{otherwise}
\end{cases}
\end{equation}

\end{document}


\documentclass{memoir}
\usepackage{notestemplate}

%\logo{~/School-Work/Auxiliary-Files/resources/png/logo.png}
%\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
%\department{Department of Mathematics}
%\title{Class Notes}
%\subtitle{Based on MATH xxx}
%\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
%\date{\today}

\begin{document}

% \maketitle

% Notes taken on 05/10/21

\section{Solvable Groups}
\label{sec:solvable_groups}

\begin{defn}[Radical Extension]
	A field \(K\) is said to be a \textbf{radical extension} of a field \(F\) if there is a chain of fields
	\begin{align*}
		F = F_0 \subset F_1 \subset F_2 \subset \ldots\subset F_n = K
	\end{align*}
	such that, for each \(i=1,\ldots,n\), \(F_i = F_{i-1}(\alpha_i)\) and some power of \(\alpha_i\) is in \(F_{i-1}\).
\end{defn}
Let \(f \in F[x]\). The equation \(f(x) = 0_F\) is \textbf{solvable by radicals} if there exists a radical extension of \(F\) that contains a splitting field of \(f(x)\). This is equivalent to the notion of there existing a "formula" for the solutions.

\begin{defn}[Solvable]
	A group \(G\) is said to be \textbf{solvable} if it has a chain of subgroups
	\begin{align*}
		\langle e \rangle = G_n \triangleleft \ldots \triangleleft G_1 \triangleleft G_0 = G
	\end{align*}
	such that each quotient group \(G_{i-1} / G_i\) is abelian.
\end{defn}
Notice that all abelian groups are solvable.
\begin{prop}
	For \(n\geq 5\) the group \(S_n\) is not solvable.
\end{prop}
\begin{thm}
	Every homomorphic image of a solvable group \(G\) is solvable.
\end{thm}

Our goal is to prove the Galois Criterion. That is, let \(f \in F[x]\). \(f(x) = 0_F\) is solvable by radicals if and only if the Galois group of \(f(x)\) is a solvable group.

\begin{lemma}
	Let \(F\) be a field and \(\eta \) a primitive \(n\)-th root of unity in \(F\). Then \(F\) contains a primitive \(d\)-th root of unity for every positive \(d\mid n\).
\end{lemma}

This combined with the next two theorems will allow us to prove the Galois Criterion.

\begin{thm}
	Let \(F\) be a field of characteristic zero and \(\eta \) a primitive \(n\)-th root of unity in some field extension of \(F\). Then \(K = F(\eta )\) is a normal extension of \(F\) and \(\textrm{Gal}_F(K)\) is abelian.
\end{thm}

\begin{thm}
	Let \(F\) be a field of characteristic zero that contains a primitive \(n\)-th root of unity. If \(\alpha \) is a root of \(x^{n}-c \in F[x]\) in some extension field of \(F\), then \(K = F(\alpha )\) is a normal extension of \(F\) and \(\textrm{Gal}_F(K)\) is abelian.
\end{thm}

\begin{lemma}
	Let \(F,E,K\) be fields of characteristic zero with
	\begin{align*}
		F\subset E\subset K = E(\alpha )\quad \alpha^k \in E
	\end{align*}
	If \(K\) is finite-dimensional over \(F\) and \(E\) is normal over \(F\), then there exists a field extension \(L\) of \(K\) which is a radical extension of \(E\) and a normal extension of \(F\).
\end{lemma}

\begin{thm}[Galois Criterion]
Let \(f \in F[x]\). \(f(x) = 0_F\) is solvable by radicals if and only if the Galois group of \(f(x)\) is a solvable group.

\end{thm}

We can use this to show that there is no formula for the solutions of all fifth-degree polynomials, as there are fifth-degree polynomials whose Galois group is \(S_5\).

\begin{thm}
	Let \(F\) be a field of characteristic zero and \(f(x) \in F[x]\). If \(f(x) = 0_F\) is solvable by radicals, then there is a normal radical field extension of \(F\) that contains the splitting field of \(f(x)\).
\end{thm}

\begin{thm}
	Let \(K\) be a normal radical field extension of \(F\) and \(E\) an intermediate field, all of characteristic zero. If \(E\) is normal over \(F\), then \(\textrm{Gal}_F(E)\) is a solvable group.
\end{thm}

\end{document}

Abstract Algebra - Introductory/Algebra II/Notes/source/Lecture31 - SolvableGrps_GalCriterion.tex


%%%%%%%%%%%%%%%%%%%%%% BFS and DFS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{BFS and DFS}

\begin{frame}
  \frametitle{BFS and DFS}

  \begin{quote}
    \textcolor{blue}{``For fundamental achievements in the design and analysis of algorithms and data structures.''}

    \hfill {--- Turing Award 1986}
  \end{quote}

  \begin{columns}
    \begin{column}{0.50\textwidth}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/Tarjan}
        \caption{{\scriptsize Robert Endre Tarjan (April 30, 1948).}}
        \label{fig:Tarjan}
      \end{figure}
    \end{column}

    \begin{column}{0.50\textwidth}
      \begin{figure}
        \includegraphics[scale=0.50]{figure/bfs_dfs/hopcroft}
        \caption{{\scriptsize John Edward Hopcroft (October 7, 1939)}.}
        \label{fig:hopcroft}
      \end{figure}
    \end{column}
  \end{columns}

\end{frame}

     %%%%%%%% Exploring types of edges on both BFS and DFS %%%%%%%
\subsection{Exploring Types of Edges on Both BFS and DFS}

\begin{frame}
  \frametitle{Types of edges on DFS}

  \begin{columns}
    \begin{column}{0.50\textwidth}
      \textcolor{purple}{DFS on digraph:}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/dfsdi}
        \caption{{\scriptsize DFS on digraph.}}
        \label{fig:dfsdi}
      \end{figure}
    \end{column}

    \begin{column}{0.50\textwidth}
      \textcolor{purple}{DFS on undirected graph  \\ ([\textcolor{blue}{$P_{380}$ 7.28}]):}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/dfsundirected}
        \caption{{\scriptsize DFS on undirected graph.}}
        \label{fig:dfsundirected}
      \end{figure}
    \end{column}
  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Types of edges on BFS}

  \begin{columns}
    \begin{column}{0.50\textwidth}
      \textcolor{purple}{BFS on digraph:}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/bfsdi}
        \caption{{\scriptsize BFS on digraph.}}
        \label{fig:bfsdi}
      \end{figure}
    \end{column}

    \begin{column}{0.50\textwidth}
      \textcolor{purple}{BFS on undirected graph:}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/bfsundirected}
        \caption{{\scriptsize BFS on undirected graph.}}
        \label{fig:bfsundirected}
      \end{figure}
    \end{column}
  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Cycle detection problems}

  \begin{table}
    \begin{tabular}{|c|c|c|}
      \hline
                & Undirected graph          & Directed graph ([\textcolor{blue}{$P_{379}$ 7.17}])           \\ \hline
      BFS       & \tabincell{c}{\includegraphics[scale=0.20]{figure/bfs_dfs/bfsundirected} \\ (\textcolor{red}{Cross edge})}
                & \tabincell{c}{\includegraphics[scale=0.20]{figure/bfs_dfs/bfsdi} \\ (\textcolor{blue}{Back edge?})}            \\ \hline
      DFS       & \tabincell{c}{\includegraphics[scale=0.20]{figure/bfs_dfs/dfsundirected} \\ \textcolor{red}{Back edge}}
                & \tabincell{c}{\includegraphics[scale=0.20]{figure/bfs_dfs/dfsdi} \\ \textcolor{red}{Back edge}}                              \\ \hline
    \end{tabular}
  \end{table}

\end{frame}

\begin{frame}
  \frametitle{Cycle detection problems}

  \begin{center}
    \textcolor{purple}{Using BFS on undirected graph for cycle detection:}
  \end{center}

  \begin{columns}
    \begin{column}{0.50\textwidth}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/bfsdi}
        \caption{{\scriptsize BFS on digraph with back edges.}}
        \label{fig:bfsundirected}
      \end{figure}
    \end{column}

    \begin{column}{0.50\textwidth}
      \begin{figure}
        \includegraphics[scale=0.30]{figure/bfs_dfs/bfsdinoback}
        \caption{{\scriptsize BFS on digraph without back edges.}}
        \label{fig:bfsdinoback}
      \end{figure}
    \end{column}
  \end{columns}

\end{frame}

    %%%%%%%% Exploring types of edges on both BFS and DFS %%%%%%%
\subsection{Exploring DFS's Active Intervals}

\begin{frame}
  \frametitle{Exploring DFS's active intervals}

    \begin{figure}
      \includegraphics[scale=0.40]{figure/bfs_dfs/activeinterval}
      \caption{{\scriptsize Active intervals in DFS on digraph.}}
      \label{fig:activeinterval}
    \end{figure}

\end{frame}

\begin{frame}
  \frametitle{Exploring DFS's active intervals}

  \begin{enumerate}
    \item Who is ancestor ?
    \item How many descendants ?
      \pause
    \item topological sorting (on digraph)
    \item strongly-connected components (on digraph)
    \item biconnected components (on undirected graph)
  \end{enumerate}

\end{frame}

\subsubsection{DAG and Topological Sorting}
\begin{frame}
  \frametitle{DAG and topological sorting}

  \begin{enumerate}
    \setlength{\itemsep}{0.20cm}

    \item absence of back edges = acyclicity (DAG)
    \item topological sorting
      \begin{itemize}
        \item source, sink
      \end{itemize}
  \end{enumerate}

  \begin{figure}
    \includegraphics[scale=0.40]{figure/bfs_dfs/DAG}
    \caption{{\scriptsize Example of DAG.}}
    \label{fig:activeinterval}
  \end{figure}

\end{frame}

%    \item critical path
%      \begin{itemize}
%        \item longest path in DAG
%      \end{itemize}

%\begin{frame}
%  \frametitle{DAG and Topological Sorting}
%
%  \begin{columns}
%    \begin{column}{0.40\textwidth}
%      \begin{figure}
%        \includegraphics[scale=0.30]{figure/bfs_dfs/longestpath}
%        \caption{{\scriptsize DAG with distance layers.}}
%        \label{fig:longestpath}
%      \end{figure}
%    \end{column}
%
%    \begin{column}{0.60\textwidth}
%      \begin{figure}
%        \includegraphics[scale=0.30]{figure/bfs_dfs/longestpathalg}
%        \caption{{\scriptsize Dag longest path algorithm.}}
%        \label{fig:longestpathalg}
%      \end{figure}
%    \end{column}
%
%  \end{columns}
%
%    \pause
%    \textcolor{purple}{In general graphs, longest path problem is NP-hard.}
%
%\end{frame}

\begin{frame}
  \frametitle{DAG and topological sorting}

  \begin{itemize}
    \item critical path $\to$ longest path in DAG
  \end{itemize}

  \pause
  \vspace{0.50cm}

  \textcolor{purple}{Note: In general graphs, longest path problem is NP-hard !}

\end{frame}

\subsubsection{Strongly Connected Components of a Digraph}

\begin{frame}
  \frametitle{SCC of digraph}

  \textcolor{purple}{``Two-trier'' structure of digraph} ([\textcolor{blue}{$P_{379}$ 7.22}]):

  \begin{figure}
    \includegraphics[scale=0.35]{figure/bfs_dfs/SCCDI}
    \label{fig:scc}
  \end{figure}

  \pause

{\small

  \begin{enumerate}
    \item ``The node that has \textcolor{red}{highest} finishTime in DFS must lie in a \textcolor{red}{source SCC''};
    \pause
    \item What we need is sink! So try \textcolor{red}{$G^R$}; ([\textcolor{blue}{$P_{380}$ 7.26}])
    \item When a sink SCC is found (by connectivity), just delete it and continue \textcolor{red}{recursively}.
  \end{enumerate}
}

\end{frame}

\subsubsection{Biconnected Components of an Undirected Graph}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \textcolor{purple}{Algorithms for biconnected components:}
  \begin{description}
    \setlength{\itemsep}{0.20cm}
    \item [Brute force:] try every vertex and check connectivity: $O \big( n(m+n) \big )$.
    \item [Clever approach:] single DFS making use of \textcolor{blue}{\emph{back edges and active intervals}}.
  \end{description}

\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

    \begin{columns}
      \begin{column}{0.50\textwidth}
         \begin{figure}
            \begin{center}
              \includegraphics[scale=0.30]{figure/bfs_dfs/dfsbiconn}
              \caption{What if DFS tree is the entirety of the graph ?}
              \label{fig:dfsbiconn}
            \end{center}
          \end{figure}
      \end{column}

      \begin{column}{0.50\textwidth}
          \begin{figure}
            \begin{center}
              \includegraphics[scale=0.30]{figure/bfs_dfs/backedgebiconn}
              \caption{Back edge, cycle, no articulation vertices.}
              \label{fig:backedgebiconn}
            \end{center}
          \end{figure}
      \end{column}

    \end{columns}

\end{frame}

%\begin{frame}
%  \frametitle{Biconnected components of an undirected graph}
%
%  \textcolor{purple}{Important information:}
%
%  \textcolor{blue}{the extend to which back edges link chunks of the DFS tree back to ancestor nodes:}
%  \begin{enumerate}
%    \setlength{\itemsep}{0.20cm}
%
%    \item $v$ is first discovered.
%        \[
%          back = discoverTime(v).
%        \]
%    \item back edge $vw$.
%        \[
%          back = min(back, discoverTime(w)).
%        \]
%    \item backtracking from $w$ to $v$.
%        \[
%          back = min(back, wback).
%        \]
%  \end{enumerate}
%
%\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \begin{figure}
    \begin{center}
      \includegraphics[scale=0.40]{figure/bfs_dfs/bicon1}
      \caption{{\scriptsize Three cases of articulations (1).}}
      \label{fig:bicon1}
    \end{center}
  \end{figure}

\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \begin{figure}
    \begin{center}
      \includegraphics[scale=0.40]{figure/bfs_dfs/bicon2}
      \caption{{\scriptsize Three cases of articulations (2).}}
      \label{fig:bicon2}
    \end{center}
  \end{figure}

\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \begin{figure}
    \begin{center}
      \includegraphics[scale=0.40]{figure/bfs_dfs/bicon3}
      \caption{{\scriptsize Three cases of articulations (3).}}
      \label{fig:bicon3}
    \end{center}
  \end{figure}

\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \begin{figure}
    \begin{center}
      \includegraphics[scale=0.30]{figure/bfs_dfs/articulationex}
      \caption{Three cases of articulations.}
      \label{fig:articulations}
    \end{center}
  \end{figure}

\end{frame}

\begin{frame}
  \frametitle{Biconnected components of an undirected graph}

  \textcolor{purple}{Q:} How the reachability relation impacts whether $v$ is an articulation vertex ?
  \begin{enumerate}
    \item Root cut-nodes ([\textcolor{blue}{$P_{382}$ 7.37}]).
    \item Bridge cut-nodes.
    \item Parent cut-nodes.
  \end{enumerate}

  \pause
  \vspace{0.40cm}

    Q : $\textrm{Back } \ge \textrm{discoverTime} \lbrack v \rbrack$ ([\textcolor{blue}{$P_{383}$ 7.42}]).

  \pause

    A : $\textrm{Back } = \textrm{discoverTime} \lbrack v \rbrack$. It just detects Bridge cut-nodes.

\end{frame}