\documentclass[11pt]{article}
\usepackage{listings}
\usepackage{hyperref}
\usepackage{fullpage}
\usepackage{amsmath}
\usepackage{url}

\lstdefinelanguage{pbcc}{
morekeywords={transform, if, else, for, from, to, using, case, do, where,
              return, or, in_training, secondary, primary, priority, generator,
              template, assert, while, config, MatrixRegion1D, MatrixRegion2D,
              MatrixRegion3D, cell, region, row, col},
sensitive=true,
morecomment=[l]{\#},
morecomment=[l]{//},
morestring=[b]',
morestring=[b]",
}
\lstset{language=pbcc}
\lstset{numbers=left}
\lstset{tabsize=2}
\lstset{commentstyle=\textit}
\lstset{literate=
{REFTransform}{{\ref{sec:transform}}}{3}
{REFRule}     {{\ref{sec:rule}}}{3}
{REFBody}     {{\ref{sec:body}}}{3}
}

\begin{document}

\title{PetaBricks Language Specification}
\author{Jason Ansel}
\maketitle
\tableofcontents

\newpage
\section{Syntax}

\subsection{Minimal PetaBricks Program}

Figure~\ref{ex1} is an example of a (near) minimal PetaBricks program. It
copies the matrix A to the matrix B.

\begin{figure}[h]
\begin{lstlisting}
transform Copy2D                                                 REFTransform
to B[w,h]                                                        REFTransform
from A[w,h]                                                      REFTransform
{
  to(B.cell(x,y) b)                                              REFRule
  from(A.cell(x,y) a)                                            REFRule
  {
    b=a;                                                         REFBody
  }
}
\end{lstlisting}
This defines the transform Copy2D, which can be invoked as follows:

\begin{lstlisting}
MatrixRegion2D a,b;
...
Copy2D(a,b);
\end{lstlisting}
  \caption{
  A minimal PetaBricks program.  This invocation will result in the rule,
  defined on lines~5-9, being invoked on every pair of elements in $a$
  and $b$ in an undefined order.
  \label{ex1}
  }
\end{figure}

\subsection{Transform Structure}

The high level structure of a transform is as follows.

\begin{lstlisting}
Transform Header  [REFTransform]
{
  Rule Header     [REFRule]
  {
    Rule Body     [REFBody]
  }

  Rule Header     [REFRule]
  {
    Rule Body     [REFBody]
  }

  ...
}
\end{lstlisting}

\subsection{Transform Header}
\label{sec:transform}

The header of a transform consists of an unordered, whitespace separated
list of clauses.  The minimal set of clauses are TRANSFORM, FROM, and TO.
All of the allowed clauses are defined as follows:

\begin{figure}[h]
\begin{lstlisting}
transform Sum1Da
to Total
from A[n]
{
  //Compute the total sequentially
  to(Total t) from(A a) {
    t=0;
    for(int i=0; i<n; ++i)
      t+=a.cell(i);
  }

  //Divided and conquer
  to(Total t) from(A.region(0,n/2) left, A.region(n/2,n) right) {
    t=Sum1Da(left)+Sum1Da(right);
  }
}
\end{lstlisting}
  \caption{
  Sum all of the elements in an array.
  \label{ex2}
  }
\end{figure}

\begin{figure}[h]
\begin{lstlisting}
transform Sum1Db
to Total
using Partial[n/2]
from A[n]
{
  //sum two neighboring elements
  to(Partial.cell(i) p) from(A.cell(2*i) a1, A.cell(2*i+1) a2){
    p=a1+a2;
  }

  //lower priority rule to handle odd sized arrays
  secondary to(Partial.cell(i) p) from(A.cell(2*i) a1){
    p=a1;
  }

  //recursively call self to finish summing
  to(Total t) from(Partial p){
    Sum1Db(t,p);
  }

  //optionally skip Partial and just compute the output directly
  //this serves as the base case for the recursive calls
  to(Total t) from(A a){
    Sum1Da(t,a);
  }
}
\end{lstlisting}
  \caption{
  Sum all of the elements in an array, by making an intermediate array of size n/2 or by calling Sum1Da.
  \label{ex3}
  }
\end{figure}

\subsubsection{TRANSFORM Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & Yes                    \\
\bf Available In Version & 1.0        \\
\bf Syntax & \tt transform IDENTIFIER[\ref{IDENTIFIER}] \\
\bf Example & \tt transform Foo       \\
\bf Used in & \tt Figure~\ref{ex1}      \\
\hline
\end{tabular}

~

\noindent
Specifies the externally visible name of the transform.

\subsubsection{FROM Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & Yes                    \\
\bf Available In Version & 1.0        \\
\bf Syntax & \tt from MATRIXDEFLIST[\ref{MATRIXDEFLIST}] \\
\bf Example & \tt from A[w,h], B[w]     \\
\bf Used in & \tt Figure~\ref{ex2}      \\
\hline
\end{tabular}

~

\noindent
\noindent Defines the inputs of the transform.  The sizes of the outputs
are given as expressions containing unbound variables.  The compiler and
runtime must find a valid assignment for these unbound variables to match
the sizes of the inputs given.  The values assigned to these variables are
visible both in rule headers and in rule bodies.

\subsubsection{TO Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & Yes                    \\
\bf Available In Version & 1.0        \\
\bf Syntax & \tt to MATRIXDEFLIST[\ref{MATRIXDEFLIST}]     \\
\bf Example & \tt to C[w,h], D[w]     \\
\bf Used in & \tt Figure~\ref{ex2}      \\
\hline
\end{tabular}

~

\noindent Defines the outputs of the transform.  The sizes of the outputs
are given as expressions containing unbound variables that are set based on the size
of the inputs.

\subsubsection{USING Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt using MATRIXDEFLIST[\ref{MATRIXDEFLIST}]     \\
\bf Example & \tt using E[w,h], F[w]     \\
\bf Used in & \tt Figure~\ref{ex3}      \\
\hline
\end{tabular}

~

\noindent Defines intermediate data used during the transform execution. This
data is temporary and neither an input nor an output.  The sizes are given
as expressions containing unbound variables that are set based on the size
of the inputs.

\subsubsection{GENERATOR Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 2.0            \\
\bf Syntax & \tt generator IDENTIFIERLIST[\ref{IDENTIFIERLIST}]\\
\bf Example & \tt generator Bar\\
\bf Used in & \tt Figure~\ref{ex4}      \\
\hline
\end{tabular}

~

\noindent Specifies the name of another transform (or list of transforms)
that should be used to generate inputs to this transform during training.
The outputs of the other transforms are paired left to right with the inputs
of this transform.  The other transforms inputs are seeded with random data
of the desired input size.

If this clause is not used, transforms will be given random data during
training.

\begin{figure}[h]
\begin{lstlisting}
transform Permute
to B[n]
from A[n], Indices[n]
generator PermuteGenerator
{
  to(B.cell(i) b) from(A a, Indices.cell(i) idx){
    assert idx < n;
    b=a.cell(idx);
  }
}

//generate random test data for Permute
transform PermuteGenerator
to A[n], Indices[n]
from A[n]
{
  to(Indices.cell(i) idx) {
    idx=randint(0, n-1);
  }
}
\end{lstlisting}
  \caption{
  Example usage for generator keyword.  Computes $B[i]=A[Indicies[i]]$.
  \label{ex4}
  }
\end{figure}

\subsubsection{TEMPLATE Clause}

\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 2.0            \\
\bf Syntax & \tt template < IDENTIFIER(INT, INT), ... > \\
\bf Example & \tt template < precision(1,5) > \\
\bf Used in & \tt Figure~\ref{ex5}      \\
\hline
\end{tabular}

~

\noindent Defines a parameter to the transform that requires independent
training for each value.  This causes N transforms to be generated, one for
each possible value.  The first INT specifies the minimum value while the
second specifies the maximum value.  The new transforms can be called with
the syntax: Name$<$ParamValue$>$.

\begin{figure}[h]
\begin{lstlisting}
template < reps(1,5) >
transform Foo
to B[n]
from A[n]
{
  to(B b) from(A a) {
    Bar(b, a);
    for(int i=1; i<reps; ++i)
      Bar(b, b);
  }
}

MatrixRegion1D a,b;
...
Foo<3>(b, a);
\end{lstlisting}
\caption{
Example usage for template keyword.
\label{ex5}
}
\end{figure}

\subsubsection{Default Values}

\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 2.0            \\
\bf Example & \tt from IN[h,w]=OUT[h,w], iterations=5 \\
\hline
\end{tabular}

In the {\tt FROM} clause, the programmer may specify default values for transform inputs.
These default values work the same way as default values in {\tt C++}.  This provides an easy way for defining in place transforms.

\subsection{Rule Header}
\label{sec:rule}

The header for a rule consists of an unordered, whitespace-separated list
of clauses.  The meanings of these clauses are defined below.  The only
required clause is TO.

\subsubsection{TO Clause}
\label{TO}
\begin{tabular}{| l | l |}
\hline
\bf Required & Yes                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt to( MATRIXARGLIST[\ref{MATRIXARGLIST}] ) \\
\bf Example & \tt to( A.cell(x,y) a, B.region(0, 0, x, y) b)\\
\hline
\end{tabular}

~

\noindent Specifies the outputs for the rule.

The coordinates in the TO and FROM clause must correspond to an affine
transformation of each other.

\subsubsection{FROM Clause}
\label{FROM}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt from( MATRIXARGLIST[\ref{MATRIXARGLIST}] ) \\
\bf Example & \tt from( A.cell(x,y) a, B.region(0, 0, x, y) b)\\
\hline
\end{tabular}

~

\noindent Specifies the inputs for the rule.

The coordinates in the TO and FROM clause must correspond to an affine
transformation of each other.

\subsubsection{USING Clause}
\label{USING}
\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 2.0             \\
\bf Syntax & \tt using( MATRIXDEFLIST[\ref{MATRIXDEFLIST}] ) \\
\bf Example & \tt using( T1[10], T2[w+h, w+h] ) \\
\hline
\end{tabular}

~

\noindent Defines temporary buffers required by the body of the rule.
PetaBricks will automatically allocate and reuse these temporary buffers.

\subsubsection{HINT Clause}
\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt hint( EXPRESSION[\ref{EXPRESSION}] )   \\
\bf Example & \tt hint( n )           \\
\hline
\end{tabular}

~

\noindent Provide a hint to to autotuner that the decision to use this
rule should be based on comparing the given EXPRESSION to some constant.
Note that as of version 2.0 this language feature should no longer be needed
except in special cases.

\subsubsection{WHERE Clause}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 2.0             \\
\bf Syntax & \tt where EXPRESSION[\ref{EXPRESSION}] \\
\bf Example & \tt where x<10               \\
\hline
\end{tabular}

~

\noindent Limits the range where this rule may be applied.

\subsubsection{PRIORITY Clause}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt priority(int)             \\
\bf Example & \tt priority(1)              \\
\hline
\end{tabular}

~

\noindent Specify the priority for the given rule.  This is a clean way for
specifying handling of corner cases.  For a given cell, only rule with a
minimum priority may be applied.  The default priority is $1$.

\subsubsection{PRIMARY Keyword}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt primary             \\
\hline
\end{tabular}

~

\noindent Equivalent to {\tt priority(0)}

\subsubsection{SECONDARY Keyword}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt secondary \\
\hline
\end{tabular}

~

\noindent Equivalent to {\tt priority(2)}

\subsubsection{ROTATABLE Keyword}
\begin{tabular}{| l | l |}
\hline
\bf Required & No                          \\
\bf Available In Version & 1.0             \\
\bf Syntax & \tt rotatable \\
\hline
\end{tabular}

~

TODO: Define this more precisely.

\noindent Syntactic sugar for defining four rules with the coordinate
system rotated by $\frac \pi 2$ in each consecutive one.

\subsubsection{Return-style}

\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 1.0        \\
\hline
\end{tabular}

~

PetaBricks allows a shorter version of rule headers that eliminates
the need for a TO clause.  It is similar to function style return syntax.

\begin{minipage}{\linewidth}
\begin{lstlisting}
B.cell(x,y) from(A.cell(x,y) a) {
  return a;
}
\end{lstlisting}
\end{minipage}
Is desugared to:

\begin{minipage}{\linewidth}
\begin{lstlisting}
to(B.cell(x,y) _rv) from(A.cell(x,y) a) {
  _rv=a;
  return;
}
\end{lstlisting}
\end{minipage}

\subsubsection{OR Keyword}

\begin{tabular}{| l | l |}
\hline
\bf Required & No \\
\bf Available In Version & 2.0        \\
\hline
\end{tabular}

~

PetaBricks provides an ``OR'' keyword to eliminate the need for redundant
rule headers.  ``OR'' simply means repeat the last rule header. For example:

\begin{minipage}{\linewidth}
\begin{lstlisting}
to(B b) from(A a) {
  Foo(b,a);
} or {
  Bar(b,a);
}
\end{lstlisting}
\end{minipage}
Is desugared to:

\begin{minipage}{\linewidth}
\begin{lstlisting}
to(B b) from(A a) {
  Foo(b,a);
}
to(B b) from(A a) {
  Bar(b,a);
}
\end{lstlisting}
\end{minipage}

\subsection{Rule Body}
\label{sec:body}

Rule bodies are standard {\tt C++} code with some preprocessing performed.
All variables defined in the corresponding rule header are visible from the
rule body.

\subsubsection{MatrixRegion Class}
Inputs an output matrix regions are of the template type {\tt
MatrixRegion<N,T>}, where $N$ is the number of dimensions and $T$ is the
element type.  The typedefs: {\tt MatrixRegion0D}, {\tt MatrixRegion1D},
{\tt MatrixRegion2D}, ... are provided to refer to output matrix regions of each
dimensionality.  Additionally the typedefs {\tt ConstMatrixRegion0D}, {\tt
ConstMatrixRegion1D}, {\tt ConstMatrixRegion2D}, ... refer to matrix
regions with read only data and are used for inputs to a rule.

This class is by reference, so using it never copies data. To make a copy,
one must explicitly call {\tt MatrixRegion::allocate} and then copy.
MatrixRegion supports numerous operations not discussed here.  The API
documentation for this class can be found here:

\noindent
\url{http://kleptocracy.csail.mit.edu/pbapi/d9/db0/classpetabricks_1_1MatrixRegion.html}

\subsubsection{Configuration/Tunable Parameters}

The user may specify configuration parameters with:
\begin{verbatim}
config(name, initial, min, max);
\end{verbatim}
in the file header.  Both min and max are optional and default to 0 and
MAX\_INT.  These parameters are stored in the program configion file, but are
{\bf not} autotuned.  The value may be changed by hand, with {\tt configtool}
(included in the scripts folder), and at runtime with {\tt name.setValue(int)}.
The current value can be accessed with either {\tt name} or {\tt name.value()}.

The user may specify tunable parameters with:
\begin{verbatim}
tunable(name, initial, min, max);
\end{verbatim}
in the file header.  This is the same as a {\tt config} except that autotuner
will attempt to determine an optimal value automatically.

\subsubsection{Training Runs}

Some transforms require a training phase to determine configuration parameters
such as the number of iterations.  The autotuner supports this through the
API function {\tt PetabricksRuntime::isTrainingRun()}, or the conditional macro
{\tt in\_training}.  On the first execution with a given configuration,
calling this method will return {\tt true} and as a side effect the timing
results will be discarded and the test re-run.  On subsequent executions
this method will return {\tt false} and have no side effects.

\begin{figure}[h]
\begin{lstlisting}
transform Foo
to B[n]
from A[n]
{
  to(B b) from(A a) {
    config(reps);
    in_training {
      // experimentally determine how many reps are needed
      // for convergence in the current configuration
      reps = 0;
      while(! isConverged(b,a) ){
        reps++;
        Bar(b, b, a);
      }
    }else{
      for(int i=1; i<reps; ++i){
        Bar(b, b, a);
      }
    }
  }
}
\end{lstlisting}
\caption{
Example usage for training runs
\label{ex6}
}
\end{figure}

\subsubsection{Clean Abort}

Transforms can abort execution (in the event of an invalid configuration or
input) using the API function {\tt PetabricksRuntime::abort()}.  During training
this will cleanly end the current trail and remove the source configuration
from consideration.

\subsection{Definitions}
\subsubsection{MATRIXDEFLIST}
\label{MATRIXDEFLIST}

A comma separated list of {\tt MATRIXDEF[\ref{MATRIXDEF}]}.

\subsubsection{MATRIXDEF}
\label{MATRIXDEF}

Declares a new matrix of a given (symbolic) size.

Of the format: {\tt NAME} [ {\tt DIMENSIONS} ]

Where {\tt NAME} is an identifier and {\tt DIMENSIONS} is a comma separated
list of list of symbolic expressions.

\subsubsection{MATRIXARGLIST}
\label{MATRIXARGLIST}

A comma separated list of {\tt MATRIXARG[\ref{MATRIXARG}]}.

\subsubsection{MATRIXARG}
\label{MATRIXARG}

Declares an argument to a rule that references a matrix defined by a MATRIXDEF.

Of the format: {\tt MATRIX.REGIONSPEC } {\tt NAME}

Where MATRIX is name of the matrix the data is from.  REGIONSPEC is the part
of the matrix to get and NAME is the identifier defined by this statement.

\subsubsection{REGIONSPEC}

One of:

\begin{itemize}
\item \tt cell(EXPRESSION...)
\item \tt region(EXPRESSION...)
\item \tt row(EXPRESSION)
\item \tt col(EXPRESSION)
\item \tt all()

\end{itemize}

\subsubsection{IDENTIFIERLIST}
\label{IDENTIFIERLIST}

A comma separated list of {\tt IDENTIFIER[\ref{IDENTIFIER}]}.

\subsubsection{IDENTIFIER}
\label{IDENTIFIER}

A name matching the regular expression: {\tt [a-zA-Z\_][a-zA-Z0-9\_]*}

\subsubsection{EXPRESSION}
\label{EXPRESSION}

An algebraic expression consisting of identifiers, integers, floats, and the operators: [*/()+-]

Unless stated otherwise, expressions must correspond to affine transformation
of the output coordinates.

\section{Semantics}

\subsection{Semantics for Rule Execution}

The header (defined Sec.~\ref{sec:rule}) of each rule contains a set of
free variables used to reference an explicit list input and output regions.
A rule may be {\em instantiated} by binding these free variables to values
determined by the compiler and runtime.  The compiler may bind these free
variables to any value such that:
\begin{itemize}
\item All values are non-negative integers.
\item All regions referenced exist (i.e. not out of bounds access).
\item All input regions referenced have already been computed.
\item Any WHERE clauses evaluate to true.
\end{itemize}

Once the inputs and outputs are bound to an explicit set of cells, rule
execution follows a semantics and syntax similar to that of {\tt C++} with
the additional features defined in Section~\ref{sec:body}.

\subsection{Semantics for Transform Execution}

The semantics for the execution of a transform revolve around executing
rules in parallel.  This section defines a internal state of an executing
transform and an operational semantics over this internal state for how a
running transform may progress from an initial state to a completion state.

The following definitions are required:
\begin{itemize}
\item
Let $r[\alpha]$ represent rule $r$ instantiated with free variables
bound to the set of definitions $\alpha$.
\item
Let $\mbox{depends}(r[\alpha])$ be the set of cells $r[\alpha]$
depends on (all the regions specified in the FROM[\ref{FROM}] clause when
free variables are bound using the mapping $\alpha$).
\item
Let $\mbox{provides}(r[\alpha])$ be the set of cells $r[\alpha]$
outputs on (all the regions specified in the TO[\ref{TO}] clause when free
variables are bound using the mapping $\alpha_j$).
\item Let $T$ be a static reference of a transform, consisting of a set of rules.
\item Let $R$ be a set of rule instances $r_0[\alpha_0] ... r_n[\alpha_n]$
\item Let $D$ be a set of cells.  Each element in this contains both the $x,y$ coordinate and the matrix in which the cell belongs.
\item Let $\langle T, R, D_0, D_1, D_2 \rangle$ represent the internal state for a running transform where:
  \begin{itemize}
  \item $T$ is the transform being executed.
  \item $R$ are the rules that may currently executing in parallel.  (Initially $\epsilon$)
  \item $D_0$ are cells already computed.  (Initially those defined by the FROM clause.)
  \item $D_1$ are cells that can be computed, but need not be. (Initially those defined by the USING clause.)
  \item $D_2$ are cells that must be computed.  (Initially those defined by the TO clause.)
  \end{itemize}
\end{itemize}

The following transition rules govern the progression of the internal state
of a transform during execution.  This semantics intentionally gives a lot of
freedom to the PetaBricks runtime.  Specifically,  the order of computation
is defined only as producers-before-consumers, data may be computed more
than once, and unfinished computations may be arbitrarily discarded.

~

\noindent
{\it
Transform initialization:}

~

\begin{tabular}{c}
Transform $T$ is called with inputs $D_0$ and outputs $D_2$.
\\
$D_1$ is the intermediate (USING) data required by $T$.
\\\hline
$\rightarrow \langle T, \epsilon, D_0, D_1, D_2 \rangle$
\end{tabular}

~

\noindent
{\it
Rule execution begins:}

~

\begin{tabular}{c}
$r \in T$ \\
$\alpha$ is a legal assignment for free variables in $r$ \\
depends$(r[\alpha]) \subset D_0$
\\\hline
$
\langle T, R, D_0, D_1, D_2 \rangle
\rightarrow
\langle T, R \cup \{r[\alpha]\}, D_0, D_1, D_2 \rangle$
\end{tabular}

~

\noindent
{\it
Rule execution ends:}

~

\begin{tabular}{c}
$r[\alpha]$ has finished executing
\\
$D_0' = D_0 \cup {\mbox{provides}(r[\alpha])}$
\\
$D_1 = D_1' \cup {\mbox{provides}(r[\alpha])}$
\\
$D_2 = D_2' \cup {\mbox{provides}(r[\alpha])}$
\\\hline
$
\langle T, R \cup \{r[\alpha]\}, D_0, D_1, D_2 \rangle
\rightarrow
\langle T, R, D_0', D_1', D_2' \rangle$
\end{tabular}

~

\noindent
{\it
Rule execution aborted:}

~

\begin{tabular}{c}
$
\langle T, R \cup \{r[\alpha]\}, D_0, D_1, D_2 \rangle
\rightarrow
\langle T, R, D_0, D_1, D_2 \rangle$
\end{tabular}

~

\noindent
{\it
Transform completion:}

~

\begin{tabular}{c}
$R=\epsilon$
\\
$D_2=\epsilon$
\\\hline
$
\langle T, R, D_0, D_1, D_2 \rangle
\rightarrow$
\end{tabular}

\end{document}


\section{Tests}
The following are worked free-response questions from three years of AP Calculus BC tests\footnote{These tests questions are owned by the College Board. The questions and College Board's sample responses are available on their website.}.

\input{./additional_materials/2018_questions.tex}
\input{./additional_materials/2018_answers.tex}
\input{./additional_materials/2017_questions.tex}
\input{./additional_materials/2017_answers.tex}
\input{./additional_materials/2016_questions.tex}
\input{./additional_materials/2016_answers.tex}


\documentclass[main.tex]{subfiles}

\begin{document}
\subsection{Explain how decision problems can be formalized by the notion of language of finite words. Develop an example of a language that formalizes a problem}
\emph{cf : \cite{2018}}

First, we need to define these concepts :
\par\textit{Decision problems :} the set of inputs for which the answer is yes.
    In other words, an instance of a decision problem is whether or not
an input A is part of the solution package. It is important to distinguish
between an instance and a problem (the former refers to a specific input whereas the
second takes as a reference all possible inputs).
\par\textit{Language of finite words:} The language of an M machine is defined as : L(M) = A. If A is the set
of all the words (strings) that the M machine accepts.
We can say that:
\begin{itemize}
    \item M accepts A
    \item M recognizes A
\end{itemize}

\todo[inline]{add examples}

\subsection{Define the notion of configuration of a Turing machine. Define the notion of computation of a deterministic Turing machine M on a word $w$.}
%Renato - 25/12
%NOT FINISHED
\emph{cf Sipser, pp. 140-141}

A \emph{configuration} of a TM consists of three items --- the current state, tape contents
and head location. For a state $q$ and two strings $u$ and $v$ over the tape alphabet $\Gamma$ we could write a configuration $uqv$ where : $uv$ is the current content of the tape,
$q$ is the current state and the head location is the first symbol of $v$.

$1011q_701111$ represents a configuration when the tape is $101101111$, the current state is $q_7$ and the head is on the second $0$ (given by the position of $q_7$ in $1011q_701111$.)

\subsection{Are nondeterministic Turing machines more powerful than deterministic Turing machines as deciders ?Justify your answer by proving that every nondeterministic Turing machine has an equivalent deterministic Turing machine}
\emph{cf Sipser, pp. 150}

A nondeterministic Turing machine is defined in the expected way. At any point
in a computation the machine may proceed according to several possibilities.
The transition function for a nondeterministic Turing machine has the form
\todo[inline]{add equation}
\par The computation of a nondeterministic Turing machine is a tree whose branches
correspond to different possibilities for the machine. \todo[inline]{add schema}
If some branch of the computation leads to the accept state, the machine accepts its input.
\par Now we show that nondeterminism does not affect the power of the Turing machine model.
\par\rule{\textwidth}{0.4pt}
\textbf{THEOREM 3.16} \textit{Every nondeterministic Turing machine has an equivalent deterministic Turing
machine.}
\par\textbf{PROOF IDEA} We can simulate any nondeterministic TM N with a deterministic TM D. The idea behind the simulation is to have D try all possible branches
of N’s nondeterministic computation. If D ever finds the accept state on one of
these branches, D accepts. Otherwise, D’s simulation will not terminate.
We view N’s computation on an input w as a tree. Each branch of the tree
represents one of the branches of the nondeterminism. Each node of the tree
is a configuration of N. The root of the tree is the start configuration. The
TM D searches this tree for an accepting configuration. Conducting this search
carefully is crucial lest D fail to visit the entire tree. A tempting, though bad,
idea is to have D explore the tree by using depth-first search. The depth-first
search strategy goes all the way down one branch before backing up to explore
other branches. If D were to explore the tree in this manner, D could go forever
down one infinite branch and miss an accepting configuration on some other
branch. Hence we design D to explore the tree by using breadth first search
instead. This strategy explores all branches to the same depth before going on
to explore any branch to the next depth. This method guarantees that D will
visit every node in the tree until it encounters an accepting configuration.
\par\textbf{PROOF } The simulating deterministic TM D has three tapes. By Theorem 3.13 this arrangement is equivalent to having a single tape. The machine D
uses its three tapes in a particular way, as illustrated in the following figure. Tape
1 always contains the input string and is never altered. Tape 2 maintains a copy
of N’s tape on some branch of its nondeterministic computation. Tape 3 keeps
track of D’s location in N’s nondeterministic computation tree.
\begin{center}
    \includegraphics[scale=0.47]{images/figure317.png}
\end{center}
Let’s first consider the data representation on tape 3. Every node in the tree
can have at most \textit{b} children, where b is the size of the largest set of possible
choices given by \textit{N’}s transition function. To every node in the tree we assign an
address that is a string over the alphabet $\sum_{b}$ = \{1, 2, ..., b\}. We assign the address 231 to the node we arrive at by starting at the root, going to its 2nd child,
going to that node’s 3rd child, and finally going to that node’s lst child. Each
symbol in the string tells us which choice to make next when simulating a step in
one branch in N’s nondeterministic computation. Sometimes a symbol may not
correspond to any choice if too few choices are available for a configuration. In
that case the address is invalid and doesn’t correspond to any node. Tape 3 contains a string over . It represents the branch of N’s computation from the root
to the node addressed by that string, unless the address is invalid. The empty
string is the address of the root of the tree. Now we are ready to describe D.
\begin{enumerate}[label=\textbf{\arabic*}]
    \item Initially tape 1 contains the input $w$, and tapes 2 and 3 are empty.
    \item Copy tape 1 to tape 2.
    \item Use tape 2 to simulate N with input $w$ on one branch of its nondeterministic computation. Before each step of N consult the next symbol on tape 3
to determine which choice to make among those allowed by N’s transition
function. If no more symbols remain on tape 3 or if this nondeterministic
choice is invalid, abort this branch by going to stage 4. Also go to stage 4
if a rejecting configuration is encountered. If an accepting configuration is
encountered, accept the input.
    \item Replace the string on tape 3 with the lexicographically next string. Simulate the next branch of N’s computation by going to stage 2.
\end{enumerate}

\subsection{Given a Turing machine $M$ and a word $w$, give the computation of $M$ on $w$. Does $M$ accept $w$}
\emph{cf Sipser, pp. 141}

The start configuration of $M$  on input $w$ is the configuration $q_0$ $w$, which
indicates that the machine is in the start state $q_0$ with its head at the leftmost
position on the tape. In an accepting configuration the state of the configuration
is $q_{accept}$. In a rejecting configuration the state of the configuration is $q_{reject}$.
Accepting and rejecting configurations are halting configurations and do not
yield further configurations. Because the machine is defined to halt when in the
states $q_{accept}$ and $q_{reject}$, we equivalently could have defined the transition function to have the more complicated form \todo[inline]{add equation}
where Q’ is Q
without $q_{accept}$ and $q_{reject}$.
\begin{center}
    \rule{0.3\textwidth}{0.4pt}
\end{center}
\par A Turing machine $M$ \textbf{accepts} input $w$ if a sequence of configurations $C_{1},C_{2}, ..., C_{k}$ exists, where
\begin{enumerate}[label=\textbf{\arabic*}]
    \item $C_{1}$ is the start configuration of $M$ on input $w$
    \item each $C_{i}$ yields $C_{i+1}$, and
    \item $C_{k}$ is an accepting configuration
\end{enumerate}

\subsection{Define when a Turing machine recognizes a language $L$ and when it decides a language $L$. What is the fundamental difference between those two notions ? Why do we use deciders to formalize the intuitive notion of algorithm}
\emph{cf Sipser, pp. 142}

The collection of strings that $M$ accepts is the language of $M$, or the language recognized by $M$, denoted $L(M)$.
\begin{mytheo*}{}
Call a language Turing-recognizable if some Turing machine
recognizes it.
\end{mytheo*}

When we start a Turing machine on an input, three outcomes are possible.
The machine may accept, reject, or loop. By loop we mean that the machine simply
does not halt. Looping may entail any simple or complex behaviour that never
leads to a halting state.
A Turing machine M can fail to accept an input by entering the $q_{reject}$ state
and rejecting, or by looping. Sometimes distinguishing a machine that is looping
from one that is merely taking a long time is difficult.
\par For this reason we prefer Turing machines that halt on all inputs; such machines never loop. These machines are called deciders because they always make a decision to accept or reject.
A decider that recognizes some language also is said to decide that language.
\begin{mytheo*}{}
Call a language Turing-decidable or simply decidable if some
Turing machine decides it.
\end{mytheo*}

\subsection{Define the notion of enumerator for a language. Prove that a language is Turing-recognizable if and only if it has an enumerator}
\emph{cf Sipser, pp. 152}
\par Loosely defined, an enumerator is a Turing machine with an attached printer. The Turing machine can use that printer as an output device to print strings. Every time the Turing machine wants to add a string to the list, it sends the string to the printer. The following figure depicts a
schematic of this model.
\begin{center}
    \includegraphics[scale=0.47]{images/figure320.png}
\end{center}
An enumerator E starts with a blank input tape. If the enumerator doesn’t
halt, it may print an infinite list of strings. The language enumerated by $E$
is the collection of all the strings that it eventually prints out. Moreover, $E$
may generate the strings of the language in any order, possibly with repetitions.
Now we are ready to develop the connection between enumerators and Turing-recognizable languages.

\par\rule{\textwidth}{0.4pt}
\textbf{THEOREM 3.2} \textit{A language is Turing-recognizable if and only if some enumerator enumerates it.}
\par\textbf{PROOF}
First we show that if we have an enumerator $E$ that enumerates a language $A$, a TM $M$ recognizes $A$. The $TM$ M works in the following way.
$M$ = “On input $w$:
\begin{enumerate}[label=\textbf{\arabic*}]
    \item Run $E$. Every time that $E$ outputs a string, compare it with $w$.
    \item If $w$ ever appears in the output of $E$, accept"
\end{enumerate}

Clearly, $M$ accepts those strings that appear on $E$’s list.
Now we do the other direction. If TM $M$ recognizes a language $A$, we can
construct the following enumerator $E$ for $A$. Say that $s_1, s_2, s_3, ...$ is
a list of all possible strings in $\sum$*.
\par$E$ = "Ignore the input.
\begin{enumerate}[label=\textbf{\arabic*}]
    \item Repeat the following for $i = 1, 2, 3, ...$.
    \item Run $M$ for $i$ steps on each input.  $s_1, s_2, ..., s_i$
    \item If any computations accept, print out the corresponding $s_j$.”
\end{enumerate}
If $M$ accepts a particular string $s$, eventually it will appear on the list generated
by $E$. In fact, it will appear on the list infinitely many times because $M$ runs
from the beginning on each string for each repetition of step 1. This procedure
gives the effect of running $M$ in parallel on all possible input strings.
\subsection{Define the notion of multitape Turing machine. Prove that every multitape Turing machine has an equivalent single tape Turing machine}
\emph{cf Sipser, pp. 148}
\par A multitape Turing machine is like an ordinary Turing machine with several
tapes. Each tape has its own head for reading and writing. Initially the input
appears on tape 1, and the others start out blank. The transition function is
changed to allow for reading, writing, and moving the heads on some or all of
the tapes simultaneously. Formally, it is \todo[inline]{add equation}
where $k$ is the number of tapes. The expression
\todo[inline]{add equation}

means that, if the machine is in state $q_i$ and heads 1 through $k$ are reading symbols $a_1$ through $a_k$, the machine goes to state $q_j$, writes symbols $b_1$ through $b_k$,
and directs each head to move left or right, or to stay put, as specified.
Multitape Turing machines appear to be more powerful than ordinary Turing
machines, but we can show that they are equivalent in power. Recall that two
machines are equivalent if they recognize the same language.
\par\rule{\textwidth}{0.4pt}
\textbf{THEOREM 3.13} \textit{Every multitape Turing machine has an equivalent single-tape Turing machine.}
\par\textbf{PROOF} We show how to convert a multitape TM $M$ to an equivalent singletape TM $S$. The key idea is to show how to simulate $M$ with $S$.
Say that $M$ has $k$ tapes. Then $S$ simulates the effect of $k$ tapes by storing
their information on its single tape. It uses the new symbol \# as a delimiter to
separate the contents of the different tapes. In addition to the contents of these
tapes, $S$ must keep track of the locations of the heads. It does so by writing a tape
symbol with a dot above it to mark the place where the head on that tape would
be. Think of these as “virtual” tapes and heads. As before, the “dotted” tape
symbols are simply new symbols that have been added to the tape alphabet.\\
The following figure illustrates how one tape can be used to represent three tapes.
\begin{center}
    \includegraphics[scale=0.47]{images/figure314.png}
\end{center}
$S$ = "On input $w = w_1, ..., w_n$ :
\begin{enumerate}[label=\textbf{\arabic*}]
    \item Firsts $S$ puts its tape into the format that represent all $k$ tapes of $M$. The formatted tape contains \todo[inline]{Add equation}
    \item To simulate a single move, $S$ scans its tape $t$ from the first \#, which marks the left-hand end, to the $(k+1)$st \#, which marks the right-hand end, in order to determine the symbols under the virtual heads. Then $S$ makes a second pass to update the tapes according to the way that $M$'s transition function dictates.
    \item If at any point $S$ moves one of the virtual heads to the right onto $a$ \#, this action signifies that M has moved the corresponding head onto the previously unread blank portion of that tape. So $S$ writes a blank symbol on this tape cell and shifts the tape contents, from this cell until the rightmost $\#$, one unit to the right. Then it continues the simulation as before."
\end{enumerate}

\subsection{What is the Church-Turing thesis ? Give a list of arguments in favor of this thesis ? Why is it not a theorem}
\emph{cf Sipser, pp. 155}
\par\textbf{HILBERT’S PROBLEMS}
\par In 1900, mathematician David Hilbert delivered a now-famous address at the
International Congress of Mathematicians in Paris. In his lecture, he identified twenty-three mathematical problems and posed them as a challenge for the
coming century. The tenth problem on his list concerned algorithms.
Before describing that problem, let’s briefly discuss polynomials. A polynomial is a sum of terms, where each term is a product of certain variables and a constant called a coefficient. For example,
\todo[inline]{add equation}
is a term with coefficient 6, and
\todo[inline]{add equation}
is a polynomial with four terms over the variables $x$, $y$ and $z$ For this discussion,
we consider only coefficients that are integers. A root of a polynomial is an
assignment of values to its variables so that the value of the polynomial is 0.
This polynomial has a root at $x = 5$, $y = 3$, and $z = 0$. This root is an integral
root because all the variables are assigned integer values. Some polynomials have
an integral root and some do not. \\
Hilbert’s tenth problem was to devise an algorithm that tests whether a polynomial has an integral root. He did not use the term algorithm but rather
\begin{quoting}[font=itshape, begintext={``}, endtext={''}]
a process according to which it can be determined by a finite number of operations.
\end{quoting}
Interestingly, in the way he phrased this problem, Hilbert explicitly
asked that an algorithm be “devised.” Thus he apparently assumed that such an
algorithm must exist, someone need only find it. \\
As we now know, no algorithm exists for this task; it is algorithmically unsolvable. For mathematicians of that period to come to this conclusion with their
intuitive concept of algorithm would have been virtually impossible. The intuitive concept may have been adequate for giving algorithms for certain tasks, but
it was useless for showing that no algorithm exists for a particular task. Proving
that an algorithm does not exist requires having a clear definition of algorithm.
Progress on the tenth problem had to wait for that definition.
\par \par The \textbf{definition} came in the 1936 papers of Alonzo Church and Alan Turing. Church used a notational system called the $\lambda$-calculus to define algorithms.
Turing did it with his “machines.” These two definitions were shown to be
equivalent. This connection between the informal notion of algorithm and the
precise definition has come to be called the Church-Turing thesis.
The Church-Turing thesis provides the definition of algorithm necessary to
resolve Hilbert’s tenth problem. In 1970, Yuri Matijasevic, building on work of
Martin Davis, Hilary Putnam, and Julia Robinson, showed that no algorithm exists for testing whether a polynomial has integral roots.
\begin{center}
    \includegraphics[scale=0.45]{images/figure322.png}
\end{center}
\todo[inline]{Need more info}
\subsection{Why don’t we use the notions of finite automata or context-free grammars to formalize algorithms}
\emph{cf Sipser, pp. 138}
\par First, here are the differences between a Turing machine and a finite automata :
\begin{enumerate}
    \item A Turing machine can both write on the tape and read from it.
    \item The read-write head can move both to the left and to the right.
    \item The tape is infinite.
    \item The special states for rejecting and accepting take effect immediately.
\end{enumerate}

\todo[inline]{Not in the book, need research}
\end{document}


\section{Variable}

Variable is a symbolic name that refers to certain value. Variable is like a jar with name on it: you can put (\emph{assign} is a proper term) new value to the jar, look inside of the jar, or even empty one.

\subsection{Naming a Variable}

Name of the variable should be easily understandable and concise. It is strongly discouraged to name a variable with only one or two letters. (e.g. \code{I}, \code{A1})

TBASIC is somewhat pedantic about the variable name: you are only allowed to use \textbf{alphabets, numbers and underscores}, and \textbf{the name cannot start with a number}. For example, \code{USER_INPUT3} is valid but \code{3INPUT} is not. In some version or implementation of the TBASIC, those bad names would actually work for some extent, but it \emph{will} break some other parts, so don't use illegal names.


\section{Introduction} \slabel{intro}

The Rayleigh-Taylor instability has been the subject of considerable study since its characterization by Lord Rayleigh in the 19th century~\cite{Rayleigh1883}.
Despite this, many aspects of the non-linear growth remain poorly understood.
Analytic models based on potential flow have been reasonably effective for flows with a large density jump, \ie an Atwood number, $\Delta \rho / \sum \rho$, near unity~\cite{Layzer1955, Layzer1955}.
However, recent experiments at low Atwood number have demonstrated a significant departure from the potential flow limit:
Rayleigh-Taylor bubbles are seen accelerating past the terminal velocity predicted by potential flow models~\cite{Wilkinson2007}.
The acceleration persists beyond times which are experimentally accessible, so numerous efforts have been made to compute the late time flow numerically~\cite{Ramaprabhu2012,Wei2012}.
Ultimately, the goal is to use these numerical experiments to inform a simple model that captures the key features of the flow, as potential flow models do for high Atwood number flows.

This study concerns the dynamics of the single-mode Rayleigh-Taylor instability (smRTI), where the interface between a heavy fluid and a light fluid is perturbed with a single wavelength $\lambda$ and corresponding wavenumber $k$.
If the Atwood number is low, then at early times the interface grows exponentially with a rate given by a linear approximation~\cite{Duff1962}:
\begin{equation} \elabel{duff0}
\gamma = \sqrt{\frac{A g k}{1 + \pi^{-1/2} k \delta} + \nu^2 k^4} - (\nu + D) k^2,
\end{equation}
where $A$ is the Atwood number,
$g$ is the local acceleration,
$\delta$ is the interface thickness,
$\nu$ is the kinematic viscosity, and
$D$ is the diffusivity.
As the amplitude approaches the wavelength, the linear growth saturates.
At unit Atwood number, the non-linear regime is described by potential flow, in which Layzer~\cite{Layzer1955} found the interface approaches a terminal velocity proportional to the root of the acceleration and wavelength: $v_L \sim \sqrt{g \lambda}$:
Potential flow models have been extended $A < 1$, with the most successful model by Goncharov~\cite{Goncharov2002}:
\begin{equation} \elabel{goncharov}
v_G = \frac{1}{\sqrt{\pi}} \sqrt{\frac{A g \lambda}{1+A}}
\end{equation}

Experiments by Wilkinson and Jacobs~\cite{Wilkinson2007} show that after reaching the velocity given by \eref{goncharov}, low Atwood bubbles unexpectedly accelerate a second time.
This is termed `re-acceleration`, with the terminal velocity replaced by a `stagnation velocity`.
Re-acceleration was not present in popular potential flow and buoyancy-drag models, and attempts to capture re-acceleration have been the emphasis of recent model development in the low Atwood number regime.

Experiments have thus far been unable to observe more than the onset of the re-acceleration phase.
Therefore, the community has turned to numerical studies to compute late-time dynamics.
At least two such efforts have been under-taken, one by Ramaprabu \etal~\cite{Ramaprabhu2012} and one by Wei and Livescu~\cite{Wei2012}, leading to slightly different conclusions.
In the study by Ramaprabhu, the flow accelerates to around twice the stagnation velocity and then decelerates back to the stagnation velocity, indicating that re-acceleration may be a transient.
In the study by Wei and Livescu, the flow re-accelerates and then breaks up into many small pockets of buoyant fluid, which themselves continue to accelerate at nearly a fixed rate.

Buoyancy-drag models have been proposed for the multi-mode and single-mode dynamics of the bubble front.
They balance the buoyant force of the bubble with a drag force to predict the front velocity as a function of the characteristic length.
Ramaprabhu proposes an additional forcing term based on the development of a vortex ring at the bubble tip, but relies on observation of the mean vorticity experimentally or in numerical simulations.
There is a need for a predictive model for re-acceleration that relies only on the initial conditions.

In \sref{model}, we propose a simple buoyancy-drag model for the dynamics of the smRTI.
In \sref{exp}, we present a battery of numerical trials.
In \sref{late}, we describe the regimes present in late time bubble trajectories.
In \sref{fit}, we fit the unconstrained model coefficients to the numerical results..
Finally, in \sref{conc}, we assess the current state of smRTI and highlight outstanding questions.


\documentclass[notoc,notitlepage]{tufte-book}
% \nonstopmode % uncomment to enable nonstopmode

\usepackage{classnotetitle}

\title{Learning Haskell}
\author{Johnson Ng}
\subtitle{Personal notes to learning Haskell}
\credentials{}
\institution{}

\input{latex-classnotes-preamble.tex}
\usepackage{listings}
\lstset{frame=tb,
  language=Haskell,
  aboveskip=3mm,
  belowskip=3mm,
  frame=single,
  showstringspaces=false,
  columns=flexible,
  basicstyle={\normalfont\small\ttfamily},
  numbers=none,
  numberstyle=\tiny\color{base16-eighties-magenta},
  keywordstyle=\color{base16-eighties-blue},
  commentstyle=\color{base16-eighties-gray},
  rulecolor=\color{base16-eighties-gray},
  stringstyle=\color{mauve},
  breaklines=true,
  breakatwhitespace=true,
  tabsize=3
}

\begin{document}
\hypersetup{pageanchor=false}
\maketitle
\hypersetup{pageanchor=true}
\tableofcontents

\nocite{rwh}
\nocite{lyah}

\chapter{Getting Started}%
\label{chp:getting_started}
% chapter getting_started

\section{Installation}%
\label{sec:installation}
% section installation

For Arch-based distributions, and who prefers to manage their packages\sidenote{Not necessarily for Haskell packages, but just for \texttt{cabal} or \texttt{stack}, while managing Haskell packages using either of these two.}, see the \href{https://wiki.archlinux.org/index.php/Haskell}{Arch Wiki}.

For most systems, see the \href{https://www.haskell.org/platform/}{official download site} for an installation guide.

% section installation (end)

\section{Introduction via \texttt{ghci}}%
\label{sec:introduction_via_ghci}
% section introduction_via_ghci

Launch \texttt{ghci}\sidenote{\texttt{ghci} stands for ``GHC Interactive''} via the terminal. You should see something along the lines of the following:

\begin{lstlisting}
  GHCi, version 6.8.2: http://www.haskell.org/ghc/    :? for help
  Loading package base ... linking ... done.
  Prelude>
\end{lstlisting}

The prompt is \texttt{Prelude>}, but since it gets longer as stuff are loaded, we shall simplify that and use \texttt{ghci>}. You can achieve the same in your prompt by running \texttt{:set prompt "ghci>"}.

\texttt{ghci} comes with some of the familiar cli editing, including the \hlnotea{up} arrow key to get the last line of input, and \hlnotea{tab completion}.

\subsection{Basic Arithmetic}%
\label{sub:basic_arithmetic}
% subsection basic_arithmetic

Basic arithmetic is similar to most other languages. We write them in \hldefn{infix form}, where the operator appears between its operands.

\begin{lstlisting}
  ghci> 3 + 7
  10
  ghci> 49 * 23
  1127
  ghci> 1923 - 333
  1590
  ghci> 5 / 2
  2.5
\end{lstlisting}

Since we explicitly stated that, it means that there is also another way of writing these expressions. In particular, we can write them in \hldefn{prefix form}.

\begin{lstlisting}
  ghci> (+) 3 7
  10
  ghci> (*) 49 23
  1127
  ghci> (-) 1923 333
  1590
  ghci> (/) 5 2
  2.5
\end{lstlisting}

\begin{warning}[Negative Numbers]
  It is often necessary to enclose a negative number in parenthesis, just like we do in mathematics. In Haskell, \texttt{-} is a unary operator, and is in fact its only unary operator, and it cannot be mixed with infix operators.
  \begin{lstlisting}
    ghci> 2 + -3

    <interactive>:1:0: error:
        Precedence parsing error
            cannot mix `+' [infixl 6] and prefix `-' [infixl 6] in the same infix expression
  \end{lstlisting}
  A similar problematic example is the following, despite having a different error message:
  \begin{lstlisting}
    ghci> 2*-3

    <interactive>:1:1: error:
        Variable not in scope: (*-) :: Integer -> Integer -> t
  \end{lstlisting}
\end{warning}

% subsection basic_arithmetic (end)

\subsection{Boolean logic, operators, and value comparisons}%
\label{sub:boolean_logic_operators_and_value_comparisons}
% subsection boolean_logic_operators_and_value_comparisons

The values of \hldefn{Boolean logic} in Haskell are \texttt{True} and \texttt{False}. Capitalization of the names matter.

There is also
\begin{itemize}
  \item \texttt{(\&\&)} as logical ``and''; and
  \item \texttt{(||)} as logical ``or''.
\end{itemize}

\begin{lstlisting}
  ghci> True && False
  False
  ghci> False || True
  True
\end{lstlisting}

\begin{warning}
  While some programming languages treat the number zero as \text{False}, this is not the case in Haskell, nor does Haskell consider non-zero values to be \texttt{True}
  \begin{lstlisting}[escapechar=\%]
    ghci> False && 0

    <interactive>:18:10: error:
    %•% No instance for (Num Bool) arising from the literal `0'
    %•% In the second argument of `(&&)', namely `0'
      In the expression: False && 0
      In an equation for `it': it = False && 0
  \end{lstlisting}
  Here's a breakdown of the error message:
  \begin{itemize}
    \item ``\textnormal{\texttt{No instance for (Num Bool) arising from the literal `0'}}'' tells us that \texttt{ghci} tried to treat the numeric value \texttt{0} as a \texttt{Bool} type, but it failed to do so.
    \item ``\textnormal{\texttt{In an equation for `it': it = False \&\& 0}}'' refers to a shortcut in \texttt{ghci} that we shall visit later.
  \end{itemize}
\end{warning}

The \hldefn{comparison operators} in Haskell are similar to those in C and many other languages.

\begin{lstlisting}
  ghci> 1 == 1
  True
  ghci> 2 < 3
  True
  ghci 4 >= 3.99
  True
\end{lstlisting}

The ``is not equal to'' operator in Haskell is \texttt{(/=)}.

\begin{lstlisting}
  ghci> 2 /= 3
  True
\end{lstlisting}

The \hldefn{logical negation} for Haskell is \texttt{not}.

\begin{lstlisting}
  ghci> not True
  False
\end{lstlisting}

% subsection boolean_logic_operators_and_value_comparisons (end)

\subsection{Operator precedence and associativity}%
\label{sub:operator_precedence_and_associativity}
% subsection operator_precedence_and_associativity

Like in written algebra and other languages that use infix operators, Haskell has operator precedence. We can use parenthesis to explicitly group parts of an expression, and precedence allows us to omit a few parenthesis. For example, we know that multiplication precedes addition, and in Haskell:

\begin{lstlisting}
  ghci> 1 + ( 4 * 4 )
  17
  ghci> 1 + 4 * 4
  17
\end{lstlisting}

In Haskell, operators are assigned numeric precedence values, with 1 being the lowest and 9 the highest. An operator with a higher precedence is applied before one that has a lower precedence. We can use \texttt{ghci} to inspect the precedence levels of individual operators using the \texttt{:info} command.

\begin{lstlisting}
  ghci> :info (+)
  class Num a where
    (+) :: a -> a -> a
    ...
          -- Defined in `GHC.Num'
  infixl 6 +
  ghci> :info (*)
  class Num a where
    ...
    (*) :: a -> a -> a
    ...
          -- Defined in `GHC.Num'
  infixl 7 *
\end{lstlisting}

The information we want here is in the line ``\texttt{infixl 6 +}'', which says that the \texttt{(+)} operator has precedence 6. Similarly, the \texttt{(*)} operator has precedence 7.

Haskell also defines \hldefn{associativity} of operators\sidenote{See \url{https://en.wikipedia.org/wiki/Operator_associativity}}. The \texttt{(+)} and \texttt{(*)} are left associative, which is shown as \texttt{infixl} in the \texttt{ghci} output above. The right associative operator is displayed with \texttt{infixr}. An example of a right associative operator is

\begin{lstlisting}
  ghci> :info (^)
	(^) :: (Num a, Integral b) => a -> b -> a       -- Defined in `GHC.Real'
	infixr 8 ^
\end{lstlisting}

Left associativity means that \texttt{1 + 2 + 3} is interpreted as \texttt{(1 + 2) + 3}, while right associativity means that \texttt{1\^{}2\^{}3} is interpreted as \texttt{1\^{}(2\^{}3)}.

The combination of precedence and associativity rules are usually referred to as \hldefn{fixity} rules.

\begin{note}
  It is sometimes better to leave at least some parentheses in place, even when you are sure that Haskell will parse your expressions correctly by fixity rules. This helps future readers to read and understand the code better.
\end{note}

% subsection operator_precedence_and_associativity (end)

% section introduction_via_ghci (end)

% chapter getting_started (end)

\appendix

\backmatter

\pagestyle{plain}

\bibliography{references}

\printindex

\end{document}


\appendix

\chapter{Some prerequisite topics}

The topics presented in this section are important concepts in mathematics and therefore should be
examined.


\documentclass{article}

\usepackage{graphicx}
\usepackage{subcaption}

\begin{document}
	\section{ NOT Gate}
   The NOT gate is called a logical inverter.It has only one input. It reverses the original input (A) to give an inverted output C.C = NOT A

	\begin{figure}[h!]
		\centering
		\begin{subfigure}[b]{0.4\linewidth}
			\includegraphics[width=\linewidth]{NOT.png}
		\end{subfigure}
			\begin{subfigure}[b]{0.4\linewidth}
		\includegraphics[width=\linewidth]{NOT table.png}
	\end{subfigure}

	\end{figure}
\end{document}


\documentclass[9pt, a4paper, oneside, reqno]{amsart}

\usepackage[final]{pdfpages}
\usepackage{wrapfig}

\usepackage{enumitem}
\usepackage{parskip}
\usepackage{fancyhdr}
\usepackage{color}
\usepackage{multicol}
\renewcommand{\thefootnote}{\fnsymbol{footnote}}

\newcommand{\hint}[1]{\footnote{\raggedleft\rotatebox{180}{Hint: #1\hfill}}}

\pagestyle{fancy}

\newlist{questions}{enumerate}{1}
\setlist[questions, 1]{label = \bf Q.\arabic*., itemsep=1em}

\lhead{\scshape Apurva Nakade}
\rhead{\scshape Honors Single Variable Calculus}
\renewcommand*{\thepage}{\small\arabic{page}}
\title{Problem Set 12}

\begin{document}

\maketitle
\thispagestyle{fancy}

\section*{Part 1 - Infinite series}
\begin{questions}

	\item 		In this problem $ a_n \ge 0$.
	\begin{enumerate}
		\item Show that if there is a number $ c < 1$, and an integer $ N$ such that $ a_n < c^n$ for all $ n > N$, then $ \sum \limits_{n=0}^\infty a_n$ converges.
		\item Show that if there is a number $ c > 1$, and an integer $ N$ such that $ a_n > c^n$ for all $ n > N$, then $ \sum \limits_{n=0}^\infty a_n$ diverges.
		\item Show that if $ \lim \limits_{n\rightarrow \infty} \sqrt[n]{a_n} = r $ then,
		      \begin{quote}
			      if $ r < 1$ then $ \sum \limits_{n=0}^\infty a_n$ converges and if $ r> 1$ then $ \sum \limits_{n=0}^\infty a_n$ diverges.
		      \end{quote} This is called the \textbf{root test}.
		\item Find examples of positive $ a_n$ and $ b_n$ with $ \lim \limits_{n\rightarrow \infty} \sqrt[n]{a_n} = 1 = \lim \limits_{n\rightarrow \infty} \sqrt[n]{b_n}$ such that $\sum \limits_{n=0}^\infty a_n$ converges and $\sum \limits_{n=0}^\infty b_n$ diverges.
		\item This part is surprising difficult to make rigorous, you can give an approximate proof instead.

		      Show that if $ \lim \limits_{n\rightarrow \infty} \dfrac{a_n}{a_{n-1}} = r$ then $ \lim \limits_{n\rightarrow \infty} \sqrt[n]{a_n} = r $.

		      (And hence, in theory, the root test is stronger than the ratio test, in practice though the ratio test is easier to use.)
	\end{enumerate}

	\item Q.1 of Ch.23 on Pages 482-484. Note that there are \textbf{20} (small) problems in this question.
\end{questions}

\newpage
\section*{Part 2 - Absolute convergence}
\begin{questions}[resume]
	\item Let $ {a_n}$ be a decreasing sequence with $ a_n > 0$. Define
	\begin{align*}
		s_n & = a_1 - a_2 + a_3 - a_4 + \cdots + (-1)^{n-1}a_n
	\end{align*}
	\begin{enumerate}
		\item Show that the sequence $ b_n = s_{2n}$ is an increasing sequence.
		\item Show that $ a_1 > b_n$ for all $ n$, hence conclude that $ b_n$ converges.
		\item Show that the sequence $ c_n = s_{2n-1}$ is a decreasing sequence.
		\item Show that $ 0 < c_n$ for all $ n$, hence conclude that $ c_n $ converges.
		\item Show that if $ \lim \limits_{n \rightarrow \infty} a_n = 0$ then $$ \lim \limits_{n \rightarrow \infty} b_n = \lim \limits_{n \rightarrow \infty} s_n = \lim \limits_{n \rightarrow \infty} c_n$$
		      (It's ok to give an approximate argument for this last part.)
	\end{enumerate}

	\item This question is a freebie.

	The theorems from this chapter are extremely important, but the proofs are a bit convoluted and it is too late in the semester to spend significant time on them, if you have time you should read their proofs and understand the discussions in the book.

	For this problem simply write down the statements of Theorems from 5 to 9.
\end{questions}

\newpage
\section*{Part 3 - Integral Computations}
Last problem set on Integrals!
\begin{questions}[resume]
	\item For this week do Q.7 i)-ix) on Pg.381 and Q.11 problems - i), ii) on Pg.383.

\end{questions}

\end{document}


\documentclass[letterpaper, 10pt, twocolumn]{article}
\usepackage[margin=1.5cm]{geometry}

\usepackage{graphicx}
\usepackage[colorlinks=true]{hyperref}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}

\setlength{\parindent}{0em}
\setlength{\parskip}{0em}

\title{\textbf{10-701 Machine Learning Review}}
\author{HMW-Alexander}

\begin{document}

\maketitle

\section{Intro}

\subsection{What is Machine Learning?}

Algorithms that improve their knowledge towards some task with data.

Goal: improve knowledge with more data.

\begin{figure}[!h]
	\centering
	\includegraphics[width=0.4\textwidth]{./img/01_intro/machinelearning.png}
\end{figure}

\subsection{Three Axes of ML}

\subsubsection{Data}

\begin{itemize}
	\item Fully observed
	\item Partially observed
	\begin{itemize}
		\item Systematically
		\item Missing data
	\end{itemize}
\end{itemize}

\subsubsection{Algorithms}

\begin{itemize}
	\item Model-based Methods\footnote{refer to generative model (hw2)}
	\begin{itemize}
		\item Probabilistic Model of the data
		\item Parametric Models: fixed-size
		\item Nonparametric Models:	grow with the data
	\end{itemize}
	\item Model-free Methods: No distribution model assumption
\end{itemize}

\subsubsection{Knowledge/Tasks}

\begin{itemize}
	\item Prediction: estimate output given input.
	\begin{itemize}
		\item Classification: discrete labels
		\item Regression: continuous labels
	\end{itemize}
	\item Description (unsupervised learning)
\end{itemize}

\newpage

\section{Parametric Models: from data to models}

\subsection{A model for coin flips}

Bernoulli distribution:
\begin{equation}
\left\{\begin{array}{rcl}
P(X=1) & = & \theta \\
P(X=0) & = & 1-\theta \\
\end{array}
\right.
\end{equation}
$$P(X) = \theta^{X}(1-\theta)^{1-X}$$

Flips are i.i.d. (independent, identically distributed)

Choose $\theta$ that maximizes the probability of observed data:
\begin{equation}
\begin{array}{rcl}
Probability~of~Data & = & \mathbb{P}(X_1,X_2,\dots,X_n;\theta) \\
					& = & \prod_{i=1}^{n} P(X_i) \\
					& = & \theta^{n_h}(1-\theta)^{n-n_h}
\end{array}
\end{equation}

\subsection{Maximum Likelihood Estimator(MLE)}

\begin{equation}
\begin{array}{rcl}
\hat{\theta} & = & \arg\max_{\theta} \mathbb{P}(X_1,\dots,X_n;\theta) \\
			 & = & \arg\max_{\theta} \{\theta^{n_h}(1-\theta)^{n-n_h}\} \\
			 & = & \arg\max_{\theta} \{n_h\log\theta+(n-n_h)\log(1-\theta)\} \\
			 & = & \frac{n_h}{n}
\end{array}
\end{equation}

\subsubsection{Consistency}

\begin{itemize}
	\item Estimator $\hat{\theta}$ converges (in probability) to the true value $\theta$ with more and more sample $n\rightarrow\infty$.
	\item For Bernoulli distribution, $\hat{\theta}=\frac{1}{n}\sum_{i=1}^{n}{X_i} \rightarrow \theta$ in probability as $n \rightarrow \infty$ by the \textbf{Law of Large Numbers}\footnote{It does not apply to distributions for whom Expected values do not exist. One example of such a distribution is the Cauchy distribution where the mean and the variance are undefined.}.
\end{itemize}

\subsubsection{Unbiasedness}

\begin{itemize}
	\item Expectation $\mathbb{E}[\hat{\theta}]$ of the estimator $\hat{\theta}$ equals to the true value $\theta$.
	\item For Bernoulli example:
	\begin{equation}
	\begin{array}{rcl}
	\mathbb{E}(\hat{\theta}) & = & \mathbb{E}(\frac{n_1}{n}) \\
	& = & \mathbb{E}(\frac{\sum_{i=1}^{n}X_i}{n}) \\
	& = & \frac{1}{n}\sum_{i=1}^{n}{\mathbb{E}(X_i)} \\
	& = & \mathbb{E}(X_1) \\
	& = & \theta
	\end{array}
	\end{equation}
\end{itemize}

\subsection{Gaussian Distribution MLE}

Gaussian Distribution:
$$P(x|\mu,\sigma)=\frac{1}{\sigma\sqrt{2\pi}}\exp(-\frac{(x-\mu)^2}{2\sigma^2})=\mathcal{N}(\mu,\sigma^2)$$
\begin{itemize}
	\item Affine transformation:
	\begin{itemize}
		\item $X \sim \mathcal{N}(\mu,\sigma^2)$
		\item $Y=aX+b \sim \mathcal{N}(a\mu+b,a^2\sigma^2)$
	\end{itemize}
	\item Sum of Gaussians:
	\begin{itemize}
		\item $X \sim \mathcal{N}(\mu_X,\sigma^2_X)$, $Y \sim \mathcal{N}(\mu_Y,\sigma^2_Y)$
		\item $Z=X+Y \sim \mathcal{N}(\mu_X+\mu_Y,\sigma^2_X+\sigma^2_Y)$
	\end{itemize}
\end{itemize}

MLE for Gaussian mean and variance:
\begin{itemize}
	\item $\hat{\mu}_{MLE}=\frac{1}{n}\sum_{i=1}^{n}{x_i}$
	\item $\hat{\sigma}^2_{MLE}=\frac{1}{n}\sum_{i=1}^{n}{(x_i-\hat{\mu})^2}$
\end{itemize}

\subsubsection{The Biased Variance of a Gaussian}

The unbiased variance estimator:
$\hat{\sigma}^2_{unbiased}=\frac{n}{n-1}\hat{\sigma}^2_{MLE}$

Proof:
\begin{equation}
\begin{array}{rcl}
\mathbb{E}(\sigma^2_{MLE}) & = & \mathbb{E}(\frac{1}{n}\sum_{i=1}^{n}{(x_i-\hat{\mu})^2}) \\
%& = & \frac{1}{n}\mathbb{E}(\sum_{i=1}^{n}{(x_i^2-2x_i\hat{\mu}+\hat{\mu}^2)}) \\
%& = & \frac{1}{n}\mathbb{E}(\sum_{i=1}^{n}{x_i^2}-\sum_{i=1}^{n}{2x_i\hat{\mu}}+\sum_{i=1}^{n}{\hat{\mu}^2}) \\
%& = & \frac{1}{n}\mathbb{E}(\sum_{i=1}^{n}{x_i^2}-2n\hat{\mu}^2+n\hat{\mu}^2) \\
& = & \frac{1}{n}\mathbb{E}(\sum_{i=1}^{n}{x_i^2}-n\hat{\mu}^2) \\
& = & \frac{1}{n}\sum_{i=1}^{n}{\mathbb{E}(x_i^2)-\mathbb{E}(\hat{\mu}^2)}\\
& = & \mathbb{E}(x_i^2)-\mathbb{E}(\hat{\mu}^2) \\
& = & (\sigma^2(x_i)+\mathbb{E}(x_i)^2)-(\sigma^2(\hat{\mu})+\mathbb{E}(\hat{\mu})^2) \\
& = & \sigma^2(x_i) - \sigma^2(\hat{\mu}) \\
& = & \sigma^2(x_i) - \sigma^2(\frac{1}{n}\sum_{i=1}^{n}{x_i}) \\
& = & \sigma^2(x_i) - \frac{1}{n^2}\sigma^2(\sum_{i=1}^{n}{x_i}) \\
& = & \sigma^2(x_i) - \frac{1}{n^2}n\sigma^2(x_i) \\
& = & \frac{n-1}{n}\sigma^2(x_i)
\end{array}
\end{equation}

\subsection{Convergence Rates of Estimator}

\subsubsection{Simple Bound (Hoeffding's Inequality)}

$$P(|\hat{\theta}-\theta^*|\geq \epsilon) \leq 2\exp(-2n\epsilon^2)$$

\subsection{PAC* (Probably Approximate Correct) Learning}

\subsection{Computational Issues of MLE}

When number of parameters, or number of samples n is large, computing the MLE is a large-scale optimization problem.

\newpage

\section{Parametric Models: Prior Information}

\subsection{Bayesian Learning}

Given a prior knowledge to estimate the model.

Bayesian Learning:
$$P(\theta|\mathcal{D}) = \frac{P(\mathcal{D}|\theta)P(\theta)}{P(\mathcal{D})}$$
or equivalently
$$P(\theta|\mathcal{D}) \propto P(\mathcal{D}|\theta)P(\theta)$$
\begin{itemize}
	\item $P(\theta|\mathcal{D})$: posterior
	\item $P(\mathcal{D}|\theta)$: likelihood
	\item $P(\theta)$: prior
\end{itemize}
Likelihood measures the fitness between data and parameters, Prior is the knowledge how possible the parameters to be.

\subsection{Conjugate Priors}

\begin{itemize}
	\item Closed-form representation of posterior
	\item Prior $P(\theta)$ and Posterior $P(\theta|D)$ have the same algebraic form as a function of $\theta$
\end{itemize}

For Binomial(Bernoulli), conjugate prior is Beta distribution:
\begin{itemize}
	\item $P(D|\theta) = \theta^{\alpha_H}(1-\theta)^{\alpha_T}$
	\item $P(\theta) = \frac{\theta^{\beta_H-1}(1-\theta)^{\beta_T-1}}{B(\beta_H,\beta_T)} \sim Beta(\beta_H,\beta_T)$
	\item $P(\theta|D) \sim Beta(\beta_H+\alpha_H,\beta_T+\alpha_T)$
	\item Mode of Beta distribution $Beta(\alpha_H,\alpha_T)$: $\frac{\alpha_H-1}{\alpha_H+\alpha_T-2}$
\end{itemize}

\subsection{Maximum A Posteriori Estimation (MAP)}

Choose $\theta$ that maximizes a posterior probability:
\begin{equation}
\begin{array}{rcl}
\hat{\theta}_{MAP} & = & \arg\max_{\theta} P(\theta|D) \\
				   & = & \arg\max_{\theta} P(D|\theta)P(\theta)
\end{array}
\end{equation}

\subsection{Regularized MLE}

Constrained MLE:
$$\max_{\theta}\log\mathbb{P}(D;\theta)$$
$$s.t. \mathcal{R}(\theta) \leq C$$

Regularized MLE:
$$\max_{\theta} \{\log\mathbb{P}(D;\theta)+\lambda\mathcal{R}(\theta)\}$$

\begin{itemize}
	\item $l_2$ regularization: (Ridge?)
	$$\mathcal{R}(\theta) = ||\theta||_2^2 = \sum_{j=1}^{p}\theta_j^2$$
	\item $l_1$ regularization: (Lasso)
	$$\mathcal{R}(\theta) = ||\theta||_1 = \sum_{j=1}^{p}|\theta_j|$$
\end{itemize}

\newpage

\section{Linear Regression}

\subsection{Bayes Optimal Rule}

$$f^* = \arg\min_f \mathbb{E}[loss(Y,f(X))]$$

\subsection{Linear Regression}

$$\hat{f}_n^L = \arg\min_{f\in \mathcal{F}_L} \frac{1}{n}\sum\limits_{i=1}^{n}(f(X_i)-Y_i)^2$$
$$f(X) = X\beta$$
$$\hat{\beta} = \arg\min_{\beta}\frac{1}{n}(A\beta-Y)^T(A\beta-Y)$$
If $(A^TA)$ is invertible,
$$\hat{\beta} = (A^TA)^{-1}A^TY$$

\subsection{Regularized Least Squares}

Guarantee solution uniqueness by adding a regular constraint.

Ridge Regression:
\begin{equation}
\begin{array}{rcl}
\hat{\beta}_{MAP} & = & \arg\min_{\beta}\sum_{i=1}^{n}(Y_i-X_i\beta)^2+\lambda ||\beta||_2^2 \\
				  & = & \arg\min_{\beta}(A\beta-Y)^T(A\beta-Y) + \lambda||\beta||_2^2 \\
				  & = & (A^TA+\lambda I)^{-1}A^TY
\end{array}
\end{equation}

Lasso Regression:
$$\hat{\beta}_{MAP} = \arg\min_{\beta}\sum_{i=1}^{n}(Y_i-X_i\beta)^2+\lambda ||\beta||_1 $$
More sparse solution.

\newpage

\section{Logistic Regression}

Assumes the following functional form for $P(Y|X)$
$$P(Y=0|X)=\frac{1}{1+\exp(w_0+\sum_i w_iX_i)}$$

\subsection{Linear Classifier}

Decision boundary: $w_0+\sum_i w_iX_i = 0$

\subsection{Training Logistic Regression}

$$\hat{w}_{MLE} = \arg\max_w \prod_{i=1}^{n} P(X_i,Y_i|w)$$
$P(X)$ and $P(X|Y)$ are unknown. Discriminative philosophy\footnote{Don't waste effort learning $P(X)$, focus on $P(Y|X)$, that's all that matters for classification.}
$$\hat{w}_{MCLE} = \arg\max_w \prod_{i=1}^{n} P(Y_i|X_i,w)$$

\subsubsection{Conditional Log Likelihood}

$$P(Y=0|X,w) = \frac{1}{1+\exp{w_0+\sum_i w_iX_i}}$$
$$P(Y=1|X,w) = \frac{\exp(w_0+\sum_i w_iX_i)}{1+\exp(w_0+\sum_i w_iX_i)}$$

\begin{equation}
\begin{array}{rcl}
l(w) & = & \ln{\prod_i {P(y^i|x^i,w)}} \\
	 & = & \sum_i [y^i(w_0+\sum_j^d w_jx_j^i \\
	 &   & - \ln(1+\exp(w_0+\sum_j^d w_jx_j^i))]
\end{array}
\end{equation}

\begin{itemize}
	\item no-closed form solution
	\item $l(w)$ is concave function of $w$
\end{itemize}

\subsubsection{Gradient Ascent for LR}

$$\frac{\partial l(w)}{\partial w_0} = \sum_i[y^i - P(Y^i=1|x^i,w)]$$
$$\frac{\partial l(w)}{\partial w_j} = \sum_i x_j^i[y^i - P(Y^i=1|x^i,w)]$$

\newpage

\section{Naive Bayes Classifier}

\subsection{Optimal Classification}

Optimal predictor: $f^*=\arg\min_f P(f(X)\neq Y)$

Optimal classifier:
\begin{equation}
\begin{array}{rcl}
f^*(x) & = & \arg\max_{Y=y} P(Y=y|X=x) \\
	   & = & \arg\max_{Y=y} P(X=x|Y=y)P(Y=y)
\end{array}
\end{equation}

\begin{itemize}
	\item Class conditional density: $P(X=x|Y=y)$
	\item Class prior: $P(Y=y)$
\end{itemize}

Naive Bayes Classifier is a model based approach: to model these two terms.

\subsection{Gaussian Bayes Classifier}

\begin{itemize}
	\item $P(Y=y)=p_y$ ($K-1$ if $K$ labels)
	\item $P(X=x|Y=y \sim N(\mu_y,\Sigma_y)$ ($\frac{Kd+Kd(d+1)}{2}=O(Kd^2)$ if $d$ features)
\end{itemize}

Binary classification:

$$P(X=x|Y=y) = \frac{1}{\sqrt{(2\pi)^d|\Sigma_y|}}\exp(-\frac{(x-\mu_y)^T\Sigma_y^{-1}(x-\mu_y)}{2})$$

$$\frac{P(Y=1|X=x)}{P(Y=0|X=x)}$$
$$=\sqrt{\frac{|\Sigma_0|}{|\Sigma_1|}}\exp(-\frac{(x-\mu_1)^T\Sigma_1^{-1}(x-\mu_1)}{2}-\frac{(x-\mu_0)^T\Sigma_0^{-1}(x-\mu_0)}{2})$$
$$\cdot\frac{\theta}{1-\theta}$$

If $\Sigma_0=\Sigma_1$, then quadratic part cancels out and equation is linear.

\subsection{Naive Bayes Classifier}

Naive assumption: Features are independent given class:
$$P(X_1,\dots,X_d|Y)=\prod_{i=1}^{d}P(X_i|Y)$$

$$f_{NB}=\arg\max_y P(x_1,\dots,x_d|y)P(y)=\arg\max_y \prod_{i=1}^{d}P(X_i|Y)P(y)$$

$P(X_i=x_i|Y=y) \sim N(\mu_i,\sigma_i^2)$ (2Kd)

Issues with NB:
\begin{itemize}
	\item Features are not conditionally independent.
	\item Insufficient data $\rightarrow$ MLE to be 0. Typically use MAP estimates.
\end{itemize}

\subsection{Gaussian Naive Bayes vs. Logistic Regression}

\begin{itemize}
	\item Both learn a linear boundary.
	\item NB makes more restrictive assumptions and has higher asymptotic error.
	\item NB converges faster to its less accurate asymptotic error.
\end{itemize}

\newpage

\section{Decision Theory: From Model to Answers; Empirical Risk Minimization}

Use decision theory to characterize the knowledge we seek (through appropriate performance measures)

\subsection{Performance Measure}

\begin{itemize}
	\item $loss(Y,f(X))$: measure of closeness between true label $Y$ and prediction $f(X)$.
	\item Risk: $R(f)=E_{XY}[loss(Y,f(X))]$
	\item Bayes optimal rule: $$f^*(P) = \arg\min_f\mathbb{E}_{(X,Y)~P}[loss(Y,f(X))]$$
\end{itemize}

E.g.
\begin{itemize}
	\item 0/1 loss: $1_{\{f(X)\neq Y\}}$ $\rightarrow$ probability of error: $P(f(X) \neq Y)$ $\rightarrow$ $f^*(P)=\mathbb{I}(P(Y=1|X)>1/2)$
	\item square loss: $(f(X)-Y)^2$ $\rightarrow$ mean square error: $\mathbb{E}[(f(X)-Y)^2]$ $\rightarrow$ $f^*(P)=\mathbb{E}(Y|X)$
\end{itemize}

\subsection{Empirical Risk Minimization}

$$\hat{f}_n = \arg\min_f \frac{1}{n}\sum_{i=1}^{n}[loss(Y_i,f(X_i))]$$
$$\frac{1}{n}\sum_{i=1}^{n}[loss(Y_i,f(X_i))] \xrightarrow[Numbers]{Law~of~Large} \mathbb{E}_{XY}[loss(Y,f(X))]$$

\begin{itemize}
	\item Computational tractability: 0/1 loss $\rightarrow$ not convex.
	\item Statistical Considerations: consistent and rate of convergence.
\end{itemize}

\end{document}


%%
%% This is file `mcmthesis-demo.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% mcmthesis.dtx  (with options: `demo')
%% !Mode:: "TeX:UTF-8"
%% -----------------------------------
%%
%% This is a generated file.
%%
%% Copyright (C)
%%     2010 -- 2015 by latexstudio
%%     2014 -- 2016 by Liam Huang
%%
%% This work may be distributed and/or modified under the
%% conditions of the LaTeX Project Public License, either version 1.3
%% of this license or (at your option) any later version.
%% The latest version of this license is in
%%   http://www.latex-project.org/lppl.txt
%% and version 1.3 or later is part of all distributions of LaTeX
%% version 2005/12/01 or later.
%%
%% This work has the LPPL maintenance status `maintained'.
%%
%% The Current Maintainer of this work is Liam Huang.
%%
\documentclass{mcmthesis}
\mcmsetup{CTeX = false,   % 使用 CTeX 套装时，设置为 true
        tcn = 0000, problem = A,
        sheet = true, titleinsheet = true, keywordsinsheet = true,
        titlepage = true, abstract = true}
\usepackage{palatino}
\usepackage{lipsum}
\title{The \LaTeX{} Template for MCM Version \MCMversion}
\author{\small \href{http://www.latexstudio.net/}
  {\includegraphics[width=7cm]{mcmthesis-logo}}}
\date{\today}
\begin{document}
\begin{abstract}
\lipsum[1]
\begin{keywords}
keyword1; keyword2
\end{keywords}
\end{abstract}
\maketitle
\section{Introduction}

\lipsum[2]
\begin{itemize}
\item minimizes the discomfort to the hands, or
\item maximizes the outgoing velocity of the ball.
\end{itemize}
We focus exclusively on the second definition.

\begin{itemize}
\item the initial velocity and rotation of the ball,
\item the initial velocity and rotation of the bat,
\item the relative position and orientation of the bat and ball, and
\item the force over time that the hitter hands applies on the handle.
\end{itemize}
\lipsum[3]
\begin{itemize}
\item the angular velocity of the bat,
\item the velocity of the ball, and
\item the position of impact along the bat.
\end{itemize}
\lipsum[4]
\emph{center of percussion} [Brody 1986], \lipsum[5]

\begin{Theorem} \label{thm:latex}
\LaTeX
\end{Theorem}
\begin{Lemma} \label{thm:tex}
\TeX .
\end{Lemma}
\begin{proof}
The proof of theorem.
\end{proof}

\subsection{Other Assumptions}
\lipsum[6]
\begin{itemize}
\item
\item
\item
\item
\end{itemize}

\lipsum[7]

\section{Analysis of the Problem}
\begin{figure}[h]
\small
\centering
\includegraphics[width=12cm]{mcmthesis-aaa.eps}
\caption{aa} \label{fig:aa}
\end{figure}

\lipsum[8] \eqref{aa}
\begin{equation}
a^2 \label{aa}
\end{equation}

\[
  \begin{pmatrix}{*{20}c}
  {a_{11} } & {a_{12} } & {a_{13} }  \\
  {a_{21} } & {a_{22} } & {a_{23} }  \\
  {a_{31} } & {a_{32} } & {a_{33} }  \\
  \end{pmatrix}
  = \frac{{Opposite}}{{Hypotenuse}}\cos ^{ - 1} \theta \arcsin \theta
\]
\lipsum[9]

\[
  p_{j}=\begin{cases} 0,&\text{if $j$ is odd}\\
  r!\,(-1)^{j/2},&\text{if $j$ is even}
  \end{cases}
\]

\lipsum[10]

\[
  \arcsin \theta  =
  \mathop{{\int\!\!\!\!\!\int\!\!\!\!\!\int}\mkern-31.2mu
  \bigodot}\limits_\varphi
  {\mathop {\lim }\limits_{x \to \infty } \frac{{n!}}{{r!\left( {n - r}
  \right)!}}} \eqno (1)
\]

\section{Calculating and Simplifying the Model  }
\lipsum[11]

\section{The Model Results}
\lipsum[6]

\section{Validating the Model}
\lipsum[9]

\section{Conclusions}
\lipsum[6]

\section{A Summary}
\lipsum[6]

\section{Evaluate of the Mode}

\section{Strengths and weaknesses}
\lipsum[12]

\subsection{Strengths}
\begin{itemize}
\item \textbf{Applies widely}\\
This  system can be used for many types of airplanes, and it also
solves the interference during  the procedure of the boarding
airplane,as described above we can get to the  optimization
boarding time.We also know that all the service is automate.
\item \textbf{Improve the quality of the airport service}\\
Balancing the cost of the cost and the benefit, it will bring in
more convenient  for airport and passengers.It also saves many
human resources for the airline. \item \textbf{}
\end{itemize}

\begin{thebibliography}{99}
\bibitem{1} D.~E. KNUTH   The \TeX{}book  the American
Mathematical Society and Addison-Wesley
Publishing Company , 1984-1986.
\bibitem{2}Lamport, Leslie,  \LaTeX{}: `` A Document Preparation System '',
Addison-Wesley Publishing Company, 1986.
\bibitem{3}\url{http://www.latexstudio.net/}
\bibitem{4}\url{http://www.chinatex.org/}
\end{thebibliography}

\begin{appendices}

\section{First appendix}

\lipsum[13]

Here are simulation programmes we used in our model as follow.\\

\textbf{\textcolor[rgb]{0.98,0.00,0.00}{Input matlab source:}}
\lstinputlisting[language=Matlab]{./code/mcmthesis-matlab1.m}

\section{Second appendix}

some more text \textcolor[rgb]{0.98,0.00,0.00}{\textbf{Input C++ source:}}
\lstinputlisting[language=C++]{./code/mcmthesis-sudoku.cpp}

\end{appendices}
\end{document}

%%
%% This work consists of these files mcmthesis.dtx,
%%                                   figures/ and
%%                                   code/,
%% and the derived files             mcmthesis.cls,
%%                                   mcmthesis-demo.tex,
%%                                   README,
%%                                   LICENSE,
%%                                   mcmthesis.pdf and
%%                                   mcmthesis-demo.pdf.
%%
%% End of file `mcmthesis-demo.tex'.


\chapter{Lecture 20 Jun 18th 2018}%
\label{chp:lecture_20_jun_18th_2018}
% chapter lecture_20_jun_18th_2018

\section{Finite Abelian Groups (Continued 2)}%
\label{sec:finite_abelian_groups_continued_2}
% section finite_abelian_groups_continued_2

\subsection{p-Groups (Continued 2)}%
\label{sub:p_groups_continued_2}
% subsection p_groups_continued_2

Recall that we had the following subgroup of a group $G$.
\begin{equation*}
  G^{(m)} = \{ g \in G : g^m = 1 \}.
\end{equation*}
We discussed about the Primary Decomposition, \cref{thm:primary_decomposition}, and then arrived at \cref{propo:p_gp_broken_down}. With these, we can have the following theorem:

\begin{thm}[Finite Abelian Groups are Isomorphic to a Direct Product of Cyclic Groups]
\label{thm:finite_abelian_groups_are_isomorphic_to_a_direct_product_of_cyclic_groups}
Let $G \neq \{1\}$ be a finite abelian $p$-group. Then $G$ is isomorpic to a direct product of cylic groups.
\end{thm}

\begin{proof}
  By \cref{propo:p_gp_broken_down}, there is a cyclic group $C_1$ and a subgroup $B_1$ of $G$, such that $G \cong C_1 \times B_1$. Since $B_1 \leq G$, we have that $\abs{B_1} \, \Big| \, \abs{G}$, and so by \cref{thm:lagrange_s_theorem}, $B_1$ is also a $p$-group. If $B_1 \neq \{1\}$, then by \cref{propo:p_gp_broken_down}, there exists a cyclic group $C_2$ and a $B_2 \leq B_1$ such that $B_1 \cong C_2 \times B_2$.

  By continuing this line of argument, we can get $C_1, C_2, ...$ until we get to some $C_k$ with $B_k = \{1\}$, for some $k \in \mathbb{N}$. Then
  \begin{equation*}
    G \cong C_1 \times C_2 \times \hdots \times C_k
  \end{equation*}
  as required. \qed
\end{proof}

\begin{remark}
  We can verify that the decomposition of a finite abelian $p$-group into a direct product of cyclic groups is in fact unique up to their orders.\sidenote{This is the bonus question on A4. It will be included once the assignment is over.}
\end{remark}

Combining the above remark, \cref{thm:primary_decomposition} and \cref{thm:finite_abelian_groups_are_isomorphic_to_a_direct_product_of_cyclic_groups}, we have the following theorem.

\begin{thm}[Finite Abelian Group Structure]
\index{Finite Abelian Group Structure}
\label{thm:finite_abelian_group_structure}
  If $G$ is a finite abelian group, then
  \begin{equation*}
    G \cong C_{p_1^{n_i}} \times \hdots \times C_{p_k^{n_k}}
  \end{equation*}
  where $C_{p_i^{n_i}}$ is a cyclic group of order $p_i^{n_i}$, where $1 \leq i \leq k$. The numbers $p_i^{n_i}$ are uniquely determined up to their order.\sidenote{Note that the $p_i$'s do not have to be unique.}
\end{thm}

\begin{remark}
  Note that if $p_1$ and $p_2$ are distinct primes, then
  \begin{equation*}
    C_{p_1^{n_1}} \times C_{p_2^{n_2}} \cong C_{p_1^{n_1} p_2^{n_2}},
  \end{equation*}
  the cyclic group of order $p_1^{n_1} p_2^{n_2}$. Thus, by combining suitable prime factors together, for a finite abelian group $G$, we can also write
  \begin{equation*}
    G \cong \mathbb{Z}_{m_1} \times \mathbb{Z}_{m_2} \times \hdots \times \mathbb{Z}_{m_r},
  \end{equation*}
  where $m_i \in \mathbb{N}$, $i \leq 1 \leq r$, $m_1 > 1$ and
  \begin{equation*}
    m_1 \, \big| \, m_2 \, \big| \hdots \big| \, m_r
  \end{equation*}
\end{remark}

\begin{eg}
  Conder an abelian group $G$ with order $48$. Since $48 = 2^4 \cdot 3$, an abelian group of order $48$ is isomorphic to $H \times \mathbb{Z}_3$, where $H$ is an abelian group of order $2^4$. The options for $H$ are:
  \begin{gather*}
    \mathbb{Z}_{2^4} \qquad \mathbb{Z}_{2^3} \times \mathbb{Z}_2 \qquad \mathbb{Z}_{2^2} \times \mathbb{Z}_{2^2} \\
    \mathbb{Z}_{2^2} \times \mathbb{Z}_2 \times \mathbb{Z}_2 \qquad \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_2
  \end{gather*}
  Therefore, we have the following possible decompositions of $G$:
  \begin{align*}
    G &\cong \mathbb{Z}_{2^4} \times \mathbb{Z}_3 \cong \mathbb{Z}_{48} \\
    G &\cong \mathbb{Z}_{2^3} \times \mathbb{Z}_2 \times \mathbb{Z}_3 = \mathbb{Z}_2 \times \mathbb{Z}_{24} \\
    G &\cong \mathbb{Z}_{2^2} \times \mathbb{Z}_{2^2} \times \mathbb{Z}_3 = \mathbb{Z}_4 \times \mathbb{Z}_{12} \\
    G &\cong \mathbb{Z}_{2^2} \times \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_3 = \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_{12} \\
    G &\cong \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_3 = \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_2 \times \mathbb{Z}_6
  \end{align*}
\end{eg}

% subsection p_groups_continued_2 (end)

% section finite_abelian_groups_continued_2 (end)

\section{Rings}%
\label{sec:rings}
% section rings

\subsection{Rings}%
\label{sub:rings}
% subsection rings

\begin{defn}[Ring]\index{Ring}
\label{defn:ring}
  A set $R$ is a ring if $\forall a, b, c \in R$,
  \marginnote{\noindent As daunting as this definition seems, it is much easier to remember if we think of $R$ being an \hlnoteb{abelian group under addition}, \hlnoteb{``almost'' a group under multiplication}, save the fact that the \hlimpo{multiplicative inverse of an element does not necessarily exist}, and with the \hlnoteb{distributive law}. }
  \begin{enumerate}
    \item $a + b \in R$
    \item $a + b = b + a$
    \item $a + (b + c) = (a + b) + c$
    \item $\exists 0 \in R \enspace a + 0 = a = 0 + a$
    \item $\exists (-a) \in R \enspace a + (-a) = 0 = (-a) + a$
    \item $ab \in R$
    \item $a(bc) = (ab)c$
    \item $\exists 1 \in R \enspace 1 \cdot a = a = a \cdot 1$
    \item $a ( b + c ) = ab + ac$ and $(b + c) a = ba + ca$
  \end{enumerate}
  We call $1$ as the \hldefn{Unity} of $R$, $0$ as the \textcolor{base16-eighties-blue}{Zero}\index{Zero of a Ring} of $R$, and $-a$ as the \hlnoteb{negative} of $a$.

  The ring $R$ is called a \hldefn{Commutative Ring} if it also satisfies the following:
  \begin{enumerate}
    \setcounter{enumi}{9}
    \item $ab = ba$.
  \end{enumerate}
\end{defn}

\begin{eg}
  $\mathbb{Z}, \mathbb{Q}, \mathbb{R}$ and $\mathbb{C}$ are commutative rings with the zero being $0$, and unity being $1$.
\end{eg}

\begin{eg}
  For $n \in \mathbb{N}, \, n \geq 2$, $\mathbb{Z}_n$ is a commutative ring with the zero being $[0]$, and unity being $[1]$.
\end{eg}

\begin{eg}
  The set $M_n(\mathbb{R})$ is a ring using matrix addition and matrix multiplication, with zero being the zero matrix $0$, and unity being the identity matrix $I$. We also know that $M_n(\mathbb{R})$ is not commutative.
\end{eg}

\begin{warning}
  Note that since $(R, \cdot)$ is not a group, we no longer have the liberty of using \cref{propo:cancellation_laws}, i.e. we do not have left or right cancellation. For example, in $\mathbb{Z}$, $0 \cdot x = 0 \cdot y \notimply x = y$.
\end{warning}

% subsection rings (end)

% section rings (end)

% chapter lecture_20_jun_18th_2018 (end)


%% Project Proposal for BMB497F as taught during FA15 at Penn State.
%%
%% The document layout is based on the document originally found at:
%% prancer.physics.louisville.edu/classes/308/project_proposal/proposal.tex

\documentclass[dvips,12pt]{article}

\usepackage[pdftex]{graphicx}
\usepackage{url}
\usepackage{nth}

\setlength{\parindent}{0em}
\setlength{\parskip}{0.7em}

\begin{document}

\title{A De Bruijn Representation of a Repeat Family:\\Project Proposal}
\author{Sindhuja Parimalarangan, Matthew Allan, Scott Cheloha}
\date{October 12, 2015}

\maketitle

\section{Introduction}

De Bruijn graphs are used in computational biology to assemble
collections of short reads into genome sequences. This approach is
confounded by repetitive elements, which create cycles in the
graph. However, we believe we can actually take advantage of the
properties of De Bruijn graphs in order to represent a set of related
repeat elements and to derive a consensus (i.e. average) form of the
repeat element.

For our project, we will implement a De Bruijn graph representation of a
repeat family.

\section{Motivation}

While other representations of repeat families exist, there is plenty of
room for experimentation toward a better, perhaps standard,
representation.

For this project, as mentioned above, we will use De Bruijn graphs as a
basis for developing such a representation.  De Bruijn graphs are
presently used to assemble short reads.  We want to explore whether it
is possible to alter and/or augment the De Bruijn structure to
better represent repeat elements.

For instance, existing consensus sequences give the probability of
finding a base at a given position, but do not give the probability of
finding a given base in the position immediately \emph{after} said base.
De Bruijn graphs could show, e.g., that an Adenine at position 1 is
almost always followed by an Adenine at position 2, even if the
probabilities of finding an Adenine at positions 1 or 2 is only 10\% for
either position.

\section{Goals}

In order of decreasing priority, we want to:
\begin{enumerate}
\item Construct a De Bruijn graph representation of a chosen repeat
  family;
\item Develop a representation for a set of related repeat elements and
  derive a consensus (i.e. average) form for the repeat element; and
\item Design and implement an optimized algorithm for representing an
  input repeat family using De Bruijn graphs.
\end{enumerate}

\section{Resources}

The following subsections list the resources we anticipate will be
necessary to achieve the prenominated goals.  The lists will grow with
our understanding of the problem.

\subsection{Data}

At a minimum we will use the following datasets:
\begin{itemize}
\item GenBank
\item LINE/SINE element sequences of indeterminate source
\end{itemize}

\subsection{Hardware}

We will probably need time on a cluster of some capacity in order to
execute our software on larger datasets.

\subsection{Software}

Exactly what form the output of this project will take with regard to
implementation language, use of preexisting libraries, if any, etc., is
unclear.  For now, at least while researching for the project and
preparing the final report, we believe we will use the following:
\begin{itemize}
\item UCSC Genome Browser,
  for lots of things;
\item Galaxy,
  for data analysis;
\item RepeatMasker,
  as a reference implementation of some aspects of the project;
\item Velvet and related software,
  for sequence assembly;
\item RepeatFinder, RECON, and RepeatGluer,
  for finding and processing repeats;
\item Some sort of graph visualization library, for algorithm design and
  the final report; and
\item The GIMP, for manipulating those visualizations.
\end{itemize}

\section{Responsibilities}

Each team member will be partly responsible for determining how to use
De Bruijn graphs to represent a consensus sequence for our target repeat
families.  Each team member will also be partly responsible for deciding
upon which axes (computational efficiency, output accuracy, quality of
output representation, etc.) we will compare our results to those of
RepeatMasker.

In addition, each team member will be responsible for a more specific
partition of tasks tuned to their interests and abilities.

\textbf{Sindhuja Parimalarangan} will:
\begin{itemize}
\item Design and implement a software workflow for collecting, parsing,
  and storing data;
\item Optimize the algorithm vis a vis data structures and external
  libraries to mitigate redundant searching; and
\item Implement both the graph construction and consensus
  sequence-finding algorithms.
\end{itemize}

\textbf{Matthew Allan} will:
\begin{itemize}
\item Identify repeat target repeat families and document why they are
  suitable for the project;
\item Determine what RepeatMasker \emph{does} in order to determine a
  consensus sequence so that we something of a reference;
\item Run RepeatMasker as needed; and
\item Explain any differences between RepeatMasker's output and that of
  our implementation.
\end{itemize}

\textbf{Scott Cheloha} will:
\begin{itemize}
\item Scour the literature for De Bruijn graph construction algorithms;
\item Determine which of those algorithms exist in a form we can use
  (legally, architecturally, etc.) and which we must implement
  ourselves; and
\item Implement both the graph construction and consensus
  sequence-finding algorithms.
\end{itemize}

\section{Timeline}

On \textbf{November 9, 2015}, we will have:
\begin{enumerate}
\item Identified our target family/families of repeats;
\item Generated De Bruijn graphs for the same using some preexisting
  implementation;
\item Obtained any needed datasets and stored them (possibly on the
  cluster) in a suitable format;
\item Run RepeatMasker and learned what we can from it; and
\item Finalized the high-level details of the algorithm we intend to
  implement.
\end{enumerate}

On \textbf{December 12, 2015}, we will have:
\begin{enumerate}
\item A clean, optimized implementation of our algorithm;
\item The results of our algorithm in a legible format; and
\item Determined how (or how \emph{not}) to make a consensus sequence
  from De Bruijn graphs that is more informative than a table of letter
  frequencies.
\end{enumerate}

\end{document}