%%%%%latex preamble%%%%%

\documentclass{article}
%% maxwidth is the original width if it is less than linewidth
%% otherwise use linewidth (to make sure the graphics do not exceed the margin)
\usepackage{alltt}
\usepackage[sc]{mathpazo}
\usepackage{amsmath, amsthm, amssymb}
\usepackage{graphicx}
\usepackage[T1]{fontenc}
\usepackage{geometry}
%\geometry{verbose,tmargin=2.5cm,bmargin=2.5cm,lmargin=1.5cm,rmargin=1.5cm}
\setcounter{secnumdepth}{2}
\setcounter{tocdepth}{2}
\usepackage{url}
\usepackage[unicode=true,pdfusetitle,
  bookmarks=true,bookmarksnumbered=true,bookmarksopen=true,bookmarksopenlevel=2,
breaklinks=false,pdfborder={0 0 1},backref=false,colorlinks=false]
{hyperref}
\hypersetup{pdfstartview={XYZ null null 1}}
\usepackage{float}
\usepackage{bm}
\usepackage{tikz}
 %changes default sectioning commands
%\usepackage{breakurl}
\renewcommand{\thesubsection}{(\alph{subsection})}
\renewcommand{\thesubsubsection}{\roman{subsection}.}
\usepackage{lastpage}
\usepackage{fancyhdr}
\pagestyle{fancy}
\lhead{}
\chead{\leftmark}
\rhead{}
\lfoot{Aaron Gonzales; Data Mining}
\cfoot{Homework 1}
\rfoot{Page \thepage\ of \pageref{LastPage}}
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}

\begin{document}

\title{Homework 1, Data Mining, Fall 2014}
\author{Aaron Gonzales}
\maketitle

given the following vectors:

\begin{align}
	x = \{1, 3, 5, 7, 6, 8, 5, 2\}  \\
	y = \{7, 11, 9, 6, 4, 3, 1, 3\}  \\
	z = \{3, 5, 7, 9, 8, 7, 6, 5\}
\end{align}

\section{wavelet transform each vector}
An un-normalized 8-point Haar matrix \[H_8\] is shown below

\[ H_{8} =
\begin{pmatrix}
	1&1&1&1&1&1&1&1 \\
	1&1&1&1&-1&-1&-1&-1 \\
	1&1&-1&-1&0&0&0&0& \\
	0&0&0&0&1&1&-1&-1 \\
	1&-1&0&0&0&0&0&0& \\
	0&0&1&-1&0&0&0&0 \\
	0&0&0&0&1&-1&0&0& \\
	0&0&0&0&0&0&1&-1
\end{pmatrix}
\]
and the vector
\[ \{ \sqrt{8}, \sqrt{8} ,2, 2, \sqrt{2}, \sqrt{2}, \sqrt{2}, \sqrt{2} \} \]
and will be used to normalize each vector after transformation.

\subsection{vector x}

\[
\begin{pmatrix}
	1&1&1&1&1&1&1&1 \\
	1&1&1&1&-1&-1&-1&-1 \\
	1&1&-1&-1&0&0&0&0& \\
	0&0&0&0&1&1&-1&-1 \\
	1&-1&0&0&0&0&0&0& \\
	0&0&1&-1&0&0&0&0 \\
	0&0&0&0&1&-1&0&0& \\
	0&0&0&0&0&0&1&-1
\end{pmatrix}
*
\begin{pmatrix}
  1 \\
  3 \\
  5 \\
  7 \\
  6 \\
  8 \\
  5 \\
  2
\end{pmatrix}
=
\begin{pmatrix}
	1 + 3 + 5  + 7 + 6 + 8 + 5 + 2 \\
	1 + 3  + 5 + 7 - 6 - 8 -5 -2 \\
	1 + 3 + 5 + 7 \\
	6+ 8 -5 -2 \\
	1-3 \\
	5-7 \\
	6-8 \\
	5-2
\end{pmatrix}
\]
\begin{align*}
	=&\\
	\{ 37, -4, -8, 7, -2, -2, -2, 3 \} \\
	\odot&\\
\{ \sqrt{8}, \sqrt{8},2, 2, \sqrt{2}, \sqrt{2}, \sqrt{2}, \sqrt{2}, \} \\
    =&\\
	\{74 \sqrt{ 2 } , -8 \sqrt{ 2 } , -16 , 14 , -2 \sqrt{ 2 } , -2 \sqrt{ 2 } , -2 \sqrt{ 2 } , 3 \sqrt{ 2 }  \} \\
	\{104.652, -11.3137, -16., 14., -2.82843, -2.82843, -2.82843, 4.24264 \}
\end{align*}

\subsection{vector y}

\[
\begin{pmatrix}
	1&1&1&1&1&1&1&1 \\
	1&1&1&1&-1&-1&-1&-1 \\
	1&1&-1&-1&0&0&0&0& \\
	0&0&0&0&1&1&-1&-1 \\
	1&-1&0&0&0&0&0&0& \\
	0&0&1&-1&0&0&0&0 \\
	0&0&0&0&1&-1&0&0& \\
	0&0&0&0&0&0&1&-1
\end{pmatrix}
*
\begin{pmatrix}
  7 \\
  11 \\
  9 \\
  6 \\
  4 \\
  3 \\
  1 \\
  3
\end{pmatrix}
=
\begin{pmatrix}
  7+11+9+6+4+3+1+3 \\
  7+11+9+6 -4 -3 -1 -3 \\
  7+11 -9-6 \\
  4+3 -1 -3 \\
  7-11 \\
  9-6 \\
  4-3 \\
  1-3

\end{pmatrix}
\]
\begin{align*}
	= \{ 44, 22, 3, 3, -4, 3, 1, -2 \} \\
	\odot&\\
\{ \sqrt{8}, \sqrt{8},2, 2, \sqrt{2}, \sqrt{2}, \sqrt{2}, \sqrt{2}, \} \\
    =&\\
	\{ 88 \sqrt{2}, 44 \sqrt{2}, 6, 6, -4 \sqrt{2}, 3 \sqrt{2}, \sqrt{2}, -2 \sqrt{2} \} \\
	\{ 124.451, 62.2254, 6., 6., -5.65685, 4.24264, 1.41421, -2.82843 \}
\end{align*}

\subsection{vector z}

\[
\begin{pmatrix}
	1&1&1&1&1&1&1&1 \\
	1&1&1&1&-1&-1&-1&-1 \\
	1&1&-1&-1&0&0&0&0& \\
	0&0&0&0&1&1&-1&-1 \\
	1&-1&0&0&0&0&0&0& \\
	0&0&1&-1&0&0&0&0 \\
	0&0&0&0&1&-1&0&0& \\
	0&0&0&0&0&0&1&-1
\end{pmatrix}
*
\begin{pmatrix}
  1 \\
  3 \\
  5 \\
  7 \\
  6 \\
  8 \\
  5 \\
  2
\end{pmatrix}
=
\begin{pmatrix}
  3 + 5 + 7 + 9 + 8 + 7 + 6 + 5 \\
  3 + 5 + 7 + 9 + -9 + -7 + -6 + -5 \\
  3 + 3 + -7 + -9 \\
  8 + 7 + -6 + -5 \\
  3 + -5 \\
  7 + -9 \\
  8 + -7 \\
  6 + -5
\end{pmatrix}
\]
\begin{align*}
  = \{ 50, -2, -8, 4, -2, -2, 1, 1 \}\\
	\odot&\\
\{ \sqrt{8}, \sqrt{8},2, 2, \sqrt{2}, \sqrt{2}, \sqrt{2}, \sqrt{2}, \} \\
    =&\\
	\{100 \sqrt{2}, -4 \sqrt{2}, -16, 8, -2 \sqrt{2}, -2 \sqrt{2}, \sqrt{2}, \sqrt{2}\}\\
	\{141.421, -5.65685, -16, 8, -2.82843, -2.82843, 1.41421, 1.41421\}
\end{align*}

\section{fourier transform each vector}

(Defintion from Wikipedia)

The transformation $W$ of size $N\times N $  can be defined as
\[W = \left(\frac{\omega^{jk}}{\sqrt{N}}\right)_{j,k=0,\ldots,N-1} \]
, or equivalently:

\[ W = \frac{1}{\sqrt{N}}
	\begin{pmatrix}
		1&1&1&1&\cdots &1 \\
		1&\omega&\omega^2&\omega^3&\cdots&\omega^{N-1} \\
		1&\omega^2&\omega^4&\omega^6&\cdots&\omega^{2(N-1)}\\
		1&\omega^3&\omega^6&\omega^9&\cdots&\omega^{3(N-1)}\\
		\vdots&\vdots&\vdots&\vdots&\ddots&\vdots\\
		1&\omega^{N-1}&\omega^{2(N-1)}&\omega^{3(N-1)}&\cdots&\omega^{(N-1)(N-1)}\\
	\end{pmatrix}
\]
where $ \omega = e^{-\frac{2\pi i}{N}} $

In our case, with an $8\times8$ matrix, we get a transformation matrix of:
\[ W=
	\frac{1}{\sqrt{8}}
	\begin{pmatrix}
		\omega^0     & \omega^0   &\omega^0   & \ldots & \omega^0        \\
		\omega^0     & \omega^1   &\omega^2   & \ldots & \omega^7        \\
		\omega^0     & \omega^2   &\omega^4   & \ldots & \omega^{14}     \\
		\omega^0     & \omega^3   &\omega^6   & \ldots & \omega^{21}     \\
		\omega^0     & \omega^4   &\omega^8   & \ldots & \omega^{28}     \\
		\omega^0     & \omega^5   &\omega^{10}   & \ldots & \omega^{35}  \\
		\vdots       & \vdots     & \vdots       & \ddots & \vdots       \\
		\omega^0     & \omega^7   &\omega^{14}   & \ldots  & \omega^{49} \\
	\end{pmatrix}
\]
where $\omega = e^{-\frac{2 \pi i}{8}} = \frac{1}{\sqrt{2}} -
\frac{i}{\sqrt{2}}$

where $ \omega = e^{-2\pi i} = 1 $  is Euler's identity, allowing us to reduce
the $8 \times 8$ matrix as follows:

\[
  \frac{1}{\sqrt{8}}
	\begin{pmatrix}
		1 & 1                      & 1    & 1                       &  1  & 1             \\
		1 & e^{\frac{- \pi i}{4}}  & -i   & e^{\frac{-3 \pi i}{4}}  & -1  & e^{\frac{-5 \pi i}{4}}        \\
		1 & -i                     & -1   & i                       &  1  & -i        \\
		1 & e^{\frac{-3 \pi i}{4}} &  i   &  e^{\frac{- \pi i}{4}}  & -1  & e^{\frac{-7 \pi i}{4}}      \\
		1 & -1                     & 1    &  -1                     &  1  & -1         \\
		1 & e^{\frac{-5 \pi i}{4}} &  -i  &  e^{\frac{-7 \pi i}{4}} & -1  & e^{\frac{- \pi i}{4}}         \\
		1 & i                      & -1   &  -i                     &  1  & i        \\
		1 & e^{\frac{-7 \pi i}{4}} & i    &  e^{\frac{-5 \pi i}{4}} & -1  &e^{\frac{-3 \pi i}{4}}
	\end{pmatrix}
\]

\subsection{Vector x}
\[ x = \{1, 3, 5, 7, 6, 8, 5, 2\} \]
\[
  \frac{1}{\sqrt{8}}
	\begin{pmatrix}
		1 & 1                      & 1  & 1                      & 1  & 1             \\
		1 & e^{\frac{- \pi i}{4}}  & -i & e^{\frac{-3 \pi i}{4}} & -1 & e^{\frac{-5 \pi i}{4}}        \\
		1 & -i                     & -1 & i                      & 1  & -i        \\
		1 & e^{\frac{-3 \pi i}{4}} & i  & e^{\frac{- \pi i}{4}}  & -1 & e^{\frac{-7 \pi i}{4}}      \\
		1 & -1                     & 1  & -1                     & 1  & -1         \\
		1 & e^{\frac{-5 \pi i}{4}} & -i & e^{\frac{-7 \pi i}{4}} & -1 & e^{\frac{- \pi i}{4}}         \\
		1 & i                      & -1 & -i                     & 1  & i        \\
		1 & e^{\frac{-7 \pi i}{4}} & i  & e^{\frac{-5 \pi i}{4}} & -1 & e^{\frac{-3 \pi i}{4}}
	\end{pmatrix}
	*
	\begin{pmatrix}
		1 \\
		3 \\
		5 \\
		7 \\
		6 \\
		8 \\
		5 \\
		2
	\end{pmatrix}
	=  \\
	\begin{pmatrix}
		1      + 3                           + 5                            + 7                           + 6                             + 8                          + 5                            + 2 \\
		1      + 3 e^{\frac{- \pi i}{4}}     + -5i                          + 7e^{\frac{-3 \pi i}{4}} -6  + 8 e^{\frac{-5 \pi i}{4}}      + 5i                         + 2 e^{\frac{-7 \pi i}{4}}  \\
		1 - 3i + -5                          + 7i                           + 6 - 8i - 5                  + 2i \\
		1      + 3e^{\frac{- 3 \pi i}{4}}    + 5i                           + 7 e^{\frac{- \pi i}{4}} -6  + 8 e^{\frac{- 7\pi i}{4}} - 5i + 2e^{\frac{-5 \pi i}{4}} \\
		1 - 3  + 5 - 7                       + 6 -8                         + 5 -2 \\
		1      + 3e^{\frac{-5\pi i}{4}}  -5i + 7 e^{\frac{-7 \pi i}{4}}  -6 + 8 e^{\frac{- \pi i}{4}}     + 5                             + 2 e^{\frac{-3\pi i}{4}} \\
		1      + 3i -5 -7i                   + 6                            + 8i -5 -2i \\
		1      + 3e^{\frac{-7\pi i}{4}}      + 5i                           + 7 e^{\frac{-5\pi i}{4}} - 6 + 8 e^{\frac{-3\pi i}{4}} -5i   + 2 e^{\frac{- \pi i}{4}}
	\end{pmatrix}
\]
\[
	=
	\begin{pmatrix}
	    37 \\
		-5         + 3 e^{-i \pi/4} + 2 e^{i \pi/4}  + 7 e^{-3 i \pi/4} + 8 e^{ 3 i \pi/4} \\
		-3 - 2i \\
		-5         + 7 e^{-i \pi/4} + 8 e^{ i \pi/4} + 3 e^{-3 i \pi/4} + 2 e^{ 3 i \pi/4} \\
		-3 \\
		-5 i       + 8 e^{-i \pi/4} + 7 e^{ i \pi/4} + 2 e^{-3 i \pi/4} + 3 e^{ 3 i \pi/4} \\
		-3         + 2i \\
		-5         + 2 e^{-i \pi/4} + 3 e^{ i \pi/4} + 8 e^{-3 i \pi/4} + 7 e^{ 3 i \pi/4}

	\end{pmatrix}
	=
	\begin{pmatrix}
		37 \\
		-12.07106 \\
		-3 -2i \\
		2.07106 \\
		-3 \\
		7.07106 - 5 i \\
		-3 + 2i \\
		-12.07106
	\end{pmatrix}
	*
   \frac{1}{\sqrt{8}}
\]
\begin{align*}
  \{13.0815, -4.26776, -1.06066-0.707107 i, 0.73223,\\
  -1.06066, 2.5-1.76777 i, -1.06066+0.707107 i, -4.26776\}
\end{align*}

\subsection{y vector}

\[ y = \{7, 11, 9, 6, 4, 3, 1, 3\}  \]
\[
  \frac{1}{\sqrt{8}}
	\begin{pmatrix}
		1 & 1                      & 1  & 1                      & 1  & 1             \\
		1 & e^{\frac{- \pi i}{4}}  & -i & e^{\frac{-3 \pi i}{4}} & -1 & e^{\frac{-5 \pi i}{4}}        \\
		1 & -i                     & -1 & i                      & 1  & -i        \\
		1 & e^{\frac{-3 \pi i}{4}} & i  & e^{\frac{- \pi i}{4}}  & -1 & e^{\frac{-7 \pi i}{4}}      \\
		1 & -1                     & 1  & -1                     & 1  & -1         \\
		1 & e^{\frac{-5 \pi i}{4}} & -i & e^{\frac{-7 \pi i}{4}} & -1 & e^{\frac{- \pi i}{4}}         \\
		1 & i                      & -1 & -i                     & 1  & i        \\
		1 & e^{\frac{-7 \pi i}{4}} & i  & e^{\frac{-5 \pi i}{4}} & -1 & e^{\frac{-3 \pi i}{4}}
	\end{pmatrix}
	*
	\begin{pmatrix}
		7 \\
		11 \\
		9 \\
		6 \\
		4 \\
		3 \\
		1 \\
		3
	\end{pmatrix}
	=  \\
	\begin{pmatrix}
		7 + 11                        + 9   + 6                        + 4   + 3                        + 1    + 3 \\
		7 + 11 e^{\frac{- \pi i}{4}}  + -9i + 6e^{\frac{-3 \pi i}{4}}  + -4  + 3 e^{\frac{-5 \pi i}{4}} + i   + 3 e^{\frac{-7 \pi i}{4}}  \\
		7 + -11i                      + -9  + 6i                       + 4   + -3i                      + -1   + 3i \\
		7 + 11e^{\frac{- 3 \pi i}{4}} + 9i  + 6 e^{\frac{- \pi i}{4}}  + -4  + 3 e^{\frac{- 7\pi i}{4}} + -i + 3e^{\frac{-5 \pi i}{4}} \\
		7 + -11                       + 9   + -6                       + 4   + -3                       + 1    + -3 \\
		7 + 11e^{\frac{-5\pi i}{4}}   + -9i + 6 e^{\frac{-7 \pi i}{4}} + -4  + 3 e^{\frac{- \pi i}{4}}  + 1    + 3 e^{\frac{-3\pi i}{4}} \\
		7 + 11i                       + -9  + -6i                      + 4   + 3i                       + -1   + -3i \\
		7 + 11e^{\frac{-7\pi i}{4}}   + 9i  + 6 e^{\frac{-5\pi i}{4}}  + -4  + 3 e^{\frac{-3\pi i}{4}}  + -i  + 3 e^{\frac{- \pi i}{4}}
	\end{pmatrix}
\]
\[
	=
	\begin{pmatrix}
	  44 \\
	  (3-8 i)   + 11 e^{-(i \pi)/4} + 3 e^{(i \pi)/4}  + 6 e^{-(3 i \pi)/4}   + 3 e^{(3 i \pi)/4} \\
	  1 - 5i \\
	  (3        + 8 i)             + 6 e^{-(i \pi)/4} + 3 e^{( i \pi)/4} + 11 e^{-(3 i \pi)/4}  + 3 e^{ 3 i \pi)/4} \\
	  -2 \\
	  (4-9 i)   + 3 e^(-(i \pi)/4)  + 6 e^{(i \pi)/4}  + 3 e^{-(3 i \pi)/4}   + 11 e^{(3 i \pi)/4} \\
	  1         + 5i \\
	  (3        + 8 i)             + 3 e^{-(i \pi)/4} + 11 e^{(i \pi)/4}     + 3 e^{-(3 i \pi)/4}   + 6 e^{(3 i \pi)/4}

	\end{pmatrix}
	=
	\begin{pmatrix}
	  44 \\
	  6.53553- 15.77817i \\
	  1 - 5i \\
	  -0.53553 + 0.22182 i \\
	  -2  \\
	  0.46446 - 1.221825 i \\
	  1 + 5i \\
	  6.5355 + 15.7781 i \\
	\end{pmatrix}
	*
   \frac{1}{\sqrt{8}}
 \]
 \begin{align*}
   \approx
   \{15.5563, 2.31066-5.57843 i, 0.353553-1.76777 i,\\
	 -0.189338+0.0784252 i, -0.707107, 0.164211-0.43198 i, \\
   0.353553+1.76777 i, 2.31065+5.5784 i\}
 \end{align*}

\subsection{z vector}

\[	z = \{3, 5, 7, 9, 8, 7, 6, 5\}  \]
\[
  \frac{1}{\sqrt{8}}
	\begin{pmatrix}
		1 & 1                      & 1  & 1                      & 1  & 1             \\
		1 & e^{\frac{- \pi i}{4}}  & -i & e^{\frac{-3 \pi i}{4}} & -1 & e^{\frac{-5 \pi i}{4}}        \\
		1 & -i                     & -1 & i                      & 1  & -i        \\
		1 & e^{\frac{-3 \pi i}{4}} & i  & e^{\frac{- \pi i}{4}}  & -1 & e^{\frac{-7 \pi i}{4}}      \\
		1 & -1                     & 1  & -1                     & 1  & -1         \\
		1 & e^{\frac{-5 \pi i}{4}} & -i & e^{\frac{-7 \pi i}{4}} & -1 & e^{\frac{- \pi i}{4}}         \\
		1 & i                      & -1 & -i                     & 1  & i        \\
		1 & e^{\frac{-7 \pi i}{4}} & i  & e^{\frac{-5 \pi i}{4}} & -1 & e^{\frac{-3 \pi i}{4}}
	\end{pmatrix}
	*
	\begin{pmatrix}
		3 \\
		5 \\
		7 \\
		9 \\
		8 \\
		7 \\
		6 \\
		5
	\end{pmatrix}
	=  \\
	\begin{pmatrix}
		3 + 5                       + 7   + 9                        + 8  + 7                        + 6   + 5 \\
		3 + 5 e^{\frac{- \pi i}{4}} + -7i + 9e^{\frac{-3 \pi i}{4}}  + -8 + 7 e^{\frac{-5 \pi i}{4}} + 6i  + 5 e^{\frac{-7 \pi i}{4}}  \\
		3 + -5                      + -7  + 9i                       + 8  + -7i                      + -6  + 5i \\
		3 + 5^{\frac{- 3 \pi i}{4}} + 7i  + 9 e^{\frac{- \pi i}{4}}  + -8 + 7 e^{\frac{- 7\pi i}{4}} + -6i + 5e^{\frac{-5 \pi i}{4}} \\
		3 + -5                      + 7   + -9                       + 8  + -7                       + 6   + -5 \\
		3 + 5^{\frac{-5\pi i}{4}}   + -7i + 9 e^{\frac{-7 \pi i}{4}} + -8 + 7 e^{\frac{- \pi i}{4}}  + 6   + 5 e^{\frac{-3\pi i}{4}} \\
		3 + 5                       + -7  + -9i                      + 8  + 7i                       + -6  + -5i \\
		3 + 5^{\frac{-7\pi i}{4}}   + 7i  + 9 e^{\frac{-5\pi i}{4}}  + -8 + 7 e^{\frac{-3\pi i}{4}}  + -6i + 5 e^{\frac{- \pi i}{4}}
	\end{pmatrix}
\]
\[
	=
	\begin{pmatrix}
	  50 \\
	  (-5-i)+5 e^{-(i \pi)/4}+5 e^{ i \pi)/4}+9 e^{-(3 i \pi)/4}+7 e^{3 i \pi)/4} \\
	  -7 + 7i \\
	  (-5+i)+5^{-(3 i \pi)/4}+9 e^{-(i \pi)/4}+7 e^{(i \pi)/4}+5 e^{(3 i \pi)/4} \\
	  -2 \\
	  3+5^{1/4 (-5 \pi i)}-7 i+9 e^{1/4 (-7 \pi i)}-8+7 e^{-(\pi i)/4}+6+5 e^{1/4 (-3 \pi i)} \\
	  3 - 7i \\
	  (-5+i)+5^{-(7 i \pi)/4}+5 e^{-(i \pi)/4}+7 e^{-(3 i \pi)/4}+9 e^{(3 i \pi)/4}

	\end{pmatrix}
	=
	\begin{pmatrix}
	  50 \\
	  -9.242 - 2.4142 i \\
	  -7 + 7i \\
	  1.9824 + 3.7269 i \\
	  -2 \\
	  9.7774 - 9.1583 i \\
	  3 - 7i \\
	  -13.6165 - 1.6663 i

	\end{pmatrix}
	*
   \frac{1}{\sqrt{8}}
\]
\begin{align*}
  \{ 17.6777, -3.26754-8535.49 i, -2.47487+2.47487 i,\\
  0.700884+1.31766 i, -0.707107, 3.45683-3.23795 i, \\
  1.06066-2.47487 i, -4.81416-0.589126 i \}
\end{align*}

\section{minkowski distance}

The Minkowski distance of order $p$ between two points

 \[ P =(x_1,x_2,\ldots,x_n)\text{ and }Q=(y_1,y_2,\ldots,y_n) \in \mathbb{R}^n
 \]

is defined as:

\[ \left(\sum_{i=1}^n |x_i-y_i|^p\right)^{1/p} \]
(Wikipedia)

and can be written as:
\[d(i,j) = \sqrt[h]{|x_{i1} - x_{j1}|^h +  |x_{i2} - x_{j2}|^h + \dots + | x_{ip} - x_{jp}|^h } \]

where $i = \{x_{i1}, x_{i_2}, \ldots, x_{ip} \}$ and  $j = \{x_{j1}, x_{j_2},
\ldots, x_{jp} \} $ are two $p$-dimensional data objects and $h$ is the order
(from class slides).

\subsection{raw vectors}
Between x and y:
\[ x = \{1, 3, 5, 7, 6, 8, 5, 2\}  \] \\
\[ 	y = \{7, 11, 9, 6, 4, 3, 1, 3\}  \] \\

\[ \sqrt{ | 1-7 | + |3 -11| + |5-9| +
  |7-6| + |6-4| + |8-3| + |5-1| + |2-3|} \]
  \[ =\sqrt{31} \]
  \[ \approx 5.5677\]

\[ \sqrt{ | 1-7 |^2 + |3 -11|^2 + |5-9|^2 +
  |7-6|^2 + |6-4|^2 + |8-3|^2 + |5-1|^2 + |2-3|^2} \]
  \[ = \sqrt{163} \]
  \[ \approx 12.767 \]

  Between x and z:

\[ x = \{1, 3, 5, 7, 6, 8, 5, 2\}  \] \\
\[ z = \{3, 5, 7, 9, 8, 7, 6, 5 \} \]

\[ \sqrt{ | 1-3 | + |3 -5| + |5-7| +
  |7-9| + |6-8| + |8-7| + |5-6| + |2-5|} \]
  \[ = \sqrt{15} \]
  \[ \approx 3.8729 \]

\[ \sqrt{ | 1-3 |^2 + |3 -5|^2 + |5-7|^2 +
  |7-9|^2 + |6-8|^2 + |8-7|^2 + |5-6|^2 + |2-5|^2} \]
  \[ = \sqrt{13} \]
  \[ \approx 5.56776 \]

  between y and z:

\[ y = \{7, 11, 9, 6, 4, 3, 1, 3\}  \] \\
 \[ z = \{3, 5, 7, 9, 8, 7, 6, 5 \} \]

\[ \sqrt{ | 7-3 | + |11 -5| + |9-7| +
  |6-9| + |4-8| + |3-7| + |1-6| + |3-5|} \]
  \[ = \sqrt{30} \]
  \[ \approx 5.477 \]

\[ \sqrt{ | 7-3 |^2 + |11 -5|^2 + |9-7|^2 +
  |6-9|^2 + |4-8|^2 + |3-7|^2 + |1-6|^2 + |3-5|^2} \]
  \[ =3 \sqrt{14} \]
  \[ \approx 11.2249 \]

\subsection{1st 3 wavlet coeff.}
X : $\{104.652, -11.3137, -16 \}$ \\
Y: $ \{ 124.451, 62.2254, 6 \}$  \\
Z: $ \{141.421, -5.65685, -16 \} $

Between x and y:\\
\subsubsection{h =1}
\[ \sqrt{ | 104.652-124.451 | + |-11.3137 -62.2254| + |-16-6| } \]
  \[ \approx 10.7396 \]
\subsubsection{h =2}
\[ \sqrt{ | 104.652-124.451 |^2 + |-11.3137 -62.2254|^2 + |-16-6|^2 } \]
  \[ \approx 79.27 \]

  between y and z:\\
Y: $ \{ 124.451, 62.2254, 6\}$  \\
Z: $ \{141.421, -5.65685, -16\} $ \\

\subsubsection{h=1}
\begin{multline*}
	\sqrt{ | 124.451 + 141.451| + | 62.2254 - -5.65685 | + | 6 - -16|}  \\
\end{multline*}
\[ \approx 18.8622 \]

\subsubsection{h=2}
\begin{multline*}
	\sqrt{ | 124.451 + 141.451|^2 + | 62.2254 - -5.65685 |^2 + | 6 - -16|^2}  \\
\end{multline*}
\[ \approx 275.311 \]

X : $\{104.652, -11.3137, -16\}$ \\
Z: $ \{141.421, -5.65685, -16\} $ \\
\subsubsection{h=1}
\begin{multline*}
	\sqrt{ | 104.652 - 141.421 | + | -11.3137 - -5.65658| + |-16 -16|} \\
\end{multline*}
\[ \approx 8.627 \]

\subsubsection{h=2}
\begin{multline*}
	\sqrt{ | 104.652 - 141.421 |^2 + | -11.3137 - -5.65658|^2 + |-16 -16|^2 } \\
\end{multline*}
\[ \approx 49.0710 \]
\subsection{largest three wavelet coefficients of x}

\subsection{largest three wavelet coefficients of y}

\subsection{which one of the above approximations matches the ordering of the distances from
the raw vectors?}

\end{document}


\documentclass[jair, twoside,11pt,theapa]{article}
\usepackage{jair, theapa, rawfonts, amssymb}
\usepackage{graphicx}

\ShortHeadings{GPU Accelerated Multilayer Perceptron}{Robinson}
\firstpageno{1}

\begin{document}
\title{GPU Accelerated Multilayer Perceptron}
\author{\name Max Robinson \email max.robinson@jhu.edu \\
       \addr Johns Hopkins University,\\
       Baltimore, MD 21218 USA
}

\maketitle

%\begin{abstract}
%\label{Abstract}
%\end{abstract}

\section{Introduction}
\label{Introduction}
\footnotetext{Some pieces of this project writeup are derived from previous work by Max Robinson for CS499}
Neural networks are an algorithm and concept that is popular and has proven very effective in the realm of Artificial Intelligence and Machine Learning.
The multilayer perceptron is an particular instantiation of a feedforward neural network that has been shown to be very effective.
It has been shown that a network with one hidden layer can be used as a universal approximator given a sufficient number of nodes in the hidden layer \cite{MLFFNetworks}.
With the capabilities that neural networks can provide, creating working and efficient implementations of multilayer perceptrons is a useful capability to have.

The goal of this project and paper is to implement and describe a GPU accelerated implementation of a multilayer perceptron using CUDA.
The program implemented was shown to successfully be able to learn to classify data well, all while running on a GPU.
While not the fastest overall implementation of a multilayer perceptron, the implementation was shown to have some computational benefits while needing improvements in others.

The rest of the paper describes the background of neural networks, the implementation, program, and experiments run to validate and compare the implementation.

\section{Background}
\label{Background}
This section describes the details behind how Multilayer perceptrons and the back propagation algorithms work.

\subsection{Feedforward Neural network and Multilayer Perceptrons}
A feedforward multilayer perceptron neural network is a structure for a learning algorithm that uses connected nodes in a directed manner to solve the problems of classification or regression.
Specifically feedforward neural networks are usually multilayered.
These networks are typically structured as layers of nodes with non-linear activations functions that are connected to another layer of nodes.

When connecting nodes from layer to layer, every node from a layer is connected to every other node in the next layer.
This is known as being a "fully connected" layer.
Each connection from one node to another has a weight $w_{ij}$.
A bias node with the value of 1 is also introduced at each layer, but no nodes connect to this bias, as it serves only to help serve as a threshold value \cite{BackProp}.

For a node, a non-linear activation function is used to determine and constrain the value of the inputs before passing it on.
It is important that this activation function be non-linear as this is what allows a network to learn non-linear models.

For this implementation, the non-linear activation function used is the logistic function.
To calculate the value of a given node in the network, a linear combinations of the input values multiplied by the weight on the connection is taken as shown in Equation \ref{Forward Pass} \cite{BackProp}.

\begin{equation}
\label{Forward Pass}
x_j = \sum_{i}{y_i w_{ij}}
\end{equation}

In Equation \ref{Forward Pass}, $x_j$ is the jth unit or node and $y_i$ is the ith input into $j$ and $w_{ij}$ is the weight between unit $i$ and $j$.
This linear combination is then put through the nodes activation function \cite{BackProp} show in Equation \ref{activation}.

\begin{equation}
\label{activation}
y_j = \frac{1}{1+e^{-x_j}}
\end{equation}

In Equation \ref{activation}, $y_j$ is the output of node $j$, with input value of $x_j$ as described above. Again, notice that the activation function is non-linear.

To get the total error for the network, which is how difference the output of the network is from the expected output of the network, is described using a form of squared error. Formally $E$ can be described by Equation \ref{error}\cite{BackProp}.

\begin{equation}
\label{error}
E = \frac{1}{2} \sum_{c}\sum_{j}(y_{j,c}-d_{j,c})^2
\end{equation}

In Equation \ref{error}, $c$ is an index over the dataset, and $j$ is an index over the output units.
For example, if there could be two output nodes of the network $y$ is the output of the network for a given output node.
Then, $d$ is the desired state for that output node.
This sum gives the total error of the state on a forward pass through the network.
More simply $ E = \frac{1}{2}(y_{j}-d_{j})^2$ is describes the error at a given node $j$ given an predicted output $y_j$ and desired output $d_j$.

\subsection{Back Propagation Algorithm}
\label{Back Propagation}
To train a feedforward neural network a technique called back propagation is used, which was developed by Rumelhart, Hinton, and Williams \shortcite{BackProp}.
Back propagation is used to incrementally update all of the weights in the neural network based on the error of the output of the network.
This technique uses gradient-descent by following the partial derivative of the error function with respect to the inputs and outputs to optimize the network.
This update is done in two parts, first calculating the update rule for the output layer, and then calculating the weight updates for the hidden layers.
The update rule for the output layer is described in Equation \ref{fullBP}.

\begin{equation}
\label{fullBP}
\Delta w_{ij} = \alpha (d_j - y_j)y_j(1-y_j)x_{ij}
\end{equation}
\begin{equation}
\label{delta_k}
\Delta w_{ij} = \alpha \delta_k x_{ij}, \delta_k = (d_j - y_j)(1-y_j)y_j
\end{equation}

Here, alpha is the learning rate, $d_j$ is the correct output for node $j$ and $y_j$ is the estimated output of the network for the jth output node, and $x_{ji}$ was the input value of the node before the activation function was applied. This $\Delta w_{ji}$ is then applied to all nodes $j$ connect to output node $i$. This follows gradient descent because the first and second parts of the equation are the derivatives of the error function with respect to the outputs and the outputs with respect the activation function respectively \cite{BackProp}.

\begin{equation}
\label{contribError}
\sum_{k \epsilon downstream(j)} \delta_k w_{ij}
\end{equation}

This sum is then combined with the derivative of the nodes activation function.
This gives the total derivative update described by Equation \ref{delta J}.

\begin{equation}
\label{delta J}
\delta_j = (y_j)(1-y_j)\sum_{k \epsilon downstream(j)} \delta_k w_{ij}
\end{equation}

\begin{equation}
\label{hiddenUpdate}
\Delta w_{ji} = \alpha \delta_j x_{ji}
\end{equation}

The update rule for weights for hidden layers is then described by Equation \ref{hiddenUpdate}.
The update rule in Equation \ref{hiddenUpdate} applies to all hidden nodes and weights.
These updates are applied to the network following the typical gradient descent pattern.
Once the network has finished training, the forward propagation part of the process of the network is used to get estimated outputs.

\section{GPU Implementation with CUDA}
\label{GPU}
Implementing neural networks can typically be done in two fashions.
The first is to loop through the weights and the inputs at each layer and calculate the output for each node one by one for the next layer.
Similarly for the back propagation, the error terms and appropriate delta values for each weight can be calculated individually for each weight.
This approach works but tends to be slow since loops must be run to calculate all elements involved individually.

The second approach, and the one taken for this implementation, is to represent the weights and layer outputs in the networks as matrices.
Representing the weights in this fashion allows for use of matrix multiplication to speed up calculations of network output.
Additionally, the vector and matrix approach enables implementation on GPU architecture since many vector and matrix operations can be parallelized on a GPU.
For this implementation, CUDA was chosen as the API to use to implement a multilayer perceptron on a GPU.

\subsection{Network Architecture}
\label{Network Arch Section}
To take advantage of multiple cores and threads on the GPU, the network architecture was structured into two parts: three weight matrices, and four vectors to represent layer outputs plus the input vector.
The weight matrices were stored as column based matrices to be compatible with CUDA.

Each row in the matrix represents all of the weights from an input node (e.g. A) to all the nodes in the next layer.
More importantly, each column in the matrix represent all the weights that are inbound on a single node in the next layer.
In Equation \ref{networkArch}, $H1$ has inputs from all input nodes with weights $A_{H1}$, $B_{H1}$, $C_{H1}$, $D_{H1}$

\begin{equation}
\label{networkArch}
\left[
\begin{array}{cccc}
A_{input} & B_{input} & C_{input} & D_{input}
\end{array}
\right]
%
\left[
\begin{array}{ccc}
A_{H1} & A_{H2} & A_{H3} \\
B_{H1} & B_{H2} & B_{H3} \\
C_{H1} & C_{H2} & C_{H3} \\
D_{H1} & D_{H2} & D_{H3} \\
\end{array}
\right]
%
=
\left[
\begin{array}{ccc}
H1_{output} & H2_{output} & H3_{output} \\
\end{array}
\right]
\end{equation}

Equation \ref{networkArch} shows an example how a 1x4 input vector multiplied by a 4x3 matrix of weights provides a 1x3 output vector for the next layer.
The 1x4 input vector is representative of a layer in the network.
The 4x3 matrix represents the weights that connect the input nodes from the input vector to the next layer of 3 nodes.

\subsection{Computation}
To compute and train a multilayer perceptron for this implementation, there are two parts to the computation: the forward pass and the back propagation.
All computation takes place on the GPU device, except for where explicitly stated that it does not.
\subsubsection{Forward Pass}
\label{FPSection}
Once the network has been created as a set of matrices the forward pass of the neural network becomes a series of matrix multiplications, and kernel calls.
The matrix multiplication takes an input vector and a weight matrix and produces the next layers raw output.
The raw output of the next layer is equivalent to the $x_j$ in Equation \ref{Forward Pass}.
The CUDA cuBLAS library \textit{cublasSgemm} function is used to compute the matrix multiplication efficiently.

After the $x_j$s have been computed the activation function still needs to be applied to get the final output of the layer.
To compute efficiently compute the output of the activation function in parallel, a custom CUDA kernel is applied to apply the activation function seen in Equation \ref{activation}.
This kernel applies the activation function to each data point individually in parallel.

After both the matrix multiplication and activation function have been applied, the output of that layer is complete.
The output from the layer just computed is then used as input into the next layer and the same steps are executed.

Once all layers have been computed the last layer represents the output values of the network.
During training, all intermediate layer output values are held onto as they are required for updating the network during back propagation.

\subsubsection{Back Propagation}
For the implementation the Back Propagation algorithm is chosen to make updates to the network.
The loss function implemented is a squared error loss function and all deltas and error functions are based on a gradient descent based approach based on the squared error loss function.
The equations to describe the update functions are found in Equations \ref{fullBP} - \ref{hiddenUpdate}

To implement back propagation on a GPU several kernels are used to calculate the necessary pieces step by step in parallel.
The first step is to calculate the $\delta_k$ error terms for the last layer.
This is done in parallel using a custom kernel that applies the $\delta_k$ function seen in Equation \ref{delta_k}.
The results provides the necessary components to update the last weight matrix.

For each hidden layer, starting from the layers closest to the output layer, the contribution of each node to the error for each output node is calculated.
The contribution to error is specified by Equation \ref{contribError}.
This is calculated by iterating through the hidden layer and calculating the dot product of that row of the weight matrix and the $\delta_k$s that were previously calculated.
The \textit{cublasSdot} function is used to compute the dot product.
After the contribution to error is computed the $\delta_j$ values or error values for hidden nodes is computed with a custom kernel.
The custom kernel applies Equation \ref{delta J} to each node output in parallel.
These steps are done for each hidden layer.

Once all error terms have been calculated the weights of the network are updated.
Updating the weights of the network applies Equations \ref{hiddenUpdate} and \ref{delta_k}.
The weights are updated by enqueuing a series of custom kernels that update each column of the weight matrix with the appropriate $x_{ji}$ value as each column of the weight matrix is associated with a node.
Weights are updated in place in the matrix.

\section{The Program}
The implementation of the Multilayer perceptron is compiled to an executable call ``network.exe".
To run the program the user must specify input parameters to the program.
Table \ref{ProgramOptions} specifies the command line parameters available to specify when running the program.
Figure \ref{HelpOutput} shows the program output if the program is run with no parameters.

\begin{table}
\centering
\caption{Program CLI options}
\label{ProgramOptions}
\begin{tabular}{|l|l|l|l|}
\hline
Option & Value & Explanation & Required \\ \hline
archFile    & string & file path to arch file & yes \\ \hline
weights    & string & \small{file path to weights file} & no \\ \hline
training    & string & file path to training data & no \\ \hline
groundTruth    & string & file path to ground truth output data  & yes, if training \\ \hline
evaluation    & string &evaluation file path for data to evaluate & yes, if not training \\ \hline
output    & string & output File path for weights & no\\ \hline
epochs    & int & number of training epochs & no\\ \hline
alpha    & float & learning parameter & no\\ \hline
\end{tabular}
\end{table}

\begin{figure}[h]
    \centering
    \caption{Program help output}
    \label{HelpOutput}
    \fbox{%
    \begin{minipage}{\textwidth}
        Usage is: ./network.exe --archFile $<$$>$ --weights $<$optional$>$ --training $<$trainingDataFile$>$ --groundTruth $<$gtFile$>$ --evaluation $<$dataFileForEval$>$ --output $<$networkWeightSaveFile$>$ --alpha $<$.1$>$ --epochs $<$200$>$
    \end{minipage}
    }
\end{figure}

To run the program in any way, the program requires an architecture file.
This file specifies what the network architecture for the multilayer perceptron is.
The architecture file is a one line csv file that has four values specifying, in order, the size of the input layer, first hidden layer, second hidden layer, and output layer.
Currently the program only supports multilayer perceptrons with this two hidden layer architecture.
The size of the layers can be any size less than or equal to 1024 and greater than 0, though smaller values are suggested.

A weights file can be given to the network to specify a a set of weights to use for each layer.
The file is a three line csv file.
The first line should have $inputSize * hiddenlayer1Size$ number of float values.
The second line should have $hiddenlayer1Size * hiddenlayer1Size$ number of float values.
The third line should have $hiddenlayer2Size * outputLayerSize$ number of float values.
Each values is read into the weight matrix in column order.

If no weights file is specified a random set of weights is created to fit the architecture of the network.
The weights are initialized on the GPU using the cuRAND CUDA library.
The weights are then shifted using a kernel function to be between the values of $-0.5$ and $0.5$.

When training a network, the ``--training" flag and the ``--groundTruth" flags must have values that point to a training file and corresponding ground truth file.
The training file must be a csv file where each line is the size of the input layer to the network.
The ground truth file must be a csv file where each line is the size of the output layer in the network.
Ground Truth files should can not be larger than what can be loaded into host memory.
The ground truth and training file should match up on a line by line basis (i.e. line 5 in training has a correspond line 5 in the ground truth file).

Training stops once the number of specified epochs is reached.
If the number of epochs is not specifies, training runs for a default of 100 epochs.
An epoch consists of a forward pass and back propagating through the network once for each data point in the dataset.
For every forward pass through the network during training there is a back propagation.

When training, the program outputs the average error per epoch as well as statistics about how quickly each part of the training process is.
Figure \ref{TrainingOutput} shows a sample output of the network while training.

\begin{figure}[h]
    \centering
    \caption{Sample program output during training}
    \label{TrainingOutput}
    \fbox{%
    \begin{minipage}{\textwidth}
        Average Error for Epoch \#8: 0.010930 \\
        Epoch Time: 128.874817ms \\
        Average Times of compute:  \\
        Iteration: 0.6860ms Forward Pass: 0.0374ms BackProp: 0.6172ms
    \end{minipage}
    }
\end{figure}

The program is also able to use the network to provide output for a dataset.
If an evaluation file is provided, the network reads in the evaluation file and only performs the forward pass of the network for each data point.
Evaluation files should be structured the same as training data files, and must not be larger than what can be loaded into host memory.
The network output is then collected and saved in csv format to a ``ResultsFile.txt".
Each line of the results file is an output from the network that corresponds to that line of the evaluation file as input data.

At the end of any execution, evaluation or training, if an output file is specified by the ``--output" option, the weights of the network are written to that file.
If no file is specified, the weights are written to a file called ``weights\_\{\}.txt" with ``\{\}" replaced by the current linux epoch time.

\section{Experiment}
\label{Experiment}
To test that the multilayer perceptron works a network was training on a dataset built for face recognition to see how accurate it can be.
Additionally, the speed of the implemented network is compared the that of a commercial CPU based implementation, TensorFlow (https://www.tensorflow.org/).

\subsection{Accuracy}
For our accuracy test, the dataset consists of 1x128 vectors that were created using OpenFace \cite{openface} on a celebrity faces image dataset \cite{celebrityFaces}.
The images first were passed through a face detection algorithm implemented using dlib, an open source c++ algorithm library.
After faces were detected, OpenFace was used to embed the faces into a 1x128 dimensional vector.

The dataset was labeled to as a classification problem were the goal is to be able to identify two celebrities and categories those who are not the celebrities as ``other".
The two celebrities to identify were Anderson Cooper and Scarlett Johansson.
The dataset was label as a 1's hot encoding dictating that the output layer of the network is size 3.

To train the network, the dataset was shuffled and two-thirds split off to be used for training while one-third is used for evaluation.
The network had a network architecture of 128, 15, 10, 3.
The network was trained for 300 epochs and a learning parameter of $alpha = .1$.

\subsection{Speed}
To compare the speed of the multilayer perceptron implementation in this paper, a multilayer perceptron was implemented using TensorFlow on a CPU.
The implementations are compared with two different network architectures.
One network has an architecture of 128, 15, 10, 3, and the other has an architecture of 128, 300, 200, 3.

The TensorFlow implementation for a multilayer perceptron is similar to the GPU implementation in this paper but there are two key differences.
First, the TensorFlow implementation uses a softmax cross entropy loss function, while the GPU implementation uses squared error.
Second, the TensorFlow implementation uses a slightly different optimization function, the Adam optimizer \cite{AdamOptimizer}.
The Adam optimizer is a gradient-based optimization technique that uses momentum as well as other parameters to help with the optimization process.

Speed for each multilayer perceptron implementation is compared by looking at the average speed to complete an epoch and the average time to feedforward and back prop through the network.

\section{Results and Analysis}
\label{Results}
This section details the results and analysis for each experiment run.

\subsection{Accuracy}
\label{Accuracy}
To measure accuracy, the error rate is calculated from the predicted classifications and the known correct classifications for the evaluation set.
Equation \ref{errorRate} describes how to calculate error rate.

\begin{equation}
\label{errorRate}
error rate = \frac{\# of misclassified test instances}{total number of test instances}
\end{equation}

The network program itself does not directly calculate error rates, but instead reports the results per data point to a file.
To calculate the error rate a small python program is used to compare the output data from the network to the ground truth test data.
Network outputs are interpreted using an argmax approach for a one's hot encoding.
Of the outputs from the network nodes, the node with the highest values is considered to be equal to 1, and all other values are 0.

After processing the network outputs, the error rate was about $0.02128$ or about 2\%.
There were 94 data points in the test set, so this means that 2 out of the 94 evaluation data points were misclassified.
This shows that the network was definitely able to learn to distinguish the two celebrities from the others most of the time.

\subsection{Speed}

The GPU implementation in this paper, called ``network.exe", and the TensorFlow implementation were trained on the same set of data as the accuracy experiment.
Average compute time is the time taken to do a forward pass through the network, calculate errors, and do a back propagation on a single training data point.
The FP time is the time it takes to complete only the forward pass through the network.
The overall average epoch time is the time taken to do a forward pass and back propagation for every data point in the training set once.
The back propagation time is not explicitly reported due to constraints in the TensorFlow implementation that does not give insight in the back propagation portion of the algorithm.

Table \ref{speedComparison} has the results for running the training and capturing timing metrics for training.
The results show that for overall epoch time for both network architectures, the TensorFlow implementation is quite a bit faster.
For the smaller network it is about two times faster and for the larger network it is about 23 times faster overall.
However, for the forward pass time, network.exe was about three times faster than the TensorFlow implementation.
This means that most of the disparity in compute times is in the back propagation portion of the algorithm.

Given the above described implementation and the complexity of training a multilayer perceptron this sort of makes sense.
The simplest part of training a multilayer perceptron is the forward pass.
As described above in sections \ref{Network Arch Section} and \ref{FPSection}, the weights are represented as matrices and the forward pass is a series of matrix multiplications.
Matrix multiplications on the GPU are done using an optimized parallelized CUDA library.
This would make it the most likely to run faster than on a CPU even with basic linear algebra libraries.
The highly tested and developed CUDA libraries provide great optimization and speed in this area.

The back propagation part of the algorithm is where the most custom code and kernels are run.
This is where the biggest time differences show.
There are a few likely reasons for this GPU implementation to be slower.
One is data transfer and memory allocations taking place on the GPU.
Second is a not having optimized weight update and error calculations.
Third is that TensorFlow is also using a compiler called XLA (https://www.tensorflow.org/xla) which is an accelerated linear algebra compiler to optimize.

When running the multilayer perceptron, the GPU implementation described in this paper ends up allocating and deallocating memory on the GPU frequently.
For example, calculating the errors for each layer in the network and using them for back propagation requires allocation of addition GPU memory to hold the errors.
The allocated space is also then deallocated to ensure no memory is leaked.
There are several places in the code where space is allocated and deallocated that might slow down the computation process.

In addition, data must be moved to and from the GPU for each new data point.
This requires copying data where the TensorFlow implementation does not need to.

The back propagation implementation for this paper is also probably not optimized properly.
There are several places where loops are used to help calculate various parts or enqueue computation.
For example, to calculate the contribution to error, a loop is used to call a cublasSdot function.
Using a loop to call the cublasSdot function, while better than doing a looped dot product, is likely not as optimized as possible.

The ``weightUpdate" kernel is also enqueued using a loop.
While each kernel is enqueued and run in parallel, there might be a more efficient way to update all the weights in the network.
Rather than enqueuing multiple kernels, it might be possible to write one kernel to update all the weights at once.
The difficult part of this is that each column in the kernel must have the same $x_{ji}$ applied to it.

Lastly, the TensorFlow API is a highly optimized commercial library even for implementation on a CPU.
TensorFlow uses a compiler XLA (https://www.tensorflow.org/xla), that optimizes linear algebra operations on all different hardware, including CPUs.
These optimizations enable the forward pass and back propagation to be as fast as possible and creates machine code that optimizes loops and kernel launches (https://developers.googleblog.com/2017/03/xla-tensorflow-compiled.html).

%While effective and implement the functions of a multilayer perceptron, might not be as optimized as a highly tested and accelerated CPU implementation.

% why so much slower?
% Three main reasons
% Moving data to and from the GPU (input data)
% allocating and dealocationg memory in the back prop portion of the training (forward pass = 0.0423ms)
    % poor implemenation on the update weight loop
% highly tuned comercial system that uses XLA (cpu accelerated linear algebra)

\begin{table}
    \centering
    \caption{Program CLI options}
    \label{speedComparison}
    \begin{tabular}{|l|l|l|l|l|}
        \hline
        Implementation & Network Arch. & Avg Epoch Time & Avg Compute Time & FP Time\\ \hline
        network.exe    &  128,15,10,3 & 127.4913ms & 0.6365ms & 0.0426ms \\ \hline
        TensorFlow     &  128,15,10,3  & 64.4191ms &  0.3302ms & 0.14902ms\\ \hline
        network.exe    &  128,300,200,3 & 1914.3486ms & 10.1816ms &  0.0558ms \\ \hline
        TensorFlow     &  128,300,200,3 & 84.9011ms & 0.4649ms & 0.1597ms \\ \hline
    \end{tabular}
\end{table}

%\section{Future Work}
%\label{Future}

\section{Conclusion}
\label{Conclusion}
The multilayer perceptron GPU implementation described in this paper is an effective implementation capable of learning to classify data points.
Computation for the multilayer perceptron takes place almost entirely on the GPU.
It utilizes both built in CUDA libraries and custom kernel execution to complete computation.
The program implementation allows for users to specify several command line parameters to enable custom execution of the program.

When comparing the speed of the implementation, it is generally slower to train than the reference TensorFlow implementation.
The GPU implementation is faster for computing the output of the network, however.
While the program is effective and implements the functions of a multilayer perceptron, it is not as optimized as a highly tested and accelerated CPU implementation.

With a few changes and updates to the GPU implementation, it is hopeful that this implementation might be able to close the gap in speed, while remaining an effective and working GPU accelerated multilayer perceptron.

\vskip 0.2in
\bibliography{ProjectWriteup}
\bibliographystyle{theapa}

\end{document}


\documentclass[letterpaper,9pt,twoside,printwatermark=false]{pinp}

%% Some pieces required from the pandoc template
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}

% Use the lineno option to display guide line numbers if required.
% Note that the use of elements such as single-column equations
% may affect the guide line number alignment.

\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{longtable}

% The geometry package layout settings need to be set here...
\geometry{layoutsize={0.95588\paperwidth,0.98864\paperheight},%
          layouthoffset=0.02206\paperwidth,%
		  layoutvoffset=0.00568\paperheight}

\definecolor{pinpblue}{HTML}{185FAF}  % imagecolorpicker on blue for new R logo
\definecolor{pnasbluetext}{RGB}{101,0,0} %

\title{Assignment 3 - Sampling Distributions. Due September 28, 11:59pm 2018}

\author[a]{EPIB607 - Inferential Statistics}

  \affil[a]{Fall 2018, McGill University}

\setcounter{secnumdepth}{5}

% Please give the surname of the lead author for the running footer
\leadauthor{Bhatnagar and Hanley}

% Keywords are not mandatory, but authors are strongly encouraged to provide them. If provided, please include two to five keywords, separated by the pipe symbol, e.g:
 \keywords{  Sampling distribution |  Standard error |  Normal distribution |  Quantiles |  Percentiles |  Z-scores  }

\begin{abstract}
Because the results of random samples include an element of chance, we
can't guarantee that our inferences are correct. What we can guarantee
is that our methods usually give correct answers. We will see that the
reasoning of statistical inference rests on asking ``How often would
this method give a correct answer if I used it very many times ?'' To be
able to answer these questions we need to understand sampling
distributions and the normal curve. In this assignment you will practice
calculating quantiles and probablities from the Normal distribution. All
graphs and calculations are to be completed in an R Markdown document
using the provided template. You are free to choose any function from
any package to complete the assignment. Concise answers will be
rewarded. Be brief and to the point. Please submit both the compiled
HTML report and the source file (.Rmd) to myCourses by September 28,
2018, 11:59pm. Both HTML and .Rmd files should be saved as
`IDnumber\_LastName\_FirstName\_EPIB607\_A3'.
\end{abstract}

\dates{This version was compiled on \today}
\doi{\url{https://sahirbhatnagar.com/EPIB607/}}

\pinpfootercontents{Assignment 3 due Sepetember 28, 2018 by 11:59pm}

\begin{document}

% Optional adjustment to line up main text (after abstract) of first page with line numbers, when using both lineno and twocolumn options.
% You should only change this length when you've finalised the article contents.
\verticaladjustment{-2pt}

\maketitle
\thispagestyle{firststyle}
\ifthenelse{\boolean{shortarticle}}{\ifthenelse{\boolean{singlecolumn}}{\abscontentformatted}{\abscontent}}{}

% If your first paragraph (i.e. with the \dropcap) contains a list environment (quote, quotation, theorem, definition, enumerate, itemize...), the line after the list may have some extra indentation. If this is the case, add \parshape=0 to the end of the list environment.

\section*{Template}\label{template}
\addcontentsline{toc}{section}{Template}

The \texttt{.Rmd} template for Assignment 3 is available
\href{https://github.com/sahirbhatnagar/EPIB607/raw/master/assignments/a3/a3_template.Rmd}{here}

\section*{\texorpdfstring{The \texttt{mosaic} package
(optional)}{The mosaic package (optional)}}\label{the-mosaic-package-optional}
\addcontentsline{toc}{section}{The \texttt{mosaic} package (optional)}

The \texttt{mosaic} package provides a consistent and user-friendly
interface for descriptive statistics, plots and inference. In particular
you might find the \texttt{mosaic::xpnorm} and \texttt{mosaic::xqnorm}
functions useful for this assignment. Have a look at the
\href{https://github.com/sahirbhatnagar/EPIB607/raw/master/slides/sampling_dist/EPIB607_sampling_dist.pdf}{slides
on sampling distributions} for some examples on how to use these
functions. Remeber to install the package:

\begin{Shaded}
\begin{Highlighting}[]
\KeywordTok{install.packages}\NormalTok{(}\StringTok{"mosaic"}\NormalTok{, }\DataTypeTok{dependencies =} \OtherTok{TRUE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Then you must load the library to access its functions:

\begin{Shaded}
\begin{Highlighting}[]
\KeywordTok{library}\NormalTok{(mosaic)}
\end{Highlighting}
\end{Shaded}

\section*{In-line R code}\label{in-line-r-code}
\addcontentsline{toc}{section}{In-line R code}

For this and future assignments you may find it useful to include
calculations from \texttt{R} directly in your text. For example, in the
following code chunk I calculate \(P(Y < 2)\) where
\(Y \sim \mathcal{N}(0,1)\), and store the result in an object called
\texttt{prob\_less\_2}:

\begin{Shaded}
\begin{Highlighting}[]
\StringTok{```}\DataTypeTok{\{r\}}
\DataTypeTok{prob_less_2 <- mosaic::xpnorm(2)}

\DataTypeTok{# round to 2 digits}
\DataTypeTok{prob_less_2 <- round(prob_less_2, 2)}
\StringTok{```}
\end{Highlighting}
\end{Shaded}

To print this result verbatim in an inline R expression use
\texttt{\textasciigrave{}r\ prob\_less\_2\textasciigrave{}} in the text.

\vspace{0.1in}

You can also call the function directly without storing the result in a
code chunk using
\texttt{\textasciigrave{}r\ round(mosaic::xpnorm(2),\ 2)\textasciigrave{}}

A sample answer would be: The probability that \(Y\) is less than 2 is
0.98.

\section{Normal probability
calculations}\label{normal-probability-calculations}

Using your method of choice, calculate the following probabilities
assuming \(Y\) is a standard normal distribution
(\(Y \sim \mathcal{N}(0,1)\)):

\begin{enumerate}
\def\labelenumi{\alph{enumi})}
\tightlist
\item
  \(P(Y < -1.80)\)
\item
  \(P(Y > -1.80)\)
\item
  \(P(Y \geq 1.60)\)
\item
  \(P(-1.8 < Y \leq 1.6)\)
\end{enumerate}

\section{HDL cholesterol}\label{hdl-cholesterol}

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  US women over the age of 19 have a mean (HDL) cholesterol measure of
  55 mg/dL with a standard deviation of 15.5 mg/dL. Assume HDL follows a
  Normal distribution.

  \begin{enumerate}
  \def\labelenumii{\roman{enumii}.}
  \tightlist
  \item
    What percent of women have low values of HDL, where low is defined
    to be 40 mg/dL or less?
  \item
    HDL levels of 60 mg/dL or more are believed to be protective against
    heart disease. What percent of women have protective levels of HDL?
  \item
    What proportion of women has HDL in the range of 40-60 mg/dL?
  \item
    What proportion of women has HDL in the range of 35-65 mg/dL?
  \end{enumerate}
\end{enumerate}

\section{Osteoporosis}\label{osteoporosis}

Osteoporosis is a condition in which the bones become brittle due to
loss of minerals. To diagnose osteoporosis, an elaborate apparatus
measures bone mineral density (BMD). BMD is usually reported in
standardized form. The standardization is based on a population of
healthy young adults. The Wolrd Health Organization (WHO) criterion for
osteoporosis is a BMD 2.5 or more standard deviations below the mean for
young adults. BMD measurements in a population of people who are similar
in age and sex roughly follow a standard normal distribution.

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  What percent of healthy young adults have osteoporosis?
\item
  Woman aged 70 to 79 are, of course, not young adults. The mean BMD in
  this age is about -2 on the standard scale for young adults. Suppose
  that the standard deviation is the same for young adults. What percent
  of this older population has osteoporosis?
\item
  Likewise, osteopenia is low BMD, defined by the WHO as a BMD between 1
  and 2.5 standard deviations below the mean of young adults. What
  percent of healthy young adults have osteopenia?
\item
  The mean BMD among women aged 70 to 79 is about -2 on the standard
  scale for young adults. Suppose that the standard deviation is the
  same as for young adults. What percent of this older population has
  osteopenia?
\end{enumerate}

\section{How deep is the ocean?}\label{how-deep-is-the-ocean}

This question is based on the
\href{https://github.com/sahirbhatnagar/EPIB607/blob/master/exercises/water/students/260194225_water_exercise_epib607.pdf}{in-class
Exercise} on sampling distributions.

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  For your samples of \(n=5\) and \(n=20\) of depths of the ocean,
  calculate the

  \begin{enumerate}
  \def\labelenumii{\roman{enumii}.}
  \tightlist
  \item
    sample mean (\(\bar{y}\))
  \item
    standard error of the sample mean (\(SE_{\bar{y}}\))
  \end{enumerate}
\item
  Calculate the 68\%, 95\% and 99\% confidence intervals (CI) for both
  samples of \(n=5\) and \(n=20\). You may use the following formulas to
  calculate the confidence intervals:
\end{enumerate}

\begin{longtable}[]{@{}ll@{}}
\toprule
CI & Formula\tabularnewline
\midrule
\endhead
68\% & \(\bar{y} \pm 1 \times SE_{\bar{y}}\)\tabularnewline
95\% & \(\bar{y} \pm 2 \times SE_{\bar{y}}\)\tabularnewline
99\% & \(\bar{y} \pm 3 \times SE_{\bar{y}}\)\tabularnewline
\bottomrule
\end{longtable}

Here is some sample code to calculate the CI. Suppose that based on a
sample of \(n=20\), my \(\bar{y} = 2900\) and \(SE_{\bar{y}} = 20\).
Then my confidence intervals are

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{ybar <-}\StringTok{ }\DecValTok{2900}
\NormalTok{SEybar <-}\StringTok{ }\DecValTok{20}
\CommentTok{# 68% CI}
\NormalTok{ybar }\OperatorTok{+}\StringTok{ }\KeywordTok{c}\NormalTok{(}\OperatorTok{-}\DecValTok{1}\NormalTok{,}\DecValTok{1}\NormalTok{) }\OperatorTok{*}\StringTok{ }\NormalTok{SEybar}
\CommentTok{# 95% CI}
\NormalTok{ybar }\OperatorTok{+}\StringTok{ }\KeywordTok{c}\NormalTok{(}\OperatorTok{-}\DecValTok{2}\NormalTok{,}\DecValTok{2}\NormalTok{) }\OperatorTok{*}\StringTok{ }\NormalTok{SEybar}
\CommentTok{# 99% CI}
\NormalTok{ybar }\OperatorTok{+}\StringTok{ }\KeywordTok{c}\NormalTok{(}\OperatorTok{-}\DecValTok{3}\NormalTok{,}\DecValTok{3}\NormalTok{) }\OperatorTok{*}\StringTok{ }\NormalTok{SEybar}
\end{Highlighting}
\end{Shaded}

Note that I have provided this code in the template as well. Take a look
at the template before starting these calculations.

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\setcounter{enumi}{2}
\tightlist
\item
  What do you notice about the size of the three intervals for a given
  sample size?
\end{enumerate}

%\showmatmethods

\bibliography{pinp}
\bibliographystyle{jss}

\end{document}


\section{Discussion}
Comparing all three algorithms, there is no clear outcome which one performs the best.

From the black box methods, RISE looks to perform better, especially when looking at more example images in the linked Jupyter notebooks. It seems like
the superpixels generated by LIME do not work that well for this dataset, because the diseases do not show up as distinct uniform patches, but rather as faint color changes.

The adjustable superpixel count for the LIME output is problematic. Using only a small superpixel count works well when the network only looks at a small part of an image, but when it looks
at a big part, this will not show up until we use more superpixels. The size of the output should be decided by the method and not by the user of the method, because the user can only guess what
the correct superpixel count could be.

Grad-CAM has a strange problem where it does not generate a heat map for some images. A reason for this behaviour could not be found. If there is output,
the quality is quite high, but the marked regions are very big. In the Infiltration case (Figures \ref{lime_example_3}, \ref{rise_example_3}, \ref{grad_cam_example_3}), Grad-CAM delivers by far
the best output, especially compared to RISE.

A big advantage of Grad-CAM is its performance, which it has in common with all white box methods. The image has to go through the neural network only once, recording activations and gradients in the process which are then analyzed in the end. In comparison, black box methods have to run many modified images through the neural network, not just one.

\section{Conclusion}
Grad-CAM provides some good insights into the neural network and will be modified for image segmentation.

RISE performs better than LIME and does not use a configuration variable which is difficult to correctly set and will therefore be prioritized for the image segmentation modifications.


\documentclass{beamer}

\input{preamble.tex}

\subtitle{Part 1: The Basics}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\titlepage
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Why \LaTeX{}?}
\begin{itemize}
\item It makes beautiful documents
\begin{itemize}
\item Especially mathematics
\end{itemize}
%
\item It was created by scientists, for scientists
\begin{itemize}
\item A large and active community
\end{itemize}
%
\item It is powerful --- you can extend it
\begin{itemize}
\item Packages for papers, presentations, spreadsheets, \ldots
\end{itemize}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{How does it work?}
\begin{itemize}
\item You write your document in \texttt{plain text} with \cmd{commands} that
describe its structure and meaning.
\item The \texttt{latex} program processes your text and commands to produce a
beautifully formatted document.
\end{itemize}
\vskip 2ex
\begin{center}
\begin{minted}[frame=single]{latex}
The rain in Spain falls \emph{mainly} on the plain.
\end{minted}
\vskip 2ex
\tikz\node[single arrow,fill=gray,font=\ttfamily\bfseries,%
  rotate=270,xshift=-1em]{latex};
\vskip 2ex
\fbox{The rain in Spain falls \emph{mainly} on the plain.}
\end{center}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{More examples of commands and their output\ldots}
\begin{exampletwoup}
\begin{itemize}
\item Tea
\item Milk
\item Biscuits
\end{itemize}
\end{exampletwoup}
\vskip 2ex
\begin{exampletwoup}
\begin{figure}
\includegraphics{gerbil}
\end{figure}
\end{exampletwoup}
\vskip 2ex
\begin{exampletwoup}
\begin{equation}
\alpha + \beta + 1
\end{equation}
\end{exampletwoup}

\tiny{Image license: \href{https://pixabay.com/en/animal-apple-attractive-beautiful-1239390/}{CC0}}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Attitude adjustment}

\begin{itemize}
\item Use commands to describe `what it is', not `how it looks'.
\item Focus on your content.
\item Let \LaTeX{} do its job.
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The Basics}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Getting started}
\begin{frame}[fragile]{\insertsubsection}
\begin{itemize}
\item A minimal \LaTeX{} document:
\inputminted[frame=single]{latex}{basics.tex}
\item Commands start with a \emph{backslash} \keystrokebftt{\bs}.
\item Every document starts with a \cmdbs{documentclass} command.
\item The \emph{argument} in curly braces \keystrokebftt{\{} \keystrokebftt{\}} tells \LaTeX{} what kind of document we are creating: an \bftt{article}.
\item A percent sign \keystrokebftt{\%} starts a \emph{comment} --- \LaTeX{}
will ignore the rest of the line.
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{} with \wllogo}
\begin{itemize}
\item Overleaf is a website for writing documents in \LaTeX.
\item It `compiles' your \LaTeX{} automatically to show you the results.
\vskip 2em
\begin{center}
\fbox{\href{\wlnewdoc{basics.tex}}{%
Click here to open the example document in \wllogo{}}}
\\[1ex]\scriptsize{}
For best results, please use \href{http://www.google.com/chrome}{Google Chrome} or a recent \href{http://www.mozilla.org/en-US/firefox/new/}{FireFox}.
\end{center}
\vskip 2ex
\item As we go through the following slides, try out the examples by typing them
into the example document on Overleaf.
\item \textbf{No really, you should try them out as we go!}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Typesetting Text}
\begin{frame}[fragile]{\insertsubsection{}}
\small
\begin{itemize}
\item Type your text between \cmdbegin{document} and \cmdend{document}.
\item For the most part, you can just type your text normally.
\begin{exampletwouptiny}
Words are separated by one or more
spaces.

Paragraphs are separated by one
or more blank lines.
\end{exampletwouptiny}
\item Space in the source file is collapsed in the output.
\begin{exampletwouptiny}
The   rain       in Spain
falls mainly on the plain.
\end{exampletwouptiny}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{}: Caveats}
\small
\begin{itemize}
\item Quotation marks are a bit tricky:\\
use a backtick \keystroke{\`{}} on the left and an apostrophe \keystroke{\'{}} on the right.
\begin{exampletwouptiny}
Single quotes: `text'.

Double quotes: ``text''.
\end{exampletwouptiny}

\item Some common characters have special meanings in \LaTeX:\\[1ex]
\begin{tabular}{cl}
\keystrokebftt{\%} & percent sign              \\
\keystrokebftt{\#} & hash (pound / sharp) sign \\
\keystrokebftt{\&} & ampersand                 \\
\keystrokebftt{\$} & dollar sign               \\
\end{tabular}
\item If you just type these, you'll get an error. If you want one to appear in
the output, you have to \emph{escape} it by preceding it with a backslash.
\begin{exampletwoup}
\$\%\&\#!
\end{exampletwoup}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Handling Errors}
\begin{itemize}
\item \LaTeX{} can get confused when it is trying to compile your document. If
it does, it stops with an error, which you must fix before it will produce
any output.
\item For example, if you misspell \cmdbs{emph} as \cmdbs{meph}, \LaTeX{} will
stop with an ``undefined control sequence'' error, because ``meph'' is not
one of the commands it knows.
\end{itemize}
\begin{block}{Advice on Errors}
\begin{enumerate}
\item Don't panic! Errors happen.
\item Fix them as soon as they arise --- if what you just typed caused an error,
you can start your debugging there.
\item If there are multiple errors, start with the first one --- the cause may
even be above it.
\end{enumerate}
\end{block}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Typesetting Exercise 1}

\begin{block}{Typeset this in \LaTeX:
\footnote{\url{http://en.wikipedia.org/wiki/Economy_of_the_United_States}}}
In March 2006, Congress raised that ceiling an additional \$0.79 trillion to
\$8.97 trillion, which is approximately 68\% of GDP. As of October 4, 2008, the
``Emergency Economic Stabilization Act of 2008'' raised the current debt ceiling
to \$11.3 trillion.
\end{block}
\vskip 2ex
\begin{center}
\fbox{\href{\wlnewdoc{basics-exercise-1.tex}}{%
Click to open this exercise in \wllogo{}}}
\end{center}

\begin{itemize}
\item Hint: watch out for characters with special meanings!
\item Once you've tried,
\fbox{\href{\wlnewdoc{basics-exercise-1-solution.tex}}{%
click here to see my solution}}.
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Typesetting Mathematics}
\begin{frame}[fragile]{\insertsubsection{}: Dollar Signs}
\begin{itemize}
\item Why are dollar signs \keystrokebftt{\$} special? We use them to mark mathematics in text.\\[1ex]
\begin{exampletwouptiny}
% not so good:
Let a and b be distinct positive
integers, and let c = a - b + 1.

% much better:
Let $a$ and $b$ be distinct positive
integers, and let $c = a - b + 1$.
\end{exampletwouptiny}
\item Always use dollar signs in pairs --- one to begin the mathematics, and one
to end it.
\item \LaTeX{} handles spacing automatically; it ignores your spaces.
\begin{exampletwouptiny}
Let $y=mx+b$ be \ldots

Let $y = m x + b$ be \ldots
\end{exampletwouptiny}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{}: Notation}
\begin{itemize}
\item Use caret \keystrokebftt{\^} for superscripts and underscore \keystrokebftt{\_} for subscripts.
\begin{exampletwouptiny}
$y = c_2 x^2 + c_1 x + c_0$
\end{exampletwouptiny}
\vskip 2ex

\item Use curly braces \keystrokebftt{\{} \keystrokebftt{\}} to group
superscripts and subscripts.
\begin{exampletwouptiny}
$F_n = F_n-1 + F_n-2$     % oops!

$F_n = F_{n-1} + F_{n-2}$ % ok!
\end{exampletwouptiny}
\vskip 2ex

\item There are commands for Greek letters and common notation.
\begin{exampletwouptiny}
$\mu = A e^{Q/RT}$

$\Omega = \sum_{k=1}^{n} \omega_k$
\end{exampletwouptiny}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{}: Displayed Equations}
\begin{itemize}
\item If it's big and scary, \emph{display} it on its own line using
\cmdbegin{equation} and \cmdend{equation}.\\[2ex]
\begin{exampletwouptiny}
The roots of a quadratic equation
are given by
\begin{equation}
x = \frac{-b \pm \sqrt{b^2 - 4ac}}
         {2a}
\end{equation}
where $a$, $b$ and $c$ are \ldots
\end{exampletwouptiny}
\vskip 1em
{\scriptsize Caution: \LaTeX{} mostly ignores your spaces in mathematics, but it
can't handle blank lines in equations --- don't put blank lines in your
mathematics.}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Interlude: Environments}
\begin{itemize}
\item \bftt{equation} is an \emph{environment} --- a context.
\item A command can produce different output in different contexts.
\begin{exampletwouptiny}
We can write
$ \Omega = \sum_{k=1}^{n} \omega_k $
in text, or we can write
\begin{equation}
  \Omega = \sum_{k=1}^{n} \omega_k
\end{equation}
to display it.
\end{exampletwouptiny}
\vskip 2ex
\item Note how the $\Sigma$ is bigger in the \bftt{equation} environment, and
how the subscripts and superscripts change position, even though we used the
same commands.
\vskip 1em
{\scriptsize In fact, we could have written \bftt{\$...\$} as
\cmdbegin{math}\bftt{...}\cmdend{math}.}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Interlude: Environments}
\begin{itemize}
\item The \cmdbs{begin} and \cmdbs{end} commands are used to create many
different environments.
\vskip 2ex

\item The \bftt{itemize} and \bftt{enumerate} environments generate lists.
\begin{exampletwouptiny}
\begin{itemize} % for bullet points
\item Biscuits
\item Tea
\end{itemize}

\begin{enumerate} % for numbers
\item Biscuits
\item Tea
\end{enumerate}
\end{exampletwouptiny}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Interlude: Packages}

\begin{itemize}
\item All of the commands and environments we've used so far are built into
\LaTeX.

\item \emph{Packages} are libraries of extra commands and environments. There
are thousands of freely available packages.

\item We have to load each of the packages we want to use with a
\cmdbs{usepackage} command in the \emph{preamble}.

\item Example: \bftt{amsmath} from the American Mathematical Society.
\begin{minted}[fontsize=\small,frame=single]{latex}
\documentclass{article}
\usepackage{amsmath} % preamble
\begin{document}
% now we can use commands from amsmath here...
\end{document}
\end{minted}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{}: Examples with \bftt{amsmath}}
\begin{itemize}
\item Use \bftt{equation*} (``equation-star'') for unnumbered equations.
\begin{exampletwouptiny}
\begin{equation*}
  \Omega = \sum_{k=1}^{n} \omega_k
\end{equation*}
\end{exampletwouptiny}
\item \LaTeX{} treats adjacent letters as variables multiplied together, which
is not always what you want. \bftt{amsmath} defines commands for many common
mathematical operators.
\begin{exampletwouptiny}
\begin{equation*} % bad!
 min_{x,y} (1-x)^2 + 100(y-x^2)^2
\end{equation*}
\begin{equation*} % good!
\min_{x,y}{(1-x)^2 + 100(y-x^2)^2}
\end{equation*}
\end{exampletwouptiny}
\item You can use \cmdbs{operatorname} for others.
\begin{exampletwouptiny}
\begin{equation*}
\beta_i =
\frac{\operatorname{Cov}(R_i, R_m)}
     {\operatorname{Var}(R_m)}
\end{equation*}
\end{exampletwouptiny}
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{\insertsubsection{}: Examples with \bftt{amsmath}}
\begin{itemize}{\small
\item Align a sequence of equations at the equals sign
\begin{align*}
(x+1)^3 &= (x+1)(x+1)(x+1) \\
        &= (x+1)(x^2 + 2x + 1) \\
        &= x^3 + 3x^2 + 3x + 1
\end{align*}
with the \bftt{align*} environment.

% for whatever reason, this doesn't play well with the twoup environment
\begin{minted}[fontsize=\small,frame=single]{latex}
\begin{align*}
(x+1)^3 &= (x+1)(x+1)(x+1) \\
        &= (x+1)(x^2 + 2x + 1) \\
        &= x^3 + 3x^2 + 3x + 1
\end{align*}
\end{minted}
\item An ampersand \keystrokebftt{\&} separates the left column (before the
$=$) from the right column (after the $=$).
\item A double backslash \keystrokebftt{\bs}\keystrokebftt{\bs} starts a new
line.
}\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{Typesetting Exercise 2}

\begin{block}{Typeset this in \LaTeX:}
Let $X_1, X_2, \ldots, X_n$ be a sequence of independent and identically
distributed random variables with $\operatorname{E}[X_i] = \mu$ and
$\operatorname{Var}[X_i] = \sigma^2 < \infty$, and let
\begin{equation*}
S_n = \frac{1}{n}\sum_{i=1}^{n} X_i
\end{equation*}
denote their mean. Then as $n$ approaches infinity, the random variables
$\sqrt{n}(S_n - \mu)$ converge in distribution to a normal $N(0, \sigma^2)$.
\end{block}
\vskip 2ex
\begin{center}
\fbox{\href{\wlnewdoc{basics-exercise-2.tex}}{%
Click to open this exercise in \wllogo{}}}
\end{center}
\begin{itemize}
\item Hint: the command for $\infty$ is \cmdbs{infty}.
\item Once you've tried,
\fbox{\href{\wlnewdoc{basics-exercise-2-solution.tex}}{%
click here to see my solution}}.
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{End of Part 1}
\begin{itemize}
\item Congrats! You've already learned how to \ldots
\begin{itemize}
\item Typeset text in \LaTeX.
\item Use lots of different commands.
\item Handle errors when they arise.
\item Typeset some beautiful mathematics.
\item Use several different environments.
\item Load packages.
\end{itemize}
\item That's amazing!
\item In Part 2, we'll see how to use \LaTeX{} to write structured documents
with sections, cross references, figures, tables and bibliographies. See you
then!
\end{itemize}
\end{frame}

\end{document}


\documentclass[12]{scrartcl}
\usepackage{amssymb,amsmath,gensymb,dsfont,calc,multicol,fullpage}
\makeatletter
\newcommand\Aboxed[1]{
   \@Aboxed#1\ENDDNE}
\def\@Aboxed#1&#2\ENDDNE{%
   &
   \settowidth\@tempdima{$\displaystyle#1{}$}
   \setlength\@tempdima{\@tempdima+\fboxsep+\fboxrule}
   \kern-\@tempdima
   \boxed{#1#2}
}
\makeatother

\begin{document}

\title{Homework 20, Section 4.4: 3, 6, 14
}
\author{Alex Gordon}
\date{\today}
\maketitle
\section*{Homework}
\subsection*{3. A)}
Since $(1,3) \in R_1$ $R_1$ is not antisymmetric
\subsection*{3. B)}
Since $(1,2) \in R_2$ and  $(2,1) \in R_2$ $R_2$ is not antisymmetric
\subsection*{3. C)}
Proof. \\
Let $(x,y) \in R_3$ be given. This means that $xy +  y = y(a+1)$ is odd. If y is odd then $a + 1$ is odd. Since, they are both even though we know this is supposed to be even. This means  $(x,y) \notin R_3$ meaning $R_3$ is antisymmetric
\subsection*{6. A)}
$R_1$ is irreflexive, antisymmetric and not transitive.
\subsection*{6. B)}
$R_2$ is irreflexive, antisymmetric and transitive.
\subsection*{14. A)}
\
\\
\\
\\
\\
\\
\\
\\

\subsection*{14. B)}
\
\\
\\
\\
\\
\\
\\
\\
\subsection*{14. C)}
\
\\
\\
\\
\\
\\
\\
\\

\end{document}


\documentclass{article}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{tikz}
\usetikzlibrary{arrows}
%Week 1

\begin{document}
\begin{center}
\textbf{\huge{Week 1}}
\end{center}
\section{Communication Theory}

Communication theory is a field of information theory and mathematics that studies the technical process of information.
Communication theory consists of:
\begin{itemize}
    \item Analyzing real world communication systems using structured thinking or mathematics.
    \item Synthesizing real world communication systems from mathematics.
\end{itemize}

\section{Signals}
Signals are defined to be a function that conveys information about a phenomenon.
They can also be said to be a medium which carry information relevant to a reciever.

Electromagnetic wave, voltage or current, digital signals carrying information are some examples of signals.

\subsection{Are there any non-time based signals?}
\begin{itemize}
    \item Images: They are represented as a combination of  monochrome images in three primary colors.
    $$ Image: \mathbb{R}^2 \rightarrow \{ R, G, B\}$$
    \item Temperature at various points in a room.
    $$ Temperature:\mathbb{R}^3 \rightarrow K \quad(or\; Celsius)$$
    \item Videos:  A video signal is a sequence of images.
    \item Text or a page in a book.
\end{itemize}

\section{Communication}
Communication is the process of exchanging information. Transmission, reception and reconstruction of the transmitted signal take place in this process.
\includegraphics[width=\textwidth]{ptp.png}

Channel: Separates the source and reciever and is not directly under our control.
This leads to some disturbances or imperfections in the channel, which is called noise.

\subsection{Transmitter}
The first step in sending a message is to convert it into electronic form suitable for transmission. The transmitter consists of various components designed to convert the electrical signal to a signal for sending it through the medium.

$$\fbox{Source} \xrightarrow{info.} \fbox{Channel Coding/Amplification/Modulation/Antenna} \rightarrow \fbox {Channel} $$

The transmitter is also called TX or TRX.

\subsection{Reciever}

\tikzstyle{int}=[draw, minimum size=2em]
\tikzstyle{init} = [pin edge={to-,thin,black}]
\begin{tikzpicture}[node distance=6cm,auto,>=latex']
    \node [int] (a) {Detect/estimate signal};
    \node (b) [left of=a,node distance=5cm, coordinate] {a};
    \node [int] (c) [right of=a] {Extract info.};
    \node [coordinate] (end) [right of=c, node distance=2cm]{};
    \path[->] (b) edge node {Noisy TX signal} (a);
    \path[->] (a) edge node {Channel decoding} (c);
    \draw[->] (c) edge node {} (end) ;
\end{tikzpicture}
\newline
A receiver is a collection of electronic components and circuits that accepts the transmitted message from the channel and converts it back to an understandable form.
The dectection and estimation step is called `demodulation'.

Channel coding and decoding is mainly used in digital communication systems. The reciever is called RX.

\subsection{Features of a good communication system}
\begin{itemize}
    \item Low degradation in quality of original signal, called high fidelity.
    \item Quick reception of information, called high rate communication.
        (in digital comms., this happens via intelligent modulation, power control)
    \item Time delay between transmission and communication should be small, called latency. (eg: high latency required in video streams but not for downloads)
\end{itemize}

\subsection{Modes of Communication}
\begin{enumerate}
    \item Point-to-point communication, in which the communication process takes place over a link between a single transmitter and a receiver. eg: Wired phone call.
    \item Broadcasting, which involves the use of a single  transmitter and multiple receivers. eg: Radio/TV broadcast, wi-fi, wireless cellular downlink
    \begin{itemize}
        \item Multiple access channel, consists of multiple transmitters and a single reciever. eg: Cellular uplink.
    \end{itemize}
\end{enumerate}

\section{Information}

\subsection{What is information?}
    Information is often thought of as the resolution of uncertainity. It can also mean the arrangement of particles.

    Say that $X$ is a random quantity representing the signal of interest (at some point in time or space).

$X \in \{0,1\}$ and it can take value $0$ with a probability of $p$ and value $1$ with probability $1-p$. This statement is a general abstraction of many different real-world scenarios.
$X$ is called a random variable, while 0 and 1 together form the sample space.

How do we find the information-content (or ``surprise'', ``average uncertainty'') in X?

The information content(surprise) is inversely proportional to the probability that the event occurs. i.e.
$$ Info.\; content\; in \;event \;``X=0" \propto \frac{1}{P(X=0)}$$

Our goal is to assign a function to the above `information content'.

Suppose $X_1\, \&\, X_2$ are independant random variables and $x_1, x_2$ are some values. Then
$$ P(X_1=x_1,X_2=x_2)=P(X_1=x_1)\times P(X_2=x_2)$$

We expect the joint information content in $X_1\, \&\, X_2$ to be the sum of the individual information contents.(as product is being converted to a sum, the log function is a good choice)

The information content in the event $X=x$ is $\log( {\frac{1}{P(X=x)})}$.

\subsection{Entropy}

The average (or expected) information content for a random variable $X$, with possible outcomes $x_1, \cdots, x_n$, which occur with probability $P(x_1),\cdots, P(x_n)$ is defined as:

$$ H(X) :=- \sum_{i=1}^{n} P(x_i) \log P(x_i)$$

$H(X)$ is called the entropy of X. Entropy in information theory is directly analogous to the entropy in statistical thermodynamics.

\subsubsection{Lemma}
Suppose $x_1 \in \mathcal{X}_1\, \&\; x_2 \in \mathcal{X}_2$ are \textbf{independant} random variables. Then,
$$ H(x_1,\;x_2)=H(x_1)+H(x_2)$$

\end{document}


\documentclass[11pt, oneside]{article}
\usepackage[a4paper,bindingoffset=0.2in,%
            left=1in,right=1in,top=1in,bottom=1in,%
            footskip=.25in]{geometry}
\usepackage{graphicx}
\usepackage{enumerate}
\usepackage{url}
\usepackage{hyperref}
\usepackage{pbox}
\usepackage{CJKutf8}
\usepackage{mathtools}
\usepackage{microtype}
\usepackage[htt]{hyphenat}
\renewcommand{\vec}[1]{\mathbf{#1}}

\begin{document}
\title{Assignment 1: Regression and KNN (part III)}
\author{Xiyou Zhou, 13307130189 \\ Computer Science and Technology}
\maketitle

\section{Logistic Regression}
First we use all 1 for weight initialization. We used four kinds of preprocessing methods and different $\lambda$ values for models. The results are shown as plots below or in the appendix.

The error rate at different $\lambda$ values is as follows.

\begin{table}[h]
\centering
\caption{Training Error}
\begin{tabular}{l|lll}
\hline
preprossing & $\lambda$ = 1     & $\lambda$ = 10    & $\lambda$ = 100   \\
\hline
none       & 43.49 & 58.47 & 38.86 \\
znormalize & 10.47 & 10.38 & 9.27  \\
binarized  & 12.10 & 11.91 & 10.70 \\
log        & 6.82  & 6.88  & 54.75 \\
\hline
\end{tabular}
\end{table}

\begin{table}[h]
\centering
\caption{Test Error}
\begin{tabular}{l|lll}
\hline
preprossing & $\lambda$ = 1     & $\lambda$ = 10    & $\lambda$ = 100   \\
\hline
none       & 45.31 & 59.38 & 37.17 \\
znormalize & 11.07 & 10.94 & 9.70  \\
binarized  & 11.13 & 11.26 & 10.81 \\
log        & 7.75  & 8.14  & 56.25 \\
\hline
\end{tabular}
\end{table}

Generally speaking, as $\lambda$ increases, training error and test error drops like in z-normalize and binarized method. This is probably because the parameters are restricted to better fit the data instead of wasting gradient descents on local minimal. log preprocessing seems to reach the best performance with 6.82\% training error and 7.75\% test error, but the penalize of $\lambda$ makes the model underfit and reaches very bad accuracy. With no preprocessing, the model performance is very random and bad. I think it is because the very non-uniform distribution of feature scale and not tuned initial weight.

With preprocessing method changes, the performance varies a lot. No preprocessing gives very bad accuracy and random result. Z-normalize reaches less than 10\% error rate and training error is always less than test error. Binarized processing also reaches error rate close to 10\% but the test error is more steady than training error and their relationship changed during the change of $\lambda$. Log preprocessing method gives very unsteady results. It can reach very great accuracy at some $\lambda$ while it underfits when $\lambda$ is too large. The test accuracy is always close to training accuracy, thus more training epochs would definitely make the results more steady.

After all, log preprocessing method provided best result and converged the most quickly.

\section{Linear Regression}

Using linear least squares to compute weight for linear regression, we can see the results from plots with different preprocessing methods below or in the appendix.

The error rate at different $\lambda$ values is as follows.

\begin{table}[h]
\centering
\caption{Training Error}
\begin{tabular}{l|lll}
\hline
preprossing & $\lambda$ = 1     & $\lambda$ = 10    & $\lambda$ = 100   \\
\hline
none       & 10.57 & 10.15 & 10.15 \\
znormalize & 10.57 & 10.41 & 9.56  \\
binarized  & 7.31 & 7.31 & 7.31 \\
log        & 6.10  & 6.04  & 6.04 \\
\hline
\end{tabular}
\end{table}

\begin{table}[h]
\centering
\caption{Test Error}
\begin{tabular}{l|lll}
\hline
preprossing & $\lambda$ = 1     & $\lambda$ = 10    & $\lambda$ = 100   \\
\hline
none       & 12.11 & 11.59 & 11.52 \\
znormalize & 12.04 & 11.59 & 11.39  \\
binarized  & 7.81 & 7.81 & 7.81 \\
log        & 6.51  & 6.58  & 6.45 \\
\hline
\end{tabular}
\end{table}

As $\lambda$ increases, training error and test error drops in all preprocessing methods. In not preprocessed method, training error does not change but test error drops as result of regularization. Binarized preprocessing method provided unchanging results.

Log preprocessing method gives the best results, even better than Logistic Regression method. Binarized results is steady, the training error and test error stayed around 7\%. Other methods have error rate around 10\% and clearly larger test error than training error.

Tried the SGD plot, but result not ideal, will ask the TA later.

\section{K-Nearest Neighbors}

The plots of training and test error as K changes can be seen below or in the appendix.
Here's the error rate at K = 1, 10, 100.
\begin{table}[h]
\centering
\caption{Training Error}
\begin{tabular}{l|lll}
\hline
preprossing & K = 1     & K = 10    & K = 100   \\
\hline
none       & 0.03 & 18.24 & 25.61 \\
znormalize & 0.03 & 8.25 & 12.46  \\
binarized  & 4.47 & 10.15 & 11.39 \\
log        & 0.03  & 5.29  & 8.91 \\
\hline
\end{tabular}
\end{table}

\begin{table}[h]
\centering
\caption{Test Error}
\begin{tabular}{l|lll}
\hline
preprossing & K = 1     & K = 10    & K = 100   \\
\hline
none       & 17.97 & 22.07 & 24.67 \\
znormalize & 9.64 & 10.09 & 13.93  \\
binarized  & 14.84 & 12.04 & 12.5 \\
log        & 6.51  & 7.03  & 10.55 \\
\hline
\end{tabular}
\end{table}

As K increases, error rate generally also increases except binarized data. When K = 0, binarized preprocessing method gives a very high training error rate because there are too many data with same different bits. Other preprocessing method gives a training error rate almost equal to 0 but not 0. I think that's because some small contradiction inside dataset or float error.

With none preprocessing, log or znormalize, all training and test error rate increases as K increases. Z and log preprocessing always have larger training error and none preprocessing have test error even larger than training error at large K. In binarized preprocessing method, the error rate converges at large K, training and test error tend to be closer as K increases.

\newpage
\section{Survey}
12 hours in 3 days.

\begin{thebibliography}{9}
\bibitem{CourseraLR}
Coursera, Machine Learning, Lecture 43 - Regularized Logistic Regression
\\\texttt{https://www.coursera.org/learn/machine-learning/lecture/4BHEy/regularized-logistic-regression}

\bibitem{cmulrbook}
CMU Statistics, Undergraduate Advanced Data Analysis, Logistic Regression
\\\texttt{http://www.stat.cmu.edu/\~{}cshalizi/uADA/12/lectures/ch12.pdf}

\bibitem{uwlecppt}
University of Washington, L2 Regularization for Logistic Regression
\\\texttt{http://courses.cs.washington.edu/courses/cse599c1/13wi/slides/l2-regularization-online-perceptron.pdf}

\bibitem{stfufldl}
Stanford, UFLDL Tutorial, Logistic Regression
\\\texttt{http://ufldl.stanford.edu/tutorial/supervised/LogisticRegression/}

\bibitem{mlapp}
Machine Learning, A Probabilistic Perspective, p16-17:\\A simple non-parametric classifier: K-nearest neighbors
\\\texttt{https://www.cs.ubc.ca/\~{}murphyk/MLbook/pml-intro-22may12.pdf}
\end{thebibliography}

\newpage
\section{Appendix}

\begin{figure}[h]
\centering
\includegraphics{foo_.pdf}
\caption{Plot of Logistic Regression Training \& Test error after 100 epochs with no preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{foo_z.pdf}
\caption{Plot of Logistic Regression Training \& Test error after 100 epochs with z-normalize preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{foo_log.pdf}
\caption{Plot of Logistic Regression Training \& Test error after 100 epochs with log preprocessing}
\end{figure}

\begin{figure}
\centering
\includegraphics{foo_binarized.pdf}
\caption{Plot of Logistic Regression Training \& Test error after 100 epochs with binarized preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{linear_.pdf}
\caption{Plot of Linear Regression Training \& Test error with linear least squares and no preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{linear_z.pdf}
\caption{Plot of Linear Regression Training \& Test error with linear least squares and z-normalize preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{linear_log.pdf}
\caption{Plot of Linear Regression Training \& Test error with linear least squares and log preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{linear_binarized.pdf}
\caption{Plot of Linear Regression Training \& Test error with linear least squares and binarized preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{knn_.pdf}
\caption{Plot of K-Nearest Neighbor Classifier Training \& Test error with no preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{knn_z.pdf}
\caption{Plot of K-Nearest Neighbor Classifier Training \& Test error with z-normalize preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{knn_log.pdf}
\caption{Plot of K-Nearest Neighbor Classifier Training \& Test error with log preprocessing}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics{knn_binarized.pdf}
\caption{Plot of K-Nearest Neighbor Classifier Training \& Test error with binarized preprocessing}
\end{figure}

\end{document}