\documentclass[a4paper, 11pt, landscape]{article}

\usepackage{mathptmx} % more compact font
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{mathtools}
\usepackage{multicol}
\usepackage{enumitem}
\usepackage{paralist} % for compacter lists
\usepackage{hyperref} % for Todo's and similar things
\usepackage[left=4.5mm, right=4.5mm, top=4.5mm, bottom=6mm, landscape, nohead, nofoot]{geometry}
\usepackage[small,compact]{titlesec}
\usepackage[usenames,dvipsnames,svgnames,table]{xcolor}
\usepackage{xparse}

% compact text
\linespread{0.9}
\setlength{\parindent}{0pt}

% compact lists even more
\setdefaultleftmargin{0em}{0em}{0em}{0em}{0em}{0em}

% compact sections
\titlespacing*{\section}{0pt}{0em}{0em}
\titlespacing*{\subsection}{0pt}{0em}{0em}
\titlespacing*{\subsubsection}{0pt}{0em}{0em}

% coloured section headings for easier read
\titleformat{name=\section}[block]
  {\sffamily}
  {}
  {0pt}
  {\colorsection}
\newcommand{\colorsection}[1]{%
	\colorbox{red!10}{\parbox[t][0em]{\dimexpr\columnwidth-2\fboxsep}{\thesection\ #1}}}

\titleformat{name=\subsection}[block]
  {\sffamily}
  {}
  {0pt}
  {\subcolorsection}
\newcommand{\subcolorsection}[1]{%
	\colorbox{orange!10}{\parbox[t][0em]{\dimexpr\columnwidth-2\fboxsep}{\thesubsection\ #1}}}

\titleformat{name=\subsubsection}[block]
  {\sffamily}
  {}
  {0pt}
  {\subsubcolorsection}
\newcommand{\subsubcolorsection}[1]{%
	\colorbox{blue!10}{\parbox[t][0em]{\dimexpr\columnwidth-2\fboxsep}{\thesubsubsection\ #1}}}

% environment for multicols inside a list
\NewDocumentEnvironment{listcols}{O{2} O{0pt}}
	{%
		\bgroup %
		\setlength{\multicolsep}{#2} %
		\begin{multicols*}{#1} %
	}
	{%
		\end{multicols*} %
		\egroup %
	}

% multicols lines & spacing
\setlength{\columnsep}{0.2cm}
\setlength{\columnseprule}{0.2pt}

% No page numbers
\pagenumbering{gobble}

% math helpers
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}

\begin{document}
\begin{multicols*}{3}

\section{PCA}
$\mathbf{X} \in \mathbb{R}^{D \times N}$. $N$ observations, $K$ properties. Target: $\tilde{\mathbf{X}} \in \mathbb{R}^{K \times N}$.
\begin{compactenum}
	\item Empirical Mean: $\overline{\mathbf{x}} = \frac{1}{N} \sum_{n=1}^N \mathbf{x}_n$
	\item Center Data: $\overline{\mathbf{X}} = \mathbf{X} - [\overline{\mathbf{x}}, \ldots, \overline{\mathbf{x}}] = \mathbf{X} - \mathbf{M}$
	\item Covariance Matrix: $\boldsymbol{\Sigma} = \frac{1}{N	} \sum_{n=1}^N (\mathbf{x}_n - \overline{\mathbf{x}}) (\mathbf{x}_n - \overline{\mathbf{x}})^T = \frac{1}{N} \overline{\mathbf{X}}\overline{\mathbf{X}}^T$
	\item Eigenvalue Decomposition: $\boldsymbol{\Sigma} = \mathbf{U} \boldsymbol{\Lambda} \mathbf{U}^T$, sort eigenvalues (and eigenvectors) in descending order
	\item Select $K < D$, keep only the first $K$ eigenvalues and corresponding eigenvectors $\Rightarrow \mathbf{U}_K, \boldsymbol{\lambda}_K$
	\item Transform data onto new Basis: $\overline{\mathbf{Z}} = \mathbf{U}_K^T \overline{\mathbf{X}}$
	\item Reconstruct to original Basis: $\tilde{\overline{\mathbf{X}}} = \mathbf{U}_k \overline{\mathbf{Z}}$
	\item Reverse centring: $\tilde{\mathbf{X}} = \tilde{\overline{\mathbf{X}}} + \mathbf{M}$
\end{compactenum}

\begin{compactitem}
	\item For compression save $\mathbf{U}_k, \overline{\mathbf{Z}}, \overline{\mathbf{x}}$.
	\item $\mathbf{U}_k \in \mathbb{R}^{D \times K}, \boldsymbol{\Sigma} \in \mathbb{R}^{D \times D}, \overline{\mathbf{Z}} \in \mathbb{R}^{K \times N}, \overline{\mathbf{X}} \in \mathbb{R}^{D \times N}$
\end{compactitem}

\begin{compactdesc}
	\item[Approx. Error:] Sum of discarded eigenvalues
\end{compactdesc}

\section{Eigenvalue / -vectors}
Eigenvalue Problem: $\mathbf{Ax} = \lambda \mathbf{x}$
\begin{compactenum}
	\item solve $\operatorname{det}(\mathbf{A} - \lambda \mathbf{I}) \overset{!}{=} 0$ resulting in $\{\lambda_i\}_i$
	\item $\forall \lambda_i$:
		solve $(\mathbf{A} - \lambda_i \mathbf{I}) \mathbf{x}_i = \mathbf{0}$, $\mathbf{x}_i$ is the $i$-th eigenvector.
	\item (opt.) normalize eigenvector $q_i$: $q_i^{\text{norm}} = \frac{1}{\|q_i\|_2} q_i$.
\end{compactenum}

\section{Eigendecomposition}
$\mathbf{A} \in \mathbb{R}^{N \times N}$ with $N$ linear independent eigenvectors, then: $\mathbf{A} = \mathbf{Q} \boldsymbol{\Lambda} \mathbf{Q}^{-1}$. $\mathbf{Q} \in \mathbb{R}^{N \times N}$, $i$-th column is the $i$-th eigenvector of $\mathbf{A}$. $\boldsymbol{\Lambda}$ is diagonal and $\boldsymbol{\lambda}_{i,i}$ is the $i$-th eigenvector. Every symmetric matrix can be ``eigendecomposed''.
$\mathbf{A}^{-1} = \mathbf{Q} \boldsymbol{\Lambda}^{-1} \mathbf{Q}^{-1}$, $(\boldsymbol{\Lambda}^{-1})_{i,i} = \frac{1}{\lambda_i}$
Calculate eigenvalues $\lambda_i$ and eigenvectors $q_i$. Normalise the eigenvectors. Put eigenvectors as columns into $\mathbf{Q}$ and eigenvalues into the diagonal of $\boldsymbol{\Lambda}$.

\section{SVD}
\begin{compactitem}
	\item $\mathbf{A} = \mathbf{U} \mathbf{D} \mathbf{V}^T = \sum_{k=1}^{\operatorname{rank}(\mathbf{A})} d_{k,k} u_k (v_k)^T$
	\item $\mathbf{A} \in \mathbb{R}^{N \times P}, \mathbf{U} \in \mathbb{R}^{N \times N}, \mathbf{D} \in \mathbb{R}^{N \times P}, \mathbf{V} \in \mathbb{R}^{P \times P}$
	\item $\mathbf{U}^T \mathbf{U} = I = \mathbf{V}^T \mathbf{V}$ ($\mathbf{U}, \mathbf{V}$ columns are orthonormal)
	\item $\mathbf{U}$ columns are eigenvectors of $\mathbf{A} \mathbf{A}^T$, $\mathbf{V}$ columns are eigenvectors of $\mathbf{A}^T \mathbf{A}$, $\mathbf{D}$ diagonal elements are singular values, i.e. the square roots of the eigenvalues ($\mathbf{A}^T \mathbf{A}$ and $\mathbf{A} \mathbf{A}^T$ have the same eigenvalues)
	\item $(\mathbf{D}^{-1})_{i,i} = \frac{1}{\mathbf{D}_{i, i}}$ ($\mathbf{D} \in \mathbb{R}^{N \times P} \to \mathbf{D}^{-1} \in \mathbb{R}^{P \times N}$, i.e. don't forget to transpose)
	\item Missing columns in $\mathbf{U}$ are basis of $\operatorname{null}(A^T)$ and in $\mathbf{V}$ are basis of $\operatorname{null}(A)$. Calculate: $\mathbf{A}^T \mathbf{u} = \mathbf{0}$ or $\mathbf{A} \mathbf{v} = \mathbf{0}$ for $\mathbf{u}$ or $\mathbf{v}$.
\end{compactitem}

\begin{compactenum}
	\item calculate $\mathbf{A}^T \mathbf{A}$.
	\item calculate eigenvalues of $\mathbf{A}^T \mathbf{A}$, the square root of them are the diagonal elements of $\mathbf{D}$.
	\item calculate eigenvectors of $\mathbf{A}^T \mathbf{A}$ using the eigenvalues resulting in the columns of $\mathbf{V}$.
	\item calculate the missing matrix: $\mathbf{U} = \mathbf{A} \mathbf{V} \mathbf{D}^{-1}$. Can be checked by calculating the eigenvectors of $\mathbf{A} \mathbf{A}^T$.
	\item normalize each columns of $\mathbf{U}$ and $\mathbf{V}$.
\end{compactenum}

\subsection{Low-Rank approximation}
Using only $K$ largest eigenvalues and corresponding eigenvectors. $\tilde{\mathbf{A}}_{i, j} = \sum_{k}^K \mathbf{U}_{i, k} \mathbf{D}_{k,k} \mathbf{V}_{j, k} = \mathbf{U}_{i, k} \mathbf{D}_{k,k} (\mathbf{V}^T)_{k, j}$.
\begin{compactdesc}
	\item[Error Frobenius:] $\|\mathbf{A} - \tilde{\mathbf{A}}\|_F = \sqrt{\sum_{i > K} \sigma_i^2} = \sqrt{\sum_{i > K} \lambda_i}$
	\item[Error Euclidean:] $\|\mathbf{A} - \tilde{\mathbf{A}}\|_2 = \sigma_{K+1}$
\end{compactdesc}

\section{Basis Transform}
$\mathbf{V}, \mathbf{W}$ are two basis matrices (each column is a basis vector, invertible). Representing $\mathbf{x}$ in basis $\mathbf{V}$ by \emph{coordinate vector} $\mathbf{a}$: $\mathbf{x} = \mathbf{V} \mathbf{a} \Rightarrow \mathbf{a} = \mathbf{V}^{-1} \mathbf{x}$. Knowing $\mathbf{a}$ representing $\mathbf{x}$ in $\mathbf{V}$ we can calculate coordinate vector $\mathbf{b}$ representing $\mathbf{x}$ in $\mathbf{W}$ with: $\mathbf{x} = \mathbf{W}^{-1} \mathbf{b} = \mathbf{V}^{-1} \mathbf{a} \Rightarrow \mathbf{b} = \mathbf{W} \mathbf{V}^{-1} \mathbf{a}$.

\section{Matrix/Vector}
\begin{compactdesc}
	\item[Orthogonal:] (i.e. columns are orthonormal!) $\mathbf{A}^{-1} = \mathbf{A}^T$, $\mathbf{A} \mathbf{A}^T = \mathbf{A}^T \mathbf{A} = \mathbf{I}$, $\operatorname{det}(\mathbf{A}) \in \{+1, -1\}$, $\operatorname{det}(\mathbf{A}^T \mathbf{A}) = 1$
	\item[Inner Product:] (in $\mathbb{R}^D$) $\langle \mathbf{x}, \mathbf{y} \rangle = \mathbf{x}^T \mathbf{y} = \sum_{i=1}^{N} \mathbf{x}_i \mathbf{y}_i$.
	\begin{inparaitem}[\color{red}\textbullet]
		\item $\langle \mathbf{x} \pm \mathbf{y}, \mathbf{x} \pm \mathbf{y} \rangle = \langle \mathbf{x}, \mathbf{x} \rangle \pm 2 \langle \mathbf{x}, \mathbf{y} \rangle + \langle \mathbf{y}, \mathbf{y} \rangle$
		\item $\langle \mathbf{x}, \mathbf{y} + \mathbf{z} \rangle = \langle \mathbf{x}, \mathbf{y} \rangle + \langle \mathbf{x}, \mathbf{z} \rangle$
		\item $\langle \mathbf{x} + \mathbf{y}, \mathbf{z} \rangle = \langle \mathbf{x}, \mathbf{z} \rangle + \langle \mathbf{y}, \mathbf{z} \rangle$
		\item $\langle \mathbf{x}, \mathbf{y} \rangle = \|\mathbf{x}\|_2 \cdot \|\mathbf{y}\|_2 \cdot \cos(\theta)$
		\item If $\mathbf{y}$ is a unit vector then $\langle \mathbf{x}, \mathbf{y} \rangle$ projects $\mathbf{x}$ onto $\mathbf{y}$
	\end{inparaitem}
	\item[Outer Product:] $\mathbf{u} \mathbf{v}^T$, $(\mathbf{u} \mathbf{v}^T)_{i, j} = \mathbf{u}_i \mathbf{v}_j$
	\item[Transpose:] $(\mathbf{A}^T)^{-1} = (\mathbf{A}^{-1})^T$
	\item[Gram-Schmidt:] $\{\mathbf{w}_i\}_i$ non-orthogonal basis. $\mathbf{v}_n = \mathbf{w}_n - \sum_{i=1}^{n-1} \frac{\langle \mathbf{v}_i, \mathbf{w}_n \rangle}{\langle \mathbf{v}_i, \mathbf{v}_i \rangle} \mathbf{v}_i$ results in $\{\mathbf{v}_i\}_i$ an orthogonal basis
\end{compactdesc}

\section{Norms}
\begin{inparadesc}
	\item[\color{red}$l_0$:] $\|\mathbf{x}\|_0 := |\{i | x_i \neq 0\}|$
	\item[\color{red}Nuclear:] $\|\mathbf{X}\|_\star = \sum_{i=1}^{\min(m, n)} \sigma_i$
	\item[\color{red}Euclidean:] $\|\mathbf{x}\|_2 := \sqrt{\sum_{i=1}^{N} \mathbf{x}_i^2} = \sqrt{\mathbf{x}^T \mathbf{x}} = \sqrt{\langle \mathbf{x}, \mathbf{x} \rangle}$
	\item[\color{red}$p$-norm:] $\|\mathbf{x}\|_p := \left( \sum_{i=1}^{N} |x_i|^p \right)^{\frac{1}{p}}$
	\item[\color{red}Frobenius:] $\|\mathbf{A}\|_F :=\allowbreak \sqrt{\sum_{i=1}^{M} \sum_{j=1}^{N} |\mathbf{A}_{i, j}|^2} =\allowbreak \sqrt{\operatorname{trace}(\mathbf{A}^T \mathbf{A})} =\allowbreak \sqrt{\sum_{i=1}^{\min\{m, n\}} \sigma_i^2}$ ($\sigma_i$ is the $i$-th singularvalue), $\mathbf{A} \in \mathbb{R}^{M \times N}$
\end{inparadesc}

\section{$K$-means Algorithm}
\begin{inparadesc}
	\item[\color{red}Target:] $\min_{\mathbf{U}, \mathbf{Z}} J(\mathbf{U}, \mathbf{Z}) = \|\mathbf{X} - \mathbf{U} \mathbf{Z}\|_F^2 = \sum_{n=1}^N \sum_{k=1}^K \mathbf{z}_{k,n} \|\mathbf{x}_n - \mathbf{u}_k\|_2^2$
\end{inparadesc}
\begin{inparaenum}[\color{red} 1.]
	\item Initiate: choose randomly $K$ centroids $\mathbf{U} = [\mathbf{u}_1, \ldots, \mathbf{u}_K]$ (usually $K$ randomly chosen data points).
	\item Assign data points to clusters. $k^\star(\mathbf{x}_n) = \argmin_k \{ \|\mathbf{x}_n - \mathbf{u}_1\|_2^2, \ldots, \|\mathbf{x}_n - \mathbf{u}_K\|_2^2 \}$ gives us the cluster $k$ whose centroid $\mathbf{u}_k$ is closest to data point $\mathbf{x}_n$. Set accordingly the $k$-th element of the $n$-th column of the assignment matrix $\mathbf{Z}$ to $1$, others to $0$.
	\item Update centroids: $\mathbf{u}_k = \frac{\sum_{n=1}^N z_{k,n} \mathbf{x}_n}{\sum_{n=1}^N z_{k,n}}$ (each centroid is in the middle of all data points assigned to it's cluster).
	\item Repeat from step 2, stops if $\frac{1}{2} \|\mathbf{Z} - \mathbf{Z}^\text{new}\|_0 = 0$.
\end{inparaenum}

\section{Probability / Statistics}
\begin{compactitem}
	\item $P(x) := Pr[X = x] := \sum_{y \in Y} P(x, y)$
	\item $P(x|y) := Pr[X = x | Y = y] := \frac{P(x,y)}{P(y)},\quad \text{if } P(y) > 0$
	\item $\forall y \in Y: \sum_{x \in X} P(x|y) = 1$ (property for any fixed $y$)
	\item $P(x, y) = P(x|y) P(y)$
	\item $P(x|y) = \frac{P(y|x)P(x)}{P(y)}$ (Bayes' rule)
	\item $P(x|y) = P(x) \Leftrightarrow P(y|x) = P(y)$ (iff $X$, $Y$ independent)
	\item $P(x_1, \ldots, x_n) = \prod_{i=1}^n P(x_i)$ (iff IID)
\end{compactitem}

\section{Gaussian Mixture Models (GMM)}
\begin{compactdesc}
	\item[GMM:] $p(\mathbf{x}) = \sum_{k=1}^K \pi_k \mathcal{N}(\mathbf{x} | \boldsymbol{\mu}_k, \Sigma_k)$
	\item[Assignment variable:] $\mathbf{z}_k \in \{0, 1\}$, $\sum_{k=1}^K \mathbf{z}_k = 1$, $\operatorname{Pr}(\mathbf{z}_k = 1) = \boldsymbol{\pi}_k \Leftrightarrow p(\mathbf{z}) = \prod_{k=1}^K \boldsymbol{\pi}_k^{\mathbf{z}_k}$
	\item[Complete data distribution:] $p(\mathbf{x}, \mathbf{z}) = \prod_{k=1}^K \left( \boldsymbol{\pi}_k \mathcal{N}(\boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k) \right)^{\mathbf{z}_k}$
	\item[Posterior Probabilities:] $\operatorname{Pr}(\mathbf{z}_k = 1 | \mathbf{x}) = \frac{\operatorname{Pr}(\mathbf{z}_k = 1) p(\mathbf{x} | \mathbf{z}_k = 1)}{\sum_{l=1}^K \operatorname{Pr}(\mathbf{z}_l = 1) p(\mathbf{x} | \mathbf{z}_l = 1)} = \frac{\boldsymbol{\pi}_k \mathcal{N}(\mathbf{x} | \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)}{\sum_{l=1}^K \boldsymbol{\pi}_l \mathcal{N}(\mathbf{x} | \boldsymbol{\mu}_l, \boldsymbol{\Sigma}_l)}$
	\item[Likelihood of observed data $\mathbf{X}$:] $p(\mathbf{X} | \boldsymbol{\pi}, \boldsymbol{\mu}, \boldsymbol{\Sigma}) = \prod_{n=1}^N p(\mathbf{x}_n) = \prod_{n=1}^N \sum_{k=1}^K \pi_k \mathcal{N}(\mathbf{x}_n | \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)$
	\item[Log-likelihood:] $\ln p(\mathbf{X} | \boldsymbol{\pi}, \boldsymbol{\mu}, \boldsymbol{\Sigma}) =\break \sum_{i=1}^N \ln \left( \sum_{k=1}^K \pi_k \mathcal{N}(\mathbf{x}_i | \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k) \right)$
	\item[Negative Log-likelihood:] The smaller, the better the fit: $-\ln(p(\mathbf{X} | \mathbf{\theta})) = -\Sigma_{n=1}^N \ln\left( \Sigma_{k=1}^K \boldsymbol{\pi}_k \mathcal{N}(\mathbf{x}_n | \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k) \right)$
\end{compactdesc}

\subsection{AIC / BIC for GMM}
Trade-off: balance between data fit (i.e. likelihood $p(\mathbf{X} | \theta)$) and complexity (i.e. \# of free parameters $\kappa(\cdot)$). Heuristics:
\begin{compactdesc}
	\item[Akaike Information Criterion (AIC):] $\operatorname{AIC}(\theta | \mathbf{X}) = -\ln(p_\theta(\mathbf{X})) + \kappa(\theta)$
	\item[Bayesian Information Criterion (BIC):] $\operatorname{BIC}(\theta | \mathbf{X}) = -\ln(p_\theta(\mathbf{x})) + \frac{1}{2} \kappa(\theta) \ln(N)$
	\item[\# of free params:] (fixed covariance matrix!) $\kappa(\theta) = K \cdot D + T$ ($K$ number of clusters, $D$ is data dimension (i.e. dimension of the $\mu$'s), $T$ number of not fixed $\pi$'s.
\end{compactdesc}
Note that the first term is the log-likelihood multiplied by $-1$. Compare AIC/BIC for different $L$, smaller is better.

\section{Expectation-Maximization (EM) for GMM}
\begin{compactenum}
	\item Initialize $\boldsymbol{\pi}_k^{(0)}, \boldsymbol{\mu}_k^{(0)}, \boldsymbol{\Sigma}_k^{(0)}, t = 1$ for $k = 1, \ldots, K$
	\item E-Step: $q_{k,n} := \mathbb{E}[z_{k,n}] = \frac{\boldsymbol{\pi}_k^{(t-1)} \mathcal{N}(\mathbf{x}_n | \boldsymbol{\mu}_k^{(t-1)}, \boldsymbol{\Sigma}_k^{(t-1)})}{\sum_{j=1}^K \boldsymbol{\pi}_j^{(t-1)} \mathcal{N}(\mathbf{x}_n | \boldsymbol{\mu}_j^{(t-1)}, \boldsymbol{\Sigma}_j^{(t-1)})}$
	\item M-Step: $\boldsymbol{\mu}_k^{(t)} := \frac{\sum_{n=1}^N q_{k,n} \mathbf{x}_n}{\sum_{n=1}^N q_{k,n}}$, $\boldsymbol{\pi}_k^{(t)} := \frac{1}{N} \sum_{n=1}^N q_{k,n}$, and $\Sigma_k^{(t)} = \frac{\sum_{n=1}^N q_{k, n} (\mathbf{x}_n - \boldsymbol{\mu}_k^{(t)})(\mathbf{x}_k - \boldsymbol{\mu}_k^{(t)})^T}{\sum_{n=1}^N q_{k,n}}$
	\item Repeat from (2.) with $t = t + 1$ if not $\| \log p(\mathbf{X} | \boldsymbol{\pi}^{(t)}, \boldsymbol{\mu}^{(t)}, \boldsymbol{\Sigma}^{(t)}) - \log p(\mathbf{X} | \boldsymbol{\pi}^{(t-1)}, \boldsymbol{\mu}^{(t-1)}, \boldsymbol{\Sigma}^{(t-1)}) \| < \epsilon$
\end{compactenum}

\section{Expectation-Maximization (EM) for Inpainting}
\begin{inparaenum}[\color{red} 1.]
	\item Assume initial $\mathbf{z}^{(0)}$, $\mathbf{x}^{(0)}$ original image, $t = 1$
	\item Reconstruct image $\mathbf{x}^{(t)} = \mathbf{M}\mathbf{x}^{(t-1)} + (\mathbf{I} - \mathbf{M})\mathbf{U}\mathbf{z}^{(t-1)}$
	\item Sparse coding of $\mathbf{x}^{(t)}$: $\mathbf{z}^{(t)} \in \argmin_{\mathbf{z}} \|\mathbf{z}\|_0$ s.t. $\|\mathbf{x}^{(t)} - \mathbf{Uz}\|_2 < \sigma$
	\item Repeat (2.) \& (3.) till converging/$\mathit{maxIters}$
	\item Image reconstruction: $\hat{\mathbf{x}} = \mathbf{Mx} + (\mathbf{I} - \mathbf{M})\mathbf{x}^{(t)}$
\end{inparaenum}

\section{Haar Wavelet}
\begin{compactdesc}
	\item[Mother function:] $\psi(t)$ is $1$ for $0 \leq t < \frac{1}{2}$, $-1$ for $\frac{1}{2} \leq t < 1$, $0$ otherwise.
	\item[Haar function:] $\psi_{n,k}(t) = 2^{n/2} \psi(2^n t - k)$, $2^{n/2}$ only needed if normalized values needed!
\end{compactdesc}

\begin{compactitem}
	\item For the not normalized basis: $\mathbf{W}^{-1} = (\mathbf{W^T} \mathbf{W})^{-1} \mathbf{W}^T = \operatorname{diag}(\frac{1}{l_1}, \ldots, \frac{1}{l_n}) \mathbf{W}^T$ where $l_i$ is the norm of the $i$-th column of $\mathbf{W}$. This is valid because $\mathbf{W}$ is orthogonal!
	\item Don't forget $[1, 1, \ldots, 1]^T$: first non-normalized basis!
	\item \textbf{``Coding'' step:} Tranform $\mathbf{x}$ into coordinate vector $\mathbf{c}$
	\item \textbf{``Thresholding'' step:} Set small absolute values in $\mathbf{c}$ to $0$
\end{compactitem}

\section{Optimization}
Gradient of $f: \mathbb{R}^D \to \mathbb{R}$ is $\nabla f(\mathbf{x}) := \left( \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_1}, \ldots, \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_D} \right)^T \in \mathbb{R}^D$

\subsection{Convex Functions}
A function $f(x)$ is convex if domain of $f$ (set of input values) is convex (e.g. $\mathbb{R}^D$) and $\forall y, z, \forall t \in [0, 1]: f(t \cdot y + (1-t) \cdot z) \leq t \cdot f(y) + (1-t) \cdot f(z)$.

\subsection{Coordinate Descent}
Update one coordinate (the $d$-th), others are fixed.
\begin{inparaenum}[\color{red} 1.]
	\item Init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$
	\item For $t = 0 \ \text{to} \ \mathit{maxIters}$:
	\item choose $d$ uniformly from $1, \ldots, D$
	\item $u^\star = \argmin_{u \in \mathbb{R}} f(x_1^{(t)}, \ldots, u, \ldots, x_D^{(t)})$
	\item $\mathbf{x}_d^{(t+1)} = u^\star$ and $\mathbf{x}_i^{(t+1)} = \mathbf{x}_i^{(t)}$ for $i \neq d$
\end{inparaenum}

\subsection{Gradient Descent}
\begin{inparaenum}[\color{red} 1.]
	\item Init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$, stepsize usually $\gamma \approx \frac{1}{t}$
	\item For $t = 0 \ \text{to} \ \mathit{maxIters}$: $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})$
\end{inparaenum}

\subsection{Stochastic Gradient Descent}
\begin{inparaenum}[\color{red} 1.]
	\item Init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$, stepsize usually $\gamma \approx \frac{1}{t}$
	\item For $t = 0 \ \text{to} \ \mathit{maxIters}$:
	\item choose $n$ uniformly from $1, \ldots, D$
	\item $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f_n(\mathbf{x}^{(t)})$
\end{inparaenum}

\subsection{Projected Gradient Descent}
In the gradient update step use a projection into constraint space $Q \subseteq \mathbb{R}^D$ with $P_Q(\mathbf{x}) = \argmin_{y \in Q} \|y - x\|$: $\mathbf{x}^{(t+1)} = P_Q\left(\mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})\right)$

\subsection{Lagrangian Multipliers}
Minimize  $f(\mathbf{x})$ subject to $g_i(\mathbf{x}) \leq 0,\ i = 1, \ldots, m$ and $h_i(\mathbf{x}) = \mathbf{a}_i^T \mathbf{x} - b_i = 0,\ i = 1, \ldots, p$.
\begin{compactdesc}
	\item[Lagrangian:] $L(\mathbf{x}, \boldsymbol{\lambda}, \boldsymbol{\nu}) := f(\mathbf{x}) + \sum_{i=1}^m \lambda_i g_i(\mathbf{x}) + \sum_{i=1}^p \nu_i h_i(\mathbf{x})$
	\item[Dual function:] $d(\boldsymbol{\lambda}, \boldsymbol{\nu}) := \inf_{\mathbf{x}} L(\mathbf{x}, \boldsymbol{\lambda}, \boldsymbol{\nu}) \in \mathbb{R}$
	\item[Dual Problem:] $\max_{\boldsymbol{\lambda}, \boldsymbol{\nu}} d(\boldsymbol{\lambda}, \boldsymbol{\nu})$ s.t. $\boldsymbol{\lambda} \geq \mathbf{0}$. Result is lower bound of optimum $f(\mathbf{x}^\star)$. Also only interested in $d(\cdot)$ where it is finite (if needed add (in)equality as a ``subject to'').
\end{compactdesc}

\section{Matching Pursuit (MP): Over-Complete Dictionaries}
Sparse approximation of a vector $\mathbf{x}$ onto an over-complete dictionary $\mathcal{D}$ represented by matrix $\mathbf{U}$ using $K$ dictionary entries.
\begin{compactenum}
	\item Initialize: Coefficient vector $\mathbb{R}^{|\mathcal{D}|} \ni \mathbf{z} = \mathbf{0}$, Residual $\mathbf{r} = \mathbf{x}$
	\item While $\|\mathbf{z}\|_0 < K$ repeat the following steps
	\item Find next dictionary index: $d^\star = \argmax_d |\langle \mathbf{u}_d, \mathbf{r} \rangle|$
	\item Update coefficients: $\mathbf{z}_{d^\star} = \mathbf{z}_{d^\star} + \langle \mathbf{u}_{d^\star}, \mathbf{r} \rangle$
	\item Update residual: $\mathbf{r} = \mathbf{r} - \langle \mathbf{u}_{d^\star}, \mathbf{r} \rangle \mathbf{u}_{d^\star}$.
\end{compactenum}

\section{Matching Pursuit (MP): Inpainting}
Diagonal matrix $\mathbf{M}$ with $\mathbf{M}_{i, i} = 1$ if $i$-th pixel known, otherwise $\mathbf{M}_{i,i} = 0$. Interested in $\mathbf{z}^\star \in \argmax_{\mathbf{z}} \|\mathbf{z}\|_0$ subject to $\|\mathbf{M}(\mathbf{x} - \mathbf{Uz})\|_2 < \sigma$. $\mathbf{U}$ over-complete dictionary of known parts, $\sigma$ allowed error. Reconstructed image: $\hat{\mathbf{x}} = \mathbf{Mx} + (\mathbf{I} - \mathbf{M})\mathbf{U} \mathbf{z}^\star$.
\begin{compactenum}
	\item Initialize: Coefficient vector $\mathbf{z} = \mathbf{0}$, residual $\mathbf{r} = \mathbf{M}\mathbf{x}$.
	\item While $\|z\|_0 < K$ and $\|r\|_2 > 0$ repeat the following steps
	\begin{listcols}[2][5pt]
		\item $d^\star = \argmax_d \frac{|\langle \mathbf{M}\mathbf{u}_d, \mathbf{r}\rangle|}{\|\mathbf{M}\mathbf{u}_d\|_2}$
		\item $\mathbf{z}_{d^\star} = \mathbf{z}_{d^\star} + \frac{\langle \mathbf{M}\mathbf{u}_{d^\star}, \mathbf{r} \rangle}{\|\mathbf{M}\mathbf{u}_{d^\star}\|_2^2}$
		\item $\mathbf{r} = \mathbf{r} - \frac{\langle \mathbf{M}\mathbf{u}_{d^\star}, \mathbf{r} \rangle}{\|\mathbf{M}\mathbf{u}_{d^\star}\|_2^2} \mathbf{M}\mathbf{u}_{d^\star}$
	\end{listcols}
\end{compactenum}

\section{Coherence ($m(\cdot)$)}
\begin{compactitem}
	\item $D$ dimension of data, $L$ number of atoms, $L > D$ then over-complete dictionary
	\item $m(\mathbf{U}) = \max_{i,j:\, i \neq j} | \mathbf{u}_i^T \mathbf{u}_j |$
	\item $m(\mathbf{B}) = 0$ if $\mathbf{B}$ is orthogonal matrix and a basis
	\item $m([\mathbf{B}, \mathbf{u}]) \geq \frac{1}{\sqrt{D}}$ if atom $\mathbf{u}$ is added to orthogonal $\mathbf{B}$
\end{compactitem}

\section{Robust PCA}
\begin{compactitem}
	\item Idea: Approximate $\mathbf{X}$ with $\mathbf{L} + \mathbf{S}$, $\mathbf{L}$ is low-rank, $\mathbf{S}$ is sparse.
	\item Minimize $\|\mathbf{L}\|_\star + \lambda \|\mathbf{S}\|_1$ s. t. $\mathbf{L} + \mathbf{S} = \mathbf{X}$
	\item A perfect reconstruction is possible if $\mathbf{S}$ is not low-rank, $\mathbf{L}$ is not sparse, or $\mathbf{X}$ is not low-rank \textit{and} sparse at the same time
\end{compactitem}

\subsection{Alternating Direction Method of Multipliers (ADMM)}
$\min_{\mathbf{x}_1, \mathbf{x}_2} f_1(\mathbf{x}_1) + f_2(\mathbf{x}_2)$ s. t. $\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 = \mathbf{b}$, $f_1, f_2$ convex
\begin{inparaitem}[\color{red}\textbullet]
	\item Augmented Lagrangian: $L_p(\mathbf{x}_1, \mathbf{x}_2, \boldsymbol{\nu}) = f_1(\mathbf{x}_1) + f_2(\mathbf{x}_2) + \boldsymbol{\nu}^T (\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 - \mathbf{b}) + \frac{p}{2}\| \mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 - \mathbf{b} \|_2^2$
	\item ADMM: $\mathbf{x}_1^{(t+1)} := \argmin_{\mathbf{x}_1} L_p(\mathbf{x}_1, \mathbf{x}_2^{(t)}, \boldsymbol{\nu}^{(t)})$, $\mathbf{x}_2^{(t+1)} := \argmin_{\mathbf{x}_2} L_p(\mathbf{x}_1^{(t+1)}, \mathbf{x}_2, \boldsymbol{\nu}^{(t)})$, $\boldsymbol{\nu}^{(t+1)} := \boldsymbol{\nu}^{(t)} + p(\mathbf{A}_1 \mathbf{x}_1^{(t+1)} + \mathbf{A}_2 \mathbf{x}_2^{(t+1)} - \mathbf{b})$
	\item ADDM for RPCA: $f_1(\mathbf{L}) = \|\mathbf{L}\|_\star$, $f_2(\mathbf{S}) = \lambda \| \mathbf{S} \|_1$, $\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 = \mathbf{b} \text{ becomes } \mathbf{L} + \mathbf{S} = \mathbf{X}$, therefore $L_p(\mathbf{L}, \mathbf{S}, \boldsymbol{\nu}) = \ldots$, $\boldsymbol{\nu}$ is matrix here!
\end{inparaitem}

\section{Non-Negative Matrix Factorization (NMF)}
\begin{inparaitem}[\color{red}\textbullet]
	\item $\mathbf{X} \in \mathbb{R}^{D \times N}_{\geq 0}$, $D$ words, $N$ documents, $X$ stores word count per document
	\item Target: $\mathbf{X} \approx \mathbf{UZ}$, $\mathbf{U} \in \mathbb{R}^{D \times K}_{\geq 0}$, $\mathbf{Z} \in \mathbb{R}^{K \times N}_{\geq 0}$ (semi-NMF: $\mathbf{U}$ allows negative entries)
\end{inparaitem}

\subsection{LSI}
Approx. $\mathbf{X}$ using SVD keeping $K$ largest singular values with $\tilde{\mathbf{X}} = \mathbf{U} \boldsymbol{\Sigma}_K \mathbf{V}^T$. Interpret $(\mathbf{U} \boldsymbol{\Sigma}_K)^T$ as mapping from $D$ dim. word space to $K$ dim. latent topic space. New document $\mathbf{x} \mapsto (\mathbf{U} \boldsymbol{\Sigma}_K)^T \mathbf{x}$. Compare using inner product $\langle \tilde{\mathbf{x}}_1, \tilde{\mathbf{x}}_2 \rangle = \langle (\mathbf{U} \boldsymbol{\Sigma}_K)^T \mathbf{x}_1, (\mathbf{U} \boldsymbol{\Sigma}_K)^T \mathbf{x}_2 \rangle$: Larger value $\Rightarrow$ more similar.

\subsection{pLSI}
\begin{compactitem}
	\item $P(\mathit{word} | \mathit{doc}) = \sum_{k=1}^K P(\mathit{word} | \mathit{topic} = k) P(\mathit{topic} = k | \mathit{doc})$
	\item $P(\mathit{word}, \mathit{doc}) = P(\mathit{word} | \mathit{doc}) P(\mathit{doc}) \break= \sum_z P(\mathit{word} | \mathit{topic}) P(\mathit{doc} | \mathit{topic}) P(\mathit{topic})$
\end{compactitem}

\textbf{Matrix Factorization View:}
\begin{inparaitem}[\color{red}\textbullet]
	\item Normalize $\mathbf{X}$ resulting in $\hat{\mathbf{X}}$: $\hat{\mathbf{X}}_{d, n} := \frac{\mathbf{X}_{d,n}}{\sum_{d'=1}^D \sum_{n'=1}^N \mathbf{X}_{d', n'}}$
	\item $\hat{\mathbf{X}} \approx \mathbf{U} \mathbf{Z}$, $\hat{\mathbf{X}} \in \mathbb{R}^{D \times N}_{\geq 0}$, $D$ words, $N$ documents
	\item $\mathbf{U}_{d, k} \equiv P(\mathit{word}=d | \mathit{topic} = k)$, $\sum_{d=1}^D \mathbf{U}_{d,k} = 1\, \forall k$
	\item $\mathbf{Z}_{k,n} \equiv P(\mathit{topic} = k | \mathit{doc} = n) P(\mathit{doc} = n)$, $\sum_{k, n} \mathbf{Z}_{k,n} = 1$
	\item $(\mathbf{U}\mathbf{Z})_{d, n} \equiv P(\mathit{word = d}, \mathit{doc} = n)$
	\item Log-likelihood: $\log P(\hat{\mathbf{X}}; \mathbf{U}, \mathbf{Z}) = \sum_{d=1}^D \sum_{n=1}^N \hat{\mathbf{X}}_{d,n} \log \sum_{k=1}^K \mathbf{U}_{d, k} \mathbf{Z}_{k, n}$
\end{inparaitem}

\textbf{Expectation Maximization (EM):}
\begin{compactenum}
	\item E-Step: $P(z|d, w) = \frac{P(z)P(d|z)P(w|z)}{\sum_{z' \in \mathcal{Z}} P(z') P(d|z') P(w|z')}$
	\item M-Steps: $P(w|z) \propto \sum_{d} f(d, w) P(z|d, w)$, $P(d|z) \propto \sum_{w} f(d, w) P(z|d, w)$, $P(z) \propto \sum_{d} \sum_{w} f(d, w) P(z|d, w)$, $f(d, w)$ is frequency of term $w$ in doc $d$.
\end{compactenum}
\begin{inparaitem}[\color{red}\textbullet]
	\item $P(w|z) \in \mathbb{R}^{D \times K}$
	\item $P(d|z) \in \mathbb{R}^{N \times K}$
	\item $P(z) \in \mathbb{R}^{K}$
\end{inparaitem}

\subsection{NMF Algorithm for quadratic cost function}
$\min_{\mathbf{U}, \mathbf{Z}} J(\mathbf{U}, \mathbf{Z}) = \frac{1}{2} \|\mathbf{X} - \mathbf{U}\mathbf{Z}\|_F^2$ s.t. $\mathbf{U}_{d,k} \in [0, \infty)\ \forall d, k$ \&  $\mathbf{Z}_{k, n} \in [0, \infty)\ \forall k, n$
\begin{inparaenum}[\color{red} 1.]
	\item Init: $\mathbf{U} \gets \mathrm{rand}(D,K), \mathbf{Z} \gets \mathrm{rand}(K,N)$
	\item repeat for $\mathit{maxIters}$:
	\item Update $\mathbf{U}$: $u_{d,k} \gets u_{d,k} \frac{(\mathbf{X}\mathbf{Z}^T)_{d,k}}{(\mathbf{U}\mathbf{Z}\mathbf{Z}^T)_{d,k}}$
	\item Update $\mathbf{Z}$: $z_{k,n} \gets z_{k,n} \frac{(\mathbf{U}^T\mathbf{X})_{k,n}}{(\mathbf{U}^T\mathbf{U}\mathbf{Z})_{k,n}}$
\end{inparaenum}

\raggedcolumns
\end{multicols*}
\end{document}


% LaTeX source for ``Python for Informatics: Exploring Information''
% Copyright (c)  2010-  Charles R. Severance, All Rights Reserved

\chapter{Tuples}
\label{tuplechap}

\section{Tuples are immutable}

\index{tuple}
\index{type!tuple}
\index{sequence}

A tuple\footnote{Fun fact: The word ``tuple'' comes from the names
given to sequences of numbers of varying lengths: single,
double, triple, quadruple, quituple, sextuple, septuple, etc.}
is a sequence of values much like a list.
The values stored in a tuple can be any type, and
they are indexed by integers.
The important difference is that tuples are {\bf immutable}.
Tuples are also {\bf comparable} and {\bf hashable} so we can
sort lists of them and use tuples as key values in Python
dictionaries.

\index{mutability}
\index{hashable}
\index{comparable}
\index{immutability}

Syntactically, a tuple is a comma-separated list of values:

\beforeverb
\begin{verbatim}
>>> t = 'a', 'b', 'c', 'd', 'e'
\end{verbatim}
\afterverb
%
Although it is not necessary, it is common to enclose tuples in
parentheses to help us quickly identify tuples when we look at
Python code:

\index{parentheses!tuples in}

\beforeverb
\begin{verbatim}
>>> t = ('a', 'b', 'c', 'd', 'e')
\end{verbatim}
\afterverb
%
To create a tuple with a single element, you have to include the final
comma:

\index{singleton}
\index{tuple!singleton}

\beforeverb
\begin{verbatim}
>>> t1 = ('a',)
>>> type(t1)
<type 'tuple'>
\end{verbatim}
\afterverb
%
Without the comma Python treats \verb"('a')" as an expression
with a string in parentheses that evaluates to a string:

\beforeverb
\begin{verbatim}
>>> t2 = ('a')
>>> type(t2)
<type 'str'>
\end{verbatim}
\afterverb
%
Another way to construct a tuple is the built-in function {\tt tuple}.
With no argument, it creates an empty tuple:

\index{tuple function}
\index{function!tuple}

\beforeverb
\begin{verbatim}
>>> t = tuple()
>>> print t
()
\end{verbatim}
\afterverb
%
If the argument is a sequence (string, list, or tuple), the result
of the call to {\tt tuple} is a tuple with the elements of the sequence:

\beforeverb
\begin{verbatim}
>>> t = tuple('lupins')
>>> print t
('l', 'u', 'p', 'i', 'n', 's')
\end{verbatim}
\afterverb
%
Because {\tt tuple} is the name of a constructor, you should
avoid using it as a variable name.

Most list operators also work on tuples.  The bracket operator
indexes an element:

\index{bracket operator}
\index{operator!bracket}

\beforeverb
\begin{verbatim}
>>> t = ('a', 'b', 'c', 'd', 'e')
>>> print t[0]
'a'
\end{verbatim}
\afterverb
%
And the slice operator selects a range of elements.

\index{slice operator}
\index{operator!slice}
\index{tuple!slice}
\index{slice!tuple}

\beforeverb
\begin{verbatim}
>>> print t[1:3]
('b', 'c')
\end{verbatim}
\afterverb
%
But if you try to modify one of the elements of the tuple, you get
an error:

\index{exception!TypeError}
\index{TypeError}
\index{item assignment}
\index{assignment!item}

\beforeverb
\begin{verbatim}
>>> t[0] = 'A'
TypeError: object doesn't support item assignment
\end{verbatim}
\afterverb
%
You can't modify the elements of a tuple, but you can replace
one tuple with another:

\beforeverb
\begin{verbatim}
>>> t = ('A',) + t[1:]
>>> print t
('A', 'b', 'c', 'd', 'e')
\end{verbatim}
\afterverb
%

\section{Comparing tuples}

\index{comparison!tuple}
\index{tuple!comparison}
\index{sort method}
\index{method!sort}

The comparison operators work with tuples and other sequences.
Python starts by comparing the first element from each
sequence.  If they are equal, it goes on to the next element,
and so on, until it finds elements that differ.  Subsequent
elements are not considered (even if they are really big).

\beforeverb
\begin{verbatim}
>>> (0, 1, 2) < (0, 3, 4)
True
>>> (0, 1, 2000000) < (0, 3, 4)
True
\end{verbatim}
\afterverb
%
The {\tt sort} function works the same way.  It sorts
primarily by first element, but in the case of a tie, it sorts
by second element, and so on.

This feature lends itself to a pattern called {\bf DSU} for

\begin{description}

\item[Decorate] a sequence by building a list of tuples
with one or more sort keys preceding the elements from the sequence,

\item[Sort] the list of tuples using the Python built-in {\tt sort}, and

\item[Undecorate] by extracting the sorted elements of the sequence.

\end{description}

\label{DSU}
\index{DSU pattern}
\index{pattern!DSU}
\index{decorate-sort-undecorate pattern}
\index{pattern!decorate-sort-undecorate}
\index{Romeo and Juliet}

For example, suppose you have a list of words and you want to
sort them from longest to shortest:

\beforeverb
\begin{verbatim}
txt = 'but soft what light in yonder window breaks'
words = txt.split()
t = list()
for word in words:
   t.append((len(word), word))

t.sort(reverse=True)

res = list()
for length, word in t:
    res.append(word)

print res
\end{verbatim}
\afterverb
%
The first loop builds a list of tuples, where each
tuple is a word preceded by its length.

{\tt sort} compares the first element, length, first, and
only considers the second element to break ties.  The keyword argument
{\tt reverse=True} tells {\tt sort} to go in decreasing order.

\index{keyword argument}
\index{argument!keyword}
\index{traversal}

The second loop traverses the list of tuples and builds a list of
words in descending order of length.  The four-character words
are sorted in {\em reverse} alphabetical order, so ``what'' appears
before ``soft'' in the following list.

The output of the program is as follows:
%
\beforeverb
\begin{verbatim}
['yonder', 'window', 'breaks', 'light', 'what',
'soft', 'but', 'in']
\end{verbatim}
\afterverb
%
Of course the line loses much of its poetic impact
when turned into a Python list and sorted in
descending word length order.

\section{Tuple assignment}
\label{tuple assignment}

\index{tuple!assignment}
\index{assignment!tuple}
\index{swap pattern}
\index{pattern!swap}

One of the unique syntactic features of the Python language
is the ability to have a tuple on the left
side of an assignment statement.  This allows you to assign
more than one variable at a time when the left side is a
sequence.

In this example we have a two-element list (which is a sequence) and
assign the first and second elements of the sequence
to the variables {\tt x} and {\tt y} in a single statement.

\beforeverb
\begin{verbatim}
>>> m = [ 'have', 'fun' ]
>>> x, y = m
>>> x
'have'
>>> y
'fun'
>>>
\end{verbatim}
\afterverb
%
It is not magic, Python \emph{roughly} translates the
tuple assignment syntax
to be the following:\footnote{Python does not translate the
syntax literally.  For example, if you try this with a dictionary,
it will not work as might expect.}

\beforeverb
\begin{verbatim}
>>> m = [ 'have', 'fun' ]
>>> x = m[0]
>>> y = m[1]
>>> x
'have'
>>> y
'fun'
>>>
\end{verbatim}
\afterverb

Stylistically when we use a tuple on the left side of the assignment
statement, we omit the parentheses, but the following is an equally
valid syntax:

\beforeverb
\begin{verbatim}
>>> m = [ 'have', 'fun' ]
>>> (x, y) = m
>>> x
'have'
>>> y
'fun'
>>>
\end{verbatim}
\afterverb
%
A particularly clever application of tuple assignment allows
us to {\bf swap} the values of two variables in a single statement:

\beforeverb
\begin{verbatim}
>>> a, b = b, a
\end{verbatim}
\afterverb
%
Both sides of this statement are tuples, but
the left side is a tuple of variables; the right side is a tuple of
expressions.  Each value on the right side
is assigned to its respective variable on the left side.
All the expressions on the right side are evaluated before any
of the assignments.

The number of variables on the left and the number of
values on the right must be the same:

\index{exception!ValueError}
\index{ValueError}

\beforeverb
\begin{verbatim}
>>> a, b = 1, 2, 3
ValueError: too many values to unpack
\end{verbatim}
\afterverb
%
More generally, the right side can be any kind of sequence
(string, list, or tuple).  For example, to split an email address
into a user name and a domain, you could write:

\index{split method}
\index{method!split}
\index{email address}

\beforeverb
\begin{verbatim}
>>> addr = 'monty@python.org'
>>> uname, domain = addr.split('@')
\end{verbatim}
\afterverb
%
The return value from {\tt split} is a list with two elements;
the first element is assigned to {\tt uname}, the second to
{\tt domain}.

\beforeverb
\begin{verbatim}
>>> print uname
monty
>>> print domain
python.org
\end{verbatim}
\afterverb
%

\section{Dictionaries and tuples}

\index{dictionary}
\index{items method}
\index{method!items}
\index{key-value pair}

Dictionaries have a method called {\tt items} that returns a list of
tuples, where each tuple is a key-value
pair\footnote{This behavior is slightly different in Python 3.0.}.

\beforeverb
\begin{verbatim}
>>> d = {'a':10, 'b':1, 'c':22}
>>> t = d.items()
>>> print t
[('a', 10), ('c', 22), ('b', 1)]
\end{verbatim}
\afterverb
%
As you should expect from a dictionary, the items are in no
particular order.

However, since the list of tuples is a list, and tuples are comparable,
we can now sort the list of tuples.  Converting a dictionary
to a list of tuples is a way for us to output the contents of a
dictionary sorted by key:

\beforeverb
\begin{verbatim}
>>> d = {'a':10, 'b':1, 'c':22}
>>> t = d.items()
>>> t
[('a', 10), ('c', 22), ('b', 1)]
>>> t.sort()
>>> t
[('a', 10), ('b', 1), ('c', 22)]
\end{verbatim}
\afterverb
%
The new list is sorted in ascending alphabetical order by the key value.

\section{Multiple assignment with dictionaries}

\index{traverse!dictionary}
\index{dictionary!traversal}

Combining {\tt items}, tuple assignment, and {\tt for}, you
can see a nice code pattern for traversing the keys and values of a dictionary
in a single loop:

\beforeverb
\begin{verbatim}
for key, val in d.items():
    print val, key
\end{verbatim}
\afterverb
%
This loop has two {\bf iteration variables} because {\tt items} returns
a list of tuples and {\tt key, val} is a tuple assignment
that successively iterates through each of the key-value pairs in the
dictionary.

For each iteration
through the loop, both {\tt key} and {\tt value} are advanced to the
next key-value pair in the dictionary (still in hash order).

The output of this loop is:

\beforeverb
\begin{verbatim}
10 a
22 c
1 b
\end{verbatim}
\afterverb
%
Again, it is in hash key order (i.e., no particular order).

If we combine these two techniques, we can print out the contents
of a dictionary sorted by the \emph{value} stored in each key-value
pair.

To do this, we first make a list of tuples where each tuple is
{\tt (value, key)}.  The {\tt items} method would give us a list of
{\tt (key, value)} tuples---but this time we want to sort by value, not key.
Once we have constructed the list with the value-key tuples, it is a simple
matter to sort the list in reverse order and print out the new, sorted list.

\beforeverb
\begin{verbatim}
>>> d = {'a':10, 'b':1, 'c':22}
>>> l = list()
>>> for key, val in d.items() :
...     l.append( (val, key) )
...
>>> l
[(10, 'a'), (22, 'c'), (1, 'b')]
>>> l.sort(reverse=True)
>>> l
[(22, 'c'), (10, 'a'), (1, 'b')]
>>>
\end{verbatim}
\afterverb
%
By carefully constructing the list of tuples to have the value as the first
element of each tuple, we can sort the list of tuples and get our dictionary
contents sorted by value.

\section{The most common words}

\index{Romeo and Juliet}
Coming back to our running example of the text from \emph{Romeo and Juliet}
Act 2, Scene 2, we can augment our program to use this technique to
print the ten most common words in the text as follows:

\beforeverb
\begin{verbatim}
import string
fhand = open('romeo-full.txt')
counts = dict()
for line in fhand:
    line = line.translate(None, string.punctuation)
    line = line.lower()
    words = line.split()
    for word in words:
        if word not in counts:
            counts[word] = 1
        else:
            counts[word] += 1

# Sort the dictionary by value
lst = list()
for key, val in counts.items():
    lst.append( (val, key) )

lst.sort(reverse=True)

for key, val in lst[:10] :
    print key, val
\end{verbatim}
\afterverb
%
The first part of the program which reads the file and computes
the dictionary that maps each word to the count of words in the
document is unchanged.  But instead of simply printing out
{\tt counts} and ending the program, we construct a list
of {\tt (val, key)} tuples and then sort the list in reverse order.

Since the value is first, it will be used for the comparisons.
If there is more than one tuple with the same value, it will look
at the second element (the key), so tuples where the value is the
same will be further sorted by the alphabetical order of the key.

At the end we write a nice {\tt for} loop which does a multiple
assignment iteration and prints out the ten most common words
by iterating through a slice of the list ({\tt lst[:10]}).

So now the output finally looks like what we want for our word
frequency analysis.

\beforeverb
\begin{verbatim}
61 i
42 and
40 romeo
34 to
34 the
32 thou
32 juliet
30 that
29 my
24 thee
\end{verbatim}
\afterverb
%
The fact that this complex data parsing and analysis
can be done with an easy-to-understand 19-line Python
program is one reason why Python is a good choice as a language
for exploring information.

\section{Using tuples as keys in dictionaries}

\index{tuple!as key in dictionary}
\index{hashable}

Because tuples are {\bf hashable} and lists are not, if we want to
create a {\bf composite} key to use in a dictionary we must use a tuple as
the key.

We would encounter a composite key if we wanted to create a
telephone directory that maps
from last-name, first-name pairs to telephone numbers.  Assuming
that we have defined the variables
{\tt last}, {\tt first}, and {\tt number}, we could write
a dictionary assignment statement as follows:

\beforeverb
\begin{verbatim}
directory[last,first] = number
\end{verbatim}
\afterverb
%
The expression in brackets is a tuple.  We could use tuple
assignment in a {\tt for} loop to traverse this dictionary.

\index{tuple!in brackets}

\beforeverb
\begin{verbatim}
for last, first in directory:
    print first, last, directory[last,first]
\end{verbatim}
\afterverb
%
This loop traverses the keys in {\tt directory}, which are tuples.  It
assigns the elements of each tuple to {\tt last} and {\tt first}, then
prints the name and corresponding telephone number.

\section{Sequences: strings, lists, and tuples---Oh My!}
\index{sequence}

I have focused on lists of tuples, but almost all of the examples in
this chapter also work with lists of lists, tuples of tuples, and
tuples of lists.  To avoid enumerating the possible combinations, it
is sometimes easier to talk about sequences of sequences.

In many contexts, the different kinds of sequences (strings, lists, and
tuples) can be used interchangeably.  So how and why do you choose one
over the others?

\index{string}
\index{list}
\index{tuple}
\index{mutability}
\index{immutability}

To start with the obvious, strings are more limited than other
sequences because the elements have to be characters.  They are
also immutable.  If you need the ability to change the characters
in a string (as opposed to creating a new string), you might
want to use a list of characters instead.

Lists are more common than tuples, mostly because they are mutable.
But there are a few cases where you might prefer tuples:

\begin{enumerate}

\item In some contexts, like a {\tt return} statement, it is
syntactically simpler to create a tuple than a list.  In other
contexts, you might prefer a list.

\item If you want to use a sequence as a dictionary key, you
have to use an immutable type like a tuple or string.

\item If you are passing a sequence as an argument to a function,
using tuples reduces the potential for unexpected behavior
due to aliasing.

\end{enumerate}

Because tuples are immutable, they don't provide methods
like {\tt sort} and {\tt reverse}, which modify existing lists.
However Python provides the built-in functions {\tt sorted}
and {\tt reversed}, which take any sequence as a parameter
and return a new sequence with the same elements in a different
order.

\index{sorted function}
\index{function!sorted}
\index{reversed function}
\index{function!reversed}

\section{Debugging}

\index{debugging}
\index{data structure}
\index{shape error}
\index{error!shape}

Lists, dictionaries and tuples are known generically as {\bf data
  structures}; in this chapter we are starting to see compound data
structures, like lists of tuples, and dictionaries that contain tuples
as keys and lists as values.  Compound data structures are useful, but
they are prone to what I call {\bf shape errors}; that is, errors
caused when a data structure has the wrong type, size, or composition,
or perhaps you write some code and forget the shape of your data
and introduce an error.

For example, if you are expecting a list with one integer and I
give you a plain old integer (not in a list), it won't work.

When you are debugging a program, and especially if you are
working on a hard bug, there are four things to try:

\begin{description}

\item[reading:] Examine your code, read it back to yourself, and
check that it says what you meant to say.

\item[running:] Experiment by making changes and running different
versions.  Often if you display the right thing at the right place
in the program, the problem becomes obvious, but sometimes you have to
spend some time to build scaffolding.

\item[ruminating:] Take some time to think!  What kind of error
is it: syntax, runtime, semantic?  What information can you get from
the error messages, or from the output of the program?  What kind of
error could cause the problem you're seeing?  What did you change
last, before the problem appeared?

\item[retreating:] At some point, the best thing to do is back
off, undoing recent changes, until you get back to a program that
works and that you understand.  Then you can start rebuilding.

\end{description}

Beginning programmers sometimes get stuck on one of these activities
and forget the others.  Each activity comes with its own failure
mode.

\index{typographical error}

For example, reading your code might help if the problem is a
typographical error, but not if the problem is a conceptual
misunderstanding.  If you don't understand what your program does, you
can read it 100 times and never see the error, because the error is in
your head.

\index{experimental debugging}

Running experiments can help, especially if you run small, simple
tests.  But if you run experiments without thinking or reading your
code, you might fall into a pattern I call ``random walk programming'',
which is the process of making random changes until the program
does the right thing.  Needless to say, random walk programming
can take a long time.

\index{random walk programming}
\index{development plan!random walk programming}

You have to take time to think.  Debugging is like an
experimental science.  You should have at least one hypothesis about
what the problem is.  If there are two or more possibilities, try to
think of a test that would eliminate one of them.

Taking a break helps with the thinking.  So does talking.
If you explain the problem to someone else (or even to yourself), you
will sometimes find the answer before you finish asking the question.

But even the best debugging techniques will fail if there are too many
errors, or if the code you are trying to fix is too big and
complicated.  Sometimes the best option is to retreat, simplifying the
program until you get to something that works and that you
understand.

Beginning programmers are often reluctant to retreat because
they can't stand to delete a line of code (even if it's wrong).
If it makes you feel better, copy your program into another file
before you start stripping it down.  Then you can paste the pieces
back in a little bit at a time.

Finding a hard bug requires reading, running, ruminating, and
sometimes retreating.  If you get stuck on one of these activities,
try the others.

\section{Glossary}

\begin{description}

\item[comparable:] A type where one value can be checked to see if it is
greater than, less than, or equal to another value of the same type.
Types which are comparable can be put in a list and sorted.
\index{comparable}

\item[data structure:] A collection of related values, often
organized in lists, dictionaries, tuples, etc.
\index{data structure}

\item[DSU:] Abbreviation of ``decorate-sort-undecorate'', a
pattern that involves building a list of tuples, sorting, and
extracting part of the result.
\index{DSU pattern}

\item[gather:] The operation of assembling a variable-length
argument tuple.
\index{gather}

\item[hashable:] A type that has a hash function.  Immutable
types like integers,
floats, and strings are hashable; mutable types like lists and
dictionaries are not.
\index{hashable}

\item[scatter:] The operation of treating a sequence as a list of
arguments.
\index{scatter}

\item[shape (of a data structure):] A summary of the type,
size, and composition of a data structure.
\index{shape}

\item[singleton:] A list (or other sequence) with a single element.
\index{singleton}

\item[tuple:] An immutable sequence of elements.
\index{tuple}

\item[tuple assignment:] An assignment with a sequence on the
right side and a tuple of variables on the left.  The right
side is evaluated and then its elements are assigned to the
variables on the left.
\index{tuple assignment}
\index{assignment!tuple}

\end{description}

\section{Exercises}

\begin{ex}
Revise a previous program as follows:  Read and
parse the ``From'' lines and pull out the
addresses from the line.   Count the number of
messages from each person using a dictionary.

After all the data has been read, print
the person with the most commits by creating
a list of (count, email) tuples from the
dictionary.   Then sort the list in reverse
order and print out the person who has the most
commits.

\beforeverb
\begin{verbatim}
Sample Line:
From stephen.marquard@uct.ac.za Sat Jan  5 09:14:16 2008

Enter a file name: mbox-short.txt
cwen@iupui.edu 5

Enter a file name: mbox.txt
zqian@umich.edu 195
\end{verbatim}
\afterverb
\end{ex}
\begin{ex}
This program counts the distribution of the hour of the day for
each of the messages. You can pull the hour from the ``From''
line by finding the time string and then splitting that string
into parts using the colon character. Once you have accumulated
the counts for each hour, print out the counts, one per line,
sorted by hour as shown below.
\beforeverb
\begin{verbatim}
Sample Execution:
python timeofday.py
Enter a file name: mbox-short.txt
04 3
06 1
07 1
09 2
10 3
11 6
14 1
15 2
16 4
17 2
18 1
19 1
\end{verbatim}
\afterverb
\end{ex}

\begin{ex}
Write a program that reads a file and
prints the {\em letters} in decreasing order of frequency.  Your program
should convert all the input to lower case and only count the letters a-z.
Your program should not count spaces, digits, punctuation, or anything
other than the letters a-z.
Find text samples from several different languages and see how letter frequency
varies between languages.  Compare your results with the tables at
\url{wikipedia.org/wiki/Letter_frequencies}.

\index{letter frequency}
\index{frequency!letter}

\end{ex}


\documentclass[color=usenames,dvipsnames]{beamer}\usepackage[]{graphicx}\usepackage[]{color}
% maxwidth is the original width if it is less than linewidth
% otherwise use linewidth (to make sure the graphics do not exceed the margin)
\makeatletter
\def\maxwidth{ %
  \ifdim\Gin@nat@width>\linewidth
    \linewidth
  \else
    \Gin@nat@width
  \fi
}
\makeatother

\definecolor{fgcolor}{rgb}{0, 0, 0}
\newcommand{\hlnum}[1]{\textcolor[rgb]{0.69,0.494,0}{#1}}%
\newcommand{\hlstr}[1]{\textcolor[rgb]{0.749,0.012,0.012}{#1}}%
\newcommand{\hlcom}[1]{\textcolor[rgb]{0.514,0.506,0.514}{\textit{#1}}}%
\newcommand{\hlopt}[1]{\textcolor[rgb]{0,0,0}{#1}}%
\newcommand{\hlstd}[1]{\textcolor[rgb]{0,0,0}{#1}}%
\newcommand{\hlkwa}[1]{\textcolor[rgb]{0,0,0}{\textbf{#1}}}%
\newcommand{\hlkwb}[1]{\textcolor[rgb]{0,0.341,0.682}{#1}}%
\newcommand{\hlkwc}[1]{\textcolor[rgb]{0,0,0}{\textbf{#1}}}%
\newcommand{\hlkwd}[1]{\textcolor[rgb]{0.004,0.004,0.506}{#1}}%
\let\hlipl\hlkwb

\usepackage{framed}
\makeatletter
\newenvironment{kframe}{%
 \def\at@end@of@kframe{}%
 \ifinner\ifhmode%
  \def\at@end@of@kframe{\end{minipage}}%
  \begin{minipage}{\columnwidth}%
 \fi\fi%
 \def\FrameCommand##1{\hskip\@totalleftmargin \hskip-\fboxsep
 \colorbox{shadecolor}{##1}\hskip-\fboxsep
     % There is no \\@totalrightmargin, so:
     \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth}%
 \MakeFramed {\advance\hsize-\width
   \@totalleftmargin\z@ \linewidth\hsize
   \@setminipage}}%
 {\par\unskip\endMakeFramed%
 \at@end@of@kframe}
\makeatother

\definecolor{shadecolor}{rgb}{.97, .97, .97}
\definecolor{messagecolor}{rgb}{0, 0, 0}
\definecolor{warningcolor}{rgb}{1, 0, 1}
\definecolor{errorcolor}{rgb}{1, 0, 0}
\newenvironment{knitrout}{}{} % an empty environment to be redefined in TeX

\usepackage{alltt}
%\documentclass[color=usenames,dvipsnames,handout]{beamer}

\usepackage[sans]{../../lab1}
\usepackage{bm}

\hypersetup{pdftex,pdfstartview=FitV}

%% New command for inline code that isn't to be evaluated
\definecolor{inlinecolor}{rgb}{0.878, 0.918, 0.933}
\newcommand{\inr}[1]{\colorbox{inlinecolor}{\texttt{#1}}}
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\begin{document}

%\setlength\fboxsep{0pt}

\begin{frame}[plain]
  \centering \huge
%  \color{MidnightBlue} %\bf
  {\color{RoyalBlue}{Lab 14 -- Model Selection and Multimodel Inference}} \\
  \vspace{1cm}
  \LARGE
%  November 26 \& 27, 2018 \\
  FANR 6750 \\
  \vfill
  \large
  Richard Chandler and Bob Cooper
\end{frame}

\section{Model Fitting}

\begin{frame}[plain]
  \frametitle{Today's Topics}
  \LARGE
  \only<1>{\tableofcontents}%[hideallsubsections]}
  \only<2 | handout:0>{\tableofcontents[currentsection]}%,hideallsubsections]}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Swiss Data}
\begin{knitrout}\small
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{swissData} \hlkwb{<-} \hlkwd{read.csv}\hlstd{(}\hlstr{"swissData.csv"}\hlstd{)}
\hlkwd{head}\hlstd{(swissData,} \hlkwc{n}\hlstd{=}\hlnum{11}\hlstd{)}
\end{alltt}
\begin{verbatim}
##    elevation forest water sppRichness
## 1        450      3    No          35
## 2        450     21    No          51
## 3       1050     32    No          46
## 4        950      9   Yes          31
## 5       1150     35   Yes          50
## 6        550      2    No          43
## 7        750      6    No          37
## 8        650     60   Yes          47
## 9        550      5   Yes          37
## 10       550     13    No          43
## 11      1150     50    No          52
\end{verbatim}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Four linear models}
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{fm1} \hlkwb{<-} \hlkwd{lm}\hlstd{(sppRichness} \hlopt{~} \hlstd{forest,} \hlkwc{data}\hlstd{=swissData)}
\hlstd{fm2} \hlkwb{<-} \hlkwd{lm}\hlstd{(sppRichness} \hlopt{~} \hlstd{elevation,} \hlkwc{data}\hlstd{=swissData)}
\hlstd{fm3} \hlkwb{<-} \hlkwd{lm}\hlstd{(sppRichness} \hlopt{~} \hlstd{forest} \hlopt{+} \hlstd{elevation} \hlopt{+}
          \hlstd{water,} \hlkwc{data}\hlstd{=swissData)}
\hlstd{fm4} \hlkwb{<-} \hlkwd{lm}\hlstd{(sppRichness} \hlopt{~} \hlstd{forest} \hlopt{+} \hlstd{elevation} \hlopt{+}
          \hlkwd{I}\hlstd{(elevation}\hlopt{^}\hlnum{2}\hlstd{)} \hlopt{+} \hlstd{water,} \hlkwc{data}\hlstd{=swissData)}
\end{alltt}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model 4 -- Estimates}
  \vspace{-5pt}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlkwd{summary}\hlstd{(fm4)}
\end{alltt}
\begin{verbatim}
##
## Call:
## lm(formula = sppRichness ~ forest + elevation + I(elevation^2) +
##     water, data = swissData)
##
## Residuals:
##     Min      1Q  Median      3Q     Max
## -11.314  -3.205  -0.377   3.334  15.082
##
## Coefficients:
##                  Estimate Std. Error t value Pr(>|t|)
## (Intercept)     4.518e+01  1.286e+00  35.137  < 2e-16 ***
## forest          2.311e-01  1.276e-02  18.111  < 2e-16 ***
## elevation      -1.016e-02  2.572e-03  -3.951   0.0001 ***
## I(elevation^2)  6.103e-08  9.661e-07   0.063   0.9497
## waterYes       -3.013e+00  6.821e-01  -4.418 1.46e-05 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 4.954 on 262 degrees of freedom
## Multiple R-squared:  0.7929,	Adjusted R-squared:  0.7897
## F-statistic: 250.8 on 4 and 262 DF,  p-value: < 2.2e-16
\end{verbatim}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model 4 -- ANOVA table}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlkwd{summary.aov}\hlstd{(fm4)}
\end{alltt}
\begin{verbatim}
##                 Df Sum Sq Mean Sq F value   Pr(>F)
## forest           1  13311   13311  542.40  < 2e-16 ***
## elevation        1  10820   10820  440.89  < 2e-16 ***
## I(elevation^2)   1      7       7    0.27    0.604
## water            1    479     479   19.52 1.46e-05 ***
## Residuals      262   6430      25
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
\end{verbatim}
\end{kframe}
\end{knitrout}
\small
%\vfill
We could compute AIC using the equation
$\mathrm{AIC}=n\log(\mathrm{RSS}/n)+2K$, where RSS is the residual
sum-of-squares.
\pause
\vfill
However, we will use the more general formula: $\mathrm{AIC} =
-2\mathcal{L}(\hat{\theta} ; {\bf y}) + 2K$.
\end{frame}

\section{Model Selection}

\begin{frame}[plain]
  \frametitle{Outline}
  \huge
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}[fragile]
  \frametitle{Compute AIC for each model}
  {Sample size}
  \small
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{n} \hlkwb{<-} \hlkwd{nrow}\hlstd{(swissData)}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%{Residual sums-of-squares (from ANOVA tables)}
{log-likelihood for each model}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{logL} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlkwd{logLik}\hlstd{(fm1),} \hlkwd{logLik}\hlstd{(fm2),} \hlkwd{logLik}\hlstd{(fm3),} \hlkwd{logLik}\hlstd{(fm4))}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%{\bf Number of parameters}
{Number of parameters}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{K} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlnum{3}\hlstd{,} \hlnum{3}\hlstd{,} \hlnum{5}\hlstd{,} \hlnum{6}\hlstd{)}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%{\bf AIC}
{AIC}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{AIC} \hlkwb{<-} \hlopt{-}\hlnum{2}\hlopt{*}\hlstd{logL} \hlopt{+} \hlnum{2}\hlopt{*}\hlstd{K}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%  {\bf $\Delta$AIC}
  {$\Delta$AIC}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{delta} \hlkwb{<-} \hlstd{AIC} \hlopt{-} \hlkwd{min}\hlstd{(AIC)}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%  {\bf AIC Weights}
  {AIC Weights}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{w} \hlkwb{<-} \hlkwd{exp}\hlstd{(}\hlopt{-}\hlnum{0.5}\hlopt{*}\hlstd{delta)}\hlopt{/}\hlkwd{sum}\hlstd{(}\hlkwd{exp}\hlstd{(}\hlopt{-}\hlnum{0.5}\hlopt{*}\hlstd{delta))}
\end{alltt}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{AIC table}
  \small
%  {\bf Put vectors in data.frame}
  {Put vectors in data.frame}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{ms} \hlkwb{<-} \hlkwd{data.frame}\hlstd{(logL, K, AIC, delta, w)}
\hlkwd{rownames}\hlstd{(ms)} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlstr{"fm1"}\hlstd{,} \hlstr{"fm2"}\hlstd{,} \hlstr{"fm3"}\hlstd{,} \hlstr{"fm4"}\hlstd{)}
\hlkwd{round}\hlstd{(ms,} \hlkwc{digits}\hlstd{=}\hlnum{2}\hlstd{)}
\end{alltt}
\begin{verbatim}
##        logL K     AIC  delta    w
## fm1 -939.03 3 1884.06 266.90 0.00
## fm2 -934.07 3 1874.15 256.99 0.00
## fm3 -803.58 5 1617.16   0.00 0.73
## fm4 -803.58 6 1619.15   2.00 0.27
\end{verbatim}
\end{kframe}
\end{knitrout}
\pause
%  {\bf Sort data.frame based on AIC values}
  {Sort data.frame based on AIC values}
\begin{knitrout}\footnotesize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{ms} \hlkwb{<-} \hlstd{ms[}\hlkwd{order}\hlstd{(ms}\hlopt{$}\hlstd{AIC),]}
\hlkwd{round}\hlstd{(ms,} \hlkwc{digits}\hlstd{=}\hlnum{2}\hlstd{)}
\end{alltt}
\begin{verbatim}
##        logL K     AIC  delta    w
## fm3 -803.58 5 1617.16   0.00 0.73
## fm4 -803.58 6 1619.15   2.00 0.27
## fm2 -934.07 3 1874.15 256.99 0.00
## fm1 -939.03 3 1884.06 266.90 0.00
\end{verbatim}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Similar process using R's {\tt AIC} function}
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlkwd{AIC}\hlstd{(fm1, fm2, fm3, fm4)}
\end{alltt}
\begin{verbatim}
##     df      AIC
## fm1  3 1884.057
## fm2  3 1874.146
## fm3  5 1617.157
## fm4  6 1619.153
\end{verbatim}
\end{kframe}
\end{knitrout}
\pause
\vfill
%{\bf Notes}
{Notes}
\begin{itemize}[<+->]
%  \item {\bf R} uses {\tt logLik(fm)} instead of {\tt n*log(RSS/n)}
  \item If we had used the residual sums-of-squares instead of the
    log-likelihoods, the AIC values would have been different, but the
    $\Delta$AIC values would have been the same
  \item Either approach is fine with linear models, but log-likelihoods
    must be used with GLMs and other models fit using maximum likelihood
\end{itemize}
\end{frame}

\section{Multi-model Inference}

\begin{frame}[plain]
  \frametitle{Outline}
  \huge
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model-specific predictions}
  \footnotesize
%  {\bf Predict number of species at site 1000m high with 25\% forest
%    cover, and no water, for \alert{each} model}
  {Expected number of species at 1000m elevation, 25\% forest
    cover, and no water, \alert{for each model}}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{predData1} \hlkwb{<-} \hlkwd{data.frame}\hlstd{(}\hlkwc{elevation}\hlstd{=}\hlnum{1000}\hlstd{,} \hlkwc{forest}\hlstd{=}\hlnum{25}\hlstd{,} \hlkwc{water}\hlstd{=}\hlstr{"No"}\hlstd{)}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
%\vfill
\vspace{-7pt}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{E1} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm1,} \hlkwc{newdata}\hlstd{=predData1,} \hlkwc{type}\hlstd{=}\hlstr{"response"}\hlstd{)}
\hlkwd{as.numeric}\hlstd{(E1)} \hlcom{# remove names (optional)}
\end{alltt}
\begin{verbatim}
## [1] 37.90222
\end{verbatim}
\end{kframe}
\end{knitrout}
\pause
%\vfill
\vspace{-7pt}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{E2} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm2,} \hlkwc{newdata}\hlstd{=predData1,} \hlkwc{type}\hlstd{=}\hlstr{"response"}\hlstd{)}
\hlkwd{as.numeric}\hlstd{(E2)}
\end{alltt}
\begin{verbatim}
## [1] 42.53368
\end{verbatim}
\end{kframe}
\end{knitrout}
\pause
%\vfill
\vspace{-7pt}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{E3} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm3,} \hlkwc{newdata}\hlstd{=predData1,} \hlkwc{type}\hlstd{=}\hlstr{"response"}\hlstd{)}
\hlkwd{as.numeric}\hlstd{(E3)}
\end{alltt}
\begin{verbatim}
## [1] 40.88604
\end{verbatim}
\end{kframe}
\end{knitrout}
\pause
%\vfill
\vspace{-7pt}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{E4} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm4,} \hlkwc{newdata}\hlstd{=predData1,} \hlkwc{type}\hlstd{=}\hlstr{"response"}\hlstd{)}
\hlkwd{as.numeric}\hlstd{(E4)}
\end{alltt}
\begin{verbatim}
## [1] 40.86092
\end{verbatim}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model-averaged prediction}
%  {\bf Expected number of species at 1000m, 25\% forest cover, and no
%    water, averaged over \alert{all} 4 models}
  {Expected number of species at 1000m, 25\% forest cover, and no
    water, \alert{averaged over all 4 models}}
  \pause
  \vspace{1pt}
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{E1}\hlopt{*}\hlstd{w[}\hlnum{1}\hlstd{]} \hlopt{+} \hlstd{E2}\hlopt{*}\hlstd{w[}\hlnum{2}\hlstd{]} \hlopt{+} \hlstd{E3}\hlopt{*}\hlstd{w[}\hlnum{3}\hlstd{]} \hlopt{+} \hlstd{E4}\hlopt{*}\hlstd{w[}\hlnum{4}\hlstd{]}
\end{alltt}
\begin{verbatim}
##        1
## 40.87927
\end{verbatim}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model-averaged regression lines}
%  {\bf Predict species richness over range of forest cover, for each model}
  {Predict species richness over range of forest cover, for each model}
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{predData2} \hlkwb{<-} \hlkwd{data.frame}\hlstd{(}\hlkwc{forest}\hlstd{=}\hlkwd{seq}\hlstd{(}\hlnum{0}\hlstd{,} \hlnum{100}\hlstd{,} \hlkwc{length}\hlstd{=}\hlnum{50}\hlstd{),}
                        \hlkwc{elevation}\hlstd{=}\hlnum{1000}\hlstd{,} \hlkwc{water}\hlstd{=}\hlstr{"No"}\hlstd{)}
\hlstd{E1} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm1,} \hlkwc{newdata}\hlstd{=predData2)}
\hlstd{E2} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm2,} \hlkwc{newdata}\hlstd{=predData2)}
\hlstd{E3} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm3,} \hlkwc{newdata}\hlstd{=predData2)}
\hlstd{E4} \hlkwb{<-} \hlkwd{predict}\hlstd{(fm4,} \hlkwc{newdata}\hlstd{=predData2)}
\hlstd{Emat} \hlkwb{<-} \hlkwd{cbind}\hlstd{(E1, E2, E3, E4)}
\end{alltt}
\end{kframe}
\end{knitrout}
\pause
\vfill
%{\bf How do we model-average these vectors?}
{How do we model-average these vectors?}
\pause
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlstd{Evec} \hlkwb{<-} \hlstd{Emat} \hlopt{%*%} \hlstd{w}
\end{alltt}
\end{kframe}
\end{knitrout}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Model-averaged regression line}
\begin{knitrout}\tiny
\definecolor{shadecolor}{rgb}{0.878, 0.918, 0.933}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlkwd{plot}\hlstd{(sppRichness}\hlopt{~}\hlstd{forest,} \hlkwc{data}\hlstd{=swissData,} \hlkwc{xlab}\hlstd{=}\hlstr{"Forest cover"}\hlstd{,} \hlkwc{ylab}\hlstd{=}\hlstr{"Species richness"}\hlstd{,} \hlkwc{cex.lab}\hlstd{=}\hlnum{1.5}\hlstd{)}
\hlkwd{lines}\hlstd{(E1} \hlopt{~} \hlstd{forest, predData2,} \hlkwc{col}\hlstd{=}\hlstr{"lightgreen"}\hlstd{,} \hlkwc{lwd}\hlstd{=}\hlnum{4}\hlstd{)}
\hlkwd{lines}\hlstd{(E2} \hlopt{~} \hlstd{forest, predData2,} \hlkwc{col}\hlstd{=}\hlstr{"orange"}\hlstd{,} \hlkwc{lwd}\hlstd{=}\hlnum{3}\hlstd{)}
\hlkwd{lines}\hlstd{(E3} \hlopt{~} \hlstd{forest, predData2,} \hlkwc{col}\hlstd{=}\hlstr{"purple"}\hlstd{,} \hlkwc{lwd}\hlstd{=}\hlnum{2}\hlstd{)}
\hlkwd{lines}\hlstd{(E4} \hlopt{~} \hlstd{forest, predData2,} \hlkwc{col}\hlstd{=}\hlstr{"red"}\hlstd{,} \hlkwc{lwd}\hlstd{=}\hlnum{1}\hlstd{)}
\hlkwd{lines}\hlstd{(Evec} \hlopt{~} \hlstd{forest, predData2,} \hlkwc{col}\hlstd{=}\hlkwd{rgb}\hlstd{(}\hlnum{0}\hlstd{,}\hlnum{0}\hlstd{,}\hlnum{1}\hlstd{,}\hlnum{0.2}\hlstd{),} \hlkwc{lwd}\hlstd{=}\hlnum{10}\hlstd{)}
\hlkwd{legend}\hlstd{(}\hlnum{60}\hlstd{,} \hlnum{30}\hlstd{,} \hlkwd{c}\hlstd{(}\hlstr{"Model 1"}\hlstd{,}\hlstr{"Model 2"}\hlstd{,}\hlstr{"Model 3"}\hlstd{,}\hlstr{"Model 4"}\hlstd{,}\hlstr{"Model averaged"}\hlstd{),} \hlkwc{lty}\hlstd{=}\hlnum{1}\hlstd{,} \hlkwc{cex}\hlstd{=}\hlnum{1.2}\hlstd{,}
       \hlkwc{lwd}\hlstd{=}\hlkwd{c}\hlstd{(}\hlnum{4}\hlstd{,}\hlnum{3}\hlstd{,}\hlnum{2}\hlstd{,}\hlnum{1}\hlstd{,}\hlnum{10}\hlstd{),} \hlkwc{col}\hlstd{=}\hlkwd{c}\hlstd{(}\hlstr{"lightgreen"}\hlstd{,} \hlstr{"orange"}\hlstd{,} \hlstr{"purple"}\hlstd{,} \hlstr{"red"}\hlstd{,} \hlkwd{rgb}\hlstd{(}\hlnum{0}\hlstd{,}\hlnum{0}\hlstd{,}\hlnum{1}\hlstd{,}\hlnum{0.2}\hlstd{)))}
\end{alltt}
\end{kframe}
\end{knitrout}
%\begin{center}
\centering
  \includegraphics[width=0.75\textwidth]{figure/reglines-1} \\
%\end{center}
\end{frame}

\begin{frame}
  \frametitle{Assignment}

%  NOTES FOR 2016: Elev model gets 100\% of the weight. Try to fix that somehow.

  {\bf \large Use the {\tt jayData} from lab 12 to do the following: \\}

  \begin{enumerate}[\bf (1)]
    \normalsize
    \item Fit four linear models of jay abundance. Include elevation
      all four models. Include at least
      one interaction, and one quadratic term for elevation.
    \item Create AIC table by hand, not using \R's {\tt AIC} function
%    \item Model-average predictions of jay abundance at 25\% forest
%      cover (i.e., forest=0.25)
    \item Model-average regression lines of jay abundance and
      elevation. Plot the averaged regression line along with the
      regression lines from each model.
    \item Bonus problem with 10 extra points: Create a map showing the
      model-averaged estimates of jay
      abundance on Santa Cruz Island. Hint: use {\tt predict} with
      {\tt cruzData} supplied as the {\tt newdata} argument.
  \end{enumerate}
\end{frame}

\end{document}


% Created 2021-10-12 Tue 14:19
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\author{Dinh Duy Kha}
\date{\today}
\title{Symmetric Key Encryption}
\hypersetup{
 pdfauthor={Dinh Duy Kha},
 pdftitle={Symmetric Key Encryption},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 28.0.50 (Org mode 9.5)},
 pdflang={English}}
\begin{document}

\maketitle
\tableofcontents

\section{Stream cipher}
\label{sec:orgad774a7}
\begin{itemize}
\item In stream cipher, a short key is stretched into a long \textbf{keystream} using pseudorandom number generator.
\end{itemize}

\section{Block cipher}
\label{sec:orgdd8884b}
\end{document}


\documentclass[8 pt]{article}

\usepackage[utf8x]{inputenc}
\usepackage{dsfont}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{tensor}
\usepackage{mathtools}
\usepackage[T1]{fontenc}
%\usepackage[spanish]{babel}
\usepackage[cm]{fullpage}
\usepackage{graphicx}
%\usepackage{float}
\usepackage{bm}
\usepackage{setspace}
\usepackage{enumitem}
\usepackage{mdwlist}
\usepackage{parskip}
\usepackage{listings}
\usepackage{color}
%\usepackage{epstopdf}
\usepackage{tikz,datatool}
\usepackage{hyperref}
\usepackage{mathabx}
\usepackage{multicol}
\usepackage{eurosym}
\usepackage{caption}
\usepackage{pdfpages}

\newcommand{\HRule}{\rule{\linewidth}{0.5mm}}

\AtBeginDocument{
  \let\myThePage\thepage
  \renewcommand{\thepage}{\oldstylenums{\myThePage}}
}

\newcommand{\gra}{$^\text{o}$}
\newcommand{\dif}{\text{d}}
\newcommand{\avg}[1]{\left\langle #1 \right\rangle}
\newcommand{\ket}[1]{\left| #1 \right\rangle}
\newcommand{\bra}[1]{\left\langle #1 \right|}
\newcommand{\bket}[2]{\left\langle #1 \middle| #2 \right\rangle}
\newcommand{\der}[2]{\frac{\text{d} #1}{\text{d} #2}}
\newcommand{\prt}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\dert}[3]{\frac{\text{d}^#3 #1}{\text{d} #2^#3}}
\newcommand{\prtt}[3]{\frac{\partial^#3 #1}{\partial #2^#3}}
\newcommand{\dl}{\mathcal{L}}
\newcommand{\dha}{\mathcal{H}}
\newcommand{\vol}{\text{vol}}
\renewcommand{\vec}[1]{\pmb{#1}}

\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}

\newenvironment{Figure}
  {\par\medskip\noindent\minipage{\linewidth}}
  {\endminipage\par\medskip}

\renewcommand\thesection{}

\begin{document}

\begin{minipage}{\textwidth}
    \centering
    \Large \textbf{\textsc{Homework 5: IMC assignment}}
    \vspace{0.5cm}

    \small \textsc{Francisco García Flórez, Joris van Lammeren, Wouter Varenkamp}
    \vspace{0.5cm}

    \begin{minipage}{0.8\textwidth}
      \textbf{Abstract.} In this homework we solve several exercises related to option trading proposed in the IMC guest lecture.
    \end{minipage}
\end{minipage}

\vspace{0.5cm}

\begin{multicols*}{2}

\section{Exercise 1}

To check when are the Q1 results for ING going to be published we checked the website \url{https://www.ing.com/Investor-relations.htm}, where it shows that the date is 10th of May, in 20 natural days (15 business days) as of 20th of April.

\section{Exercise 2}

Now, looking on Euronext (data attached at the end) we see the books for options expiring in April and May, right before and after the publication of the Q1 results. Since they expire the third Friday of each month, the ones in April will be done by the 21st (1 business day from today) and the ones in May by the 19th (21 business days from today).

So, using the put-call parity

\begin{equation*}
  S = C - P + Ke^{-rT} ~~,
\end{equation*}

we can compute the asset price $S$ at each time. Using only ask prices for the options (taking the point of view of someone wanting to buy) shown in the appendix, and taking the average over the strike prices, we get

\begin{align*}
  \begin{split}
    S_c &= 14.12 \\
    S_1 &= 14.11 \\
    S_2 &= 13.847
  \end{split} ~~.
\end{align*}

Here we used $r=-10bps$ per year, and we can make a few remarks. In the first place, $S_1$ is very similar to the current asset price since it expires in one day, but in $S_2$ we can see the effect of the Q1 results.

\section{Exercises 3 \& 4}

Using $S_1$ and $S_2$ we can now compute the implied volatility, again using ask prices. Since $S_1=14.11$, we will consider options at strike prices 14.00 and 14.20, and for $S_2$ these will be 13.50 and 14.00. Using Matlab's \texttt{blsimpv} function, we can compute the following implied volatilities:

\begin{align*}
  \begin{split}
    \sigma_1^- &= 28.13 \% \\
    \sigma_1^+ &= 29.80 \% \\
    \sigma_2^- &= 30.99 \% \\
    \sigma_2^+ &= 29.74 \%
  \end{split} ~~,
\end{align*}

where the superscripts $^\pm$ refer to over the forward asset price and below, respectively.

Finally, to get the volatility at the forward, we need to compute $x$ in the following equation

\begin{equation*}
  S_i = x_i K_i^+ + (1 - x_i) K_i^- \longrightarrow x_i = \frac{S_i - K_i^-}{K_i^+ - K_i^-} ~~,
\end{equation*}

and apply it to the volatilities

\begin{equation*}
  \sigma_i^f = \sqrt{(\sigma_i^+)^2 x_i + (1 - x_i) (\sigma_i^-)^2} ~~.
\end{equation*}

Following this procedure we find

\begin{align*}
  \begin{split}
    \sigma_1^f &= 28.89 \% \\
    \sigma_2^f &= 30.61 \%
  \end{split} ~~,
\end{align*}

which then we can use to compute the jump volatility.

\section{Exercise 5}

We proceed now to compute the jump volatility using the following equation

\begin{equation*}
  \sigma_{\text{jump}} = \sqrt{\sigma_{\text{after}}^2 T - \sigma_{\text{diff}}^2 (T-1)} ~~,
\end{equation*}

where we will assume $\sigma_{\text{diff}} = \sigma_1^f$, for simplification. From here, using $T=21$, we obtain a value of

\begin{equation*}
  \sigma_{\text{jump}} = 54.64 \% \longrightarrow \text{daily move} = 3.4 \% ~~.
\end{equation*}

\section{Exercise 6}

Finally we can determine what to do in this situation. Since the analyst estimates a lower daily move (2\%), we can assume that the actual movement will be less than our value, let's take 2.7\% as a more conservative value. Using these two volatilities we find that the expected asset value is going to change between 38 and 54 cents.

Since it is already too late to buy options expiring tomorrow (ask prices are higher than usual because of the Q1 results), we can buy options expiring in May, and then following a simple $\Gamma$ hedging strategy we can estimate P\&L by

\begin{equation*}
  \text{P\&L} = \Delta \delta S + \frac{1}{2}\Gamma (\delta S)^2 ~~,
\end{equation*}

so considering the $\Delta$ and $\Gamma$ values computed from the BS model shown in the last table attached, we can see that for $K$ between 14.00 and 15.00 we have $\Delta \sim 0.7$ and $\Gamma \sim 0.2$, so we can expect a P\&L between 28 and 36 cents per option.

Now, considering a volatility of 0\%, even though it's highly unlikely, our hedging strategy wouldn't work and therefore P\&L = 0. However, a change of a 10\%, again highly unlikely, would mean $\Delta \sim 0.46$ and $\Gamma \sim 0.25$, giving a P\&L of 90 cents per option, which is of course higher than our realistic estimate.

\begin{thebibliography}{1}
\raggedright
\addcontentsline{toc}{section}{Bibliography}

\bibitem{Wilmott} P. Wilmott et al, \emph{The Mathematics of Financial Derivatives}, 1995.

\end{thebibliography}

\end{multicols*}

\includepdf[pages={-}]{pdf/ing_data.pdf}
\includepdf[pages={-}]{pdf/greeks_data.pdf}

\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Short Sectioned Assignment
% LaTeX Template
% Version 1.0 (5/5/12)
%
% This template has been downloaded from:
% http://www.LaTeXTemplates.com
%
% Original author:
% Frits Wenneker (http://www.howtotex.com)
%
% License:
% CC BY-NC-SA 3.0 (http://creativecommons.org/licenses/by-nc-sa/3.0/)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%----------------------------------------------------------------------------------------
%	PACKAGES AND OTHER DOCUMENT CONFIGURATIONS
%----------------------------------------------------------------------------------------

\documentclass[paper=a4, fontsize=11pt]{scrartcl} % A4 paper and 11pt font size
\usepackage{amsmath,amsfonts,graphicx}

\usepackage{booktabs}
%\usepackage{algpseudocode}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage{algpseudocode}
\usepackage[top=2cm, bottom=2cm, left=2cm, right=2cm]{geometry}

\usepackage{listings}
\usepackage{color}
\usepackage{xcolor}
\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}
\lstset{frame=tb,
     %language=Java,
     aboveskip=3mm,
     belowskip=3mm,
     showstringspaces=false,
     columns=flexible,
     basicstyle = \ttfamily\small,
     numbers=none,
     numberstyle=\tiny\color{gray},
     keywordstyle=\color{blue},
     commentstyle=\color{dkgreen},
     stringstyle=\color{mauve},
     breaklines=true,
     breakatwhitespace=true,
     tabsize=3
}

%\usepackage[]{algorithm2e}

\usepackage[T1]{fontenc} % Use 8-bit encoding that has 256 glyphs
\usepackage{fourier} % Use the Adobe Utopia font for the document - comment this line to return to the LaTeX default
\usepackage[english]{babel} % English language/hyphenation
\usepackage{amsmath,amsfonts,amsthm} % Math packages

\usepackage{lipsum} % Used for inserting dummy 'Lorem ipsum' text into the template

\usepackage{sectsty} % Allows customizing section commands
\allsectionsfont{\centering \normalfont\scshape} % Make all sections centered, the default font and small caps

\usepackage{fancyhdr} % Custom headers and footers
\pagestyle{fancyplain} % Makes all pages in the document conform to the custom headers and footers
\fancyhead{} % No page header - if you want one, create it in the same way as the footers below
\fancyfoot[L]{} % Empty left footer
\fancyfoot[C]{} % Empty center footer
\fancyfoot[R]{\thepage} % Page numbering for right footer
\renewcommand{\headrulewidth}{0pt} % Remove header underlines
\renewcommand{\footrulewidth}{0pt} % Remove footer underlines
\setlength{\headheight}{13.6pt} % Customize the height of the header

\numberwithin{equation}{section} % Number equations within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{figure}{section} % Number figures within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{table}{section} % Number tables within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)

\setlength\parindent{0pt} % Removes all indentation from paragraphs - comment this line for an assignment with lots of text

%----------------------------------------------------------------------------------------
%	TITLE SECTION
%----------------------------------------------------------------------------------------

\newcommand{\horrule}[1]{\rule{\linewidth}{#1}} % Create horizontal rule command with 1 argument of height

\title{
\normalfont \normalsize
\textsc{The University of Melbourne } \\ [25pt] % Your university, school and/or department name(s)
\horrule{0.5pt} \\[0.4cm] % Thin top horizontal rule
\huge COMP90038 Algorithms and Complexity SM2, 2018
Assignment 1 \\ % The assignment title
\horrule{2pt} \\[0.5cm] % Thick bottom horizontal rule
}

\author{Peiyong Wang $\,$ 955986} % Your name

\date{\normalsize\today} % Today's date or a custom date

\begin{document}

\maketitle % Print the title

\section{Problem One}
\paragraph{a} Clearly this algorithm searches in an array for a value which equals to its index, like A[9]=9.
\paragraph{b} The algorithm firstly examines whether A[lo] > lo or A[hi] < hi. If so, it means all values are bigger or smaller than their indexes, and in this situation the algorithm will return -1, indicating there is no possibility in the input array that $\exists i \in  \mathbb{N^+},\, s.t. A[i] = i $. If the input array satisfy this condition, then the algorithm will use hi and lo to compute mid and check whether A[mid] = mid. If so, return mid; if not, the algorithm will split the array into two halves and if A[mid] is bigger than mid the algorithm will continuing its search in the lower half, otherwise it will search in the higher half of the input array and recursively call the function itself.

\vbox{ }

However, to make this algorithm work fine, the input array must only have on value that equals to its key, and there should be no duplicates.
If it has many values that they all equal to their keys repectively, the algorithm will only find one. If the input array have duplicates there is a chance that there will be a A[$lo$]>$lo$ or A[$hi$]<$hi$
situation, which won't pass the first if-clause in the algorithm.
\section{Problem Two}

See Algorithm 1.

\begin{algorithm}
        \caption{Count the number of occurrences of a certain integer $x$ in an array $A[]$}
        \begin{algorithmic}[1] %每行显示行号
            \Require  $Array$ $A[\cdot]$,$Length\;of\;the\;array$ n, $Integer$ x
            \Ensure   Number of occurrence of integer x
            \Function {NumberCount}{$A[\cdot], x,n$}

            	\State $firstApperence \gets $\Call{FirstOccurrenceSearch}{$A[\cdot]$,0,$n-1$,$x$,$n$}

            	\If{$firstApperence=-1$}
            	\State \Return $firstApperence$
            	\EndIf

            	\State $lastApperence\gets$ \Call{LastOccurrenceSearch}{$A[],firstApperence,n-1,x,n$}
            	\State \Return $lastApperence-firstApperence+1$

                %\State $result \gets 0$

                %\If {$high \geqslant low$}
                    %\State $mid \gets (high + low) // 2$   \#"//" means integer division
                    %\State $result \gets result +$ \Call{MergerSort}{$Array, left, middle$}
                    %\State $result \gets result +$ \Call{MergerSort}{$Array, middle, right$}
                    %\State $result \gets result +$ \Call{Merger}{$Array,left,middle,right$}
                %\EndIf
                %\If {$mid = 0 \, OR x > A[mid-1]$}
                %	\State \Return $mid$
                %\Else
                %	\If {$x>A[mid]$}
                %		\State \Return

                 %\Return{$result$}
            \EndFunction
           \State
            \Function{FirstOccurrenceSearch}{$A[\cdot],low, high,x,n$}

            \If{$high \geqslant low$}
            \State $mid\gets(low + high)//2$ \Comment "//" means integer division
            \EndIf
            \If{$\{mid = 0 \; OR\;  x>A[mid-1]\}\; AND\; \{A[mid]=x\}$}
            \State \Return $mid$
            \Else
            	\If{$x>A[mid]$}
            	\State \Return \Call{FirstOccurrenceSearch}{$A[\cdot],mid+1,high,x,n$}
            	\Else
            	\State \Return \Call{FirstOccurrenceSearch}{$A[\cdot],low,mid-1,x,n$}
            	\EndIf
            \EndIf
            \State \Return -1

            \EndFunction

            \State
            \Function{LastOccurrenceSearch}{$A[\cdot],low,high,x,n$}

            \If{$high \geqslant low$}
            \State $mid\gets(low + high)//2$ \Comment "//" means integer division
            \EndIf
            \If{$\{mid = n-1 \; OR\;  x<A[mid-1]\}\; AND\; \{A[mid]=x\}$}
            \State \Return $mid$
            \Else
            	\If{$x<A[mid]$}
            	\State \Return \Call{LastOccurrenceSearch}{$A[\cdot],low, mid-1, x,n$}
            	\Else
            	\State \Return \Call{LastOccurrenceSearch}{$A[\cdot],mid+1, high,x,n$}
            	\EndIf
            \EndIf
            \State \Return -1

            \EndFunction

            %\Function{Merger}{$Array, left, middle, right$}
             %   \State $i\gets left$
              %  \State $j\gets middle$
               % \State $k\gets 0$
               % \State $result \gets 0$
               % \While{$i<middle$ \textbf{and} $j<right$}
                %    \If{$Array[i]<Array[j]$}
                 %       \State $B[k++]\gets Array[i++]$
                  %  \Else
                   %     \State $B[k++] \gets Array[j++]$
                    %    \State $result \gets result + (middle - i)$
                    %\EndIf
                %\EndWhile
                %\While{$i<middle$}
                %    \State $B[k++] \gets Array[i++]$
                %\EndWhile
                %\While{$j<right$}
                %    \State $B[k++] \gets Array[j++]$
                %\EndWhile
                %\For{$i = 0 \to k-1$}
                %    \State $Array[left + i] \gets B[i]$
                %\EndFor
                %\State \Return{$result$}
            %\EndFunction
        \end{algorithmic}
    \end{algorithm}

\section{Problem Three}
\paragraph{a} From Figure 3.1, which is a "state tree" that in this tree a path starts from node "start" (but not include node start) and ends at a node that in the binary tree's bottom indicates a possible collections of booleans that could satisfy the formula. For example, the array $A = [true, false, false]$ in the question can be converted to a path starts from node 1, then go to node 4 and ends at node 10. So we can tell that for the worst case scenario of the algorithm stated in the question, we need to traverse all the paths in the state tree. The time complexity of binary tree traversal is $O(n+n-1)=O(n)$, where $n$ is the number of nodes in the binary tree. If we let $n=number\;of\;elements\;in\;array\;A$, then the total number of nodes in the binary tree is $1 + \sum_{i=0}^n 2\cdot 2^i = 2^{n+2}-1 $. So the worst case time complexity is $O(2^{n+2}-1)=O(2^n)$

\begin{figure}[htbp!]
		\centering
		\includegraphics[width=1.0\textwidth]{fig1.png}
		\caption{State Tree}%\label{book}
		\vspace{-1em}
\end{figure}

\paragraph{b}
From paragraph A, we can easily see that:
\begin{align}
	\begin{split}
		C(1) &= 2
	\end{split}
\end{align}

\begin{align}
	\begin{split}
		C(n) &= 2C(n-1)
	\end{split}
\end{align}

Then we can have that:

\begin{align}
	\begin{split}
		C(n) &= 2C(n-1)\\
		&= 2(2(C(n-2)))\\
		&= \cdots\\
		&= 2^{n-1}C(1)\\
		&= 2^n
	\end{split}
\end{align}

So the time complexity is $\Theta (2^n) $.

\section{Problem Four}

\paragraph{a}
See Algorithm 2.
\begin{algorithm}
	\caption{Follow a given route linked list to move to the destination and then go back.}
	\begin{algorithmic}[1]
		\Require head of the linked list of route
		\Function{GoAndBack}{$head$}
			\State $p\gets head$  \Comment Starting from the home base
			\State $goBack\gets$ empty stack
			%\State $i\gets 0$
			\While{$p$ is not null}
				\State \Call{Move}{$p$.val}
				\State $goBack$.push($p.val$) \Comment Push the current position to the stack
				\State $p\gets p$.next
				%\State $i\gets i+1$
            \EndWhile
            \State $hashBack\gets$ \{ "N":"S","E":"W","S":"N","W":"E" \}\Comment This is a hash table that stores the opposite directons, like a dictionary in Python, eg: $hashBack$("N")="S".
			\While{$goBack$ is not null}
				\State \Call{Move}{$hashBack$($goBack$.pop)} \Comment Pop the top element of the stack,use the hash table to find its opposite direction, and call the Move function
			\EndWhile

		\EndFunction
	\end{algorithmic}

\end{algorithm}

\paragraph{b} See Algorithm 3.
\begin{algorithm}
	\caption{Use BFS to search the shortest route from home base to goal}
	\begin{algorithmic}[1]
		\Require Graph adjacency matrix $A[\cdot,\cdot]$, start, goal
		\Ensure Route list
		\Function{ShortestPath}{$A[\cdot,\cdot],start, goal$}
			\State $pathOfNodes\gets$\Call{BFSPaths}{$start$}
			\State $reverseRoute\gets$ empty list []
			\State $currentVortex\gets goal$
			\While{$pathOfNodes[currentVortex]\neq -1$}
				\State $reverseRoute$.append($currentVortex$)
				\State $currentVortex\gets pathOfNodes[currentVortex]$
			\EndWhile
			\State $routeList\gets $empty list []
			\State $j\gets reverseRoute$.length
			\While{$j\geq 1$}
				$routeList$.append($A[reverseRoute[j],reverseRoute[j-1]]$)
            \EndWhile
            \State $routeLinkedList\gets$ empty linked list with head q and length is the same as $routeList$
            \For{$k = 0; k < routeList.length; k++$}
                \State $q.value\gets routeList[k]$
                \State $q\gets q.next$
            \EndFor
			\State \Return $routeLinkedList$
		\EndFunction
        \State

		\Function{BFSPaths}{start}
			\For{each vortex $v$}
				\State $flag[v]\gets false$
				\State $pathList[v]\gets -1$ \Comment Maintain a list that records what is the previous node of a node in the BFS searching path
			\EndFor
			\State $Q\gets empty\; queue$
			\State $flag[start]\gets true$
			\State \Call{Enqueue}{$Q$,$start$}
			\While{$Q$ is not empty}
				\State $v\gets$ \Call{Dequeue}{$Q$}
				\For{each $w$ adjacent to $v$}
					\If{$flag[w]=true$}
						\State $flag[w]\gets true$
						\State $pathList[w]\gets v$
						\State \Call{Enqueue}{$Q, w$}
					\EndIf
				\EndFor
			\EndWhile
			\State \Return $pathList$
		\EndFunction
	\end{algorithmic}
\end{algorithm}

\end{document}

%\begin{align}
%\begin{split}
%(x+y)^3 	&= (x+y)^2(x+y)\\
%&=(x^2+2xy+y^2)(x+y)\\
%&=(x^3+2x^2y+xy^2) + (x^2y+2xy^2+y^3)\\
%&=x^3+3x^2y+3xy^2+y^3
%\end{split}
%\end{align}

%\begin{tabbing}
%\hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \=\kill
%\>$Euclid(m,n)=$ \\
%\>\> {\bf while} n$ \neq $ 0 \\
%\>\>\> r $ \leftarrow $ $m$ mod $n$  \\
%\>\>\>  m $\leftarrow$n\\
%\>\>{\bf return} m
%\end{tabbing}

%Python code:
%\begin{lstlisting}[language = python]
%def gcd(m,n):
%	while n != 0:
%		r = m % n
%		m = n
%		n = r
%	return m
%\end{lstlisting}

%\paragraph{Heading on level 4 (paragraph)}

%\begin{tabbing}
%	\hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \= \hspace*{.25in} \=\kill
%	{\bf function} find (A,x,n)\\
%	\> j $\leftarrow$ 0\\
%	\> {\bf while} j < n\\
%	\>\> {\bf if} A[j]=x\\
%	\>\>\>  {\bf return} j  \\
%	\>\> j $\leftarrow$ j+1\\
%	\> {\bf return} -1
%\end{tabbing}

%\begin{figure}[htbp!]
%		\centering
%		\includegraphics[width=0.6\textwidth]{lec26.png}
%		\caption{Linked List}%\label{book}
%		\vspace{-1em}
%\end{figure}

%\begin{align}
%A =
%\begin{bmatrix}
%A_{11} & A_{21} \\
%A_{21} & A_{22}
%\end{bmatrix}
%\end{align}

%\begin{algorithm}
 %       \caption{Count the number of occurrences of a certain integer $x$ in an array $A[]$}
  %      \begin{algorithmic}[1] %每行显示行号
   %         \Require  $Array$ A[],$Length\;of\;the\;array$ n, $Integer$ x
    %        \Ensure   Number of occurrence of integer x
     %       \Function {NumberCount}{$A[], x,n$}
      %
       %     	\State $firstApperence \gets $\Call{FirstOccurrenceSearch}{$A[]$,0,$n-1$,$x$,$n$}

        %    	\If{$firstApperence=-1$}
         %   	\State \Return $firstApperence$
          %  	\EndIf

           % 	\State $lastApperence\gets$ \Call{LastOccurrenceSearch}{$A[],firstApperence,n-1,x,n$}
            %	\State \Return $lastApperence-firstApperence+1$

                %\State $result \gets 0$

                %\If {$high \geqslant low$}
                    %\State $mid \gets (high + low) // 2$   \#"//" means integer division
                    %\State $result \gets result +$ \Call{MergerSort}{$Array, left, middle$}
                    %\State $result \gets result +$ \Call{MergerSort}{$Array, middle, right$}
                    %\State $result \gets result +$ \Call{Merger}{$Array,left,middle,right$}
                %\EndIf
                %\If {$mid = 0 \, OR x > A[mid-1]$}
                %	\State \Return $mid$
                %\Else
                %	\If {$x>A[mid]$}
                %		\State \Return

                 %\Return{$result$}
            %\EndFunction
           %\State
            %\Function{FirstOccurrenceSearch}{$A[],low, high,x,n$}

      %      \If{$high \geqslant low$}
      %      \State $mid\gets(low + high)//2$ \Comment "//" means integer division
      %      \EndIf
      %      \If{$\{mid = 0 \; OR\;  x>A[mid-1]\}\; AND\; \{A[mid]=x\}$}
      %      \State \Return $mid$
      %      \Else
      %      	\If{$x>A[mid]$}
      %      	\State \Return \Call{FirstOccurrenceSearch}{$A[],mid+1,high,x,n$}
      %      	\Else
      %      	\State \Return \Call{FirstOccurrenceSearch}{$A[],low,mid-1,x,n$}
      %      	\EndIf
      %      \EndIf
      %      \State \Return -1
      %
      %      \EndFunction
      %
      %      \State
      %      \Function{LastOccurrenceSearch}{$A[],low,high,x,n$}
      %
      %      \If{$high \geqslant low$}
      %      \State $mid\gets(low + high)//2$ \Comment "//" means integer division
      %      \EndIf
      %      \If{$\{mid = n-1 \; OR\;  x<A[mid-1]\}\; AND\; \{A[mid]=x\}$}
      %      \State \Return $mid$
      %     \Else
       %     	\If{$x<A[mid]$}
        %    	\State \Return \Call{LastOccurrenceSearch}{$A[],low, mid-1, x,n$}
         %   	\Else
          %  	\State \Return \Call{LastOccurrenceSearch}{$A[],mid+1, high,x,n$}
           % 	\EndIf
            %\EndIf
            %\State \Return -1

           % \EndFunction

            %\Function{Merger}{$Array, left, middle, right$}
             %   \State $i\gets left$
              %  \State $j\gets middle$
               % \State $k\gets 0$
               % \State $result \gets 0$
               % \While{$i<middle$ \textbf{and} $j<right$}
                %    \If{$Array[i]<Array[j]$}
                 %       \State $B[k++]\gets Array[i++]$
                  %  \Else
                   %     \State $B[k++] \gets Array[j++]$
                    %    \State $result \gets result + (middle - i)$
                    %\EndIf
                %\EndWhile
                %\While{$i<middle$}
                %    \State $B[k++] \gets Array[i++]$
                %\EndWhile
                %\While{$j<right$}
                %    \State $B[k++] \gets Array[j++]$
                %\EndWhile
                %\For{$i = 0 \to k-1$}
                %    \State $Array[left + i] \gets B[i]$
                %\EndFor
                %\State \Return{$result$}
            %\EndFunction
      %  \end{algorithmic}
    %\end{algorithm}