% TEMPLATE FOR STANDARD QUIZ
% by laura

\documentclass[11pt,epsfig]{article}
\usepackage{amsmath}
\usepackage{mdframed}
\usepackage{graphicx}
\usepackage{multirow}
\def\infinity{\rotatebox{90}{8}}
\oddsidemargin=0in
\evensidemargin=0in
\textwidth=6.3in
\topmargin=-0.5in
\textheight=9in
\renewcommand{\arraystretch}{2}
\parindent=0in
\pagestyle{empty}

\input{notestpoints}

\begin{document}

%%%(change to appropriate class and semester)
\textbf{Revise(Chapter 5): CIS 2033 Sprint 2015: Computational Prob and Stat}

%%%(change to appropriate quiz type and date)
\textbf{Shanshan Zhang {(tuf14438@temple.edu)} }

% problem
\section{Notations}
$X, Y, Z$: Random Variables.\\
$k,a,b$: specific whole number (e.g. 0,1,2,...). \\
$x,y$: specific real number (e.g. 1.2, 0.5, ...). \\
$p(k)$: Probability mass function for discrete random variable $X$. It calculate for any specific whole number $k$, the probability of $P(X=k)$.\\
$f(x)$: Probability density function for continuous random variable $X$.\\
$F(x)=P(X\le x)$: Distribution function, sometimes it's called cumulative distribution function.\\
\begin{tabular}{|l|}
\hline
If $X$ is discrete: $F(k) = \sum_{y\le k}p(y)$, where $y$ is any possible value for $X$ that is less or equal than $k$.\\
If $X$ is continuous: $F(x)=\int	^x_{-\infinity}f(x)dx$\\
\hline
\end{tabular}

\subsection{Important distributions}
\begin{tabular}{ |l|l|l|p{6cm}| }
%\caption{Important Distributions}
\hline
& \bf{Notation }& \bf{$p(k)$ or $f(x)$} & \bf{$F(k)$ or $F(x)$}\\ \hline
\multirow{3}{*}{Discrete} & $X\sim Ber(p)$ & $p(1) = p;p(0)=1-p$ & $F(k)=0,(k<0);F(k)=1-p,(0\le k <1);F(k)=1,(k>1)$\\ \cline{2-4}
 & $X\sim Bin(n,p)$ & $p(k)={n\choose k} p^k(1-p)^{n-k},k=0,1,...,n$& $F(k)=\sum_{y\le k}p(y),k=0,1,...,n$\\  \cline{2-4}
 & $X\sim Geo(p)$ & $p(k)=(1-p)^{k-1}p,k=1,...$& $F(k)=1-(1-p)^{k},k=0,1,...$\\ \cline{2-4}
 & $X\sim Pois(\mu)$ & $p(k)=\frac{\mu^k}{k!}{e^{-k}},k=0,1,...$ & $F(k)=\sum_{y\le k}p(y),k=0,1,...$\\ \hline
\multirow{4}{*}{Continuous} & $X\sim Unif(\alpha,\beta)$ & $f(x)=\frac{1}{\beta-\alpha},for~x\in (\alpha,\beta)$& $F(x)=\frac{x-\alpha}{\beta-\alpha}, for~ x \in (\alpha,\beta)$\\ \cline{2-4}
 & $X\sim Exp(\lambda)$ & $f(x)=e^{-\lambda x}, for~ x \in [0,\infinity)$& $F(x)= 1 - e^{-\lambda x},for~ x \in [0,\infinity)$\\ \cline{2-4}
 & $X\sim Par(\alpha)$ & $f(x)=\frac{\alpha}{x^{\alpha+1}},for ~x\in [1,\infinity)$& $F(x)=1-\frac{1}{x^\alpha},for ~x\in [1,\infinity)$\\ \cline{2-4}
& $X\sim N(\mu,\sigma^2)$&$f(x)=\frac{1}{\sqrt{(2\pi\sigma^2)}}e^{-\frac{(x-\mu)^2}{2\sigma^2}},x\in (-\infinity,+\infinity)$&NO explicit form\\ \hline
\end{tabular}
\newpage
\section{Table of Basic Integrals}
\subsection{Basic Forms}

\begin{equation}
\int x^n dx = \frac{1}{n+1}x^{n+1},\hspace{1ex}n\neq -1
\end{equation}

\begin{equation}
\int \frac{1}{x}dx = \ln |x|
\end{equation}

\begin{equation}
\int u dv = uv - \int v du
\end{equation}

\begin{equation}
\int \frac{1}{ax+b}dx = \frac{1}{a} \ln |ax + b|
\end{equation}

\subsection{Integrals of Rational Functions}

\begin{equation}
\int \frac{1}{(x+a)^2}dx = -\frac{1}{x+a}
\end{equation}

\begin{equation}
\int (x+a)^n dx = \frac{(x+a)^{n+1}}{n+1}, n\ne -1
\end{equation}

\begin{equation}
\int x(x+a)^n dx = \frac{(x+a)^{n+1} ( (n+1)x-a)}{(n+1)(n+2)}
\end{equation}

\begin{equation}
\int \frac{1}{1+x^2}dx = \tan^{-1}x
\end{equation}

\begin{equation}
\int \frac{1}{a^2+x^2}dx = \frac{1}{a}\tan^{-1}\frac{x}{a}
\end{equation}

\begin{equation}
\int \frac{x}{a^2+x^2}dx = \frac{1}{2}\ln|a^2+x^2|
\end{equation}

\begin{equation}
\int \frac{x^2}{a^2+x^2}dx = x-a\tan^{-1}\frac{x}{a}
\end{equation}

\begin{equation}
\int \frac{x^3}{a^2+x^2}dx = \frac{1}{2}x^2-\frac{1}{2}a^2\ln|a^2+x^2|
\end{equation}

\begin{equation}
\int \frac{1}{ax^2+bx+c}dx = \frac{2}{\sqrt{4ac-b^2}}\tan^{-1}\frac{2ax+b}{\sqrt{4ac-b^2}}
\end{equation}

\begin{equation}
\int \frac{1}{(x+a)(x+b)}dx = \frac{1}{b-a}\ln\frac{a+x}{b+x}, \text{ } a\ne b
\end{equation}

\begin{equation}
\int \frac{x}{(x+a)^2}dx = \frac{a}{a+x}+\ln |a+x|
\end{equation}

\begin{equation}
\int \frac{x}{ax^2+bx+c}dx = \frac{1}{2a}\ln|ax^2+bx+c|
-\frac{b}{a\sqrt{4ac-b^2}}\tan^{-1}\frac{2ax+b}{\sqrt{4ac-b^2}}
\end{equation}
\end{document}


\documentclass[pdf]{beamer}
\usetheme{AnnArbor}         % tema
\usecolortheme{orchid}      % cores
\mode<presentation>{}
\usepackage{multirow}
\usepackage{hyperref}
\usepackage{color}
\usepackage{epsfig}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amscd}
\usepackage[brazilian]{babel}
%\usepackage[utf8]{inputenc}
\usepackage{float}
\usepackage{algpseudocode}
\usepackage{subfig}
\usepackage{listings}
\usepackage{tikz}

\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}
\algnewcommand{\LineComment}[1]{\State \(\triangleright\) #1}
\def  \R   {{\mathbb R}}
\def  \Z   {{\mathbb Z}}
%\def  \N   {{\mathbb N}}
\def  \Q   {{\mathbb Q}}
\def \Ev {{\mathbb E}}
\def \Pr {{\mathbb P}}
\def \f {{\mathbf f}}
\def \u {{\mathbf u}}
\def \y {{\mathbf y}}
\def \x {{\mathbf x}}
\def \m {{\mathbf m}}
\def \gu {{\bar{g}}}

\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}

%% preamble
\title{The title}
\subtitle{The subtitle}
\author{your name}
\begin{document}

\lstset{frame=tb,
	language=Python,
	aboveskip=3mm,
	belowskip=3mm,
	showstringspaces=false,
	columns=flexible,
	basicstyle={\small\ttfamily},
	numbers=none,
	numberstyle=\tiny\color{gray},
	keywordstyle=\color{blue},
	commentstyle=\color{dkgreen},
	stringstyle=\color{mauve},
	breaklines=true,
	breakatwhitespace=true,
	tabsize=3
}

{\setbeamertemplate{footline}{}
	\frame{\titlepage}}

\section*{Outline}\begin{frame}{Outline}\tableofcontents
\end{frame}
%% title frame
\begin{frame}
\titlepage
\end{frame}

%% normal frame
\section{Introduction}
\begin{frame}{Introduction}
Some introduction.
\end{frame}

\begin{frame}{Introduction}
\begin{block}{Bayesian theory}
Building blocks
\begin{itemize}
	\item Prior probability $p(\theta)$
	\item Likelihood $p(\mathcal{D}|\theta)$
\end{itemize}

Posterior probability
\begin{equation*}
 p(\mathcal{\theta}|\mathcal{D}) = \frac{p(\mathcal{D}|\theta) p(\theta)}{\int p(\mathcal{D}|\theta') p(\theta') d \theta'}
\end{equation*}
\pause
Using posterior probability:
\begin{equation*}
\int_\Theta f(\theta) p(\theta|\mathcal{D},M) d \theta
\end{equation*}
\end{block}
\pause
\begin{block}{}
	\centering{\textbf{Bayesian theory requires integration!}}
\end{block}

\end{frame}

\begin{frame}{Introduction}
\begin{block}{Approximate inference}
Ways to integrate:
\begin{itemize}
	\item Monte Carlo
	\begin{equation*}
	\int_\Theta f(\theta) p(\theta|\mathcal{D}) d\theta \approx \frac{1}{N} \sum_{i=1}^N f(\theta_i), \, \theta_i \sim p(\theta|\mathcal{D})
	\end{equation*}
	\item Approximate distribution
	\begin{equation*}
	p(\theta|\mathcal{D}) \approx q(\theta), \quad \int_\Theta  f(\theta) q(\theta) d\theta
	\end{equation*}
\end{itemize}

\pause
\begin{block}{}
\centering{Usual methods demands many evaluations of $p(\mathcal{D}|\theta) p(\theta)$. However, \textit{this is not always feasible.}}
\end{block}

\end{block}
\end{frame}

\begin{frame}
\begin{block}{}
In science, there are many cases that $p(\mathcal{D}|\theta)$ demands the computation of a forward model $g(\theta)$, which comes from an expensive simulation.
  \begin{figure}[h]
	\centering
	\includegraphics[height=0.3\paperheight]{figs/simulation_figure.jpg}
	%\includegraphics[height=6cm]{figCoordEsf02}
\end{figure}
\pause
This requires approximate inference methods "on a budget". In this work, one such method is developed, based on preexisting work. We name it
\pause
\begin{block}{}
\centering{\textit{Boosted Variational Bayesian Monte Carlo (BVBMC)}.}
\end{block}
\end{block}
\end{frame}

\begin{frame}{Introduction}
\begin{block}{BVBMC schema}
  \begin{figure}[h]
	\centering
	\includegraphics[height=.5\linewidth]{figs/diagram1a.png}
	%\includegraphics[height=6cm]{figCoordEsf02}
\end{figure}
\end{block}
\end{frame}
\begin{frame}{Introduction}
\begin{block}{An illustration of BVBMC}
	\centering{
	 \begin{tikzpicture}
	\node<1> (exfig1) {\includegraphics[width=.7\linewidth]{figs/exampleintro1}};
	\node<2> (exfig2) {\includegraphics[width=.7\linewidth]{figs/exampleintro2}};
	\node<3> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro3}};
	\node<4> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro4}};
	\node<5> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro5}};
	\node<6> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro6}};
	\node<7> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro7}};
	\node<8> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro8}};
	\node<9> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro9}};
	\node<10> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro10}};
	\node<11> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro11}};
	\node<12> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro12}};
	\node<13> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro13}};
	\node<14> (exfig3) {\includegraphics[width=.7\linewidth]{figs/exampleintro14}};

	\end{tikzpicture}
}
\end{block}
\end{frame}

\section{Gaussian Processes}
\begin{frame}{Gaussian Processes}
\begin{block}{Definition}
Gaussian processes (GP): distribution over functions $f:\mathcal{X} \to \mathbb{R}$ such that $f(\mathbf{x}) = (f(x_1),\ldots,f(x_n))$ follows a multivariate normal distribution. A GP is completely defined by:
\begin{itemize}
	\item $m(x) := \Ev[f(x)]$, mean function.
	\item $k(x,x') := \Ev[f(x),f(x')]$, covariance function or kernel.
\end{itemize}
such that $f(\mathbf{x}) \sim \mathcal{N}(m(\mathbf{x}),K(\mathbf{x},\mathbf{x}))$.
\end{block}
\pause
\begin{block}{Gaussian process regression}
Given $\mathcal{D} = {(x,y)}_{i=1}^N$, a Gaussian process regression is made by assuming $p(y|x) = p(y|f(x))$, with $f$ following a prior $GP(m,k)$.
\end{block}

\end{frame}

\begin{frame}{Gaussian Processes}
\begin{block}{Posterior GP}
If $p(y|f(x)) = \mathcal{N}(f(x),\sigma_n^2)$, $f|\mathcal{D} \sim GP(m_\mathcal{D},k_\mathcal{D})$, where
\begin{equation*}
\begin{split}
 m_\mathcal{D}(x) & := m(x) + K(x^\star,\mathbf{x}) (K(\mathbf{x},\mathbf{x})+\sigma^2_n)^{-1} (\mathbf{y} - m(\mathbf{x})) \\
 k_\mathcal{D}(x,x') & := k(x,x') - K(x,\mathbf{x})(K(\mathbf{x},\mathbf{x})+\sigma_n^2)^{-1}K(\mathbf{x},x)
\end{split}
\end{equation*}
Reduces to deterministic measurement when $\sigma_n^2 = 0$. More general $p(y|f(x))$ must resort to explicit marginalization.
\end{block}
\end{frame}

\begin{frame}{Gaussian Processes}
\begin{block}{Example case}

\begin{figure}
	\centering
	\subfloat[GP prior]{\label{gprex1a}\includegraphics[width=0.5\textwidth]
		{figs/gprex1a.png}}
	\subfloat[GP posterior]{\label{gprex1b}\includegraphics[width=0.5\textwidth]
		{figs/gprex1b.png}}

\end{figure}

\end{block}
\end{frame}

\begin{frame}{Gaussian Processes}
\begin{block}{Kernels}
The exigence that $K(\mathbf{x},\mathbf{x})$ limits which functions can be kernels. Some examples of kernels in $\mathbb{R}$ are:
\begin{itemize}
	\item $k_{SQE}(x,x') = \theta_0 \exp \left(-\frac{1}{2} \frac{(x-x')^2}{l^2} \right)$
	\item $k_{\text{Matern},3/2}(x,x') = \theta_0 \left( \sqrt{3} \frac{(x-x')}{l} \right) \exp \left(-\sqrt{3} \frac{(x-x')}{l}\right)$
\end{itemize}
Kernels in $\mathbb{R}^D$ can be constructed by changing $\frac{(x-x')}{l}$ for $\sqrt{\sum_{i=1}^D \frac{(x_i-x_i')}{l_i}}$.

If $k_1$,$k_2$ are kernels, the following, among others are kernels: $k_1(x,x')+k_2(x,x')$,$k_1(x,x')k_2(x,x')$,$k_1(x,x')k_2(y,y')$,$k_1(f(y),f(y'))$.
\end{block}
\pause
\begin{block}{Mean functions}
In general,they are less important than kernels, since the latter determines the structure of the posterior GP. However, \textit{outside the sampling area the GP prediction defaults to the mean}, which may be of importance.
\end{block}
\end{frame}

\begin{frame}{Gaussian processes}
\begin{block}{Handling hyperparameters}
\begin{equation*}
\begin{split}
\log p(\mathcal{D}|M,\sigma_n) = & -\frac{1}{2}(\mathbf{y} - m(\mathbf{x}))^T (K(\mathbf{x},\mathbf{x}) + \sigma_n \mathbf{I})^{-1} (\mathbf{y} - m(\mathbf{x})) + \\
&-\frac{1}{2} \log \det (K(\mathbf{x},\mathbf{x}) + \sigma_n \mathbf{I}) - \frac{1}{2} N \log(2\pi).
\end{split}
\end{equation*}
Inference can be done either by MLE, MAP, or integration techniques.
\end{block}
\begin{block}{Scaling}
The bottleneck of GP regression: $(K(\mathbf{x},\mathbf{x}) + \sigma_n \mathbf{I})^{-1}$. Cost is $\mathcal{O}(N^3)$.

In online learning, each new sample is incorporated in $\mathcal{O}(N^2)$.
\end{block}

\end{frame}

\section{Bayesian Monte Carlo}
\begin{frame}{Bayesian Monte Carlo}
\begin{block}{Integrating a GP}
\begin{equation*}
 Z = \int f(x) p(x) dx
\end{equation*}
If $f \sim GP(m,k)$, given $\mathcal{D} = \{(x_i,f(x_i))\}_{i=1}^N$, $Z_\mathcal{D} = \int f_\mathcal{D}(x) p(x) dx$ is Gaussian:
\begin{equation*}
\begin{split}
& \Ev[Z_{\mathcal{D}}] = \int m(x) p(x) dx - \mathbf{z}^T K^{-1} (\mathbf{f}-m(\mathbf{x})), \quad
\Var[Z_{\mathcal{D}}] = \Gamma - \mathbf{z}^T K^{-1} \mathbf{z}, \\
& z_i = \int k(x,x_i) p(x) dx, \quad \Gamma = \int \int k(x,x') p(x) p(x') dx dx'.
\end{split}
\end{equation*}
\pause
\begin{block}{}
\centering{Name Bayesian \textit{Monte Carlo} is misleading.}
\pause

\centering{Treating $f$ as a random variable may be philosophically odd.}
\end{block}
\end{block}
\end{frame}
\begin{frame}{Bayesian Monte Carlo}
\begin{figure}
	\centering
	\includegraphics[width=0.8\linewidth]{figs/exbmc.png}
\end{figure}
\end{frame}
\begin{frame}{Bayesian Monte Carlo}
\begin{block}{Kernel integral terms}
In the general case, they can be estimated by Monte Carlo.
When $p(x)$ is Gaussian or a mixture of Gaussians:
\begin{itemize}
	\item Analytically tractable when $k(x,x')$ is the SQE kernel.
	\item Efficiently tractable when $k(x,x') = k(x_1,y_1) \ldots k(x_D,y_D)$.
\end{itemize}
\begin{block}{Active sampling}
	Given $\{(x_1,f(x_1),\ldots,x_N,f(x_N)\}$, $x_{N+1}$ may be chosen by optimizing acquisition functions.
\begin{equation*}
\alpha^N_{\text{MMLT}}(x) = e^{2 m_\mathcal{D}(x) + k_\mathcal{D}(x,x)} \left(e^{k_\mathcal{D}(x,x)}-1\right).
\end{equation*}
\end{block}
\end{block}
\end{frame}

\section{Variational Inference}
\begin{frame}{Variational Inference}
\begin{block}{Back to approximating posteriors $p(\theta|\mathcal{D}) \approx q(\theta;\lambda)$}

Variational Inference: given $g(\theta)$, seeks minimization of $D_{KL}(q(\cdot;\lambda)||g)$.

Given unnormalized $\gu$, this is equivalent to maximizing the evidence lower bound (ELBO)
\begin{equation*}
\mathcal{L}(\lambda) =
\int \log \gu(\theta) q(\theta) d\theta - \int \log q(\theta) q(\theta) d\theta
\end{equation*}
The family of variational posteriors $q(\theta;\lambda)$ must be easy to treat, in order for the approximation to be useful.

\end{block}
\begin{block}{$D_{KL}(q(\cdot;\lambda)||g)$ vs $D_{KL}(g||q(\cdot;\lambda))$}
$D_{KL}(q(\cdot;\lambda)||g) \neq D_{KL}(g||q(\cdot;\lambda))$: two minimization objectives. Gives two different algorithms (the second one, \textit{expectation propagation}, is not treated here).

\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Illustration}
\begin{figure}
	\centering
	\subfloat[$D_{KL}(q||g)$ ]{\label{vixep1a}\includegraphics[width=0.45\textwidth]
		{figs/klil3a.png}}
	\subfloat[$D_{KL}(g||q)$]{\label{vixep1b}\includegraphics[width=0.45\textwidth]
		{figs/klil3b.png}}
\end{figure}
\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Mean field variational inference}
	Consider factorized proposals $q(\theta) = q(\theta_1)\ldots q(\theta_D)$.

	Training by coordinate descent
	\begin{equation*}
	q^{*}_j(\theta_j;q_{-j}) \propto \exp \Ev_{\theta_{-j} \sim q_{-j}} [ \log \gu(\theta)].
	\end{equation*}

\end{block}
\begin{block}{Generic variational inference}
Uses stochastic gradient descent to find $q(\theta;\lambda)$.

REINFORCE: $\nabla \mathcal{L}(\lambda) = \Ev_{q(\theta;\lambda)} \left[ \left( \log \left( \frac{\gu(\theta)}{q(\theta;\lambda)}\right) + C\right) \nabla_{\lambda} \log q(\theta;\lambda) \right]$

Reparametrization: $\nabla \mathcal{L}(\lambda) = \nabla \left( \Ev_{r(\epsilon)} \left[\log \frac{ \gu(s(\epsilon;\lambda))}{q(s(\epsilon;\lambda);\lambda)}\right]\right) \approx \frac{1}{K} \sum_{i \in [K], \epsilon_i \sim r(\epsilon)} \nabla \left( \log \frac{ \gu(s(\epsilon;\lambda))}{q(s(\epsilon;\lambda);\lambda)} \right).$
\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Mixture of Gaussians}
$q_k(\theta;\lambda) = \sum_{i=1}^k w_i f_i(\theta) = \sum_{i=1}^k w_i \mathcal{N}(\theta;\mu_,\Sigma_i)$. Analytical mean and covariance. Samples can be easily generated.

Covariance parameterizations:
\begin{itemize}
	\item $\Sigma_i = \text{diag}(\sigma_{i,1}^2,\ldots,\sigma_{i,D}^2)$
	\item $\Sigma_i = \u_i \u_i^T + \text{diag}(\sigma_{i,1}^2,\ldots,\sigma_{i,D}^2)$
\end{itemize}

Weights parameterizations $w_i(\nu_i) = \frac{\phi(\nu_i)}{\sum_{i=1}^k \phi(\nu_k)}$. $\phi$ can be:
\begin{itemize}
	\item $\phi(\nu) = \exp(\nu)$
	\item $\phi(\nu) = \text{softplus}(\nu) = \log(1+\exp(\nu))$
\end{itemize}
\begin{equation*}
\mathcal{L}(\lambda) = \sum_{i=1}^k w_i(\nu_i) \Ev_{\epsilon \sim \mathcal{N}(0,I)}\left[\log \frac{ \gu(s(\epsilon;\mu_i,\sigma_i))}{q_k(s(\epsilon;\mu_i,\sigma_i);\lambda)}\right]
\end{equation*}

\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Boosting mixtures}
Problem: no way to know how many mixtures is needed. Adding mixtures sequentially can become costly. One solution: boosting.
$q_{i-1}(\theta) = \sum_{j=1}^{i-1} w_j f_j(\theta)$

$q_{i}(\theta) = \sum_{j=1}^{i-1} (1-w_{i}) w_j f_j(\theta) + w_{i} f_i(\theta)$

How to find $w_{i}$ and $f_i(\theta) = \mathcal{N}(\theta;\mu_{i},\Sigma_{i})$?
\begin{itemize}
	\item Optimize jointly $\mathcal{L}_{i}(w_{i},\mu_{i},\Sigma_{i})$
	\item Seek good proposal $f_i(\theta)$ and optimize $\mathcal{L}_{i}(w_{i})$ via it's derivative
\begin{equation*}
\begin{split}
	\mathcal{L}'_{i}(w_{i}) & = \int \log (\gu(\theta)) (f_{i}(\theta) - q_{i-1}(\theta)) d\theta - \\
	& \qquad{} \int \log((1-w_{i}) q_{i-1}(\theta) + w_{i} f_{i}(\theta)) (f_i(\theta) - q_{i-1}(\theta)) d\theta.
\end{split}
\end{equation*}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Gradient boosting of mixtures}

\begin{equation*}
\begin{split}
f_i = \argmin_{f} \nabla D_{KL}(q_{i-1} || g) \cdot f = \argmin_{f} \int \log \frac{q_{i-1}(\theta)}{g(\theta)} f(\theta) d\theta.
\end{split}
\end{equation*}
Problem: degenerate solution. Needs regularization.

Maximization objective for mixture of Gaussians:
\begin{equation*}
\begin{split}
\text{RELBO}(\mu_i,\Sigma_i) = & \int \log(\gu(\theta)) \mathcal{N}(\theta|\mu_i,\Sigma_i) d\theta - \\
& \int \log(q_{i-1}(\theta)) \mathcal{N}(\theta|\mu_i,\Sigma_i) d\theta + \frac{\lambda}{4} \log |\Sigma|,
\end{split}
\end{equation*}
Estimated by the reparameterization trick.
\end{block}

\end{frame}

\begin{frame}{Variational Inference}
\begin{figure}
	\begin{algorithmic}[1]\label{vbalgorithm}
		\Procedure{VariationalBoosting}{$\log \gu,\mu_0$,$\Sigma_0$}
		\LineComment{$\mu_0,\Sigma_0$ the are initial boosting values}
		\State $w_0 := 1.0$
		\For{$t=1,...,T$}
		\State $\mu_{t},\Sigma_{t} := \argmax RELBO(\mu_{t},\Sigma_{t})$ \Comment{Using reparameterization}
		\State $w_{t} := \argmax \mathcal{L}_i(w_i)$ \Comment{Using $\mathcal{L}'_t(w_t)$ for gradient descent}
		\For{$j=0,...,t-1$}
		\State $w_{j} \gets (1-w_t)w_j$
		\EndFor
		\EndFor
		\State \Return $\{(\mu_t,\Sigma_t,w_t)\}_{t=1}^T$
		\EndProcedure
	\end{algorithmic}
\end{figure}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Variational Bayesian Monte Carlo (VBMC)}
$\mathcal{L}(\lambda) =
 \int \log \gu(\theta) q(\theta;\lambda) d\theta - \int \log q(\theta;\lambda) q(\theta;\lambda) d\theta$

Use Bayesian Monte Carlo:
$\mathcal{L}_\mathcal{D}(\lambda) =
\int \log \gu_\mathcal{D}(\theta) q(\theta;\lambda) d\theta - \int \log q(\theta;\lambda) q(\theta;\lambda) d\theta$

\begin{block}{}
\begin{equation*}
\begin{split}
\text{Maximize } \Ev[\mathcal{L}_\mathcal{D}(\lambda)] & = M(\lambda) + \mathbf{z}^T \mathbf{w} - \int \log q(\theta;\lambda) q(\theta;\lambda) d\theta\\
\mathbf{w} & = K^{-1} \mathbf{y} \\
M(\lambda) & = \int m(\theta) q(\theta;\lambda) d\theta \\
\mathbf{z}_i & = \int k(x,x_i) q(\theta;\lambda) dx.
\end{split}
\end{equation*}

\end{block}
\end{block}
\end{frame}

\begin{frame}{Variational Inference}
\begin{block}{Mean function}
$m(\theta) = 0$: $\log \gu_\mathcal{D}(\theta)$ is not a log probability

Principled solution: $m(\theta) = -\frac{1}{2} \sum_{i=1}^D \frac{(\theta_i - c_i)^2}{l_i^2}$. Lends analytical $M(\lambda)$.

Ad-hoc solution: $m(\theta) = C$, with $C$ being a large negative constant.
\end{block}

\begin{block}{Active evaluation}
Just as in BMC, it is possible to do active evaluation.
Some options:
\begin{itemize}
	\item $\alpha^\mathcal{D}_{\text{US}}(\theta_{N+1}) = k_\mathcal{D}(\theta_{N+1},\theta_{N+1}) q_k(\theta_{N+1};\lambda)^2.$
	\item $\label{prospective_vbmc}
	\alpha^\mathcal{D}_{\text{PROP}}(\theta_{N+1}) = k_\mathcal{D}(\theta_{N+1},\theta_{N+1}) \exp(m_\mathcal{D}(\theta_{N+1}))q_k(\theta_{N+1};\lambda)^2$
\end{itemize}
\end{block}
\end{frame}

\section{Boosted Variational Bayesian Monte Carlo}
\begin{frame}{Boosted Variational Bayesian Monte Carlo}
\begin{block}{BVBMC}
BVBMC = VBMC + boosting + small changes
\end{block}
\begin{block}{BMC in boosted variational inference}
\begin{equation*}
\begin{split}
\text{RELBO}_\mathcal{D}(\mu_i,\Sigma_i) = & \int \Ev[\log \gu_\mathcal{D}(\theta)] \mathcal{N}(\theta|\mu_i,\Sigma_i) d\theta - \\ &\int \log(q_{i-1}(\theta)) \mathcal{N}(\theta|\mu_i,\Sigma_i) d\theta
+ \frac{\lambda}{4} \log |\Sigma_i|
\end{split}
\end{equation*}
\begin{equation*}
\begin{split}
\mathcal{L}_{i,\mathcal{D}}(w) = & \int \log \gu_\mathcal{D}(\theta)
((1-w_{i}) q_{i-1}(\theta) + w_{i} f_{i}(\theta)) d\theta - \\
& \int \log ((1-w_{i}) q_{i-1}(\theta) + w_{i} f_{i}(\theta)) ((1-w_{i}) q_{i-1}(\theta) + w_{i} f_{i}(\theta)) d\theta \\
\end{split}
\end{equation*}
\end{block}
\end{frame}

\begin{frame}{Boosted Variational Bayesian Monte Carlo}
\begin{block}{Practical considerations}
\begin{itemize}
	\item RELBO stabilization
	\begin{equation*}
	\text{RELBO}^{\delta_D}_\mathcal{D}(\mu_i,\Sigma_i) = \int \log \left(\frac{r_\mathcal{D}(\theta)}{q_{i-1}(\theta)+\delta_D} \right) \mathcal{N}(\theta;\mu_i,\Sigma_i) d \theta + \log |\Sigma_i|.
	\end{equation*}
	\item Output scaling
	\begin{equation*}
	\tilde{y}_i = (y_i-m_y)/\sigma_y, \, \tilde{\mathcal{D}} = \{x_i,\tilde{y}_i\}, \,\sigma_y \log g_\mathcal{\tilde{D}}(x) + \mu_y
	\end{equation*}
	\item Component pruning: discard negligible components
	\item Initialization: either large covariance or maximize ELBO for first Gaussian component.
	\item Mean function: $m(\theta) = C$ found to be more stable.
\end{itemize}

\end{block}

\end{frame}

\begin{frame}
\begin{block}{Practical considerations}
	\begin{itemize}
	\item Periodic joint parameter updating: sometimes maximize $\Ev[\mathcal{L}_\mathcal{D}(\lambda)]$ for all parameters in $\sum_{i=1}^k w_k \mathcal{N}(\theta;\mu_k,\Sigma_k)$.
	\item Product of Matern kernels:
	\begin{equation*}
	 k_{\text{PMat},\nu}(x,x';\theta,l) = \theta \prod_{d=1}^D k_{\text{Matern},\nu}(|x_i-x_i'|;l_d).
	\end{equation*}
	Is integrated in BVBMC by Gauss-Hermite quadrature. Found to be more stable than the SQE kernel.
	\item More acquisition functions:
	\begin{equation*}
	\alpha^\mathcal{D}_{MMLT}(x_{m+1}) = e^{2 m_\mathcal{D}(x) + k_\mathcal{D}(x,x)} \left(e^{k_\mathcal{D}(x,x')}-1\right).
	\end{equation*}
	\begin{equation*}
	\alpha^\mathcal{D}_{MMLT_P}(x_{m+1}) = e^{2 m_\mathcal{D}(x) + k_\mathcal{D}(x,x)} \left(e^{k_\mathcal{D}(x,x')}-1\right)q_k(\theta_{N+1};\lambda)^2.
	\end{equation*}
	\end{itemize}
\end{block}
\end{frame}

\section{Experiments}
\begin{frame}
\begin{block}{1-d mixture of Gaussians}
\begin{figure}
	\centering
	\includegraphics[width=0.6\linewidth]{figs/targetexil1a.png}
\end{figure}
\begin{displaymath}
f(x) = \sum_{i=1}^{12} w_i \mathcal{N}(x;\mu_i,\sigma^2_i),
\end{displaymath}
\centering{$w_i = \frac{1}{12}$, $\mu_i \sim \mathcal{N}(0,\sqrt{5})$, $\sigma^2_i = 1$}.
\end{block}
\end{frame}

\begin{frame}

\end{frame}

\end{document}


\documentclass{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amsthm}
\title{Statistical Machine Learning \\ Reading Assignment 3 Report}

\linespread{1.25}

\newtheorem{prop}{property}

\author{Alireza Sadeghi - Seyed Mohsen Shojaee}

\begin{document}

\maketitle

\section{Introduction}
The Bayesian approach to statistical problems has yielded very promising results. However in the context
of nonparametric problem is has been quite unsuccessful. One reason for this problem is the unsatisfied need for a manageable prior.
There are two essential properties for a prior distribution to work well in a nonparametric setting.
The Bayesian approach to statistical problems has yielded very promising results. However in the context
of nonparametric problem is has been quite unsuccessful. One reason for this problem is the unsatisfied need for a manageable prior.
There are two essential properties for a prior distribution to work well in a nonparametric setting.
\begin{itemize}
  \item The support of prior should be large to allow for many possible posteriors.
\item Posterior on observed data should be manageable analytically.
\end{itemize}
Dirichlet distribution introduced in \cite{fegusen} a prior
(probably the first one) for which the two properties above are simultaneously satisfied.

There are two alternate ways to define Dirichlet process. One by
describing the distribution on any finite subset and rely on
Kolmogorov's consistency theorem for the existence of the process.
The second way is by generalizing Dirichlet distribution to infinite dimension.

In this section, we first review the Dirichlet distribution. Then we provide two formal definitions and finally
give some insights about capabilities and shortcomings of Dirichlet Process i.e. the problems it can and can not model.

\subsection{Dirichlet Distribution}
A k-dimensional Dirichlet distribution is essentially a distribution over all probability measures on k possible outcomes.
For Example a 6-dimensional Dirichlet can be used to model the probability distribution of dices.
The formal definition is as follows.
Let $X_1, X_2, \ldots, X_k$ be Gamma random variables with parameter $(\alpha_i, 1)$. A Dirichlet distribution with
parameter $(\alpha_1, \ldots, \alpha_n)$  is defined as distribution of $ Y = (Y_1, \ldots, Y_n)$, where
\begin{equation}
  Y_i = \frac{Z_i}{\sum_{i=1}^k Z_j}
\end{equation}
As can be seen in the above equation, $Y$ will lie on the $k-$simplex;
thus interpreted as a probability distribution over a sample space with $k$ members.

\section{Dirichlet Process}
We now turn to definition of Dirichlet process. Let $ \mathcal{X}$  be a measurable space
and $ \mathcal{X} $  a $\sigma-$field on it. The process is defined by specifying the joint distribution of any
Sequence $A_1, \ldots, A_n \in \mathcal{A}$.

To this end, it is sufficient to specify the distribution on every measurable portioning $B_1, \ldots, B_n$.
(We say portioning $(B_1, \ldots, B_n)$ to be measurable if $\forall i,j: B_i \in \mathcal{A}$ and $B_i \cap B_j = \Phi$ and $\bigcup{j=1}^n B_j = \mathcal{X}$ )
From these distributions the distribution for arbitrary sequence $A_1, \ldots, A_n \in \mathcal{A}$ can be uniquely determined.
We are almost ready to give a formal definition for Dirichlet distribution, it only remains
the parameter of the process which should be a non-null finite measure on $\mathcal{X}$. We use $\alpha$ to denote the parameter.

We say a random process $Q$ is a Dirichlet process with parameter $\alpha$ if for every finite measurable portioning $B_1, \ldots, B_n$
the distribution of $(P(B_1), \ldots, P(B_n))$ is  Dirichlet with parameter $(\alpha(B_1), \ldots, \alpha(B_n))$.

An interesting property of Dirichlet distribution that can be considered a version of large support desired property is stated below.

Let $Q$ be a Dirichlet distribution, then for every fixed probability measure $T$ on $\mathcal{X}$ sequence of measurable sets
$A_1, \ldots, A_n \in \mathcal{A}$ and for every $\epsilon > 0$:
\begin{equation}
  \mathbb{P}(|P(A_i) - T(A_i) <\epsilon) > 0
\end{equation}

The following theorem ensures the posterior manageability for the Dirichlet process too,

\textbf{Therom 1} Let $P$ be a Dirichlet process on $(\mathcal{X},\mathcal{A})$ with parameter $\alpha $ and let $X_1,\ldots, X_n$
be a sample of size $n$ from $P$.
Then conditional distribution of $P$ given $ X_1, \ldots , X_n$, is a Dirichlet process with parameter $\alpha + \sum_{1}^n \delta_{X_i}$.

\section{Sampling}

Let $y$ denote the observations or data, and let $\theta$ denote the parameter or set of parameters by which the data are to be summarized. Bayesian methods combine prior evidence on the parameters contained in the density $p(\theta)$ with the likelihood $p(y|\theta)$ to produce the entire posterior density $p(\theta|y)$ of $p(\theta)$.

From the posterior density one may extract any information, not simply "the most likely value" of a parameter, as with maximum likelihood (ML) estimators. However, until the advent of Monte Carlo Markov Chain methods it was not straightforward to sample from the posterior density, except in cases where it was analytically defined. Monte Carlo Markov Chain (MCMC) methods are iterative sampling methods that allow sampling from the posterior distribution, meaning $p(\theta|y)$.

Although MCMC methods can be encompassed within the broad class of \textit{Monte Carlo} methods, they must be distinguished from conventional Monte Carlo methods that generate independent simulations $\{ u_(1), u_(2), \ldots u_(T) \}$ from a target density $\pi(u)$. From such simulations the expectation of a function $g(u)$ under $\pi(u)$, namely
    $$ \mathbb{E}_{\pi}[g(u)] = \int{g(u)\pi(u)du}$$
    is estimated by taking the average of the function $g(y)$ evaluated at the sampled $u^{t}$, namely
    $$ \mathbb{E} = \sum_{t=1}^{T} g(u^{(t)})/T $$

Under independent sampling the Monte Carlo estimator \textit{g} tends to $\mathbb{E}_{\pi}[g(u)]$ as $T \rightarrow \infty$.
However, suppose we were to take $\pi(\theta) = p(\theta|y)$ as the density we wanted to find expectation from. We cannot use conventional independent Monte Carlo sampling, as this form of sampling from a posterior density $p(\theta|y)$ is not usually feasible.

When suitably implemented, MCMC methods offer an effective way to generate samples from the joint posterior distribution, $p(\theta|y)$. The "target density" for MCMC samples is the posterior density $\pi(\theta) = p(\theta|y)$, and MCMC sampling is specially relevant when the posterior cannot be stated exactly in analytic form, e.g. when the prior density assumed for $\theta$ is not conjugate with the likelihood $p(y|\theta)$.

\subsection{Forward Sampling in Bayesian Networks}
    Our goal in forward sampling is to compute the marginalized probability distribution over the desired random variable. So our goal is to compute terms of the form $p(Y=y)$. In order to do so, we generate samples from the Bayesian Network and then compute the fraction in which $Y=y$.

    Sampling from a Bayesian Network is done in a top-down manner. We start from the root nodes (nodes that have no parents associated with them), and sample each variable from a node's CPD (conditional probability distribution). We then move down the Bayesian Network, sampling subsequent nodes, with respect to the previously sampled parents. Upon getting to the bottom of the graphical model, we have successfully sampled a full realization of the join distribution of parameters. This process can be repeated to generate more samples and thus arrive at more accurate estimates of our favored distribution.

    With a little help from the discipline of \textit{Learning Theory}, we can introduce several interesting bounds on the number of samples, required to estimate the value of a parameter, while satisfying specific precision requirements. Two of the most important bounds are:

    \begin{itemize}
        \item \textit{Hoeffding's Bound (Additive)}
            \begin{equation}
                P_D (T_D \notin [p-\epsilon, p+\epsilon]) \leq 2e^{-2M\epsilon^2}
            \end{equation}
            \begin{equation}
                M \geq \frac{\ln(2/\delta)}{2\epsilon^2}
            \end{equation}
        \item \textit{Chernoff's Bound (Multiplicative)}
            \begin{equation}
                P_D (T_D \notin [p(1-\epsilon), p(1+\epsilon)]) \leq 2e^{-Mp\epsilon^2/3}
            \end{equation}
            \begin{equation}
                M \geq 3\frac{\ln(2/\delta)}{p\epsilon^2}
            \end{equation}
    \end{itemize}

    Where in both bounds, $M$ is the number of samples, $\delta$ is equal to the right hand side of the (1) and (3) inequalities and $T_D$ is the estimator for the parameter.

\subsection{Rejection Sampling in Bayesian Networks}
    Our goal in rejection sampling is to compute the marginalized probability distribution over a desired random variable, while taking into account and observed evidence, so our goal actually is to compute terms of the form $P(Y=y | E=e)$.

    When we have some evidence and we want to do queries with respect to those evidences, we need to take a slightly different approach. The approach we take in this configuration of the problem is called \textit{Rejection Sampling}.

    In rejection sampling, we start by sampling the Bayesian Network exactly as we did in normal forward sampling, and finish by computing the fraction where $Y=y$. The only difference is that we throw away all those samples that conflicts our observed evidence.

    As obvious, in this case more computational power is required since the number of required samples grows exponentially with the number of observed variables. (since we're keeping a sample with only probability p(e)).

\subsection{Markov Chain}

A Markov chain defines a probabilistic transition model $T(x \rightarrow x')$ over states while satisfying the following property:
$$ \forall x: \sum_{x'} T(x \rightarrow x') = 1$$

\subsubsection{Markov Property}
    For each $n \geq 1$, if $A$ is an event depending only on any subset of $\{ X_{n-1}, X_{n-2}, \ldots, 0\}$, then, for any states i and j in S,
    $$ P(X_{n+1}=j|X_n=i \quad and \quad A) = P(X_{n+1} = j| X_n=i)$$
    More generally, for each $n \geq 1$ and $m \geq 1$, if A is as defined previously, then for any states i and j in S:
    $$ P(X_{n+m}=j|X_n=i \quad and \quad A) = P(X_{n+m}=j|X_n=i)$$

\subsubsection{Temporal Dynamics}
    The probability of being in state $x'$ at the time step $t+1$, is the sum over all states, from which one can get to $x'$, multiplied by the probability of the transition occurring, namely,
$$ P^{(t+1)}(X^{(t+1)} = x') = \sum_x P^{(t)}(X^{(t)} = x)T(x \rightarrow x')$$

\subsubsection{Stationary Distribution}
    Ultimately, as the process evolves, the probability distribution nearly equalizes. Meaning: $$ P^(t) (x') = p^(t+1)(x') = \sum_x P^{(t)}(x)T(x \rightarrow x')$$
    So, we introduce a notion of a \textit{Stationary Distribution} as follows $$ \pi(x') = \sum_x \pi(x)T(x \rightarrow x')$$
    But not all Markov Chains converge to a stationary distribution, actually only \textit{Regular Markov Chains} will eventually arrive at a stationary distribution. We define a Markov chain to be \textit{Regular} in case there exists a k, such that, for every x, x', the probability of getting from x to x' in exactly k steps is a $ > 0$. If this condition holds, we know that the Markov chain converges to a unique stationary distribution regardless of the start state. A Markov chain is regular if both the following conditions hold (Sufficient conditions):

    \begin{itemize}
        \item Every two states are connected
        \item For every state, there exists a self-transition
    \end{itemize}

\subsubsection{MCMC}
    Our goal using sampling, as explained before is to find the probability distribution $P(X \in S)$ when P is too hard to be compute analytically or to sample directly from. In this case we construct a Markov chain, whose unique stationary distribution is P. Since we only want to use samples that are sampled from a distribution close to P, we only start collecting samples after the chain has run long enough to be \textit{mixing}. So in summary:

    \begin{itemize}
        \item For $c=1, \ldots, C$
        \begin{itemize}
            \item Sample $X^{(c, 0)}$ from $P^{(0)}$
        \end{itemize}
        \item Repeat until mixing
        \begin{itemize}
            \item For $c=1, \ldots, C$
            \begin{itemize}
                \item Generate $X^{(c, t+1)}$ from $T(x^{(c, t)} \rightarrow x')$
            \end{itemize}
            \item Compare window statistics in different chains to determine mixing (It's a heuristic method to check whether the Markov chain has started to mix)
            \item $t=t+1$
        \end{itemize}
    \end{itemize}

    Now we know the Markov chain is mixing (we have transitioned in the Markov chain long enough to be sure we've reached its stationary distribution) and with that, we do actual MCMC as follows:

    \begin{itemize}
        \item Repeat until sufficient samples are obtained
        \begin{itemize}
            \item $D = \emptyset$
            \item For $c=1, \ldots, C$
            \begin{itemize}
                \item Generate $X^{(c, t+1)}$ from $T(x^{(c, t)} \rightarrow x')$
                \item $D = D \bigcup {X^{(c, t+1)}}$
            \end{itemize}
            \item $t = t+1$
        \end{itemize}
        \item Let $D = {x[1], \ldots, x[M]}$
        \item Estimate $E_P[f] = \frac{1}{M}\sum_{m=1}^M f(x[m])$
    \end{itemize}

\subsection{Summary of MCMC}
\begin{itemize}
    \item Pros:
    \begin{itemize}
        \item Very general purpose
        \item Often easy to implement
        \item Good theoretical guarantees when $t \rightarrow \infty$
    \end{itemize}
    \item Cons:
    \begin{itemize}
        \item Lots of tunable parameters / design choices
        \item Can be quite slow to converge
        \item Difficult to tell whether it's working (detecting mixing)
    \end{itemize}
\end{itemize}

\section{Mixture of Dirichlet Process Models}
In this section we first introduce mixtures of Dirichlet process models and then  briefly review three MCMC Sampling Algorithms for DPMM from \cite{neal}.

Consider that $y_i'$s are the data we want to model and there is mixture assumption in data that data points come from a number of mixtures. The
 data and underlying mixtures can be modeled by Dirichlet process:
 \begin{align*}
   y_i | \theta_i &\sim f_{\theta_i} \\
   \theta_i | G &\sim^{iid} G \\
   G | \nu, M &\sim DP(M, G_\nu)\\
   (\nu, M) &\sim \pi.
 \end{align*}
In the equation above, $f$ is the family of distribution for each mixture e.g. Gaussian, and $\theta_i'$s are the parameters of
the i-th mixture. These parameters are themselves governed with a Dirichlet Process $G$ and there is a prior $(\nu, M)$ on $G$.
We also introduce the notation of cluster membership indicators, $s_i=j$ iff $i \in S_j$ the j-th cluster.

We now present three MCMC sampling algorithms for the model above, with the conjugate prior assumption. Escobar (1988) proposed the first posterior Gibbs sampler for the DPM model,
based on transition probabilities that update $ \theta_i$ by draws from the complete
conditional posterior $P(\theta_i | \theta_{-i}, y)$. However, this Gibbs sampler suffers from a slowly mixing Markov chain. The reason for this
is that $\theta$'s are updated independently. While in natural data  there are often groups of observations that with high probability are associated with the same
$\theta$. Since the algorithm cannot change the $\theta$ for more than
one observation simultaneously, changes to the values of $\theta$ in such
group can occur only rarely, as they require passage through low probability
intermediate state in which observations in the group have different
values for $\theta$. This problem can be avoided if we first integrate
out the $\theta$ and sample the marginal distribution where cluster assignments $s_i$ are
sampled instead of mixture parameters $\theta$. In this model
new values for $\theta_i$ which we denote by $\theta_i^*$ are drawn form $P(\theta_i^* | s,y)$ which can be modeled by:
\begin{equation}
  P(\theta_i^* | s,y) \propto G(\theta_i^* ) \prod_{i \in S_j} f_{\theta_i^* }(y_i)
\end{equation}
The posterior presented above for $\theta_i^* $ is a parametric model with prior $G(\theta)$.
To complete this model we should define the posteriors for cluster assignments.

\end{document}


\section{Sequential Implementation}
\label{sec:seq}

The initial implementation is straight-forward, following the explanation given on \autoref{subsec:how_it_works}. There are some differences however. Instead of base-10 digits, groups of bits were used. Each digit will consist of a group of bits, allowing for better performance (by using bitwise operators), and also to give the possibility of easily varying the amount of bits per digit.

With a smaller amount of bits per digit, less buckets will be used (number of buckets is equal to $2^g$, where $g$ is the number of bits). Consequentely, more iterations will be necessary. On the other hand, a bigger value of $g$ will require more buckets, increasing the fragmentation of the data and memory overhead.


\documentclass[11pt, oneside]{article}   	% use "amsart" instead of "article" for AMSLaTeX format

% \usepackage{draftwatermark}
% \SetWatermarkText{Draft}
% \SetWatermarkScale{5}
% \SetWatermarkLightness {0.9}
% \SetWatermarkColor[rgb]{0.7,0,0}

\usepackage{geometry}                		% See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   		% ... or a4paper or a5paper or ...
%\geometry{landscape}                		% Activate for for rotated page geometry
%\usepackage[parfill]{parskip}    		% Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}				% Use pdf, png, jpg, or eps� with pdflatex; use eps in DVI mode
								% TeX will automatically convert eps --> pdf in pdflat
								% TeX will automatically convert eps --> pdf in pdflatex
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{mathrsfs}
\usepackage{hyperref}
\usepackage{url}
\usepackage{subcaption}
\usepackage{authblk}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage[export]{adjustbox}
\usepackage{fixltx2e}
\usepackage{hyperref}
\usepackage{alltt}
\usepackage{color}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{float}
\usepackage{bigints}
\usepackage{braket}
\usepackage{siunitx}

\theoremstyle{definition}
\newtheorem{thm}{Theorem}[section]
% \newtheorem{defn}[thm]{Definition}
\newtheorem{definition}{Definition}[section]

\newcommand{\veq}{\mathrel{\rotatebox{90}{$=$}}}
\DeclareMathOperator{\bda}{\Big \downarrow}

\DeclareMathOperator{\E}{\mathbb{E}}
\newcommand{\argmax}{\operatornamewithlimits{argmax}}
\newcommand{\argmin}{\operatornamewithlimits{argmin}}

\title{A Few Notes on Cayley's Theorem}
\author{David Meyer \\ dmm@\{1-4-5.net,uoregon.edu\}}

\date{Last update: \today}							% Activate to display a given date or no date

\begin{document}
\maketitle

\section{Introduction}
Group Theory is the study of symmetry.  Cayley's Theorem is a fundamental theorem in Group Theory, and the topic of these notes.

\bigskip
\noindent
Before diving into Cayley's Theorem, a couple of notes:
\begin{itemize}
\item The Symmetric Group $Sym(G)$ or sometimes $S_n$, where $n = |G|$ ($G$ is finite), is the set of all bijections from $G$ to itself with function composition as
the group operation.  That is, $Sym(G) = S_n = S_{|G|} = \{f: G \rightarrow G \mid f \text{ is an bijection}\}$.
\item We use the symbol $\simeq$ (or sometimes $\cong$) to mean that the groups $G$ and $H$ are \emph{isomorphic}. That is,
$G \simeq H \implies \exists f \mid f: G \rightarrow H$ where $f$ is a bijection and a homomorphism. See Equation \ref{eqn:homomorphism}.
 \item To show that $f$ is one-to-one, show that $f(x) = f(y) \implies x = y$.
 \item To show that $f$ is onto, pick an arbitrary $h \in H$ and show that $\exists g \in G \mid f(g) = h$.
\end{itemize}

\bigskip
\noindent
Recall also that if we have two groups $(G,*)$ and $(H,\cdot)$ we say that $(G,*)$  is \emph{isomorphic} to $(H,\cdot)$ if there exists a bijection $f:G \rightarrow H$
which satisfies the \emph{homomorphism} property:

\bigskip
\begin{equation}
f(x * y) = f(x) \cdot f(y) \;\;\; \forall  x,y \in G
\label{eqn:homomorphism}
\end{equation}

\bigskip
\noindent
That is, $f$ is a bijection (one-to-one and onto) and $f$ is also a homomorphism.

\bigskip
\noindent
Any bijective function $f$ which satisfies Equation \ref{eqn:homomorphism} is called a \emph{group} isomorphism from $G$ to $H$. The basic idea of
$(G,*)$ being isomorphic to $(H,\cdot)$ is that $(G,*)$ and $(H,\cdot)$ are "algebraically equivalent". That is, there is a one-to-one correspondence
between elements of $G$ and elements of $H$ where the outcomes of operations on elements of $G$ are matched with the outcomes of the
corresponding operations on the corresponding elements of $H$.

\section{Cayley's Theorem}

\begin{thm}
\textbf{Cayley's Theorem: }If $G$ is a group then there exists a subgroup $H$ of $Sym(G)$ such that $G$ is isomorphic to $H$.
\label{thm:caley}
\end{thm}

\noindent
\textbf{Proof:} Suppose that $G$ is a group.  Then to prove Cayley's Theorem we need to find a subgroup $H$ of $Sym(G)$ and a bijective homomorphism
$f: G \rightarrow H$. My roadmap for the proof looks like

\bigskip
\begin{enumerate}
\item Define $\phi_{a}: G \rightarrow G$ for each $a \in G$ and show that $\phi_{a}$ is a bijection
\item Define $H = \{\phi_{a} \mid a \in G\}$ and show that $H$ is a subgroup of $Sym(G)$
\item Define $f:G \rightarrow H$ and show that $f$ is both a bijection and a homomorphism
\end{enumerate}

\bigskip
\noindent
BTW, a nice thing about the proof of Cayley's theorem is that it is a \emph{constructive} proof: the statement of the theorem is that a certain group $H$ exists.
In the course of the proof of the theorem one can actually show not only that such an $H$ exists but also how to actually find it. We'll see an example of this
below (Section \ref{subsub:klein}).

\subsection{Define $\phi_{a}: G \rightarrow G$ for each $a \in G$ and show that $\phi_{a}$ is a bijection}
To start, for each fixed element $a \in G$ define $\phi_{a}: G \rightarrow G$ by the map $x \mapsto ax$. That is

\begin{equation}
\phi_{a}(x) = ax \quad \forall x \in G
\label{eqn:phi}
\end{equation}

\bigskip
\noindent
Luckily it turns out that each $\phi_{a}$ is a bijection.  To see this we need to show that $\phi_a$ is one-to-one and onto.
First,  consider that $\phi_a$  is one-to-one since

\begin{equation}
\begin{array}{llll}
\phi_{a}(x)
&=&                 \phi_{a}(y)                        &\quad  \mathrel{\#} \text{to show $\phi_a$ is 1-to-1 show } \phi_a(x) = \phi_a(y) \Rightarrow x = y      \\
&\Rightarrow& ax = ay                            &\quad  \mathrel{\#} \text{definition of $\phi_a(x)$ (Equation \ref{eqn:phi})}                                                                                \\
&\Rightarrow& a^{-1}(ax) = a^{-1}(ay)     &\quad  \mathrel{\#} \text{multiply by $a^{-1}$};  a \in G \text{ \& $G$ a group} \Rightarrow a^{-1} \in G \\
&\Rightarrow&  (a^{-1}a)x = (a^{-1}a)y    &\quad  \mathrel{\#} \text{multiplication is associative}                                                                             \\
&\Rightarrow&  x = y                                &\quad  \mathrel{\#} a^{-1}a = 1
\end{array}
\label{eqn:1-to-1}
\end{equation}

\bigskip
\noindent
So $\phi_{a}$ is one-to-one.

\bigskip
\noindent
\textbf{Aside on cancellation laws:} Note that in (\ref{eqn:1-to-1}) we used the fact that $a \in H$ and that $H$ is a group so $a^{-1} \in H$.
Here we have  $a^{-1}a = 1$, which essentially gives us a \emph{cancellation law}\footnote{Note that having a cancellation law is equivalent to saying
there are no \emph{zero divisors}.}; in  (\ref{eqn:1-to-1}) this allows us to "cancel" the $a$ on both sides. Now, what if we don't have
access to multiplicative inverses? We might be faced with this situation if we have a ring, where we don't in general have multiplicative
inverses\footnote{A ring with multiplicative inverses is called a division ring (or skew field).  Example: the quaternions.}. So if we don't  have multiplicative
inverses how do we go about showing that something is one-to-one?

\bigskip
\noindent
One  approach is to factor out $a$ and note that by assumption, $a \neq 0$
so something else must be. For example

\begin{equation*}
\begin{array}{llll}
\phi_{a}(x)
&=&                 \phi_{a}(y)                         &\quad  \mathrel{\#} \text{to show $\phi_a$ is 1-to-1 show that } \phi_a(x) = \phi_a(y) \Rightarrow x = y \\
&\Rightarrow& ax = ay                            &\quad  \mathrel{\#} \text{definition of $\phi_a(x)$ (\text{Equation \ref{eqn:phi})}}            \\
&\Rightarrow& ax-ay = 0                         &\quad  \mathrel{\#} \text{subtract $ay$ from both sides} \\
&\Rightarrow&  a (x-y) = 0                      &\quad  \mathrel{\#} \textit{factor out a}                            \\
&\Rightarrow&  x - y     = 0                       &\quad  \mathrel{\#} a \neq 0 \text{ by assumption so } x - y = 0  \\
&\Rightarrow&  x = y                               &\quad  \mathrel{\#} \text{so $\phi_{a}$ is one-to-one}
\end{array}
\end{equation*}

\bigskip
\noindent
Getting back to showing that $\phi_{a}$ is a bijection, we next need to show that $\phi_{a}$ is onto. To do this pick an arbitrary $y \in G$ (here $G$ is the range).
Then $a^{-1}y \in G$ (here $G$ is the domain) and so $\phi_{a}(a^{-1}y) = a(a^{-1}y)$. Since multiplication is associative we have
$\phi_{a}(a^{-1}y) = a(a^{-1}y) = (aa^{-1})y = y$. So $\phi_{a}$ is onto and hence $\phi_{a}$ is a bijection.

\subsection{Define $H = \{\phi_{a} \mid a \in G\}$ and show that $H$ is a subgroup of $Sym(G)$}
Now we can define $H = \{\phi_a \mid a \in G\}$. Since each element of $H$ is a bijection from $G$ to $G$ and since $Sym(G)$ is the set of all
bijections from $G$ to $G$ we know that $H \subseteq G$. To show that $H$ is a subgroup of $Sym(G)$ we also need to show that $H$ is closed under function
composition and  inversion.

\bigskip
\noindent
To show closure under function composition we need to show that $\alpha, \beta \in H \Rightarrow \alpha \circ \beta \in H$.  To see this consider
$\alpha, \beta \in H$. Then there exists $a \in G$ such that $\alpha = \phi_{a}$. Similarly there exists $b \in G$ such that $\beta = \phi_b$. So we
know that

\begin{equation}
\alpha \circ \beta = \phi_a \circ \phi_b
\label{eqn:ab}
\end{equation}

\bigskip
\noindent
and so for any $x \in G$ we have

\bigskip
\begin{equation}
\begin{array}{rcll}
(\alpha \circ \beta)(x)
&=& (\phi_a \circ \phi_b)(x)                &\quad  \mathrel{\#}  \alpha \circ \beta = \phi_a \circ \phi_b \text{ (Equation \ref{eqn:ab})} \\
&=& \phi_{a}(\phi_{b}(x))                    &\quad  \mathrel{\#} \text{definition of function composition}          \\
&=& \phi_{a}(bx)                                 &\quad  \mathrel{\#} \phi_b(x) = bx \text{ (definition of $\phi_b$)}  \\
&=& a(bx)                                           &\quad  \mathrel{\#} \phi_a(x) = ax \text{ (definition of $\phi_a$)}  \\
&=& (ab)x                                           &\quad  \mathrel{\#} \text{multiplication is associative}                 \\
&=& \phi_{ab}(x)                                 &\quad  \mathrel{\#} \phi_{g}(x) =gx  \text{ where $g=ab$}
\end{array}
\label{eqn:function_composition}
\end{equation}

\bigskip
\noindent
So $\alpha \circ \beta = \phi_a \circ \phi_b = \phi_{ab}$. Since $ab \in G$ ($G$ is closed under multiplication) we know that $\phi_{ab} \in H$. Now
we have $\phi_{ab} \in H$ and $\alpha \circ \beta = \phi_{ab}$ which together imply that  $\alpha \circ \beta \in H$. So $H$ is closed under function composition.

\bigskip
\noindent
To show that $H$ is closed under inversion we need to show that $\alpha \in H \Rightarrow \alpha^{-1} \in H$.
To see this consider $\alpha \in H$.  Then there exists an $a \in G$ such that $\alpha = \phi_a$.
Since $a \in G$ and since $G$ is a group, $a^{-1} \in G$ and so $\phi_{a^{-1}} \in H$. Note further that for any $x \in G$

\bigskip
\begin{equation}
\begin{array}{rcll}
(\phi_{a^{-1}} \circ \phi_a)(x)
&=& \phi_{a^{-1}}(\phi_a(x))        &\quad  \mathrel{\#} \text{definition: $(f \circ g)(x) = f(g(x))$}        \\
&=& \phi_{a^{-1}}(ax)                   &\quad  \mathrel{\#} \phi_a(x) = ax                                                \\
&=& a^{-1}(ax)                             &\quad  \mathrel{\#} \text{definition: } \phi_{a^{-1}}(x) = a^{-1}(x)  \\
&=& (a^{-1}a)x                             &\quad  \mathrel{\#} \text{multiplication is associative}                  \\
&=& x                                           &\quad  \mathrel{\#} a^{-1}a = 1
\end{array}
\label{eqn:comp1}
\end{equation}

\bigskip
\noindent
and

\bigskip
\begin{equation}
\begin{array}{rcll}
(\phi_{a} \circ \phi_{a^{-1}})(x)
&=& \phi_{a}(\phi_{a^{-1}}(x))      &\quad  \mathrel{\#} \text{definition: $(f \circ g)(x) = f(g(x))$}  \\
&=& \phi_{a}(a^{-1}x)                   &\quad  \mathrel{\#} \phi_{a^{-1}}(x) = a^{-1}x                          \\
&=& a(a^{-1}x)                             &\quad  \mathrel{\#} \text{definition: }  \phi_{a}(x) = ax             \\
&=& (a a^{-1})x                            &\quad  \mathrel{\#} \text{multiplication is associative}             \\
&=& x                                           &\quad  \mathrel{\#} aa^{-1} = 1
\end{array}
\label{eqn:comp2}
\end{equation}

\bigskip
\noindent
Recall that if a function $f$ is a bijection we know $(f^{-1} \circ f)(x) = (f \circ f^{-1})(x) = x$.  From (\ref{eqn:comp1})
and  (\ref{eqn:comp2}) we see that  $\phi_{a^{-1}}$ is the inverse of $\phi_a$. More specifically $\phi_{a^{-1}} = \phi^{-1}_{a}$.
Since $\alpha = \phi_a$,  $\alpha^{-1} = \phi^{-1}_a = \phi_{a^{-1}}  \in H$. So $H$ is closed under inversion.

\subsection{Define $f:G \rightarrow H$ and show that $f$ is a homomorphic bijection}
We still need to show a homomorphic bijection $f$ from $G$ to $H$. One way to do this is to define $f(g) = \phi_{g}$ for all $g \in G$.
Then to show that $f$ is a bijection we need to show that $f$ is both  one-to-one and onto.

\bigskip
\noindent
To see that $f$ is one-to-one consider

\begin{equation*}
\begin{array}{rcll}
f(a)
&=& f(b)                                                  &\quad  \mathrel{\#} \text{to show $f$ is 1-to-1 show that } f(a) = f(b) \Rightarrow a = b \\
&\Rightarrow& \phi_a(x) = \phi_b(x)       &\quad  \mathrel{\#} \text{definition of $f(g): f(g) = \phi_g \text{ for all } g \in G$} \\
&\Rightarrow& \phi_a(a) = \phi_b(a)       &\quad  \mathrel{\#} \text{evaluate at $a \in G$} \\
&\Rightarrow& aa = ba                           &\quad  \mathrel{\#} \text{definition of $\phi_{g}: \phi_{g}(x) = gx  \text{ for all } g \in G$} \\
&\Rightarrow& (aa)a^{-1} = (ba)a^{-1}    &\quad  \mathrel{\#} \text{multiply by $a^{-1}$}; a \in G \text{ and $G$ a group} \Rightarrow a^{-1} \in G \\
&\Rightarrow& a(aa^{-1}) = b(aa^{-1})    &\quad  \mathrel{\#} \text{multiplication is associative}            \\
&\Rightarrow& a = b                               &\quad  \mathrel{\#} aa^{-1} = 1    \\
\end{array}
\end{equation*}

\bigskip
\noindent
So $f$ is one-to-one.

\bigskip
\noindent
To show that $f$ is onto, choose a $\alpha \in H$. Then there exists an $a \in G$ such that $\alpha = \phi_a$. However we know that
$f(a) = \phi_a$ and  $\phi_a = \alpha$ so we know that  $f(a) = \alpha$. So $f$ in onto and since we saw that $f$ is one-to-one, $f$ is a bijection.

\bigskip
\noindent
Finally, to show that $f$ is also a homomorphism we want to show that $f(ab) = f(a) \circ f(b)$. To see this consider that for any $a,b \in G$ we have

\bigskip
\begin{equation*}
\begin{array}{rcll}
f(ab)
&=& \phi_{ab}                          &\quad  \mathrel{\#} \text{definition of $f$}                                           \\
&=& \phi_{a} \circ \phi_{b}       &\quad  \mathrel{\#} \text{Equation  \ref{eqn:function_composition}}   \\
&=& f(a) \circ f(b)                     &\quad  \mathrel{\#} \text{definition of $f$}
\end{array}
\end{equation*}

\bigskip
\noindent
So  $f$ is a homomorphism.

\bigskip
\noindent
This completes the proof of  Cayley's Theorem.

\section{Examples}

% \newpage

\subsection{($\mathbb{Z}_4,+) \rightarrow (G,\cdot)$}
Let $(\mathbb{Z}_4,+)$ be the set $\mathbb{Z}_4 = \{0,1,2,3\}$ with addition modulo 4 and let $(G,\cdot)$ be the set $G = \{1,-1,i,-i\}$ (the fourth roots of unity)
with the usual multiplication on $\mathbb{C}$. Then $(\mathbb{Z}_4,+) \simeq (G,\cdot)$. To see that $\mathbb{Z}_4$ is isomorphic to $G$, let
$f: \mathbb{Z}_4 \rightarrow G$ be the bijection

\begin{equation*}
\begin{array}{lccr}
& 0 &\longrightarrow&  1 \\
& 1 &\longrightarrow&  i \\
& 2 &\longrightarrow&  -1 \\
& 3 &\longrightarrow&  -i \\
\end{array}
\end{equation*}

\bigskip
\noindent
Here are the Cayley tables for $\mathbb{Z}_4$ and $G$:

\bigskip
\begin{table}[ht]
\begin{minipage}{0.5\linewidth}
\centering
\begin{tabular}{c | c c c c c}
+ & 0 & 1  & 2 & 3 \\
\cline{1-5}
0 & 0 & 1 & 2 & 3 \\
1 & 1 & 2 & 3 & 0 \\
2 & 2 & 3 & 0 & 1 \\
3 & 3 & 0 & 1 & 2
\end{tabular}
\caption{$\mathbb{Z}_4$}
\label{tab:z4}
\end{minipage}
%
 \begin{minipage}{0.5\linewidth}
\centering
\begin{tabular}{r | r r r r}
$\cdot$ & $1$ & $i$  & $-1$ & $-i$ \\
\cline{1-5}
$1$   & $1$   & $i$     & $-1$   & $-i$  \\
$i$    & $i$    & $-1$   & $-i$    & $1$  \\
$-1$  & $-1$  & $-i$    & $1$    & $i$    \\
$-i$   &  $-i$  & $1$     & $i$     & $-1$
\end{tabular}
\caption{$G$}
\label{tab:G}
\end{minipage}
\end{table}

\bigskip
\noindent
To show that $f$ is an isomorphism we need to show that $f$ is a homomorphism, that is, that
$f(x+y) = f(x) \cdot f(y)$.  Since there are only $n^2 = 4^2 = 16$ values for $f(x+y)$  we can just
enumerate them:

\begin{equation*}
\begin{array}{lllll}
f(0+0) &= f(0) &= 1  &= 1 \cdot 1     &=  f(0) \cdot f(0) \\
f(0+1) &= f(1) &= i   &= 1 \cdot i      &=  f(0) \cdot f(1) \\
f(0+2) &= f(2) &= -1 &= 1 \cdot -1   &=  f(0) \cdot f(2)  \\
f(0+3) &= f(3) &= -i  &= 1 \cdot  -i    &=  f(0) \cdot f(3) \\
f(1+0) &= f(1) &= i   &=  i \cdot 1      &=  f(1) \cdot f(0) \\
f(1+1) &= f(2) &= -1 &=  i \cdot i       &=  f(1) \cdot f(1)  \\
f(1+2) &= f(3) &= -i  &=  i \cdot  -1    &=  f(1) \cdot f(2) \\
f(1+3) &= f(0) &= 1  &=  i \cdot -i      &=  f(1) \cdot f(3) \\
f(2+0) &= f(2) &= -1 &=  -1 \cdot 1    &=  f(2) \cdot f(0)  \\
&& \qquad \vdots                                                           \\
f(3+3) &= f(2) &= -1 &=  -i \cdot -i   &=  f(3) \cdot f(3)  \\
\end{array}
\end{equation*}

\bigskip
\noindent
So the bijection $f: \mathbb{Z}_4 \rightarrow G$ above is a homomorphism and hence $f$ is a group isomorphism.

% \newpage

\subsection{The Klein 4-group}
\label{subsub:klein}

The Klein 4-group is the group $K = \{e,a,b,c\}$ where $e$ is the identity element and the group operation
is defined by the Cayley table below (Table \ref{tab:klein}).

\bigskip
\begin{table}[H]
\centering
\begin{tabular}{c | c c c c c}
& $e$ & $a$ & $b$ & $c$ \\
\cline{1-5}
$e$ & $e$ & $a$ & $b$ & $c$ \\
$a$ & $a$ & $e$ & $c$ & $b$ \\
$b$ & $b$ & $c$ & $e$ & $a$ \\
$c$ & $c$ & $b$ & $a$ & $e$
\end{tabular}
\caption{The Klein 4-group Operation}
\label{tab:klein}
\end{table}

\bigskip
\noindent
Here $K$ is \emph{not} isomorphic to $\mathbb{Z}_{4}$. To see this notice that there are 24 bijections from $\mathbb{Z}_4$ and $K$:
$|K| = |\mathbb{Z}_4|  = 4$ so  there are $n! = 4! = 24$ possible bijections from $\mathbb{Z}_4$ to $K$. Since we need
$f(0) = e$ that leaves $3! = 6$  bijections that could be homomorphisms.  For example, consider the bijection

\begin{equation*}
\begin{array}{ccr}
0 &\longrightarrow&  e \\
1 &\longrightarrow&  a \\
2 &\longrightarrow&  c \\
3 &\longrightarrow&  b
\end{array}
\end{equation*}

\bigskip
\noindent
This bijection is not a homomorphism since $f(1+3) = f(4) = f(0) = e$ while $f(1) \cdot f(3) = ab = c$, so $f(1+3) \neq f(1) \cdot f(3)$.

\bigskip
\noindent
One way to see that $\mathbb{Z}_4$ is not isomorphic to $K$ is to recognize that every element of $K$ satisfies the equation $x \cdot x =e$
(a key property of the Klein 4-group). However not every element of $\mathbb{Z}_4$ satisfies the equation $x + x = 0$.

\bigskip
\noindent
This gives a clue as to how to prove,  by contradiction,   that $\mathbb{Z}_4$ is not isomorphic to $K$. Specifically,
suppose that $\mathbb{Z}_4$ is isomorphic to $K$. Then there exists a bijection
$f: \mathbb{Z}_4 \rightarrow K$ such that $f(x + y) = f(x) \cdot f(y)$ for all $x,y \in \mathbb{Z}_4$. Well,
we know by definition that $f(0) = e$ and since $f$ is one-to-one we also know that $f(1) \neq e$. Since
$f$ is a homomorphism we also know that

\begin{equation*}
f(1+1) = f(1) \cdot f(1)
\end{equation*}

\bigskip
\noindent
However, since $f(1) \in K$ and all elements of $K$ satisfy $x \cdot x = e$ we can conclude that $f(1) \cdot f(1) = e$, so $f(1+1) = f(2) = e$. Now we
have $f(0) = e$ \emph{and}  $f(2) = e$ which is a contradiction since we assumed that  $f$ was one-to-one. So the original assumption that
$\mathbb{Z}_4$ is isomorphic to $K$ is false.

\bigskip
\noindent
Ok, but Cayley's Theorem says there is a subgroup $H$ of $S_4$ which is isomorphic to $K$. How to find $H$? Since as noted above Cayley's Theorem is constructive,
we should be able to follow the approach used in the proof to find $H$. Here we let $H =\{\phi_{e}, \phi_{a}, \phi_{b}, \phi_{c}\}$ where, for all $x \in K$

\bigskip
\begin{equation*}
\begin{array}{llll}
\phi_{e}(x) = ex   &&&\quad  \mathrel{\#}  \phi_{e}(x) = x  \\
\phi_{a}(x) = ax  \\
\phi_{b}(x) = bx  \\
\phi_{c}(x) = cx  \\
\end{array}
\end{equation*}

\bigskip
\noindent
Now we can rewrite the Cayley table for the Klein 4-group (Table \ref{tab:klein}) as
\bigskip
\begin{equation*}
\begin{array}{llll}
\phi_{e} = \begin{pmatrix} e & a & b & c \\ e & a & b & c \end{pmatrix}  \\
\phi_{a} = \begin{pmatrix} e & a & b & c \\ a & e & c & b \end{pmatrix}  \\
\phi_{b} = \begin{pmatrix} e & a & b & c \\ b & c & e & a \end{pmatrix}  \\
\phi_{c} = \begin{pmatrix} e & a & b & c \\ c & b & a & e \end{pmatrix}
\end{array}
\end{equation*}

\bigskip
\noindent
Now, if we relabel $K$ by the bijection

\begin{equation*}
\begin{array}{l c c c c c c c c c c c c}
 e      & a       & b      & c \\
 \bda  & \bda & \bda & \bda \\
 1     & 2       & 3      & 4
\end{array}
\end{equation*}

\bigskip
\noindent
we can represent $K$ in cyclic notation:

\bigskip
\begin{equation*}
\begin{array}{llll}
\phi_{e} = \begin{pmatrix} 1 & 2 & 3 & 4  \\ 1 & 2 & 3 & 4  \end{pmatrix}  = (1)         \\
\phi_{a} = \begin{pmatrix} 1 & 2 & 3 & 4  \\ 2 & 1 & 4 & 3 \end{pmatrix}   = (12)(34) \\
\phi_{b} = \begin{pmatrix} 1 & 2 & 3 & 4  \\  3 & 4 & 1 & 2 \end{pmatrix}  = (13)(24)  \\
\phi_{c} = \begin{pmatrix} 1 & 2 & 3 & 4  \\  4 & 3 & 2 & 1 \end{pmatrix}   = (14)(23)
\end{array}
\end{equation*}

\bigskip
\noindent
Now we can see that $K \simeq H$ where $H = \{(1), (12)(34), (13)(24), (14)(23)\}$. That is,
$f:K\rightarrow H$ is the bijection

\bigskip
\begin{equation*}
\begin{array}{c c c c c}
f: &  1   & 2        & 3        & 4 \\
   & \bda & \bda     & \bda     & \bda \\
   & (1)  & (12)(34) & (13)(24) &  (14)(23)
\end{array}
\end{equation*}

\bigskip
\noindent
The Klein 4-group $K$ has many other interesting properties, including

\begin{itemize}
\item $K$ is the smallest non-cyclic group
\item $K$ is the underlying group of the four-element field
\item $K$ is the symmetry group of a non-square rectangle
\item $K$ is the group of bitwise exclusive or operations on two-bit binary values
\item $K = \mathbb{Z}_2 \times \mathbb{Z}_2$, the direct product of two copies of the cyclic group of order 2
\end{itemize}

\section{Acknowledgements}

\newpage
\bibliographystyle{plain}
\bibliography{/Users/dmm/papers/bib/qc}
\end{document}


% ----------------------------------------------------------------
% Article Class (This is a LaTeX2e document)  ********************
% ----------------------------------------------------------------
\documentclass[11pt]{article}
\usepackage[english]{babel}
\usepackage{amsmath,amsthm, amssymb}
\usepackage{amsfonts}
\usepackage{multirow}
\usepackage{threeparttable}

%Candelaria's favorite packages
\usepackage{setspace}	% Set the spacing for the document
\usepackage{xtab}		% Use for tables
\usepackage{comment}	% Use for block comments
\usepackage{rotating}	% Helpful for rotating figures
\usepackage{lscape}	% Change a page into landscape
\usepackage{hyperref}
\hypersetup{colorlinks=false,pdfborder={0 0 0},breaklinks=true}
\usepackage{graphicx}	% Load a graphic image
%\usepackage{url}		% Use for formatting URLS (nice for NatBib also)
\usepackage{booktabs}  	% Use for nice tables/ Works well with ESTOUT
\usepackage{longtable} 	% Used to break long tables over multiple pages
\usepackage{caption}
\usepackage{subcaption} 	% Used to create SubFloats
\usepackage{listings} 	% Use to enter code blocks, like a fancy verbatim

% ADD COLOR:
\usepackage{color}
\usepackage[usenames, dvipsnames, svgnames, table]{xcolor}

% Packages for the bibliography
\usepackage[longnamesfirst]{natbib}
%\usepackage{natbib}

% Set Page Margins
%\usepackage{fullpage}
\addtolength{\oddsidemargin}{-.875in}
\addtolength{\evensidemargin}{-.875in}
\addtolength{\textwidth}{1.75in}
\addtolength{\topmargin}{-.875in}
\addtolength{\textheight}{1.75in}

% Set line and table spacing
\renewcommand{\baselinestretch}{1.0}
\renewcommand{\arraystretch}{1.1}
\setlength\abovedisplayskip{0pt}
\setlength\belowdisplayskip{0pt}

% Redefine the cite command
\renewcommand\cite{\citet}

%\setlength{\parindent}{0in}

%\setlength{\leftmargin}{0pt} \setlength{\parskip}{1.1ex} \setlength{\parindent}{1em} \setlength{\itemsep}{0pt} \setlength{\footnotesep}{10pt}
%\renewcommand{\footnoterule}{\rule{3in}{.3pt}\vspace{-.3pt}}

% THEOREMS -------------------------------------------------------
\newtheorem{thm}{Theorem}[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\theoremstyle{definition}
\newtheorem{defn}[thm]{Definition}
\theoremstyle{remark}
\newtheorem{rem}[thm]{Remark}
%\numberwithin{equation}{section} %Number equations within section
% ----------------------------------------------------------------

%%%%%%TYPESET LINEAR EQUATIONS
\usepackage{environ}

\makeatletter

\newcommand{\LinearSystems@SetupLets}{%
  \let\col=&%
  \let\+=+%
  \let\-=-%
  \let\===%
}
\newcommand{\LinearSystems@SetupCatcodes}{%
  \catcode`\&=\active
  \catcode`\+=\active
  \catcode`\-=\active
  \catcode`\==\active
}
\newcommand{\LinearSystems@Setup}{}
\begingroup
  \LinearSystems@SetupCatcodes
  \gdef\LinearSystems@Setup{%
    \LinearSystems@SetupLets
    \LinearSystems@SetupCatcodes
    \newcommand&[1][0pt]{\col\hspace{##1}\col}%
    \def+{\col\+{}{}\col}%
    \def-{\col\-{}{}\col}%
    \def={\col\={}{}\col}%
  }
\endgroup

\NewEnviron{LinearSystems}[1]{\begin{alignat*}{#1}\BODY\end{alignat*}}
\let\LinearSystems@OriginalBegin\LinearSystems
\def\LinearSystems{\LinearSystems@Setup\LinearSystems@OriginalBegin}

\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%

% AUTHOR INFO

\title{Lecture 1: Algebra and Precalculus \\ September 4, 2018}%

% ----------------------------------------------------------------

%%%%%%
% SHORT CITES
%%%%%%
\shortcites{ColEtAl1966}

\begin{document}

\begin{center}
{\huge Pre-Calculus Lecture Notes\footnote{Contributor(s): Chris Candelaria, Erin Fahle, Betsy Williams, and Klint Kanopka. If you find errors, please let Klint know so that he may correct them. Thanks!}}\\[5pt]
{\Large Stanford GSE Math Camp \\[5pt]
Do Not Distribute Outside GSE}
\end{center}

\section{Software Packages of Interest}
    \subsection{Document Processing}
    	\begin{itemize}
        	\item \textbf{Microsoft Word - } Basically everyone uses this.  Some advisors are crazy about ``track changes'' on working documents.  You'll need a copy of this to make sure things are formatted in a non-awful manner.
            \item \LaTeX - This document was formatted using \LaTeX.
            Pros:
            \begin{itemize}
                \item Looks great.
                \item Handles equations really well.
                \item Flexible.
                \item Free.
            \end{itemize}
            Cons:
            \begin{itemize}
                \item Can be extremely frustrating.
                \item Not straightforward to learn.
                \item Many people can't deal with .tex files and .pdfs can be hard to collaborate on.
                \item Some journals prefer .docx submissions.
            \end{itemize}
            If you want to try it out, \LaTeX is easily accessible from native clients (TexPad, TexShop, TexWorks, etc) or compiled from a text editor of your choice (Sublime Text, Atom, VIM).  Also really easy to use online (Overleaf). Overleaf has also been steadily improving their collaboration features.
            \item \textbf{Markdown - } The README on the github repo was written in markdown. It generally has fewer options than \LaTeX, but is easier to deal with. It's also built into RSTudio (if you're an R user).
        \end{itemize}

   	\subsection{Citation Managers}
    	\begin{itemize}
        	\item \textbf{Mendeley - } Citation manager.  I use this one because it has the least-awful Linux client. Stanfod once had an instiutional license, but has since cancelled it.
            \item More than I care to mention exist (Zotero and EndNote are the other two I'm aware of).  A good comparison is available here:  https://guides.library.utoronto.ca/c.php?g=250610\&p=1671260
        \end{itemize}

    \subsection{Quantitative Software}
    	\begin{itemize}
        	\item \textbf{Excel - } Spreadsheet program on almost everyone's computer.  Pros: You probably already know how to use it.  Lots of buttons, so it's pretty easy to figure out.  Good for quick tasks.  Cons: Slow with large data sets.  Shameful to use for ``real'' work, because work done in Excel (or any other spreadsheet) is not easily reproducable.
            \item \textbf{SPSS - } Statistical Package for Social Science (I think?).  Popular amongst faculty trained in Psychology.  Pros: Lots of buttons, so it works like a super-powerful Excel.  Has support for scripting and syntax, so can be used conduct reproducable research.  Cons: Not popular among faculty not trained in Psychology.
            \item \textbf{SAS - } Statistical Analysis Software.  Pros: Good for large data sets and data cleaning.  Cons: I don't know of anyone in the GSE who uses it, so support will be thin if you're picking it up for the first time.
        	\item \textbf{Stata - } Statistical software popular among the Econ crowd.  Pros: Every class in the 400 and 430 sequence in the GSE is taught in Stata.  There is no easier way to work with large panel data sets. Commands are easy to learn and do files help you automate or reproduce tasks.  Your advisor probably uses Stata. Your advisor will probably buy you Stata.  Cons:  Not free.  Making attractive plots can be annoying, but not impossible.
            \item \textbf{R - } A (quirky) programming language that gets pitched as statistical software.  Pros: Free.  Packages exist to do pretty much whatever you'd like.  Popular in the data science community.  Cons:  Very steep learning curve, especially if you have no prior programming experience. Really weird syntax.  All arrays are 1-indexed, so if you have prior programming experience, this will cause you headaches to no end.
            \item \textbf{Python - } Interpreted programming language.  Pros: Free. Fast. Popular in the data science community and can be used for a whole bunch of non-statistical applications.  Cons: Steep learning curve.  Some of the stats plugins are even less friendly here than in R.
        \end{itemize}

    \subsection{VPN}
		\begin{itemize}
        	\item \textbf{Cisco VPN - } Worth the effort to set up if you live off-campus or travel often.  Allows access to journals, data, library materials, and anything else that requires you be on the Stanford network from anywhere in the world. Instructions available at:  uit.stanford.edu/service/vpn
        \end{itemize}
\section{Review of Algebra and Notation}

\subsection{Order of Operations}
\begin{itemize}
\item \textbf{Basic Algebraic Symbols:}
\begin{itemize}
\item Addition ($+$), subtraction ($-$), multiplication ($\times$ or $*$), division ($\div $ or $/ $)
\item Sum ($\sum $, $\sum\limits_{a \in A} $, or  $\sum\limits_{i=1}^N$), product ($\prod $,  $\prod\limits_{a\in A} $,  or $\prod \limits_{i=1}^N$)

\end{itemize}
\item \textbf{PEMDAS:}
\begin{itemize}
\item \textbf{P}lease \textbf{E}xcuse \textbf{M}y \textbf{D}ear \textbf{A}unt \textbf{S}ally
\item \textbf{P}arentheses \textbf{E}xponents \textbf{M}ultiplication \textbf{D}ivision \textbf{A}ddition \textbf{S}ubtraction
\end{itemize}
\item \textbf{Exercises:}
\begin{enumerate}
\item Evaluate $\sum\limits_{x=1}^5 x$
\item Solve for x: $50 = 5(x+2) + 5$
\item Evaluate: $\displaystyle \prod_{x=1}^3 (x-1)$
\end{enumerate}
 \end{itemize}
\subsection{Working with Units}
\begin{itemize}

\item Remember that units multiply and divide, too!
\begin{itemize}
\item Driving at 60 miles/hour for 2 hours takes you 120 miles, because  $\dfrac{hours}{hours}$ cancels out.
$$ 2~hours \times 60~miles/hour = 120~miles$$
\item A rectangular neighborhood that is 2 km by 3 km has an area of 6 square km.\\
$$2~km \times 3~km = 6~km^2$$
\item A proportion has no units. A cake recipe that uses 2 1/2 cups sugar and 3 1/2 cups flour has a ratio of 5 parts sugar to 7 parts flour, whether you double the recipe or measure it again in metric units.
$$2.5~cups:3.5~cups = 5:7$$
\end{itemize}
\end{itemize}

\subsection{Exponents}
\begin{itemize}
\item \textbf {Rules of Exponents} If $a$ and $b$ are positive numbers, $n$ is a positive integer, and $x$ and $y$ are any real numbers, then:
\begin{enumerate}
\item $a^n = \underbrace{a \times a \times a \times \cdots \times a}_n$
\item $a^0 = 1$
\item $a^{-1} = \displaystyle \frac{1}{a}$
\item $a^{x+y}=a^x a^y$
\item $a^{x-y}= \displaystyle \frac{a^x}{a^y}$
\item $(a^x)^y = a^{xy}$
\item $(ab)^x = a^x b^x$
\item $\displaystyle \left( \frac{a}{b} \right)^x = \frac{a^x}{b^x}$
\item $a^{\frac{1}{2}} = \sqrt{a}$
\item $\sqrt[x]{a} = a^{\frac{1}{x}}$
\end{enumerate}
\item \textbf{Exercises:}
\begin{enumerate}
\item Evaluate: $(5 \times 4)^2$
\item Evaluate: $((1+3)^3)^2 \times 2 + 5$
\end{enumerate}
\end{itemize}

\subsection{The Coordinate Plane:} There are four \textbf{quadrants} on a coordinate plane (labeled below). The horizontal axis is known as the x-axis and the vertical axis is known as the y-axis. Coordinates of the form (x,y) are plotted on this plane. \\
\includegraphics[scale=0.3, trim=-450 100 0 50]{img/coordplan_quadlab.pdf}
\begin{itemize}
\item Point on the coordinate plane are represented as couples of numbers, e.g. $P_1 = (x_1, y_1)$
\item \textbf{Distance Formula:} The distance between two points $P_1 = (x_1, y_1)$ and $P_2 = (x_2, y_2) $ in the x-y plane is given by:
\begin{equation*}
|P_1P_2| = \sqrt[]{(x_2-x_1)^2+(y_2-y_1)^2}
\end{equation*}
\item \textbf{Exercises:}
\begin{enumerate}
\item Can you plot the points (1,2) and (3,-2) on the coordinate plane and find the distance between them?
\item Demonstrate that the Pythagorean Theorem and the distance formula are equivalent. Remember, the PT says:  in a right triangle, where a and b are the lengths of sides next to the right angle, and c is the length of the hypotenuse, the lengths have the relationship
\end{enumerate}
\begin{equation*}
a^2 + b^2 = c^2
\end{equation*}
\end{itemize}

\subsection{Lines:}

\begin{itemize}

\item \textbf{Slope of a Line}: The slope of a line passing through the points $P_1 = (x_1, y_1)$ and $P_2 = (x_2, y_2) $ is
\begin{equation*}
m=\displaystyle \frac{\Delta y}{\Delta x}  = \displaystyle \frac{y_2 - y_1}{ x_2 - x_2} = \displaystyle \frac{rise}{run}
\end{equation*}
\item\textbf{Equation of a Line:}
\begin{enumerate}
\item \textbf{Point-Slope Form:} Given a point, $P_1 = (x_1, y_1)$, and a slope, m, you can plot the line:
\begin{equation*}
y-y_1 = m(x - x_1)
\end{equation*}
\item \textbf{Slope-Intercept Form:} Given a slope, m, and the y-intercept, (0,b), you can plot the line:
\begin{equation*}
y = mx + b
\end{equation*}

\item \textbf{Terminology:}
\begin{enumerate}
\item \textbf{Parallel:} Two lines are parallel if and only if they have the same slope.
\item \textbf{Perpendicular:} Two lines are perpendicular if and only if their slopes, $m_1$ and $m_2$, are negative reciprocals, i.e. $m_1m_2 = -1$, $m_1 =\frac{-1}{m_2}$
\end{enumerate}
\end{enumerate}

\item \textbf{Linear Regression:}
\begin{itemize}
    \item Linear relationships form the basis of much quantitative education research. For a dependent variable, $Y$, with a single independent variable, $X$, we can write:
    $$ Y = \beta_0 + \beta_1 X$$
    \item We interpret this as ``A one unit change in $X$ is associated with a $\beta_1$ unit change in $Y$.''
    \item This works for multiple independent variables, too!
    \item To learn more about this, take EDUC400B, EDUC430A, EDUC430B, or equivalent courses in other departments.
\end{itemize}

\item \textbf{Exercises:}
\begin{enumerate}
\item Plot the line through the points $(1,2)$ and $(4,3)$.
\begin{enumerate}
\item What is the slope?
\item What is the intercept?
\end{enumerate}
\item A researcher runs a regression of SAT Math score on age and finds the relationship: $$SAT_{Math,i} = 30 \times AGE_i$$.
\begin{enumerate}
\item What does this relationship mean, in words?
\item What problems do you see with the results of this regression?
\end{enumerate}
\end{enumerate}

\end{itemize}

\subsection{Polynomials:}
\begin{itemize}

\item\textbf{Definition:} A polynomial is a mathematical expression of one or more terms, where each term is a constant multiplied by one or more variables each raised to a nonnegative integer power. For example: $ax^2 + bx + c$.
\item\textbf{Terminology:}
\begin{enumerate}
\item \textbf{Degree:} The degree of a polynomial is the highest degree of its terms. The degree of a term is the sum of the exponents on the variables. For example, the degree of $ax^2 + bx + c$ is $2$.
\item \textbf{Root:} The roots of a polynomial function $f(x)$ are the values of the variable $x$ where $f(x)=0$.
\end{enumerate}
\item\textbf{Finding Roots:} To find the roots (or zeros) of the quadratic polynomial equation $y=ax^2 + bx + c$, use the \textbf{quadratic equation} \\
\begin{equation*}
x = \frac{-b \pm \sqrt[]{b^2-4ac}}{2a}
\end{equation*}

\item \textbf{Exercises:}
\begin{enumerate}
\item Find the roots of: $y=x^2 - 6x + 8$
\end{enumerate}
\end{itemize}

\section{Limits}
\begin{itemize}

\item\textbf{Definitions:}
\begin{itemize}

\item\textbf{Intuitive definition:}
The \textbf{limit} of $f(x)$, as $x$ approaches $a$, equals $L$ if, as we take values closer and closer to $a$ (just a bit bigger and a bit smaller, but not equal to $a$), we can make the values of $f(x)$ arbitrarily close to $L$.  Remember: When talking about limits, we are not considering the point $x=a$.

\item\textbf{Precise definition:}
Let $f$ be a function defined on some open interval that contains the number $a$ (the function need not be defined at $a$ itself). Then we say the limit of $f(x)$ as $x$ approaches $a$ is $L$ and write
\[\lim_{x\to a} f(x) = L \] if for every number $\varepsilon > 0$ there is a corresponding number $\delta > 0$ such that $|f(x) - L| < \varepsilon$ whenever $0 < |x-a| < \delta$.

\item\textbf{Left-hand limit of $f(x)$}
Writing $\lim_{x \to a^-}f(x)=L$ means that the left-hand limit of $f(x)$ as $x$ approaches $a$ is equal to $L$ if we can make values of $f(x)$ as close to $L$ as we like by taking $x$ to be sufficiently close to $a$ and $x$ less than $a$.

\item\textbf{Right-hand limit of $f(x)$}
Writing $\lim_{x \to a^+}f(x)=L$ means that the right-hand limit of $f(x)$ as $x$ approaches $a$ is equal to $L$ if we can make values of $f(x)$ as close to $L$ as we like by taking $x$ to be sufficiently close to $a$ and $x$ greater than $a$.

\item\textbf{Another definition}
\[\lim_{x\to a} f(x) = L \quad \text{if and only if} \lim_{x \to a^-}f(x) = L \; \text{and} \lim_{x \to a^+}f(x)=L \]
\end{itemize}

\item\textbf{The Limit Laws:}
Suppose that $c$ is a constant and that the limits
\[ \lim_{x\to a} f(x) \quad \text{and} \quad \lim_{x\to a} g(x) \] exist. Then, the following rules hold:
\begin{enumerate}
\item $\lim_{x\to a}[f(x) + g(x)] = \lim_{x\to a}f(x) + \lim_{x\to a}g(x) $
\item $\lim_{x\to a}[f(x) - g(x)] = \lim_{x\to a}f(x) - \lim_{x\to a}g(x) $
\item $\lim_{x\to a}[cf(x)] =  c\lim_{x\to a}f(x) $
\item $\lim_{x\to a}[f(x)g(x)] = \lim_{x\to a}f(x) \times \lim_{x\to a}g(x) $
\item $\displaystyle \lim_{x\to a}\frac{f(x)}{g(x)} = \frac{\displaystyle \lim_{x\to a}f(x)}{\displaystyle \lim_{x\to a}g(x)}$ if $\lim_{x\to a}g(x) \neq 0$
\item $\lim_{x\to a}[f(x)]^n = [\lim_{x\to a} f(x)]^n$, where $n$ is a positive integer.
\item $\lim_{x \to a} c = c$
\item $\lim_{x \to a} x = a$
\item $\lim_{x \to a} x^n = a^n$, where $n$ is a positive integer.
\item $\lim_{x \to a} \sqrt[n]{x} = \sqrt[n]{a}$, where $n$ is a positive integer. Note: If $n$ is even, we assume that $a>0$.
\item $\lim_{x \to a} \sqrt[n]{f(x)} = \displaystyle \sqrt[n]{\lim_{x \to a} f(x)}$, where $n$ is a positive integer. If $n$ is even, we assume that $\lim_{x \to a} f(x) >0$.
\end{enumerate}

\item\textbf{Direct Substitution Property:}
If $f$ is a polynomial or a rational function and $a$ is in the domain of $f$, then
\begin{equation*}
\lim_{x \to a} f(x) = f(a)
\end{equation*}

\item\textbf{Infinite Limits and Limits at Infinity}
\begin{itemize}

\item $\lim_{x \to a} f(x) = \infty$ means that values of $f(x)$ can be made arbitrarily large by taking $x$ sufficiently close to $a$ but not equal to $a$.

\item Let $f$ be a function defined on some interval $(a, \infty)$. Then $\lim_{x \to \infty} f(x) =L$ means that values of $f(x)$ can be made as close to $L$ as we like by taking $x$ sufficiently large.
\end{itemize}

\item\textbf{Continuity Definitions}
\begin{itemize}
\item\textbf{Main definition:}
A function $f$ is continuous at a number $a$ if
\begin{equation*}
\lim_{x \to a} f(x) = f(a)
\end{equation*}
\item\textbf{Continuous from the right:}
A function $f$ is continuous \textit{from the right} at a number $a$ if
\begin{equation*}
\lim_{x \to a^+} f(x) = f(a)
\end{equation*}
\item\textbf{Continuous from the left:}
A function $f$ is continuous \textit{from the left} at a number $a$ if
\begin{equation*}
\lim_{x \to a^-} f(x) = f(a)
\end{equation*}
\item\textbf{Continuous on an interval:}
A function is continuous on an interval if it is continuous at every number in the interval.
\end{itemize}

\item \textbf{Exercises:}
\begin{enumerate}
\item Let $y=x$. Does the limit as $x \to 4$ exist? If it exists, find the limit.
\item Let
\begin{equation*}
 y=\begin{cases}
    1, & \text{if $x<1$}\\
    x^2, & \text{if $x \geq 1$}
  \end{cases}
\end{equation*}
Does the limit as $x \to 1$ exist? If it exists, find the limit.
\item Let
\begin{equation*}
 y=\begin{cases}
    x, & \text{if $x\neq2$}\\
    4, & \text{if $x = 2$}
  \end{cases}
\end{equation*}
Does the limit as $x \to 2$ exist? If it exists, find the limit.
\end{enumerate}
\end{itemize}

\newpage
\noindent The following two sections are ``bonus" materials for you to review. I have not included exercises for these sections, but am happy to review them with you if you have questions.

\section{Sequences}
\begin{itemize}
\item A \textbf{sequence} $\{a_n\} = \{a_1, a_2, a_3, a_4, \ldots, a_n, \ldots\}$ is a list of real numbers written in a definite order, where $a_1$ is the first term, $a_2$ is the second term, and in general $a_n$ is the $n$th term. In this lecture, we will only consider infinite sequences; thus, each term $a_n$ will have a successor $a_{n+1}$. We can also write the sequence as $\{a_n\}^{\infty}_{n=1}$.

\item Examples:
\begin{enumerate}
\item $\displaystyle \{a_n\}=\left\lbrace\frac{n}{n+1}\right\rbrace = \left\lbrace\frac{1}{2}, \frac{2}{3}, \frac{3}{4}, \frac{4}{5},\dots,\frac{n}{n+1},\dots \right\rbrace$
\item $\displaystyle \{a_n\}=\left\lbrace\frac{(-1)^n(n+1)}{3^n}\right\rbrace = \left\lbrace -\frac{2}{3}, \frac{3}{9}, -\frac{4}{27}, \frac{5}{81},\dots,\frac{(-1)^n(n+1)}{3^n},\dots \right\rbrace$
\end{enumerate}

\item Sequences are like functions. Instead of $y=f(x)$ with $x$ specified over some domain, we  have $\{y_n\}={f(n)}$ with $n=1, 2, 3, \dots.$

\item There are three types of sequences: (1) those that converge to a limit; (2) those that increase without bound; (3) those that neither converge nor increase increase without bound (e.g. a sequence that alternates over the number line).

\item A sequence ${a_n}$ has the limit $L$ (i.e., $\lim_{n \to \infty} = L$) if we can make the terms $a_n$ as close to $L$ as we like by taking $n$ sufficiently large. If $\lim_{n\to\infty}$ exists, we say the sequence converges. Otherwise, the sequence diverges.

\end{itemize}

\section{Series}
\begin{itemize}
\item An \textbf{infinite series} is the expression we obtain when we add the terms of an infinite sequence $\{a_n\}^{\infty}_{n=1}$, denoted as $\displaystyle \sum_{n=1}^\infty a_n$.

\item Definition: Given a series $\displaystyle \sum_{n=1}^\infty a_n$, let $s_n$ denote its $n$th partial sum: $\displaystyle s_n = \sum_{i=1}^n a_i$. If the sequence ${s_n}$ is convergent and $\lim_{n \to \infty} s_n = s$ exists and is a real number, then the series $\displaystyle \sum_{n=1}^\infty a_n$ is called convergent. In terms of notation: $\displaystyle \sum_{n=1}^\infty = s$, where $s$ is the sum of the series. If the sequence ${s_n}$ is divergent, then the series is divergent.

\item \textbf{Geometric series:} The geometric series $\displaystyle \sum_{n=1}^\infty ar^{n-1} = a + ar + ar^2 + \cdots$ is convergent if $|r| < 1$ and its sum is $\displaystyle \sum_{n=1}^\infty ar^{n-1} = \frac{a}{1-r}$, where $|r|<1$. If $|r| \geq 1$, then the geometric series is divergent.

\end{itemize}

\end{document}


\documentclass{article}
\usepackage[T1]{fontenc}

\usepackage{amssymb}
\usepackage{courier}
\usepackage{fancyhdr}
\usepackage{fancyvrb}
\usepackage[top=.75in, bottom=.75in, left=.75in,right=.75in]{geometry}
\usepackage{graphicx}
\usepackage{lastpage}
\usepackage{listings}
\lstset{basicstyle=\small\ttfamily}
\usepackage{mdframed}
\usepackage{parskip}
\usepackage{soul}
\usepackage{tabularx}
\usepackage{textcomp}
\usepackage{upquote}
\usepackage{xcolor}

% http://www.monperrus.net/martin/copy-pastable-ascii-characters-with-pdftex-pdflatex
\lstset{
upquote=true,
columns=fullflexible,
keepspaces=true,
literate={*}{{\char42}}1
         {-}{{\char45}}1
         {^}{{\char94}}1
}
\lstset{
  moredelim=**[is][\color{blue}\bf\small\ttfamily]{@}{@},
}

% http://tex.stackexchange.com/questions/40863/parskip-inserts-extra-space-after-floats-and-listings
\lstset{aboveskip=6pt plus 2pt minus 2pt, belowskip=-4pt plus 2pt minus 2pt}

\usepackage[colorlinks,urlcolor={blue}]{hyperref}
\usepackage[capitalise,nameinlink,noabbrev]{cleveref}
\crefname{section}{Question}{Questions}
\Crefname{section}{Question}{Questions}

\begin{document}

\fancyhead[C]{\hl{Select the right page in Gradescope or we will not grade the question!}}
\fancyhead[L]{}
\fancyhead[R]{}

\fancyfoot[L]{\color{gray} C4CS -- W'17}
\fancyfoot[R]{\color{gray} Revision 1.0}
\fancyfoot[C]{\color{gray} \thepage~/~\pageref*{LastPage}}
\pagestyle{fancyplain}

\title{\textbf{Homework 14\\Profiling, Linting, and Static Analysis}}
\author{\textbf{\color{red}{Solutions}}}
\date{}
\maketitle

\section*{Submission Instructions}
Submit this assignment on \href{https://gradescope.com/}{Gradescope}.
You may find the free online tool \href{https://www.pdfescape.com}{PDFescape}
helpful to edit and fill out this PDF.
You may also print, handwrite, and scan this assignment.

\section*{Optional Reading}

\emph{Developer Survey 2017}, from Stack Overflow.

\url{https://stackoverflow.com/research/developer-survey-2017}

Some students have asked for some insights into CS in industry. Every year,
Stack Overflow does a large survey of developers. Their writeup caveats the
survey a bit, but it's worth calling out again that this is a self-selecting
group of the 50,000 people who are active or attentive enough to Stack
Overflow to see the survey and who are willing to spend the time to fill it
out. The response demographics are not necessarily representative of the
industry as a whole, nor should you feel obligated to fall into the bins that
the survey identifies. That said, I think it presents some interesting data.
They've been doing this for a few years now and looking over the history is
interesting as well.

\newpage

\section{Exploring Profiling\protect\footnote{%
  The \texttt{perf} profiling tools only work on Linux. You'll need a Linux
  environment (like your VM) for this assignment. If you primarily develop on
  a mac, XCode ships with a tool called \texttt{Instruments} that can be used
  to profile that may be worth exploring on your own.
}}

Clone a copy of
\url{https://gitlab.eecs.umich.edu/ppannuto/c4cs-f16-profiling}.

Look through \texttt{main.c} to understand what this program is doing (not
much, a ``nop'' is ``no-operation'', that is, sit idle for a cycle). The call
graph of this program is about:
{\small
\begin{verbatim}
\-- main
    \-- parent
            \-- child1
            \-- child2,child2,child2,...
    \-- no_children
\end{verbatim}
}

When run, this program will execute \texttt{16*LONG\_TIME} nop instructions.
By analyzing the code, what percentage of nop's should be attributed to each
function? Finish filling in the first column of this table:

\bigskip
\renewcommand{\arraystretch}{1.25} % space out the table rows
\textbf{
  \begin{tabular}{r r || r r r}
                  & \texttt{nop} Analysis & Profile 1 & Profile 2 & Profile 3 \\\hline
    main:         & 0\%                   &           &           &           \\
    parent:       & 6.25\%                &           &           &           \\
    child1:       & \color{violet}12.5\%  & & \color{violet} varies &         \\
    child2:       & \color{violet}50\%    & \multicolumn{3}{c}{\color{violet}\small(trends should match, can easily vary by 5-10\% on some machines though)} \\
    no\_children: & \color{violet}31.25\% &           &           &           \\
  \end{tabular}
}

\bigskip
Now compile and profile the code 3 times, recording the percentage of time
attributed by \texttt{perf} to each function.

\textbf{Fill in the remaining three columns of the table with the results from
  these profiling runs.}

\textbf{Does the trend of time attributed to each function align with your
  expectations from the \texttt{nop} analysis?}
\begin{quote}
  \textbf{(Yes or No) \color{violet} Yes}
\end{quote}

\textbf{Give one reasonable explanation for why you think the measured
  percentages from profiling don't perfectly match the \texttt{nop} analysis:}

\begin{quote}
  \color{violet}
  Plenty of reasonable explanations. Profiling sampling rate. Nothing (main)
  takes no time.
\end{quote}

\vfill

\subsection{Call-Graph Profiling}
Try running \texttt{perf record -g ./main \&\& perf report -g}

The \texttt{-g} flag enables \texttt{call-graph} profiling. This causes to
profiler to pay attention to which functions called which. This can be a
slightly more intuitive profiling view, as the leftmost column is now the
percentage of time that this function and all its children ran. I find this
view makes finding the path of functions called to get to the time-consuming
function easier. In C programs, \texttt{main} will always sort to the top as
100\% of your code runs inside of \texttt{main}. As a thought exercise, what
code executes outside of \texttt{main} in C++?

In this example from my machine, we can see that the \texttt{parent} function
takes 65\% of the time, with 8\% of that coming from code \texttt{parent}
itself, and $42+15$\% coming from child functions:

{\footnotesize
\begin{verbatim}
+   99.99%     0.00%  main     main               [.] main
+   99.99%     0.00%  main     libc-2.23.so       [.] __libc_start_main
+   99.99%     0.00%  main     [unknown]          [.] 0x07be258d4c544155
+   64.51%     7.90%  main     main               [.] parent
+   42.00%    41.77%  main     main               [.] child2
+   34.86%    34.49%  main     main               [.] no_children
+   15.13%    15.02%  main     main               [.] child1
\end{verbatim}
}

\emph{\small
(There is no question you have to answer in section 1.1)
}

\newpage
\section{Trying out Static Analysis}

Recall from the debugging homework earlier this semester, we had some buggy
code that was trying to compute prime numbers.

\begin{verbatim}
git clone https://github.com/c4cs/debugging-basics
git checkout gdb-debug-1       # The first bug
git checkout valgrind-debug    # The last bug
\end{verbatim}

Try running our static analysis tools on these two branches.

\textbf{%
  Does \texttt{cppcheck} find the first or last bug?
  Does \texttt{scan-build} find either bug?
}

\begin{quote}
  \color{violet}
  \texttt{cppcheck} can't find either bug. \texttt{scan-build} can find
  the last bug.

  The lesson is that while static analysis tools can help, they vary in what
  kinds of bugs they will find, and passing static analysis by no means
  ensures that your program is bug/crash-free.
\end{quote}

\subsection{On your own code!}

Run the static analysis tools on current or past code you've written. Recall
that by default, \texttt{cppcheck} does not enable all checks (check out the
lecture slide if you've forgotten).

\textbf{Copy one warning or error reported here:}\footnote{%
  In the unlikely case that no code you've ever written produces any warnings,
  write a small program that generates a message and use that here.
}

\begin{quote}
  \color{violet}
  (varies)
\end{quote}

\textbf{Does this warning or error impact the correctness of your program? Why
  or why not?}

\begin{quote}
  \color{violet}
  (varies -- \texttt{cppcheck} with all checks includes several style checks
  that are likely benign)
\end{quote}

\end{document}


%!TEX TS-program = xelatex
%!TEX encoding = UTF-8 Unicode

\documentclass[a4paper,12pt]{article}
\usepackage{amssymb}
\begin{document}
\title{Example 2}
\author{My name}
\date{January 5, 2011}
\maketitle
\section{What's this?}
This is our second document. It contains a title and a section with text.

Shall we call ourselves \TeX users

or \TeX{} users?

$ y = \left| x - 1 \right| $

Show a equation is, $ a = \left| x_2 - x_1 \right| $ match before word.

$$ \left|P_1P_2\right| = \sqrt{a^2 + b^2 + c^2} $$

\end{document}