\documentclass[10pt]{beamer}

\mode<presentation>
{
 \usetheme{Boadilla}
\pagestyle{empty}

\setbeamerfont*{frametitle}{size=\normalsize,series=\bfseries}
\setbeamerfont*{block}{size=\normalsize,series=\bfseries}
%\setbeamertemplate{blocks}[rounded][shadow=true]
}

\definecolor{links}{HTML}{2A1B81}
\hypersetup{colorlinks,linkcolor=,urlcolor=links}

\usepackage{etex}
% \usepackage{helvet}
\usepackage{amsmath, amssymb}
\usepackage{color}
\usepackage{asymptote}
\usepackage{mathrsfs}
\usepackage{dsfont}
\usepackage{makeidx}
\usepackage{multido}
\usepackage{dsfont}

\usepackage{pst-sigsys,pst-plot,pstricks-add}
%\usepackage{auto-pst-pdf}
\usepackage{pst-pdf}

\definecolor{links}{HTML}{2A1B81}
\hypersetup{colorlinks,linkcolor=,urlcolor=links}

\def\nn{\nonumber}

\definecolor{links}{HTML}{2A1B81}
\hypersetup{colorlinks,linkcolor=,urlcolor=links}

\newcommand{\fs}[2]{#2}

\title[]{LTI systems and the convolution operation}
\author[\textcolor{blue}{Systems and Circuits}]{\textcolor{darkblue}{Pablo M. Olmos} (olmos@tsc.uc3m.es)\\ \textcolor{darkblue}{Emilio Parrado} (emipar@tsc.uc3m.es)}
\institute{\textcolor{white}{UC3M}}

\definecolor{darkblue}{rgb}{0.0, 0.0, 0.40}
\setbeamercolor{title}{fg=darkblue}
\setbeamercolor{frametitle}{fg=darkblue}
\definecolor{darkgreen}{rgb}{0.0, 0.4, 0.0}

\AtBeginSection[]
{
  \begin{frame}<beamer>{Index}
    \tableofcontents[currentsection,currentsubsection]
  \end{frame}
}

\begin{document}
\frame{
\titlepage
\thispagestyle{empty}
\begin{center}
\includegraphics[scale=0.05]{Figures/uc3m-logo.pdf}
\end{center}
}

\section{Introduction to LTI systems}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=0.3]{Filtro_1.png}
\end{figure}

}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=1.0]{2D_1.pdf}
\end{figure}

\begin{align*}
Y[1,1]&=X[1,1]*K[1,1]+X[1,2]*K[1,2]+X[1,3]*K[1,3]\\
&+X[2,1]*K[2,1]+X[2,2]*K[2,2]+X[2,3]*K[2,3]\\
&+X[3,1]*K[3,1]+X[3,2]*K[3,2]+X[3,3]*K[3,3]
\end{align*}
}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=1.0]{2D_2.pdf}
\end{figure}

\begin{align*}
Y[1,2]&=X[1,2]*K[1,1]+X[1,3]*K[1,2]+X[1,4]*K[1,3]\\
&+X[2,2]*K[2,1]+X[2,3]*K[2,2]+X[2,4]*K[2,3]\\
&+X[3,2]*K[3,1]+X[3,3]*K[3,2]+X[3,4]*K[3,3]
\end{align*}
}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=1.0]{2D_1.pdf}
\end{figure}

\begin{align*}
Y[i,j]&=\sum_{k_1=1}^{n}\sum_{k_2=1}^{n} X[k_1,k_2]K[i-k_1,j-k_2], ~~~ K[u,q]=0 \text{ para } u>3, q>3
\end{align*}

{\bf Linear operator, the result does not depend on the position of the image}
}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=0.3]{Filtro_1.png}
\end{figure}

}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=0.3]{Filtro_2.png}
\end{figure}

}

\frame{
\frametitle{Motivation: Image Filtering}

\begin{figure}
\includegraphics[scale=0.3]{Filtro_3.png}
\end{figure}

}

\frame{
\begin{itemize}
\item The theory of LTI systems has direct applications in a wide set of technical areas:
\begin{itemize}
\item Nuclear magnetic resonance spectroscopy
\item Seismology
\item Electric circuit design
\item Control Theory
\item Any application that involves Signal Processing
\end{itemize}
\item Our goal in Systems and Circuits: predict the output of a given LTI system for a given input.
\item In future courses you will face the design of LTI systems according to certain specifications.
\end{itemize}
}

\frame{
\frametitle{Time Invariance}

\begin{block}{}
A system is time-invariant if a time shift in the input signal causes a time shift in the output signal.
\end{block}
\vspace{0.5cm}

\begin{itemize}
\item Given $y[n]=f(x[n])$, the system is time-invariant if $f(x[n-n_0])=y[n-n_0]$ $\forall~n_0$.
\item Given $y(t)=f(x(t))$, the system is time-invariant if $f(x(t-t_0))=y(t-t_0)$ $\forall~t_0$.
\end{itemize}
}

\frame{
\frametitle{Linearity}
Linear system posses the important property of superposition.

\begin{exampleblock}{}
 For any system, consider two arbitrary inputs and their respective outputs:
\begin{align}\nn
x_1(t)\rightarrow y_1(t)\\\nn
x_2(t)\rightarrow y_2(t),
\end{align}
the system is linear if
\begin{align}\nn
ax_1(t)+bx_2(t)\rightarrow ay_1(t)+by_2(t)
\end{align}
for any two complex constant $a,b\in\mathbb{C}$.
\end{exampleblock}

\begin{block}{Linear discrete-time signals}
\begin{align}\nn
ax_1[n]+bx_2[n]\rightarrow ay_1[n]+by_2[n]
\end{align}
\end{block}

}

\frame{
\frametitle{Analysis of LTI systems}

\begin{figure}
\centering\includegraphics[scale=0.3]{P6.pdf}
\end{figure}

\vspace{0.5cm}
\begin{itemize}
\item Decompose input signal by means of a linear combination of simpler signals
\end{itemize}

$$x(t) = \dots a_{-1} \phi_{-1} (t) + a_{-2} \phi_{0} (t) + a_{0} \phi_{0} (t) + a_{1} \phi_{1} (t) + a_{2} \phi_{2} (t) + \ldots $$

\begin{itemize}
\item These ``basic" signals are chosen to provide a certain degree of analytical convenience, so we can  analyze the system's properties and its response to arbitrary input signals:
\begin{itemize}
\item Delayed Impulses $\Rightarrow$ \textbf{\textcolor{darkblue}{Convolution}}
\item Complex exponential signals $\Rightarrow$ \textbf{\textcolor{darkblue}{Fourier analysis}}
\end{itemize}
\end{itemize}

}

\section{Discrete-time LTI systems}

\frame{

\begin{figure}
\centering\includegraphics[scale=0.3]{P1.pdf}
\end{figure}

\vspace{0.5cm}
Remember that any discrete-time sequence $x[n]$ can be decomposed as a linear combination of unit impulses:
\begin{block}{}
\begin{align}\nn
x[n]&=\sum_{k=-\infty}^{\infty}x[k]\delta[n-k]\\\nn
&=\ldots+x[-20]\delta[n+20]+x[-19]\delta[n+19]+\ldots\\\nn
&+x[-1]\delta[n+1]+x[0]\delta[n]+x[1]\delta[n-1]+\ldots
\end{align}
\end{block}
}

\frame{
\textbf{If the system is linear} and we are able to compute
\vspace{0.5cm}

\begin{figure}
\centering\includegraphics[scale=0.3]{P2.pdf}
\end{figure}

then we can make use of the superposition property!
}

\frame{

\begin{figure}
\centering\includegraphics[scale=0.3]{P3.pdf}
\end{figure}

\vspace{0.5cm}
\begin{exampleblock}{}
\begin{align}\nn
y[n]=\sum_{k=-\infty}^{\infty}x[k]h_{k}[n]
\end{align}
\end{exampleblock}

\vspace{0.5cm}
\begin{itemize}
\item This is just a consequence of the system linearity.
\item The problem is reduced to evaluate the system's output for any $\delta[n-k]$.
\item The problem can be further reduced  by exploiting that the system is \textbf{time-invariant}.
\end{itemize}

}

\frame{

\begin{figure}
\centering\includegraphics[scale=0.3]{P4.pdf}
\end{figure}

Therefore, for any input $x[n]$, if $h[n]$ is known then the system output is given by

\begin{exampleblock}{}
\begin{align}\nn
y[n]=\sum_{k=-\infty}^{\infty}x[k]h_{k}[n]=\sum_{k=-\infty}^{\infty}x[k]h[n-k]
\end{align}
\end{exampleblock}

}

\frame{

\begin{itemize}
\item $h[n]$ is the system's \textbf{impulse response}.
\item Any LTI system is \textbf{completely defined} by $h[n]$!
\vspace{0.3cm}
\begin{figure}
\centering\includegraphics[scale=0.3]{P5.pdf}
\end{figure}
\end{itemize}

\begin{block}{Convolution}
Given two discrete-time signals $x[n]$ and $h[n]$:
\begin{align}\nn
\sum_{k=-\infty}^{\infty}x[k]h[n-k]=x[n]\ast h[n]
\end{align}
is the convolution operation between them.
\end{block}

}

%
%\frame{
%\frametitle{Example I}
%
%\psset{xunit=0.75cm,yunit=0.75cm}
%
%\begin{tabular}{|c|c|}
%\hline
%\begin{pspicture}[showgrid](-2,-1)(4,3)
%  \rput(0,0){\psaxeslabels(0,0)(-2,0)(4,0){$n$}{}
%     \rput[tl](-2,3){$x[n]$}
%     \psstem[style=Stem,linecolor=blue](-1,1)
%            {0,1,2,0}}
%\end{pspicture}
%&
%\begin{pspicture}[showgrid](-1,-1)(5,3)
%  \rput(0,0){\psaxeslabels(0,0)(-1,0)(5,0){$n$}{}
%     \rput[tl](-1,3){$h[n]$}
%     \psstem[style=Stem,linecolor=blue](0,1)
%            {0,1,1,1,0}}
%\end{pspicture}\\
%\hline
%\end{tabular}
%
%
%}
%
%\frame{
%%\frametitle{Example I}
%
%\begin{center}
%\begin{pspicture}[](5,2)
%   \pssignal(0,1){x}{$x[n]$}
%   \psblock(2,1){a}{$h[n]$}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(4,1){y}{$y[n]$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%\end{pspicture}
%\end{center}
%
%\psset{xunit=0.75cm,yunit=0.75cm}
%
%\begin{center}
%
%\begin{pspicture}[showgrid](0,-1)(6,3)
%  \rput(0,0){\psaxeslabels(0,0)(0,0)(6,0){$n$}{}
%     \rput[tl](0,3){$y[n]$}
%     \psstem[style=Stem,linecolor=red](1,1)
%            {1,3,3,2,0}}
%\end{pspicture}
%
%\end{center}
%
%}

%\frame{
%\frametitle{Example}
%
%\psset{xunit=0.75cm,yunit=0.75cm}
%
%\begin{center}
%
%\begin{tabular}{|c|c|}
%\hline
%\begin{pspicture}[showgrid](-1,-1)(4,5)
%  \rput(0,0){\psaxeslabels(0,0)(-1,0)(4,0){$n$}{}
%     \rput[tl](-1,3){$x[n]$}
%     \psstem[style=Stem,linecolor=blue](-1,1)
%            {0,4,1,2,5}}
%\end{pspicture}
%&
%\begin{pspicture}[showgrid](-2,-1)(3,3)
%  \rput(0,0){\psaxeslabels(0,0)(-2,0)(3,0){$n$}{}
%     \rput[tl](-2,3){$h[n]$}
%     \psstem[style=Stem,linecolor=blue](-2,1)
%            {0,1,2,-1}}
%\end{pspicture}\\
%\hline
%\end{tabular}
%
%\end{center}
%
%
%
%}
%
%\frame{
%\frametitle{Example}
%
%\begin{center}
%\begin{pspicture}[](5,2)
%   \pssignal(0,1){x}{$x[n]$}
%   \psblock(2,1){a}{$h[n]$}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(4,1){y}{$y[n]$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%\end{pspicture}
%\end{center}
%
%\psset{xunit=0.35cm,yunit=0.35cm}
%
%\begin{center}
%
%\begin{pspicture}[showgrid](-2,-5)(7,10)
%  \rput(0,0){\psaxeslabels(0,0)(-2,0)(7,0){$n$}{}
%     \rput[tl](-2,10){$y[n]$}
%     \psstem[style=Stem,linecolor=red](-2,1)
%            {0,4,9,0,8,8,-5,0}}
%\end{pspicture}
%
%\end{center}
%
%}

\frame{
\frametitle{Problem}

\begin{itemize}
\item $x[n]=\alpha^{n}u[n]$ with $\alpha\in(0,1)$
\item $h[n]=u[n]$
\end{itemize}

\begin{figure}
\centering\includegraphics[scale=0.3]{P5.pdf}
\end{figure}

}

\frame{
\frametitle{Sol.}

\begin{align*}
y[n]=\left\{\begin{array}{cc}
0 & n<0\\\\
\frac{1-\alpha^{n+1}}{1-\alpha} & n\geq 0
\end{array}
\right.
\end{align*}

}

\section{Continuous-time LTI systems}

\frame{

\textbf{\color{red}{The discussion for continuous-time LTI systems is just a generalization of the discrete-time case.}}

\begin{figure}
\centering\includegraphics[scale=0.3]{P6.pdf}
\end{figure}

\vspace{0.5cm}
Remember that any continuous-time signal $x(t)$ can be decomposed as a linear combination of an infinite number of
impulses:
\begin{block}{}
\begin{align}\nn
x(t)=\int_{-\infty}^{\infty}x(\tau)\delta(t-\tau)d\tau
\end{align}
\end{block}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% EPH 20140222
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{
\textbf{{\color {red} Approximate} $x(t)$ by a combination of  scaled and equally spaced versions $\delta_{\Delta}(t)$}

\begin{figure}
\centering\includegraphics[scale=0.4]{Figures/Signal32.pdf}
\end{figure}
}

\frame{
\[
x(t) \approx x_{\Delta}(t) = \sum_{k=-\infty}^{\infty}{x(k\Delta){\color{blue}\delta_{\Delta}(t-k\Delta)}\Delta}
\]
\begin{figure}
\centering\includegraphics[scale=0.3]{P7.pdf}
\end{figure}
\begin{figure}
\centering\includegraphics[scale=0.3]{Figures/Signal33.pdf}
\end{figure}

Then applying the superposition property
\begin{figure}
\centering\includegraphics[scale=0.3]{P8.pdf}
\end{figure}
}

\frame{
%El resultado anterior todavía es inmanejable porque depende de conocer infinitas $h_{k\Delta}(t)$
%
%\vspace{0.5cm}

If the system is also {\bf time-invariant}:
\begin{figure}
\centering\includegraphics[scale=0.3]{P9.pdf}
\end{figure}

\vspace{0.5cm}

Taking the limit $\Delta \rightarrow 0$:
\begin{itemize}
\item $k\Delta \rightarrow \tau$
\item $\sum \rightarrow \int$
\item $\Delta \rightarrow d\tau$
\item $h_0(t) = h(t)$
\end{itemize}

\begin{exampleblock}{}
\begin{align}\nn
y(t) = \lim_{\Delta \rightarrow 0} y_{\Delta}(t) = \int_{-\infty}^{\infty}{x(\tau)h(t-\tau)d\tau}
\end{align}
\end{exampleblock}

}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% FIN EPH 20140222
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%
%
%\frame{
%
%If the system's output is known for $\delta(t-\tau)$ for any $\tau\in\mathbb{R}$:
%
%\begin{center}
%\begin{pspicture}[](5,2)
%   \pssignal(0,1){x}{$\delta(t-\tau)$}
%   \psblock(2,1){a}{LTI}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(4,1){y}{$h_{\tau}(t)$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%\end{pspicture}
%\end{center}
%
%Then, by the superposition property,
%
%\begin{center}
%\begin{pspicture}[](7,2)
%   \pssignal(0,1){x}{$\displaystyle x(t)=\int_{-\infty}^{\infty}x(\tau)\delta(t-\tau)d\tau$}
%   \psblock(4,1){a}{LTI}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(6,1){y}{$y(t)$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%\end{pspicture}
%\end{center}
%
%\begin{exampleblock}{}
%\begin{align}\nn
%y(t)=\int_{-\infty}^{\infty} x(\tau)h_{\tau}(t) d\tau
%\end{align}
%\end{exampleblock}
%
%}
%
%\frame{
%
%If the system is time-invariant:
%
%\begin{center}
%\begin{pspicture}[](7,2)
%   \pssignal(0,2){x}{$\delta(t)$}
%   \psblock(2,2){a}{LTI}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(5,2){y}{$h_{0}(t)=h(t)$}
%
%    \pssignal(0,1){x1}{$\delta(t-\tau)$}
%   \psblock(2,1){a1}{LTI}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(5,1){y1}{$h_{\tau}(t)=h(t-\tau)$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%   \ncline{x1}{a1}  \ncline{a1}{y1}
%\end{pspicture}
%\end{center}
%
%then, we finally obtain
%
%\begin{center}
%\begin{pspicture}[](7,2)
%   \pssignal(0,1){x}{$\displaystyle x(t)=\int_{-\infty}^{\infty}x(\tau)\delta(t-\tau)d\tau$}
%   \psblock(4,1){a}{LTI}
%   %\psblock(4,1){b}{$h[n], H(z)$}
%   \pssignal(6,1){y}{$y(t)$}
%   %-----------------
%   \psset{arrows=->}
%   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
%\end{pspicture}
%\end{center}
%
%\begin{exampleblock}{}
%\begin{align}\nn
%y(t)=\int_{-\infty}^{\infty} x(\tau)h(t-\tau) d\tau
%\end{align}
%\end{exampleblock}
%
%}

\frame{

\begin{itemize}
\item $h(t)$ is the \textbf{system's response to the impulse $\delta(t)$}.
\item Any LTI system is \textbf{completely defined} by $h(t)$!
\begin{figure}
\centering\includegraphics[scale=0.3]{P10.pdf}
\end{figure}
\end{itemize}

\begin{block}{Continuous-time Convolution}
Given two signals $x(t)$ and $h(t)$:
\begin{align}\nn
\int_{-\infty}^{\infty} x(\tau)h(t-\tau) d\tau=x(t)\ast h(t)
\end{align}
is the continuous-time convolution operation between them.
\end{block}
}

\frame{
\frametitle{Problem 43}

Consider the two signals $x(t)$ and $h(t)$ given by:

\begin{align*}
x(t)=\left\{
\begin{array}{cc}
1 & 0< t< T\\
0 & \text{otherwise}
\end{array}
\right.,
\end{align*}

\begin{align*}
h(t)=\left\{
\begin{array}{cc}
t & 0< t< 2T\\
0 & \text{otherwise}
\end{array}
\right..
\end{align*}
If $h(t)$ is the impulse response of an LTI system,  compute the system's output when $x(t)$ is the input signal.

}

\frame{
\frametitle{Sol.}

\begin{figure}
\centering\includegraphics[scale=0.3]{P10.pdf}
\end{figure}

\begin{align*}
y(t)=\left\{
\begin{array}{cc}
0 & t<0\\
\frac{t^2}{2} & 0\leq t< T\\
tT-\frac{T^2}{2} & T\leq t <2T\\
Tt+\frac{3}{2}T^2-\frac{t^2}{2} &2T\leq t< 3T\\
0 & t\geq3T
\end{array}
\right.,
\end{align*}

}

\section{Properties of the Convolution operation}

%\frame{
%
%\begin{block}{Discrete-time Convolution}
%Given two discrete-time signals $x[n]$ and $h[n]$:
%\begin{align}\nn
%\sum_{k=-\infty}^{\infty}x[k]h[n-k]=x[n]\ast h[n]
%\end{align}
%is the convolution operation between them.
%\end{block}
%
%\begin{block}{Continuous-time Convolution}
%Given two signals $x(t)$ and $h(t)$:
%\begin{align}\nn
%\int_{-\infty}^{\infty} x(\tau)h(t-\tau) d\tau=x(t)\ast h(t)
%\end{align}
%is the continuous-time convolution operation between them.
%\end{block}
%
%}

\frame{
\frametitle{Commutative property}

\begin{exampleblock}{}
\begin{align}\nn
x[n]\ast h[n]=h[n]\ast x[n]
%\Rightarrow \sum_{k=-\infty}^{\infty}x[k]h[n-k]=\sum_{k=-\infty}^{\infty}h[k]x[n-k]
\end{align}
\end{exampleblock}

\textbf{Proof.:}

Given
\begin{align}\nn
x[n]\ast h[n]=\sum_{k=-\infty}^{\infty}x[k]h[n-k]
\end{align}
define $v=n-k$, thus
\begin{align}\nn
x[n]\ast h[n]=\sum_{v=+\infty}^{-\infty}x[n-v]h[v]=\sum_{v=-\infty}^{+\infty}h[v]x[n-v]=h[n]\ast x[n]
\end{align}
}

\frame{
\frametitle{Commutative property}
Therefore,
\begin{center}
\begin{pspicture}[](5,2)
   \pssignal(0,1){x}{$x[n]$}
   \psblock(2,1){a}{$h[n]$}
   %\psblock(4,1){b}{$h[n], H(z)$}
   \pssignal(4,1){y}{$y[n]$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
\end{pspicture}
\end{center}
is equivalent to
\begin{center}
\begin{pspicture}[](5,2)
   \pssignal(0,1){x}{$h[n]$}
   \psblock(2,1){a}{$x[n]$}
   %\psblock(4,1){b}{$h[n], H(z)$}
   \pssignal(4,1){y}{$y[n]$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
\end{pspicture}
\end{center}

\begin{exampleblock}{Commutative property for the continuous-time convolution}
\begin{align}\nn
x(t)\ast h(t)=h(t)\ast x(t)\Rightarrow \int_{-\infty}^{\infty} x(\tau)h(t-\tau) d\tau=\int_{-\infty}^{\infty} h(\tau)x(t-\tau) d\tau
\end{align}
\end{exampleblock}

}

\frame{
\frametitle{Associative property}

\begin{align}\nn
x(t)\ast \Big(h(t) \ast z(t)\Big)=\Big(x(t)\ast h(t)\Big)  \ast z(t)\\\nn\\\nn
x[n]\ast \Big(h[n] \ast z[n]\Big)=\Big(x[n]\ast h[n]\Big)  \ast z[n]
\end{align}
Therefore, the following configurations are equivalent:

\begin{center}
\begin{pspicture}[](8,2)
   \pssignal(0,1){x}{$x(t)$}
   \psblock(2,1){a}{$h_1(t)$}
   \psblock(5,1){b}{$h_2(t)$}
   \pssignal(8,1){y}{$y(t)$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{b}  \ncline{b}{y}
\end{pspicture}
\vspace{0.5cm}

\begin{pspicture}[](7,2)
   \pssignal(0,1){x}{$x(t)$}
   \psblock(3,1){a}{$h_1(t)\ast h_2(t)$}
   %\psblock(4,1){b}{$h[n], H(z)$}
   \pssignal(6,1){y}{$y(t)$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
\end{pspicture}
\vspace{0.5cm}

\begin{pspicture}[](8,2)
   \pssignal(0,1){x}{$x(t)$}
   \psblock(2,1){a}{$h_2(t)$}
   \psblock(5,1){b}{$h_1(t)$}
   \pssignal(8,1){y}{$y(t)$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{b}  \ncline{b}{y}
\end{pspicture}
\end{center}

}

\frame{
\frametitle{Distributive property with respect to the sum}
\begin{align}\nn
x[n]\ast\Big(y[n]+z[n]\Big)=x[n]\ast y[n]+x[n]\ast z[n]\\\nn\\\nn
x(t)\ast\Big(y(t)+z(t)\Big)=x(t)\ast y(t)+x(t)\ast z(t)
\end{align}
The following configurations are equivalent:

\begin{center}
\begin{pspicture}[](6.5,2)
   \pssignal(0,1){x}{$x(t)$}
   \pnode(1,1){b}
   \pnode(1,2){c}
   \pnode(1,0){d}
   \psblock(3,2){e}{$h_1(t)$}
   \psblock(3,0){f}{$h_2(t)$}
   \pnode(5,2){g}
   \pnode(5,0){h}
   \pscircleop(5,1){oplus}
   \pssignal(6.25,1){i}{$y(t)$}
   \ncline{x}{b}  \ncline{b}{c}  \ncline{b}{d}
    \ncline{e}{g} \ncline{f}{h}
   \psset{style=RoundCorners ,style=Arrow}
     \ncline{c}{e} \ncline{d}{f}
   \ncline{g}{oplus}  \ncline{h}{oplus} \ncline{oplus}{i}
\end{pspicture}
\vspace{1cm}

\begin{pspicture}[](5,2)
   \pssignal(-1,1){x}{$x(t)$}
   \psblock(2,1){a}{$h_1(t)+ h_2(t)$}
   %\psblock(4,1){b}{$h[n], H(z)$}
   \pssignal(5,1){y}{$y(t)$}
   %-----------------
   \psset{arrows=->}
   \ncline{x}{a}  \ncline{a}{y}  %\ncline{b}{y}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{Convolution with an impulse signal}

Remember that any signal can be decomposed as a linear combination of an infinite number of unit impulses:

\begin{align}\nn
x[n]&=\sum_{k=-\infty}^{\infty}x[k]\delta[n-k],\\\nn
x(t)&=\int_{-\infty}^{\infty}x(\tau)\delta(t-\tau)d\tau
\end{align}

Therefore
\begin{alertblock}{}
\begin{align}
x[n]\ast \delta[n]=x[n],\nn\\\nn
x(t)\ast \delta(t)=x(t).
\end{align}
\end{alertblock}

}

\frame{
\frametitle{Convolution with a delayed  impulse (discrete-time)}

\begin{align}\nn
x[n]\ast \delta[n-n_0]=\sum_{k=-\infty}^{\infty}x[k]\delta[n-k-n_0]
\end{align}
Define $v=k+n_0$, thus
\begin{align}\nn
x[n]\ast \delta[n-n_0]&=\sum_{v=-\infty}^{\infty}x[v-n_0]\delta[n-v]\\\nn
&=x[n-n_0]\ast \delta[n]=x[n-n_0].
\end{align}

Therefore,
\begin{block}{}
\begin{align}
x[n]\ast \delta[n-n_0]=x[n-n_0],\nn\\\nn
%x(t)\ast \delta(t-t_0)=x(t-t_0).
\end{align}
\end{block}

}

\frame{
\frametitle{Convolution with a delayed  impulse (continuous-time)}

\begin{align}\nn
x(t)\ast \delta(t-t_0)=\int_{\tau=-\infty}^{\infty}x(\tau)\delta(t-\tau-t_0)\text{d}\tau
\end{align}
Define $v=\tau+t_0$ and $x'(t)=x(t-t_0)$, thus
\begin{align*}
x(t)\ast \delta(t-t_0)&=\int_{v=-\infty}^{\infty}x(v-t_0)\delta(t-v)\text{d}v\\
&=\int_{v=-\infty}^{\infty}x'(v)\delta(t-v)\text{d}v=x'(t)\ast \delta(t)=x'(t)
\end{align*}

Therefore,
\begin{block}{}
\begin{align}
x(t)\ast \delta(t-t_0)=x(t-t_0),\nn\\\nn
%x(t)\ast \delta(t-t_0)=x(t-t_0).
\end{align}
\end{block}

}

\end{document}

Slides Systems_Circuits/M_8_LTI Systems and Convolution/LTI_Convolution_First_Part.tex


\section*{Computational Statistical Experiments in \Matlab}\label{S:CompStatExps}

This book is intended as an undergraduate textbook on introductory to intermediate level Computational Statistics. The goal is to equip students with some of the most useful tools in Computational Statistics and the ability to use them effectively. This will be achieved by maintaining balance between theoretical foundation, algorithmic implementation and practical application of these tools. Hands-on experimentation is encouraged by describing algorithms in pseudocode as well as providing functional and self-contained Matlab code, together with plenty of real data examples. The book will adopt a spiral model of instruction that emphasizes motivation, clarity and concreteness of ideas at the introductory level, followed by mathematical formalism, implementational subtleties and extensions at the intermediate level.

\subsection*{Who is this book addressed to?}
This book is a guide for a {\it computational statistical experimenter}.

A {\it statistical experimenter} is a person who conducts a {\it statistical experiment} (for simplicity, from now on, these will be called ``experimenters" and ``experiments").  Roughly, an experiment is an action with an {\it empirically observable outcome} (data) that cannot necessarily be predicted with certainty, in the sense that a {\it repetition} of the experiment may result in a different outcome.  Most quantitative scientists are experimenters if they apply statistical principles to further their current understanding or {\it theory} of an {\it empirically observable real-world phenomenon} (simply, a {\it phenomenon}).  Roughly, `furthering' your understanding or theory is done by improving your mathematical model (\lq rigorous cartoon\rq) of the phenomenon on the basis of its compatibility with the {\it observed data} or {\it outcome} of the experiment.  In this sense, an experimenter attempts to learn about a phenomenon through the outcome of an experiment.  An experimenter is often a scientist or engineer, and vice versa.

Technological advances have fundamentally inter-twined computers with most experiments today.  First, our instrumentational capacity to observe an empirical phenomenon, by means of automated data gathering (sensing) and representation (storage and retrieval), is steadily increasing.  Second, our computational capability to process statistical information or to make decisions using such massive data-sets is also steadily increasing.  Thus, our recent technological advances are facilitating computationally intensive statistical experiments based on possibly massive amounts of empirical observations, in a manner that was not viable a decade ago.  Hence, a successful scientist or engineer in most specialisations today is a {\it computational statistical experimenter}, i.e.~a statistical experimenter who understands the information structures used to represent their data as well as the statistical algorithms used to process their scientific or engineering decisions.  This course is designed to help you take the first steps along this path.

Let us first demonstrate the need for a statistical experiment.  Recall that statistical inference or learning is the process of using observations or data to infer some aspect of the distribution function (DF) that generated it.  A generic question is:
\[
\text{Given realisations from $X_1, X_2, \ldots, X_n \sim$ some unknown DF $F$, how do we infer $F$} ?
\]
Some of the concrete problems involving experiments include:
\begin{itemize}
\item {\bf Simulation:} Often, it is necessary to simulate a random variable (RV) with some specific distribution to gain insight into its features or simulate whole systems, such as the air-traffic queues at Heathrow Airport, to make better management decisions.
\item {\bf Exploration:} This is the art of (visually) exploring the observed data in order to better understand the empirical phenomenon that generated the data.  Visual explorations of simulated data may provide benchmarks against which we may compare and contrast the observed data.
\item {\bf Estimation:}
\begin{enumerate}
\item {\bf Parametric Estimation:} Using samples from some unknown DF $F$ that is parameterised by some unknown $\theta$, we can estimate $\theta$ from a statistic $\widehat{\Theta}_n$ called the estimator of $\theta$ using one of several methods (maximum likelihood, moment estimation or parametric bootstrap).
\item {\bf Non-parametric Estimation of the DF:}  Based on $n$ independent and identically distributed (IID) observations from an unknown DF $F$, we can estimate it under the general assumption that $F$ belongs to the collection all DFs.
\item {\bf Confidence Sets:}  We can obtain a $1-\alpha$ confidence set for the point estimates, of the unknown parameter $\theta$  that belongs to a collection of parameters (also known as parameter space) denoted by $\BB{\Theta}$ or the unknown  DF $F$ that belongs to the collection all DFs denoted by $\{ \text{all DFs} \}$.
\end{enumerate}
\item {\bf Testing:}  Based on observations from some DF $F$ that is hypothesised as belonging to a subset $\BB{\Theta}_0$ of $\BB{\Theta}$ called the space of null hypotheses, we will learn to test (attempt to reject) the falsifiable null hypothesis that $F$ belongs to a smaller collection of parameters denoted by $\BB{\Theta}_0$.  This smaller collection is contained in the parameter space $\BB{\Theta}$.
\item {\bf Learning:}  Supervised, unsupervised learning, classification and regression \ldots
\end{itemize}

\subsection*{What is unique about our book?}

\begin{itemize}
\item Our unified and integrative approach to statistical experiments where mathematical, statistical and computational aspects naturally come together, will provide the student with the fundamentals so that they can pursue new experiments on their own.
\item Current books in the field either focus on a long list of recipes with little cohesive mathematical and algorithmic structure or ignore fundamental aspects of the field such as the rudiments of random number generation.  Our book is specifically designed to have a solid introduction to the fundamentals in the style of classical books in the foundational areas by Luc Devroye, Donald Knuth and Larry Wasserman.
\item Appealing to students of the YouTube generation is a high priority.  Various measures have been taken to keep the book interesting to the current generation of students.  These measures include:
\begin{itemize}
\item Each concept is introduced from a historical perspective to make it more concrete for the student.
\item Kinesthetic learning --- learning by doing with one's hands --- is known to reinforce mathematical and algorithmic concepts.  We have chosen several experiments that allow kinesthetic learning (Galton's quincunx, Galton's dice, Buffon's Needle, etc.).  See the accompanying Laboratory for Mathematical statistical Experiments and the devices that have been built for such purpose at \url{http://www.math.canterbury.ac.nz/~r.sainudiin/lmse}.
\item Computer animations and interactive visualisations of each fundamental concept is provided with \Matlab code.  The instructor and/or the student may use these code demos to teach and/or learn each concept.
\item Our book uses \Matlab as opposed to {\tt R} for computational statistics.  In most engineering curricula, \Matlab is used in almost all other courses.  Therefore, there is a demand for a book in computational statistics in \Matlab in our increasingly data-rich world.
\item All codes for the book will be provided and no additional toolboxes will be needed.  In effect our codes will form a toolbox that will accompany the book.
\item Our examples are taken from physical, social and biological sciences to keep the subject of interest to computational statistical experimenters across all fields of science and engineering.
\item Finally, the most unique aspect of our book is the student projects.  We encourage each student to conduct her/his own experiment, including the original concept formulation, data collection, algorithm design, coding and data analysis and writing a statistical report. Please visit:
\begin{itemize}
\item
\href{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2007.pdf}{\url{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2007.pdf}} \item
\href{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2008.pdf}{\url{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2008.pdf}}
\end{itemize}
to see the term projects completed by budding computational statistical experimenters, students of this course at our home institution.
\end{itemize}
\end{itemize}

\section*{Working Outline of Chapters}
The statistical experiment and the basic objects that make it mathematically definitive and internally consistent are introduced later. The precursor to an experiment is a mathematical model for probability in the `real world' called the {\it probability model}.  This arises from an axiomatic system based on more fundamental objects, such as sample space, outcomes, events, probability, the addition rule for mutually exclusive events, the definition of conditional probability, limits, real number system, and random variables.  These objects in turn build on the set theory, so a refresher in set theory is our first stop.  Then, we will introduce the probability model via the intuition behind this construction, and various useful notions associated with continuous and discrete random variables, such as distribution, density, mass function and expectations.  Commonly encountered random variables will serve as examples.  We will learn to simulate these random variables and visualise the realisations.  We will visit the most elementary limit theorems before conducting our own computational statistical experiments involving estimation, testing and learning.

The Table of Contents from a rough set of lecture notes (our starting point for the final book) are attached below.  We hope to have about 500 pages when we are finished with our book.


\documentclass[11pt, oneside]{article}   	% use "amsart" instead of "article" for AMSLaTeX format

%\usepackage{draftwatermark}
% \SetWatermarkText{Confidential}
% \SetWatermarkScale{5}
% \SetWatermarkLightness {0.85}
% \SetWatermarkColor[rgb]{0.7,0,0}

\usepackage{geometry}                		% See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   		% ... or a4paper or a5paper or ...
%\geometry{landscape}                		% Activate for for rotated page geometryA. G. Barto, R. S. Sutton, and C. W. Anderson
%\usepackage[parfill]{parskip}    		% Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}				% Use pdf, png, jpg, or eps� with pdflatex; use eps in DVI mode
								% TeX will automatically convert eps --> pdf in pdflatex
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage{hyperref}
\usepackage{url}
\usepackage{authblk}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{fixltx2e}
\usepackage{hyperref}
\usepackage{alltt}
\usepackage{color}
\usepackage{stix}
\usepackage{caption}

\newcommand{\argmax}{\operatornamewithlimits{argmax}}
\newcommand{\argmin}{\operatornamewithlimits{argmin}}
\newcommand\independent{\protect\mathpalette{\protect\independenT}{\perp}}
\def\independenT#1#2{\mathrel{\rlap{$#1#2$}\mkern2mu{#1#2}}}
\DeclareMathOperator{\E}{\mathbb{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}

\title{Some Notes on Kolmogorov Complexity, \\  the Algorithmic Markov Condition, and Causality}
\author{David Meyer \\ dmm@\{1-4-5.net,uoregon.edu,...\}}

\date{Last update: \today}							% Activate to display a given date or no date

\begin{document}
\maketitle

\section{Introduction}
\label{sec:intro}
The Kolmogorov complexity of a string $s$, $K(x)$,  is generally be defined as a function from finite binary strings of arbitrary length to the natural numbers
$\mathbb{N}$ \cite{Janzing:2010:CIU:1870245.1870271}.
That is,  $K: \{0,1\}^* \rightarrow  \mathbb{N}$  is a function defined on \emph{objects} represented by binary strings. As we shall see, this definition is can be extended to
other types of objects such as numbers, sets, functions and probability distributions.

\bigskip
\noindent
As a first approximation, $K(x)$ may be thought of as the length of the shortest computer program that prints $x$ and then halts. This computer program may be written in C, Java, Python or any other universal programming language. These are programming languages in which a universal Turing Machine can be implemented. Many if not most  programming languages have this property. Here we will just fix some universal language (say, python) and define Kolmogorov complexity with respect to it. The invariance theorem (Section \ref{sec:invariance_theorem}) tells us that it does not really matter which language we pick.

\bigskip
\noindent
Note that computer programs often make use of data, and frequently such data are encoded inside the program itself. An example is the bitstring "010110..." in the program

\begin{equation}
\label{eqn:print_string}
\text{print }  \text{"}\underbrace{010101\hdots01}_{\text{n times "01"}} \text{"}
\end{equation}

\bigskip
\noindent
So what is the complexity of the program shown in  \ref{eqn:print_string}? We think of the "interpreter" for this program as a \emph{Universal Machine} (in particular, a Universal Turing Machine or UTM),
so all programs have a constant (call it $C$) complexity that accounts for the UTM. What is the Kolmogorov complexity of such a program? Well, since a program to print this string and halt,
call it $p$, might be (in this case, python):

\begin{equation*}
\label{eqn:python_print_string}
\text{print "01" * n}
\end{equation*}

\bigskip
\noindent
Here we notice that we need only $O(\log n)$ bits to represent the number $n$, the number of
times the string "01" is repeated.  So for this program $K(p) = O(\log n)$. Other programs that have
$K(p) = O(\log n)$ include printing out the first $n$ digits of $\pi$ (or $e$). So while the decimal expansion
of $\pi$ might appear to be random, it has low Kolmogorov complexity (in fact,  $O(\log n)$) since the digits of $\pi$ can be
generated by a simple program where the only quantity that
is a function of $n$ is $n$ itself, and we need $O(\log n)$ bits to represent $n$.

\bigskip
\noindent
A bit of notation: We use $x$ to denote finite bitstrings $x \in \mathbb{B}^*$.  Also, let $l(x)$, the length of a given bitstring $x$, be $n$.
That is, $n \coloneqq\ l(x)$.  Boldface $\mathbf{x}$ denotes a (possibly infinite) binary string and $x_{[1:n]}$ is the $n$-bit prefix
of $\mathbf{x}$.

\section{Properties of $K(x)$}
\label{sec:properties}

\subsection{Very Simple Programs}
For very simple objects,  $K(x) = O(\log n)$.  Here $K(x)$ must be small for 'simple' or 'regular' objects $x$. For example, there exists a fixed-size program that,
when input $n$, outputs the first n bits of $\pi$ and then halts (see above). It is easy to see that this specification of $n$ takes $O(\log n)$ bits. Thus, as we saw above,
when $x$ consists of the first $n$ bits of $\pi$, i$K(x) = O(\log n)$.

\bigskip
\noindent
Note values of $n$, $K(x)$ may be quite a bit smaller than $O(\log n)$.
For example, suppose $n = 2^m$ for some $m \in \mathbb{N}$, or said differently, $m = \log n$.  Then we can describe $n$ by first describing $m$.
To do so we need to describe a program implementing the function $f(z) = 2^z$.  Note that the description of the program that computes $f(z)$ takes a constant
number of bits which does not depend on $n$. Now, we know that the description of $m$ takes $O(\log m)$ bits, and we also know that
$m = \log n$,  so if $n$ has this form we get $K(x) = O(\log m) = O(\log \log n)$.

\subsection{Completely Random Objects}
Here $K(x) = n + O(\log n)$.  Here we can think of a code or description method as a binary relation between source words, namely, strings to be encoded and code words
(the encoded versions of these strings). Without loss of generality, we can take the set of code words to be finite binary strings \cite{Cover2006}.  In these notes we'll only consider uniquely
decodable codes where the relation is one-to-one or one-to- many\footnote{We'll need this 1:1 or many:1 property counting arguments.}, indicating that given an encoding
$E(x)$ of string $x$, we can always reconstruct the original $x$. The Kolmogorov complexity of $x$, $K(x)$, can be viewed as the code length of $x$ that results from using the
Kolmogorov code $E^*(x)$: this is the code that encodes $x$ by the shortest program that prints $x$ and halts. Another way to think about this: if $K(x) \geq n$ then $x$ is "Kolmogorov random";
we in this case we might also refer to $x$ as "incompressible". See also \cite{2009arXiv0910.1691J}.

\bigskip
\noindent
One of the key insights relating to Kolmogorov codes is as follows: for any uniquely decodable code, there are no more than $2^n$ strings $x$ which can be described by $n$ bits.
The reason for this is that, as we know, there are no more than $2^n$ binary strings of length $n$. However,  the number of strings that can be described by less than $n$ bits,
that is, $2^0 + 2^1 + 2^2 + \hdots + 2^{n -1} = \sum\limits_{i = 0}^{n -1} 2^i = 2^n - 1$, and $2^n - 1 < 2^n$; thus by the Pigeon Hole Principle there is at least one string of length
$n$ which can not be mapped to a shorter string\footnote{Such a string has $K(X) \geq |x|$, that is, $x$ is not compressible.}.

\bigskip
\noindent
We can use the same argument to observe that  the fraction of strings $x$ of length $n$ with $K(x) < n - k$ is less than $2^{-k}$. Why? We know that
$2^0 + 2^1 + 2^2 + \hdots + 2^{{n -1}} < 2^n$.  Now, suppose that the $2^n$ strings are uniformly distributed (that is, all strings have equal probability,
namely, $2^{-n}$).  Then there are $2^n$ strings of length $n$, and $2^{(n-k)}/2^n = 2^{(n-k-n)} = 2^{-k}$. So no more than $2^{-k}$
of the  $2^n$ strings are compressible.  On the other hand, most of the strings of length $n$, notably the fraction $1-2^{-k}$, aren't compressible
by more than a constant.  Somewhat surprising. Said another way, we know that  $K(x) \geq n - k $ with probability at least $1 - 2^{-k}$.

\bigskip
\noindent
To avoid variability due to encoding, we encode $x$ in a prefix-free manner (see Section \ref{sec:prefix_codes} on prefix codes), which requires
$n + O(\log n)$ bits, rather than $n + O(1)$. Therefore, for all $x$ of length $n$, $K(x) \leq n+O(\log n)$. Now, as we saw
above, except for a fraction of $2^{-k}$ of these $K(x) \geq  n - k$, so we see that for the overwhelming majority of $x$, $K(x) = n + O(\log n)$.

\subsection{Invariance}
\label{sec:invariance_theorem}
It would seem that $K(x)$ depends strongly on what programming language we used in our definition of $K$. However, it turns out that for any two
universal languages $L_1$ and $L_2$

\begin{equation}
|K_1(x) - K_2(x)| \leq C
\end{equation}

\bigskip
\noindent
where $K_1$ and $K_2$ denote the respective complexities of $L_1$ and $L_2$ and $C$ is a constant that depends on
$L_1$ and $L_2$ but not on $x$ or its length. So any two universal machines differ by only a constant.

\subsection{Prefix Codes}
\label{sec:prefix_codes}

Let's start with a bit of notation:  Let $\mathcal{X}$  be some finite or countable set\footnote{Sometimes the symbol $\mathbb{B}$, for "binary" I assume, is used instead of $\mathcal{X}$.}.
As usual, the notation $\mathcal{X}^*$ (or $\mathbb{B}^*$) is used to denote the set of finite
strings or sequences over  $\mathcal{X}$ . For example, we may take $\mathcal{X}^* = \{0,1\}^* = \{\epsilon,0,1,00,01,10,11,000, \hdots\}$, where $\epsilon$ represents the
empty string, that is, the string with no characters. The length $l(x)$ of $x$ is the number of bits in the binary string $x$. For example, $l(010) = 3$ and $l(\epsilon) = 0$. If $x$ is interpreted
as an integer, we get $l(x) = \lfloor log(x + 1) \rfloor$, and for $x \geq 2$,

\begin{equation}
\lfloor \log x \rfloor  \leq l(x) \geq \lceil \log x \rceil
\end{equation}

\bigskip
\noindent
where $\lceil x \rceil$ is the smallest integer larger than or equal to $x$ and $\lfloor x \rfloor$ is the largest integer smaller than or equal to $x$, and $\log$ denotes logarithm to base two.
Kolmogorov theory is  typically be concerned with encoding finite-length binary strings by other finite-length binary strings; binary strings are only used as a convenience since observations in
any alphabet can be so encoded in binary.

\bigskip
\noindent
One of the problems we face with many codes is that in general,  we cannot uniquely recover $x$ and $y$ from their concatenated encoding $E(xy)$. To see this, let $E$ be the identity
mapping ($E(X) = x$). Then $E(11)E(11) = 1111 = E(111)E(1)$.  As we will see below, prefix-free codes are one way to address this problem. First, a bit of notation.

\bigskip
\noindent
A binary string $x$ is a proper prefix of a  binary string $y$ if we can write $y = xz$ for  $z \neq \epsilon$.
 A set $\{x,y, \hdots\}  \subseteq \{0,1\}^*$ is prefix-free if for any pair of distinct elements in the set neither is a
proper prefix of the other. A function $D : \{0,1\}^* \rightarrow N$
defines a prefix-free code if its domain is prefix-free. Decoding a prefix-free code is relatively simple: start at the beginning and decode one code word at a
time. When we come to the end of a code word, we know it is the end, since no code word is the prefix of any other code word in a prefix-free code. Clearly, prefix-free codes
are uniquely decodable: we can always unambiguously reconstruct an outcome from its encoding.  Note that prefix codes are not the only codes with this property; there are uniquely
decodable codes which are not prefix-free. In the next section, we will define Kolmogorov complexity in terms of prefix-free codes.

\subsection{Prefix-free Codes for $\mathbb{N}$}
Suppose we encode each binary string $x = x_1x_2 \hdots x_n$ as

\begin{equation}
\label{eqn:pairing}
\bar{x} = \underbrace{1111 \hdots 1}_{\text{n times}}0 x_1x_2 \hdots x_n
\end{equation}

\bigskip
\noindent
We can see that $\bar{x}$ is prefix-free:  we can determine where the code word $\bar{x}$ ends by reading it from left to right without the need to back up.
$\bar{x}$ has some interesting properties. First,
note $l(\bar{x}) = 2n + 1$; here we have encoded strings in $\mathbb{B}^*$ in a relatively simple prefix-free manner; the cost is doubling the length of the encoding.
There are, however, approaches that give us a much more efficient code. One such approach is to apply the $\bar{x}$ encoding (Equation \ref{eqn:pairing})
to $l(x)$ rather than to $x$. To do so  define $x^\prime = \overline{l(x)}x$, where $l(x)$ is interpreted as a binary string of length
$\log l(x)$, where $\log l(x) = \log n$. Then the code that maps $x$ to $x^\prime$ is a prefix-free code satisfying $\forall x \in \{0,1\}^*, \; l(x^\prime) = n + 2 \log n + 1$ (ignoring rounding errors).
This is because the length of the prefix-free code described  in Equation \ref{eqn:pairing} is $2n + 1$, here $n$ represents the length of $l(x)$, which is $\log n$. So we have the length of
$\overline{l(x)}  = 2 \log n + 1$  so that $l(x^\prime) = l(\overline{l(x)}x) = 2 \log n + 1 + |x| =  n + 2 \log n + 1$.
This code is frequently called the \emph{standard prefix-free code for the natural numbers} and $L_{\mathbb{N}}(x)$ represents the code length of $x$ under this code. That is,
$L_{\mathbb{N}}(x) = l(x^\prime) = n + 2 \log n + 1$.

\newpage
\bibliographystyle{ieeetr}
\bibliography{/Users/dmm/papers/bib/ml}

\end{document}


\documentclass{beamer}
\input{../util/beamerpreamble.tex}

\usepackage{tikz}
\usetikzlibrary{arrows,shapes}

\title[GB13604]{GB13604 - Maths for Computer Science}
\subtitle[]{Lecture 6 -- Counting Part I}
\author[Claus Aranha]{Claus Aranha\\{\footnotesize caranha@cs.tsukuba.ac.jp}}
\institute[COINS]{College of Information Science}
\date[2018-11-14]{2018-11-14\\{\tiny Last updated \today}}

\tikzstyle{vertex}=[circle,fill=black!25,minimum size=10pt,inner sep=0pt]
\tikzstyle{blue vertex}=[circle,fill=blue!100,minimum size=10pt,inner sep=0pt]
\tikzstyle{red vertex}=[circle,fill=red!100,minimum size=10pt,inner sep=0pt]
\tikzstyle{yellow vertex}=[circle,fill=yellow!100,minimum size=10pt,inner sep=0pt]
\tikzstyle{edge} = [draw,thick,-]
\tikzstyle{pedge} = [draw,thick,.]
\tikzstyle{red edge} = [draw, thick,-,red!50]
\tikzstyle{black edge} = [draw, line width=2pt,-,black!20]
\tikzstyle{weight} = [font=\smaller]

\begin{document}

\begin{frame}
  \maketitle

  \begin{center}
    {\smaller This course is based on Mathematics for Computer Science, Spring
    2015, by Albert Meyer and Adam Chlipala, Massachusetts Institute
    of Technology OpenCourseWare.}

    \includegraphics[width=0.2\textwidth]{../img/by-nc-sa}
  \end{center}
\end{frame}

\section{Introduction}

\begin{frame}
  \frametitle{Week 6 and 7 summary}

  {\larger
    {\bf Counting}

    \bigskip

    \begin{itemize}
    \item \structure{Sums and Products}
    \item \structure{Asymptotics}
    \item \alert{Counting with Bijections}
    \item \alert{Repetitions and Binomial Theorem}
    \item \alert{Pigeonhole Principle}
    \end{itemize}
  }
\end{frame}

\section{Sums and Products}

\begin{frame}
  \frametitle{Sums for Children}

  {\larger
    Next two lectures: Talking about \structure{sums} and \structure{counting}.

    \bigskip

    \begin{block}{Middle School Classwork:}
      \begin{equation*}
        89 + 102 + 115 + 128 + \ldots + 440 + 453 + 466
      \end{equation*}
    \end{block}

    \bigskip
    It is said that \alert{Gauss} found a quick solution for this
    problem at \alert{9 years old}:
    \begin{itemize}
    \item \structure{30 numbers, each 13 greater than the previous one.}
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Sums for Children}

  {\larger

    \begin{itemize}
    \item $89 + (89+13) + \ldots + (89+28\times13) +(89+29\times13)$
    \item $F + (F+d) + \ldots + (L-d) + L = A$
    \item<2-> $L + (L-d) + \ldots + (F+d) + F = A$
    \item<3-> \structure{$(F+L) + (F+L) + \ldots + (F+L) + (F+L) = 2A$}
    \end{itemize}

    \bigskip

    \begin{onlyenv}<3->
      \begin{equation}
        A = \frac{(F+L)}{2} \times (\text{\# of terms})
      \end{equation}
    \end{onlyenv}

    \begin{onlyenv}<4->
      \begin{equation}
        A = \frac{(1+n)}{2} \times (n)
      \end{equation}
    \end{onlyenv}

  }
\end{frame}

\begin{frame}
  \frametitle{Why Counting?}

  {\larger

    \begin{itemize}
    \item \structure{Counting} techniques help understand the structure of numbers:
    \item We will study three different sums:
      \begin{itemize}
      \item Arithmetic Sums
      \item Geometric Sums
      \item Harmonic Sums
      \end{itemize}
    \end{itemize}

    \bigskip

    \begin{block}{Motivation for this unit}
      Counting techniques can be used to estimate \structure{upper
        bounds} and \structure{lower bounds} of computer complexity in
      algorithms.
    \end{block}
  }
\end{frame}

\subsection{Geometric Sums}

\begin{frame}
  \frametitle{Arithmetic Sums and Geometric Sums}

  {\larger

    \begin{itemize}
    \item \structure{Arithmetic Sums}: Add a fixed value to each element\\
      $A = (a+0d) + (a+d) + (a+2d) + \ldots + (a+nd)$

      \bigskip

    \item \structure{Geometric Sums}: Multiply a fixed value to each element\\
      $G = (kx^0) + (kx^1) + (kx^2) + \ldots + (kx^n)$
    \end{itemize}

  }

\end{frame}

\begin{frame}
  \frametitle{Closed Form for the Geometric Sum}

  {\larger
    \begin{itemize}
    \item $A = 1 + 2 + 3 + \ldots + n = \frac{n(n+1)}{2}$, $G = 1 + x + x^2 + \ldots + x^n = ?$

      \bigskip

    \item<2-> $G = 1 + x + x^2 + \ldots + x^n$
    \item<2-> $xG = x + x^2 + x^3 + \ldots + x^{n+1}$

      \bigskip

    \item<3-> $G = 1 + x + x^2 + \ldots + x^n$
    \item<3-> $-xG = - x - x^2 - \ldots - x^{n+1}$

    \item<3-> $G-xG = 1 $\hspace{2cm}$-x^{n+1}$
    \item<3-> $G(1-x) = 1 - x^{n+1}$, $G = \frac{1-x^{n+1}}{1-x}$
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Closed Form for the Geometric Sum}

  {\larger
    \begin{itemize}
    \item The {\bf Proof By Induction} can show if a formula is
      correct, but does not show \alert{where the formula comes from}

      \bigskip

    \item The {\bf Perturbation Method} can be used to find
      \structure{closed forms} for sums.
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Infinite Geometric Sums}

  {\larger

    \begin{itemize}
    \item \structure{Geometric Sum}:

      \begin{equation}
        G_n = \frac{1-x^{n+1}}{1-x}
      \end{equation}

    \item \structure{Geometric {\bf Series}}:
      \begin{equation}
        \lim_{n\rightarrow\inf}G_n = \frac{1-\lim_{n\rightarrow\inf}x^{n+1}}{1-x} = \frac{1}{1-x}
      \end{equation}
      \hfill(Provided that $|x| < 1$)
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Application: Calculating Loans}

  {\larger

    {\bf Problem:} I pay you \structure{100\$ in 1 year} if you
    \alert{pay me X\$ now}

    \bigskip

    How do we calculate a fair amount for \alert{X}?
    \begin{itemize}
    \item <2-> My bank will pay me 3\% \structure{interest}\\
      \hfill{\emph{bankrate} b ::= 1.03}
    \item <3-> If I deposit \alert{X\$} now, I will have
      \alert{$b\times X$\$} in 1 year.
    \item <3-> So I will not lose money if $b\times X \geq 100$\\
      \hfill $X \geq 100/1.03 =$ \structure{97.09}
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{The future worth of money}

  {\larger
    \begin{itemize}
    \item 1\$ next year is worth \$0.9709 now.
      \bigskip

    \item \structure{r\$} last year is worth 1 today, if $r ::= 1/b$
      \bigskip

    \item $n$ in two years is worth
    \item $nr$ in one year, and it is worth
    \item $nr^2$ today.
    \end{itemize}

    \vfill

    \begin{center}
      \structure{$n$} paid in $k$ years is \alert{worth $nr^k$ today},
      where \structure{$r ::= 1/\text{bankrate}$}
    \end{center}
  }
\end{frame}

\begin{frame}
  \frametitle{Annuity}

  {\larger
    {\bf Insurance Company}: I will pay you 100\$ for 10 years,
    if you pay me \$Y now. \structure{How much should Y be?}

    \bigskip

    \begin{itemize}
    \item The Insurance company needs:
    \item $100r + 100r^2 + 100r^3 + \ldots + 100r^{10}$
    \item<2-> Insurance = $100r(1+r+r^2+\ldots+r^9)$
    \item<2-> $I = 100r \times \frac{1-r^{10}}{1-r} = 853.02$
    \end{itemize}

  }
\end{frame}

\subsection{Book Stacking}

\begin{frame}
  \frametitle{Problem: Book Stacking}

  \begin{center}
    \includegraphics[width=0.8\textwidth]{../img/bookstack1}
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{Book Stacking Problem}

  {\large

    \begin{itemize}
    \item All books have size 1.
    \item For 1 book: \structure{Center of Mass} is 0.5\\
      \hfill 1-book overhang is 0.5.
      \bigskip
    \item What about $n$ books?

    \end{itemize}

    \begin{center}
      \includegraphics[width=0.6\textwidth]{../img/bookstack2}
    \end{center}
  }
\end{frame}

\begin{frame}
  \frametitle{Book Stacking Problem}
  \begin{center}
    \includegraphics[width=0.6\textwidth]{../img/bookstack3}
  \end{center}

  {\larger $\Delta$-overhang ::= distance between CoM of $n$ books and
    CoM of the $n+1$ book.  }
\end{frame}

\begin{frame}
  \frametitle{$\Delta$-overhang}

  {\larger
    From physics, we know that:
    \begin{equation}
      \Delta = \frac{1}{2(n+1)}
    \end{equation}

    \begin{itemize}
    \item $B_n ::=$ overhang of $n$ books
    \item $B_1 = 1/2$
    \item $B_{n+1} = B_n + \frac{1}{2(n+1)}$
      \bigskip

    \item $B_n = 1/2(1+1/2+1/3+\ldots+1/n)$\\
      \hfill(This is harmonic Sum!)
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{How big can the overhang get?}

  {\larger

    It turns out that by increasing the number of $n$ books, we
    can make $1/2H_n$ as big as necessary.

    \vfill

    So there is no upper limit for the size of the overhang, if you
    position the books \alert{very carefully} and know integrals.
  }
\end{frame}

%% TODO: Include slides about Integral Theorem (for upper and lower limits)
%% TODO: Include slides about calculating bounds for Factorials

\section{Asymptotics}

\begin{frame}
  \begin{center}
    {\huge
      Asymptotics
    }
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Study of Asymptotics}

  {\larger
    \begin{itemize}
    \item How fast do expressions grow?
      \bigskip

    \item What is the maximum/minimum size of an expression?
      \bigskip

    \item How can we compare two expressions as they get more complex?
    \end{itemize}

    \bigskip

    We will look at \structure{four notations} that describe the
    relationship between the {\bf growth of functions}.

  }
\end{frame}

\begin{frame}
  \frametitle{Asymptotic Equivalence}

  {\larger

    Def: \structure{$f(n) \sim g(n)$}: \hfill(f(n) is asymptotically equal to g(n))

    \bigskip

    \begin{equation}
      \lim_{n\to\infty}\frac{f(n)}{g(n)} = 1
    \end{equation}

    {\bf Example:}
    \begin{itemize}
    \item $n^2 \sim n^2+n$ because...
    \item $\lim_{n\to\infty}\frac{n^2+n}{n} = \lim_{n\to\infty} \frac{1}{n} + 1 = 1$
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Asymptotic Equivalence}

  {\larger

    {\bf Lemma:} $\sim$ is symmetric \hfill($f \sim g \implies g\sim f$)

    \begin{itemize}
    \item Let $f \sim g$ be true, is $g \sim f$ true too?
    \item $\lim \frac{g}{f} = \lim \frac{1}{(f/g)}$
    \item $\lim \frac{1}{(f/g)} = \frac{1}{\lim(f/g)} = \frac{1}{1} = 1$
    \item $f \sim g \implies g\sim f$\hfill$\blacksquare$
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Asymptotic Equivalence}

  {\larger

    {\bf Transitivity:} Suppose $f \sim g$ and $g \sim h$. Prove $f \sim h$

    \bigskip

    \begin{equation}
      1 = \lim\frac{f}{g} = \lim\frac{f/h}{g/h} = \frac{\lim(f/h)}{\lim(g/h) = 1} = \lim\frac{f}{h}
    \end{equation}

    \vfill

    {\bf Colorary:} $\sim$ is an \structure{equivalence relation}

    \begin{itemize}
    \item \alert{important} $sim$ is a relation \alert{on functions}
    \item $f(n) \sim g(n)$ does not care about \structure{particular
      values} of f(n) or g(n). ($f \sim g$)
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Little Oh: $o(\cdot)$ -- Asymptotically Smaller}

  {\larger
    {\bf Definition:} $f(n) = o(g(n))\iff \lim_{n\to\infty}\frac{f(n)}{g(n)} = 0$

    \bigskip

    {\bf Example:}
    \begin{itemize}
    \item $n^2 = o(n^3)$
    \item $\lim_{n\to\infty}\frac{n^2}{n^3} = \lim_{n\to\infty}\frac{1}{n} = 0$
    \end{itemize}

    \bigskip

    {\bf Lemma:} $o(\cdot)$ defines a \structure{strict partial order}
  }
\end{frame}

\begin{frame}
  \frametitle{Big Oh: $O(\cdot)$ -- Asymptotic Order of Growth}

  {\larger
    {\bf Definition:} $f(n) = O(g(n))$
    \begin{equation}
      \limsup_{n\to\infty}\frac{f(n)}{g(n)} < \infty
    \end{equation}

    \vfill

    \begin{itemize}
    \item The limit of f(n)/g(n) is \structure{finite}.
    \item Could be 0, could be 1, could be something else.
    \item Why \structure{limsup} not \structure{lim}? Ignore this for now.

      \bigskip

    \item {\bf Example}: $3n^2 = O(n^2)$ because $\lim\frac{3n^2}{n^2} = \lim3 = 3$
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Why do we like $O(\cdot)$ so much?}

  {\larger
    \begin{itemize}
    \item What O() means is that \structure{constant factors} don't matter.
    \item Only \structure{rate of growth} matters.

      \bigskip

    \item When we talk about {\bf execution time}, if the hardware changes,
      only a \structure{constant factor} changes.

      \bigskip

    \item Slow algorithms will still be slow (for bigger data!) even if
      the hardware changes.
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Theta: $\Theta(\cdot)$ -- Same Order of Growth}

  {\larger

    {\bf Definition}:
    \begin{equation}
      f = \Theta(g) \iff f = O(g) \land g = O(f)
    \end{equation}

    \bigskip

    {\bf Lemma}: $\Theta$ is an equivalence relation.
  }

\end{frame}

\begin{frame}
  \frametitle{Asymptotics: Intuitive Summary}

  {\larger

    \begin{itemize}
    \item $f\sim g$ \hfill f and g nearly equal;
      \bigskip

    \item $f = o(g)$ \hfill f much less than g;
      \bigskip

    \item $f = O(g)$ \hfill f is about $\leq$ g;
      \bigskip

    \item $f = \Theta(g)$ \hfill f is about equal to g;
    \end{itemize}

  }
\end{frame}

\subsection{Asymptotic Properties}

\begin{frame}
  \frametitle{Asymptotic Properties}

  {\larger

    \begin{itemize}
    \item If $f \sim g$ or $f = o(g)$ then $f = O(g)$
      \bigskip

    \item If $f = o(g)$ then $g \neq O(f)$\\
      \hfill ($\lim f/g = 0 \implies \lim g/f = \infty$)
      \bigskip
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Big Oh $O(\cdot)$ and limsup}

  {\larger
    {\bf Alternate Definition:} $f(n) = O(g(n))$

    \begin{equation}
      \exists c,n_0, \forall n \geq n_0, f(n) \leq c\cdot g(n)
    \end{equation}

    \begin{center}
      \includegraphics[width=0.6\textwidth]{../img/bigOh}
    \end{center}

  }
\end{frame}

\begin{frame}
  \frametitle{Big Oh $O(\cdot)$ and limsup}

  {\larger
    \begin{itemize}
    \item Why is \structure{limsup} necessary in the first definition?
    \item Suppose $f \leq 2g$ then $f = O(g)$ but $f/g$ has no limit.
      \bigskip

    \item {\bf Example:} $f(n) = (1+\sin^2(\frac{n\pi}{2}))\times g(n)$
      \bigskip

    \item The \structure{limit} of $f/g$ in this expression alternates
      between 1 and 2.
    \item On the other hand, the \structure{limsup} of $f/g$ is 2.
    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{A few more facts about asymptotics}

  {\larger

    \begin{itemize}
    \item {\bf Lemma:} $x^a = o(x^b)$ if a < b\\
      \hfill because $\lim\frac{x^a}{x^b} = \lim\frac{1}{x^{a-b}}$ and $a-b > 0$
      \bigskip

    \item {\bf Lemma:} $\ln(x) = o(x^\epsilon)$ for $\epsilon > 0$\\
      \hfill (logarithms grow slower than roots)
    \end{itemize}

    {\bf Proof:}
    \begin{itemize}
    \item $1/y \leq y$ for $y \geq 1$, so $\int_1^z \frac{1}{y}dy \leq \int_1^z ydy$
    \item $\ln(z) \leq \frac{z^2}{2}$ for $z \geq 1$, so let $z =
      \sqrt{x^\delta}$, (for some $\delta > 0$)
    \item $\frac{\delta\ln(x)}{2} \leq \frac{x^\delta}{2}$, but
      $x^\delta = o(x^\epsilon)$ for $\delta < \epsilon$
    \item So $\delta\ln(x) = o(x^\epsilon)$\hfill $\blacksquare$
    \end{itemize}

  }
\end{frame}

\begin{frame}
  \frametitle{Asymptotic Blunders -- Be careful!}

  {\larger

    \begin{itemize}
    \item ``$\cdot = O(\cdot)$'' defines a \structure{binary relation}.\\
      \hfill \alert{Do not write O(x) = x!!}
      \begin{itemize}
      \item If $x = O(x)$ and $O(x) = x$...
      \item But $2x = O(x)$ and $O(x) = x$... so $2x = x$?????
      \end{itemize}

      \bigskip

    \item Big Oh is \structure{not a lower bound}.\\
      \hfill \alert{Do not write: ``f is at least $O(n^2)$''}
      \begin{itemize}
      \item If you want to say that $n^2$ is a \structure{lower bound} of f...
      \item $n^2 = O(f)$
      \end{itemize}

    \end{itemize}
  }
\end{frame}

\begin{frame}
  \frametitle{Asymptotic Blunders -- Be careful!}

  {\larger
    \begin{itemize}
    \item {\bf False Proof:} $\sum^n_{i=1}i = O(n)$\\
      \hfill(We know that $\sum^n_{i=1}i = n(n+1)/2)$
      \bigskip

    \item Any constant is O(1): $0 = O(1), 1 = O(1), 2 = O(1) \ldots$
    \item So, $i = O(1)$
    \item So, $\sum^n_{i=1}i = O(1) + O(1) + O(1) + O(1) \ldots$
    \item So, $\sum^n_{i=1}i = nO(1) = O(n)$\hfill (???)
    \end{itemize}

    \vfill

    \alert{$O(\cdot)$ is not a quantity! Do not do arithmetic with it!}

  }
\end{frame}

\section{Conclusion}
\begin{frame}
  \frametitle{End of Class -- Class Summary}

  {\larger

    \begin{itemize}
    \item Sums: Arithmetic, Geometric, Harmonic, and Closed Forms

      \bigskip
    \item Asymptotic Notation: $\sim$, o(), O(), $\Theta$()
    \end{itemize}

  }
\end{frame}

\end{document}


%
% body.tex
%
% Copyright © 2020 Libao Jin <jinlibao@outlook.com>
% Distributed under terms of the MIT license.
%
\ifx\undefined\sol
Submissions without signature will not be considered. All your work and this signed page should be together as one PDF file.
\vspace{3em}
\else
{ \large
\textbf{Honesty Clause: {\color{red}I hereby testify that all the work on this project is \underline{solely my own}. I have complied with the academic honesty guidelines as stated in the University regulations.}}

% \special{pdf:ann width 3in height 1in
%   <<
%     /T (My Signature)
%     /Subtype /Widget
%     /FT /Sig
%     /F 4
%     /Q 1
%     /MK << /BC [] >>
%   >>
% }

\begin{Form}
\TextField[width=21em,
           name = print.name,
           format = {
               var f = this.getField('print.name');
               % f.textFont = 'Verdana';
               % f.strokeColor = ['T'];
               % f.fillColor = ['T'];
               f.userName = 'Print Name'
               },
           value = ,
           charsize = 12pt]
          {Name: }
\end{Form}
\begin{Form}
\TextField[width=10em,
           format   ={AFDate_FormatEx("mm/dd/yyyy");},
           keystroke={AFDate_KeystrokeEx("mm.dd.yyyy");},
           value = ,
           charsize = 12pt]
           {Date: }
\end{Form}
}
\vspace{5em}
\fi

\textbf{Instruction}

\begin{enumerate}[label={\arabic*.}]
  \item Go to \url{https://www.overleaf.com} and sign in (required).
  \item Open \href{https://www.overleaf.com/read/jkcmttjccbdc}{template}, click \emph{Menu} (up left corner), then \emph{Copy Project}.
  \item Go to \verb|LaTeX/meta.tex| (the file \verb|meta.tex| under the folder \verb|LaTeX|) to change the section and your name, e.g.,
    \begin{itemize}
      \item change author to \verb|\author{Carl F. Gauss}|
    \end{itemize}
  \item You need to write function/script files, store results to output/plot files. Here are suggested names for function files, script files, output files, and plot files:
    \begin{table}[!hbtp]
      \centering
      \begin{tabular}{cllll}
        \toprule
        Problem & Function File                    & Script File                   & Output File         & Plot File               \\
        \midrule
        1       & \verb|cubic_spline.m|            & \verb|final_p1.m|             & \verb|final_p1.txt| & \verb|final_p1a.pdf|    \\
        1       &                                  &                               &                     & \& \verb|final_p1b.pdf| \\
        2       & \verb|backward_euler.m|       & \verb|final_p2.m|             &                     & \verb|final_p2.pdf|     \\
        3       & \verb|golden_section.m|          & \verb|final_p3.m|             & \verb|final_p3.txt| & \verb|final_p3.pdf|     \\
        3       & \& \verb|successive_parabolic.m| &                               &                     &                         \\
        \bottomrule
      \end{tabular}
    \end{table}

    Once finished, you need to upload these files to the folder \verb|src| on Overleaf. If you have different filenames, please update the filenames in \verb|\lstinputlisting{../src/your_script_name.m}| accordingly. You can code in the provided files in \href{https://libaoj.in/courses/2021s/MATH3340/final.zip}{final.zip}, and use the MATLAB script \verb|save_results.m| to generate the output files and store the graphs to \verb|.pdf| files automatically (the script filenames should be exactly same as listed above).
  \item Recompile, and download the generated \verb|.pdf| file.
  \item \textbf{\color{red}{Important}}: \underline{Enter your name and the date} in the above boxes \emph{before} you submit it on WyoCourses.
\end{enumerate}

\ifx\undefined\sol
\newpage
\fi

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Problem 1
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Problem 1}%
\label{sec:problem_1}
Consider the data in the following table:
\begin{table}[!hbtp]
  \centering
  \begin{tabular}{ccccc}
    \toprule
    $k$     & $0$   & $1$          & $2$          & $3$         \\
    \midrule
    $x_{k}$ & $0.0$ & $1.761062$   & $3.522123$   & $5.283185$  \\
    $y_{k}$ & $1.0$ & $-0.1891196$ & $-0.9284676$ & $0.5403023$ \\
    \bottomrule
  \end{tabular}
\end{table}

The values of $y$ in this table have been obtained as $y_{k} = \cos(x_{k})$. Your goal will be to create two different spline interpolants using these data points, and compare them with the original function.

\begin{enumerate}[label=(\alph*)]
  \item \label{enum:1a} Compute the usual spline interpolant $^{1} S(x)$ that uses the natural boundary conditions. Plot this interpolant versus the original function $\cos(x)$ using $100$ data points equally spaced between $x_{0}$ and $x_{3}$; make sure to also indicate the four data points on the plot using a special marker (you may use $\ast$ or $\circ$ for example).
  \item You will probably agree that the comparison at point \ref{enum:1a} above doesn't look very good. This is due to the fact that the natural boundary conditions do not match the behavior of the $\cos(x)$ function. Your task for this point is to modify your code to account for the correct second derivatives at the end points. In other words, create a new interpolant $^{2} S(x)$ that satisfies the following conditions:
    \begin{equation*}
      ^{2} S''(x_{0}) = -\cos(x_{0});
      \quad
      ^{2} S''(x_{3}) = -\cos(x_{3}).
    \end{equation*}
    You can do this by modifying the system of equations for the spline coefficients accordingly. Plot again the newly-obtained interpolant versus the original function in the same manner as above. Turn in your codes together with the two plots.
\end{enumerate}
\begin{solution}
  \quad
  \begin{itemize}
    \item Output file \verb|final_p1.txt|:
      \lstinputlisting[style=Plain]{../src/final_p1.txt}
    \item Plot files \verb|final_p1a.pdf| and \verb|final_p1b.pdf|:
      \begin{figure}[!hbtp]
        \centering
        \includegraphics[width=0.8\textwidth]{../src/final_p1a.pdf}
        \includegraphics[width=0.8\textwidth]{../src/final_p1b.pdf}
        \caption{Cubic Spline With Different Boundary Conditions}
        \label{fig:p1}
      \end{figure}
    \newpage
    \item Function file \verb|cubic_spline.m|:
      \lstinputlisting[style=MATLAB]{../src/cubic_spline.m}
    \item Script file \verb|final_p1.m|:
      \lstinputlisting[style=MATLAB]{../src/final_p1.m}
  \end{itemize}
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Problem 2
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Problem 2}%
\label{sec:problem_2}
The implicit, backward Euler method for the general first order initial value problem
\begin{equation*}
  \frac{du}{dt} = f(u, t), \quad u(t_{0}) = u_{0}
\end{equation*}
is given by
\begin{equation*}
  u_{n + 1} = u_{n} + (t_{n + 1} - t_{n}) f(u_{n + 1}, t_{n + 1}), \quad n = 0, 1, \ldots
\end{equation*}
Use this method to solve numerically the differential equation
\begin{equation*}
  \frac{du}{dt} = 2 + \sqrt{u - 2t + 3}
\end{equation*}
subject to the initial condition $u(0) = 1$. Use a constant time step $\Delta t = t_{n + 1} - t_{n} = 0.05$ and advance the solution to $t = 2$. Turn in the code and the plot of $u(t)$ versus $t$. On the plot, compare your numerical solution with the exact solution $u(t) = 1 + 4t + t^{2} / 4$. Use markers to highlight the points produced by the numerical solution.

\textbf{NOTE:} You will need to solve a nonlinear equation at each time step. Solutions that do not perform this task as required will not receive credit.
\begin{solution}
  \quad
   \begin{itemize}
    \item Plot file \verb|final_p2.pdf|:
      \begin{figure}[!hbtp]
        \centering
        \includegraphics[width=0.8\textwidth]{../src/final_p2.pdf}
        \caption{Solution to $du / dt = 2 + \sqrt{u - 2t + 3}$ using Implicit Backward Euler Method}
        \label{fig:}
      \end{figure}
    \item Function file \verb|backward_euler.m|:
      \lstinputlisting[style=MATLAB]{../src/backward_euler.m}
    % UNCOMMENT THE NEXT TWO LINES IF YOU HAVE OTHER AUXILLIARY FUNCTIONS
    % \item Function file \verb|auxilliary_function.m|:
    %   \lstinputlisting[style=MATLAB]{../src/auxilliary_function.m}
    \item Script file \verb|final_p2.m|:
      \lstinputlisting[style=MATLAB]{../src/final_p2.m}
  \end{itemize}
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Problem 3
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Problem 3}%
\label{sec:problem_3}
Use both successive parabolic interpolation and the golden section search method to find the minimum of the function $f(x) = |x^{2} - 2| + |2x + 3|$ on the interval $[-4, 0]$ with a tolerance of $10^{-8}$ and an identical accuracy. You are again requested to do this with MATLAB codes which you must show, together with the output; recall that golden search method code was discussed in detail in class. For parabolic interpolation, remember that some approximation steps may produce points that are not feasible and must be replaced. Write your code such that it reverts to the golden section search in that case.

\begin{solution}
  \quad
  \begin{itemize}
    \item Output file \verb|final_p3.txt|:
      \lstinputlisting[style=Plain]{../src/final_p3.txt}
    \item Plot file \verb|final_p3.pdf|:
      \begin{figure}[!hbtp]
        \centering
        \includegraphics[width=0.8\textwidth]{../src/final_p3.pdf}
        \caption{Minimum Obtained by Successive Parabolic Interpolation and Golden Section Search Method}
        \label{fig:}
      \end{figure}
    \item Function file \verb|successive_parabolic.m|:
      \lstinputlisting[style=MATLAB]{../src/successive_parabolic.m}
    \item Function file \verb|golden_section.m|:
      \lstinputlisting[style=MATLAB]{../src/golden_section.m}
    \item Script file \verb|final_p3.m|:
      \lstinputlisting[style=MATLAB]{../src/final_p3.m}
  \end{itemize}
\end{solution}


\documentclass{article}

\usepackage[T1]{fontenc}
\usepackage[osf]{libertine}
\usepackage[scaled=0.8]{beramono}
\usepackage[margin=1.5in]{geometry}
\usepackage{url}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{nicefrac}
\usepackage{microtype}

\usepackage{sectsty}
\sectionfont{\large}
\subsectionfont{\normalsize}

\usepackage{titlesec}
\titlespacing{\section}{0pt}{10pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}
\titlespacing{\subsection}{0pt}{5pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}
\titlespacing{\subsubsection}{0pt}{5pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}

\setlength{\parindent}{0pt}
\setlength{\parskip}{1ex}

\newcommand{\acro}[1]{\textsc{\MakeLowercase{#1}}}
\newcommand{\given}{\mid}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\data}{\mc{D}}
\newcommand{\intd}[1]{\,\mathrm{d}{#1}}
\newcommand{\inv}{^{-1}}
\newcommand{\R}{\mathbb{R}}

\DeclareMathOperator{\var}{var}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}

\begin{document}

\section*{Point estimation}

Suppose we are interested in the value of a parameter $\theta$, for
example the unknown bias of a coin.  We have already seen how one may
use the Bayesian method to reason about $\theta$; namely, we select a
likelihood function $p(\data \given \theta)$, explaining how observed
data $\data$ are expected to be generated given the value of $\theta$.
Then we select a prior distribution $p(\theta)$ reflecting our initial
beliefs about $\theta$.  Finally, we conduct an experiment to gather
data and use Bayes' theorem to derive the posterior $p(\theta \given
\data)$.

In a sense, the posterior contains all information about $\theta$ that
we care about.  However, the process of inference will often require
us to use this posterior to answer various questions.  For example, we
might be compelled to choose a single value $\hat{\theta}$ to serve as
a \emph{point estimate} of $\theta$.  To a Bayesian, the selection of
$\hat{\theta}$ is a \emph{decision,} and in different contexts we
might want to select different values to report.

In general, we should not expect to be able to select the true value
of $\theta$, unless we have somehow observed data that unambiguously
determine it.  Instead, we can only hope to select an estimate that is
``close'' to the true value.  Different definitions of ``closeness''
can naturally lead to different estimates. The Bayesian approach to
point estimation will be to analyze the impact of our choice in terms
of a \emph{loss function,} which describes how ``bad'' different types
of mistakes can be.  We then select the estimate which appears to be
the least ``bad'' according to our current beliefs about $\theta$.

\section*{Decision theory}

As mentioned above, the selection of an estimate $\hat{\theta}$ can be
seen as a decision.  It turns out that the Bayesian approach to
decision theory is rather simple and consistent, so we will introduce
it in an abstract form here.

A decision problem will typically have three components.  First, we
have a \emph{parameter space} (also called a \emph{state space})
$\Theta$, with an unknown value $\theta \in \Theta$.  We will also
have a \emph{sample space} $\mc{X}$ representing the potential
observations we could theoretically make.  Finally, we have an
\emph{action space} $\mc{A}$ representing the potential actions we may
select from.  In the point estimation problem, the potential actions
$\hat{\theta}$ are exactly those in the parameter space, so $\mc{A} =
\Theta$.  This might not always be the case, however.  Finally, we
will have a likelihood function $p(\data \given \theta)$ linking
potential observations to the parameter space.

After conducting an experiment and observing data, we are compelled to
select an action $a \in \mc{A}$.  We define a (deterministic)
\emph{decision rule} as a function $\delta\colon \mc{X} \to \mc{A}$
that selects an action $a$ given the observations $\data$.\footnote{We
  may also consider \emph{randomized} decision rules, where $\delta$
  maps observed data $\data$ to a probability distribution over
  $\mc{A}$, which we select a sample from.  This can be useful, for
  example, when facing an intelligent adversary.  Most of the
  expressions we will derive can be derived analogously for this case;
  however, we will not do so here.}  In general, this decision rule
can be any arbitrary function.  How do we select which decision rule
to use?

To guide our selection, we will define a \emph{loss function,} which
is a function $L\colon \Theta \times \mc{A} \to \R$.  The value
$L(\theta, a)$ summarizes ``how bad'' an action $a$ was if the true
value of the parameter was revealed to be $\theta$; larger losses
represent worse outcomes.  Ideally, we would select the action that
minimizes this loss, but unfortunately we will never know the exact
value of $\theta$; complicating our decision.

As usual, there are two main approaches to designing decision rules.
We begin with the Bayesian approach.

\subsection*{Bayesian decision theory}

The Bayesian approach to decision theory is straightforward.  Given
our observed data $\data$, we find the posterior $p(\theta \given
\data)$, which represents our current belief about the unknown
parameter $\theta$.  Given a potential action $a$, we may define the
\emph{posterior expected loss} of $a$ by averaging the loss function
over the unknown parameter:
\begin{equation*}
  \rho\bigl(p(\theta \given \data), a\bigr)
  =
  \mathbb{E}\bigl[L(\theta, a) \given \data\bigr]
  =
  \int_{\Theta}
  L(\theta, a) p(\theta \given \data) \intd{\theta}.
\end{equation*}
Because the posterior expected loss of each action $a$ is a scalar
value, it defines a total order on the action space $\mc{A}$.  When
there is an action minimizing the posterior expected loss, it is the
natural choice to make:
\begin{equation*}
  \delta^\ast(\data) =
  \argmin_{a \in \mc{A}}
  \rho\bigl(p(\theta \given \data), a \bigr),
\end{equation*}
representing the action with the lowest expected loss, given our
current beliefs about $\theta$.  Note that $\delta^\ast(\data)$ may
not be unique, in which case we can select any action attaining the
minimal value.  Any minimizer of the posterior expected loss is called
a \emph{Bayes action.}  The value $\delta^\ast(\data)$ may also be
found by solving the equivalent minimization problem
\begin{equation*}
  \delta^\ast(\data) =
  \argmin_{a \in \mc{A}}
  \int_{\Theta}
  L(\theta, a)
  p(\data \given \theta) p(\theta) \intd{\theta};
\end{equation*}
the advantage of this formulation is that it avoids computing the
normalization constant $p(\data) = \int p(\data \given \theta)
p(\theta) \intd{\theta}$ in the posterior.

Notice that a Bayes action is tied to a particular set of observed
data $\data$.  This does not limit its utility terribly; after all, we
will always have a particular set of observed data at hand when making
a decision.  However we may extend the notion of Bayes actions in a
natural way to define an entire decision rule.  We define the
\emph{Bayes rule,} a decision rule, by simply always selecting a (not
necessarily unique) Bayes action given the observed data.  Note that
the second formulation above can often be minimized even when
$p(\theta)$ is not necessarily a probability distribution (such priors
are called \emph{improper} but are often encountered in practice).  A
decision rule derived in this way from an improper prior is called a
\emph{generalized Bayes rule.}

In the case of point estimation, the decision rule $\delta$ may be
more naturally written $\hat{\theta}(\data)$.  Then, as above, the
point estimation problem reduces to selecting a loss function and
deriving the decision rule $\hat{\theta}$ that minimizes the expected
loss at every point.  A decision rule that minimizes posterior
expected loss for every possible set of observations $\data$ is called
a \emph{Bayes estimator.}

We may derive Bayes estimators for some common loss functions.  As an
example, consider the common loss function
\begin{equation*}
  L(\theta, \hat{\theta})
  =
  (\theta - \hat{\theta})^2,
\end{equation*}
which represents the squared distance between our estimate and the
true value.  For the squared loss, we may compute:
\begin{align*}
  \mathbb{E}\bigl[L(\theta, \hat{\theta}) \given \data \bigr]
  &=
  \int
  (\theta - \hat{\theta})^2
  p(\theta \given \data)
  \intd{\theta}
  \\
  &=
  \int
  \theta^2
  p(\theta \given \data)
  \intd{\theta}
  -
  2
  \hat{\theta}
  \int
  \theta
  p(\theta \given \data)
  \intd{\theta}
  +
  \hat{\theta}^2
  \int
  p(\theta \given \data)
  \intd{\theta}
  \\
  &=
  \int
  \theta^2
  p(\theta \given \data)
  \intd{\theta}
  -
  2
  \hat{\theta}
  \mathbb{E}[\theta \given \data]
  +
  \hat{\theta}^2
  .
\end{align*}
We may minimize this expression by differentiating with respect to
$\hat{\theta}$ and equating to zero:
\begin{equation*}
  \frac{\partial \mathbb{E}\bigl[L(\theta, \hat{\theta}) \given \data \bigr]}
       {\partial \hat{\theta}}
  =
  -2\mathbb{E}[\theta \given \data]
  +
  2\hat{\theta}
  =
  0,
\end{equation*}
from which we may derive $\hat{\theta} = \mathbb{E}[\theta \given
  \data]$.  Examining the second derivative, we see
\begin{equation*}
  \frac{\partial^2\mathbb{E}\bigl[L(\theta, \hat{\theta}) \given \data \bigr]}
       {\partial \hat{\theta}^2}
  = 2 > 0,
\end{equation*}
so this is indeed a minimum.  Therefore we have shown that the Bayes
estimator in the case of squared loss is the posterior mean
$\hat{\theta}(\data) = \mathbb{E}[\theta \given \data].$

A similar analysis shows that the Bayes estimator for the absolute
deviation loss $L(\theta, \hat{\theta}) = \lvert \theta - \hat{\theta}
\rvert$ is the posterior median, and the Bayes estimators for a
relaxed 0--1 loss:
\begin{equation*}
  L(\theta, \hat{\theta}; \varepsilon)
  =
  \begin{cases}
    0 & \lvert \theta - \hat{\theta} \rvert   < \varepsilon; \\
    1 & \lvert \theta - \hat{\theta} \rvert \geq \varepsilon,
  \end{cases}
\end{equation*}
converge to the posterior mode for small $\varepsilon$.

The posterior mode, also called the \emph{maximum a posteriori}
(\acro{MAP}) estimate of $\theta$ and written
$\hat{\theta}_{\text{\acro{MAP}}}$, is a rather common estimator used
in practice.  The reason is that optimization is almost always easier
than integration.  In particular, we may find the \acro{MAP} estimate
by maximizing the \emph{unnormalized} posterior
\begin{equation*}
  \hat{\theta}_{\text{\acro{MAP}}}
  =
  \argmax_\theta
  p(\data \given \theta)p(\theta),
\end{equation*}
where we have avoided computing the normalization constant $p(\data) =
\int p(\data \given \theta) p(\theta) \intd{\theta}.$ An important
caveat is that the \acro{MAP} estimator is not invariant to nonlinear
transformations of $\theta$!

\subsection*{Frequentist decision theory}

The frequentist approach to decision theory is somewhat different.  As
usual, in classical statistics it is not allowed to place a prior
distribution on a parameter such as $\theta$; rather, it is much more
common to use the likelihood $p(\data \given \theta)$ to hypothesize
what data might look like for different values of $\theta$, and use
this analysis to drive your action.

The frequentist approach to decision theory involves the notion of
\emph{risk functions.}  The \emph{frequentist risk} of a decision
function $\delta$ is defined by
\begin{equation*}
  R(\theta, \delta)
  =
  \int_{\mc{X}}
  L\bigl(\theta, \delta(\data)\bigr)
  p(\data \given \theta)
  \intd{\data},
\end{equation*}
that is, it represents the expected loss incurred when repeatedly
using the decision rule $\delta$ on different datasets $\data$ as a
function of the unknown parameter $\theta$.

To a Bayesian, the frequentist risk is a very strange notion: we know
the exact value of our data $\data$ when we make our decision, so why
should we average over other datasets that we haven't seen?  The
frequentist counterargument to this is typically that we might know
$\data$ but can't know $p(\theta)$!

Notice that whereas the posterior expected loss was a scalar defining
a total order on the action space $\mc{A}$, which could be extended to
naturally define an entire decision rule, the frequentist risk is a
function of $\theta$ and the entire decision rule $\delta$.  It is
very unusual for there to be a single decision rule $\delta$ that
works the best for every potential value of $\theta$.  For this
reason, we must decide on some mechanism to use the frequentist risk
to select a ``good'' decision rule.  There are many proposed
mechanisms for doing so, but we will simply quickly describe two
below.

\subsubsection*{Bayes risk}

One solution to the problem of comparing decision rules is to place a
prior distribution on the unknown $\theta$ and compute the average
risk:
\begin{equation*}
  r\bigl(p(\theta), \delta\bigr)
  =
  \mathbb{E}\bigl[R(\theta, \delta)\bigr]
  =
  \int_{\Theta}
  R(\theta, \delta)
  p(\theta)
  \intd{\theta}
  =
  \int_{\Theta}
  \int_{\mc{X}}
  L\bigl(\theta, \delta(\data)\bigr)
  p(\data \given \theta)
  p(\theta)
  \intd{\theta}
  \intd{\data}.
\end{equation*}
The function $r\bigl(p(\theta), \delta\bigr)$ is called the
\emph{Bayes risk} of $\delta$ under the prior $p(\theta)$.  Again, the
Bayes risk is scalar-valued, so we induce a total order on all
decision rules, making identifying a unique decision rule easier.  Any
$\delta$ minimizing Bayes risk is called a \emph{Bayes rule.}  We have
seen this term before!  It turns out that given a prior $p(\theta)$,
the Bayesian procedure described above for defining a decision
function by selecting an action with minimum posterior expected loss
is guaranteed to minimize Bayes risk and therefore produce a Bayes
rule with respect to $p(\theta)$.  Note, however, that it is unusual
in the Bayesian perspective to first find an entire decision rule
$\delta$ and then apply it to a particular dataset $\data$.  Instead,
it is almost always easier to minimize the expected posterior loss
only at the actual observed data.  After all, why would we need to know
what decision we would make with other data?

\subsubsection*{Admissibility}

Another criterion for selecting between decision rules in the
frequentist framework is called \emph{admissibility.}  In short, it is
often difficult to identify a single best decision rule, but it can
sometimes be easy to discard some bad ones, for example if they can be
shown to always be no better than (and sometimes worse than) another
rule.

Let $\delta_1$ and $\delta_2$ be two decision rules.  We say that
$\delta_1$ \emph{dominates} $\delta_2$ if:
\begin{itemize}
\item
  $R(\theta, \delta_1) \leq R(\theta, \delta_2)$ for all $\theta \in
  \Theta$, and
\item
  there exists at least one $\theta$ for which $R(\theta, \delta_1) <
  R(\theta, \delta_2)$.
\end{itemize}
If there is a decision rule $\delta$ that is not dominated by any
other rule, it is called \emph{admissible.} One interesting result
tying Bayesian and frequentist decision theory is the following:
\begin{itemize}
\item
  Every Bayes rule is admissible.
\item
  Every admissible decision rule is a generalized Bayes rule for some
  (possibly improper) prior $p(\theta)$.
\end{itemize}
So, in a sense, all admissible frequentist decision rules can be
equivalently derived from a Bayesian perspective.

\section*{Examples}

Here we give two quick examples of applying Bayesian decision theory.

\subsection*{Classification with 0--1 loss}

Suppose our observations are of the form $(x, y)$, where $x$ is an
arbitrary input, and $y \in \{0, 1\}$ is a binary label associated
with $x$.  In classification, our goal is to predict the label $y'$
associated with a new input $x'$. The Bayesian approach is to derive a
model giving probabilities $\Pr(y' = 1 \given x', \data)$.  Suppose
this model is provided for you.  Notice that this model is not
conditioned on any additional parameters $\theta$; we have
integrated them out via
\begin{equation*}
  \Pr(y' = 1 \given x', \data)
  =
  \int
  \Pr(y' = 1 \given x', \data, \theta)
  p(\theta \given \data)
  \intd{\theta}.
\end{equation*}

Given a new datapoint $x'$, which label $a$ should we predict?  Notice
that the prediction of a label is actually a \emph{decision.}  Here
our action space is $\mc{A} = \{0, 1\}$, enumerating the two labels we
can predict.  Our parameter space is the same: the only uncertainty we
have is the unknown label $y'$.

Let us suppose a simple loss function for this problem:
\begin{equation*}
  L(y', a) =
  \begin{cases}
    0 & a = y'; \\
    1 & a \neq y'.
  \end{cases}
\end{equation*}
This loss function, called the \emph{0--1 loss,} is common in
classification problems.  We pay a constant loss for every mistake we
make.  In this case, the expected loss of each possible action is
simple to compute:
\begin{align*}
  \mathbb{E}\bigl[L(y', a = 1) \given x', \data\bigr]
  &=
  \Pr(y' = 0 \given x', \data);
  \\
  \mathbb{E}\bigl[L(y', a = 0) \given x', \data\bigr]
  &=
  \Pr(y' = 1 \given x', \data).
\end{align*}
The Bayes action is then to predict the class with the highest
probability.  This is not so surprising.  Notice that if we change the
loss to have different costs of mistakes (so that $L(0, 1) \neq L(1,
0)$), then the Bayes action might compel us to select the less-likely
class to avoid a potentially high loss for misclassification!

\subsection*{Hypothesis testing}

The Bayesian approach to hypothesis testing is also rather
straightforward.  A \emph{hypothesis} is simply a subset of the
parameter space $\mc{H} \subseteq \Theta$.  The Bayesian approach
allows one to compute the posterior probability of a hypothesis
directly:
\begin{equation*}
  \Pr(\theta \in \mc{H} \given \data)
  =
  \int_{\mc{H}}
  p(\theta \given \data)
  \intd{\theta}.
\end{equation*}
Now, equipped with a loss function, the posterior expected loss
framework above can be applied directly to select between different
hypotheses.

See last week's notes for a brief discussion of frequentist
hypothesis testing.

\end{document}