\documentclass[12pt, answers]{exam}
%\documentclass[12pt]{exam}
\usepackage[top=1in, bottom=1in, left=1in, right=1in]{geometry}
\usepackage{setspace}
\PassOptionsToPackage{hyphens}{url}
\usepackage{tabu}
\onehalfspacing
\setlength{\parindent}{0mm} \setlength{\parskip}{1em}

% packages
\RequirePackage{amssymb, amsfonts, amsmath, latexsym, verbatim, xspace, setspace}
\RequirePackage{tikz}
% The float package HAS to load before hyperref
\usepackage{float} % for psuedocode formatting
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage{times}
\renewcommand{\ttdefault}{cmtt}
\usepackage{amsmath}
\usepackage{graphicx} % for graphics files

% for creating indented blocks
\usepackage{scrextend}
\usepackage{paralist, tabularx}

% from Denovo Methods Manual
\usepackage{mathrsfs}
\usepackage[mathcal]{euscript}
\usepackage{color}
\usepackage{array}

\usepackage[pdftex]{hyperref}
\usepackage[parfill]{parskip}
\usepackage{cancel}

\newcommand{\nth}{n\ensuremath{^{\text{th}}} }
\newcommand{\ve}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\Macro}{\ensuremath{\Sigma}}
\newcommand{\vOmega}{\ensuremath{\hat{\Omega}}}
\newcommand{\cc}[1]{\ensuremath{\overline{#1}}}
\newcommand{\ccm}[1]{\ensuremath{\overline{\mathbf{#1}}}}

%--------------------------------------------------------------------
%--------------------------------------------------------------------
\begin{document}
\begin{center}
{\bf NE 155, Topic 10, S21 \\
 Vectors and Matrices: March 2-4, 2021}
\end{center}

\setlength{\unitlength}{1in}
\begin{picture}(6,.1)
\put(0,0) {\line(1,0){6.25}}
\end{picture}

%-------------------------------------------------------------
%-------------------------------------------------------------
\section*{Vector Review}

A real $n$-dimensional vector $\vec{x}$ is an ordered set of $n$ real numbers that expresses magnitude and direction:
%
\begin{equation}
\vec{x} = (x_1, x_2, \dots, x_n) \nonumber
\end{equation}

\textbf{Properties}:
%
\begin{enumerate}
\item sum: two vectors of the same size give a new vector of that size: $\vec{x} + \vec{y} = (x_1 + y_1, x_2 + y_2, \dots, x_n + y_n)$

\item scalar multiple: $c\vec{x} = (cx_1, cx_2, \dots, cx_n)$

%\item Euclidean norm (length): $||\vec{x}|| = (x_1^2 + x_2^2 + \dots + x_n^2)^{1/2}$

\item dot product: takes two equal length vectors and results in a scalar.

Algebraically, it is the sum of the products of the corresponding entries of the two sequences of numbers: $\vec{x} \cdot \vec{y} = \sum_{i=1}^n a_i b_i = x_1 y_1 + x_2 y_2 + \dots + x_n y_n$

\ifprintanswers
Geometrically, it is the product of the magnitudes of the two vectors and the cosine of the angle between them. $\vec{x} \cdot \vec{y} = ||\vec{x}|| ||\vec{y}|| cos\theta$.
\else
\vspace{2em}
\fi

\item $||\vec{x}||^2 = \vec{x} \cdot \vec{x}$

\item distance from $\vec{x}$ to $\vec{y}$: $||\vec{x} - \vec{y}|| = ((x_1 - y_1)^2 + (x_2 - y_2)^2 + \dots + (x_n - y_n)^2)^{1/2}$

\ifprintanswers
\item commutative property: $\vec{x} + \vec{y} = \vec{y} + \vec{x}$
\else
\item
\fi

\item associative property: $(\vec{x} + \vec{y}) + \vec{z} = \vec{x} + (\vec{y} + \vec{z})$

\item distributive property: $c(\vec{x} + \vec{y}) = c\vec{x} + c\vec{y}$
\end{enumerate}

%-------------------------------------------------------------
%-------------------------------------------------------------
\section*{Matrix Review}

\begin{align}
    \ve{A} &= [a_{ij}]_{m\times n}   =    \begin{pmatrix}
      a_{11} & a_{12} & \cdots & a_{1j} & \cdots & a_{1n} \\
      a_{21} & a_{22} & \cdots & a_{2j} & \cdots & a_{2n} \\
       \vdots & \vdots & \ddots & \vdots & \ddots   & \vdots \\
      a_{i1} & a_{i2} & \cdots & a_{ij} & \cdots & a_{in} \\
      \vdots & \vdots & \ddots & \vdots & \ddots   & \vdots \\
      a_{m1} & a_{m2} & \cdots & a_{mj} & \cdots & a_{mn} \\
    \end{pmatrix} \nonumber
\end{align}
%
where $i = 1, \dots, m$ is the row index and $j = 1, \dots, n$ is the column index.

$\ve{A} \in \mathbb{R}^{m \times n}$ is an $m \times n$ real matrix\\
$\ve{A} \in \mathbb{C}^{m \times n}$ is an $m \times n$ complex matrix

\textbf{Properties}:
%
\begin{enumerate}
\item sum: $\ve{A} + \ve{B} = [a_{ij} + b_{ij}]_{m \times n}$

\item scalar multiple: $c\ve{A} = [c a_{ij}]_{m \times n}$

\ifprintanswers
\item multiplication: $\ve{C} = \ve{A}\ve{B}$;

$\ve{A} \in \mathbb{C}^{m \times n}$, and $\ve{B} \in \mathbb{C}^{n \times p}$, and $\ve{C} \in \mathbb{C}^{m \times p}$, then $c_{ij} = \sum_{k=1}^n a_{ik} b_{kj}$

$\ve{A}\ve{B} \neq \ve{B}\ve{A}$
\else
\item
\vspace{4em}
\fi

\item commutative property: $\ve{A} + \ve{B} = \ve{B} + \ve{A}$

\item associative property: $(\ve{A} + \ve{B}) + \ve{C} = \ve{A} + (\ve{B} + \ve{C})$

\item distributive property: $c(\ve{A} + \ve{B}) = c\ve{A} + c\ve{B}$

\end{enumerate}

%-------------------------------------------------------------
\subsection*{Definitions}

Given $\ve{A} \in \mathbb{C}^{m \times n}$, \ve{A} is
%
\begin{enumerate}
\item Transpose: $\ve{A} \in \mathbb{C}^{m \times n}$, and $\ve{B} \in \mathbb{C}^{n \times m} = \ve{A}^T$ from

$b_{ij} = a_{ji}$ for $i = 1, \dots, n$ and $j = 1, \dots, m$
%
\newcommand{\aaa}{1-i}
\newcommand{\aab}{2}
\newcommand{\aba}{3+2i}
\newcommand{\abb}{4}
\newcommand{\aca}{5}
\newcommand{\acb}{6+0.4i}
%
\begin{equation}
\ve{A} = \begin{pmatrix}
   \aaa & \aab \\
   \aba & \abb \\
   \aca & \acb \\
\end{pmatrix}\:, \qquad
%
\ve{A}^T = \begin{pmatrix}
   \aaa & \aba & \aca \\
   \aab & \abb & \acb \\
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\item Conjugate Transpose / adjoint, $\ve{A}^H = \cc{\ve{A}^T}$ is the complex conjugate of the transpose.

Recall that complex conjugates are a pair of complex numbers, both the same except with imaginary parts of opposite signs. For example, The conjugate of the complex number $z=a+ib$, where $a$ and $b$ are real numbers, is $\overline{z} = a - ib$.
%http://en.wikipedia.org/wiki/Complex_conjugate
\renewcommand{\aaa}{1+i}
\renewcommand{\aba}{3-2i}
\renewcommand{\acb}{6-i}
%
\begin{equation}
\ve{A}^H = \begin{pmatrix}
   \aaa & \aba & \aca \\
   \aab & \abb & \acb \\
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\item Inverse: $\ve{AA}^{-1} = \ve{A}^{-1}\ve{A} = \ve{I}$, where $\ve{I}$ is a diagonal matrix containing ones on the diagonal. If this exists, $\ve{A}$ is non-singular / invertible.
\renewcommand{\aaa}{4}
\renewcommand{\aab}{3}
\renewcommand{\aba}{3}
\renewcommand{\abb}{2}
%
\begin{equation}
\ve{A} = \begin{pmatrix}
   \aaa & \aab \\
   \aba & \abb
\end{pmatrix} \nonumber
\end{equation}
%
\renewcommand{\aaa}{-2}
\renewcommand{\abb}{-4}
%
\begin{equation}
\ve{A}^{-1} = \begin{pmatrix}
   \aaa & \aab \\
   \aba & \abb
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\ifprintanswers
\item Regular if $\ve{A}^{-1}$ exists
\else
\item
\fi

%--------------------
\item Hermitian / self-adjoint if $\ve{A} = \ve{A}^H$
%
\renewcommand{\aaa}{2}
\renewcommand{\aab}{2+i}
\newcommand{\aac}{4}
\renewcommand{\aba}{2-i}
\renewcommand{\abb}{3}
\newcommand{\abc}{i}
\renewcommand{\aca}{4}
\renewcommand{\acb}{-i}
\newcommand{\acc}{1}
%
\begin{equation}
\ve{A} = \ve{A}^H= \begin{pmatrix}
   \aaa & \aab & \aac \\
   \aba & \abb & \abc \\
   \aca & \acb & \acc \\
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\item Symmetric if $\ve{A} = \ve{A}^T$
%
\renewcommand{\aaa}{1}
\renewcommand{\aab}{2}
\renewcommand{\aba}{2}
\renewcommand{\abb}{3}
%
\begin{equation}
\ve{A} = \ve{A}^T = \begin{pmatrix}
   \aaa & \aab \\
   \aba & \abb
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\item Antisymmetric / skew-symmetric if $\ve{A}^T = -\ve{A}$ (or, skew-Hermitian if $\ve{A}^H = -\ve{A}$)
%
\renewcommand{\aaa}{0}
\renewcommand{\aab}{-2}
\renewcommand{\aba}{2}
\renewcommand{\abb}{0}
%
\begin{equation}
\ve{A} = \begin{pmatrix}
   \aaa & \aab \\
   \aba & \abb
\end{pmatrix}\:, \qquad
%
\ve{A}^T = \begin{pmatrix}
   \aaa & \aba \\
   \aab & \abb
\end{pmatrix} \nonumber
\end{equation}

%--------------------
\item Unitary if $\ve{A}\ve{A}^H = \ve{I}$
%
%http://college.cengage.com/mathematics/larson/elementary_linear/4e/shared/downloads/c08s5.pdf
\begin{equation}
\ve{A} = \frac{1}{2}\begin{pmatrix}
   1+i & 1-i \\
   1-i & 1+i
\end{pmatrix}\:,\nonumber
\end{equation}
%
\begin{equation}
\ve{AA}^H = \frac{1}{2}\begin{pmatrix}
   1-i & 1+i \\
   1+i & 1-i
\end{pmatrix}  \times
\frac{1}{2}\begin{pmatrix}
   1+i & 1-i \\
   1-i & 1+i
\end{pmatrix} =
\frac{1}{4}\begin{pmatrix}
   4 & 0 \\
   0 & 4
\end{pmatrix} = \ve{I}\nonumber
\end{equation}

%--------------------
\item Normal if $\ve{A}\ve{A}^H = \ve{A}^H\ve{A}$
%
\begin{equation}
\ve{A} = \begin{pmatrix}
   1 & 1 & 0 \\
   0 & 1 & 1 \\
   1 & 0 & 1
\end{pmatrix}\:, \qquad
\ve{A}^H = \begin{pmatrix}
   1 & 0 & 1 \\
   1 & 1 & 0 \\
   0 & 1 & 1
\end{pmatrix}\:,\nonumber
\end{equation}
%
\begin{equation}
\ve{AA}^H = \begin{pmatrix}
   1 & 1 & 0 \\
   1 & 1 & 0 \\
   0 & 1 & 1
\end{pmatrix}  \times
\begin{pmatrix}
   1 & 0 & 1 \\
   1 & 1 & 0 \\
   0 & 1 & 1
\end{pmatrix} =
\begin{pmatrix}
   2 & 1 & 1 \\
   1 & 2 & 1 \\
   1 & 1 & 2
\end{pmatrix} = \ve{A}^H\ve{A}\nonumber
\end{equation}

%--------------------
\item Orthogonal if A is real and $\ve{AA}^T = \ve{A}^T\ve{A}$, which means $\ve{A}^{-1} = \ve{A}^T$ .
%
\begin{equation}
\ve{A} = \frac{1}{\sqrt{2}}\begin{pmatrix}
   1 & 1 \\
   1 & -1
\end{pmatrix}\:, \qquad
\ve{A}^T = \frac{1}{\sqrt{2}}\begin{pmatrix}
   1 & 1 \\
   1 & -1
\end{pmatrix}\:,\qquad
\ve{A}^T\ve{A} = \frac{1}{2}\begin{pmatrix}
   2 & 0 \\
   0 & 2
\end{pmatrix} = \ve{I} \nonumber
\end{equation}

\end{enumerate}

Among complex matrices, all unitary ($\ve{AA}^H = \ve{I}$), Hermitian/self-adjoint ($\ve{A} = \ve{A}^H$), and skew-Hermitian ($\ve{A}^H = -\ve{A}$) matrices are normal (($\ve{AA}^H = \ve{A}^H\ve{A}$)).

\ifprintanswers
Likewise, among real matrices, all orthogonal ($\ve{AA}^T = \ve{A}^T\ve{A}$), symmetric ($\ve{A} = \ve{A}^T$), and skew-symmetric ($\ve{A}^T = -\ve{A}$) matrices are normal.
\else
\vspace{3em}
\fi

However, it is \textit{not} the case that all normal matrices are either unitary or (skew-)Hermitian (see example above).% http://en.wikipedia.org/wiki/Normal_matrix

%-------------------------------------------------------------
\subsection*{Equations and Special Matrices}

All of the information above is context to help us solve actual problems.

We often write systems of equations as $\ve{A}\vec{x} = \vec{b}$ from
\begin{align}
&a_{11} x_1 + a_{12} x_2 + \dots + a_{1n} x_n = b_1 \nonumber \\
&\vdots \nonumber \\
&a_{n1} x_1 + a_{n2} x_2 + \dots + a_{nn} x_n = b_n \nonumber
\end{align}

To find $\vec{x}$, we need to find a way to affect $\ve{A}^{-1}\vec{b}$. This can be done many many ways, and the first thing we'll talk about are methods for inverting the matrix.

\begin{itemize}
\item Tridiagonal matrix has entries on only the main, upper, and lower diagonal
\ifprintanswers
\item Lower triangular has entries on the diagonal and below
\item Upper triangular has entries on the diagonal and above
\else
\item
\item
\fi
\item Block Tridagonal has blocks of elements (like sub-matrices) on the diagonal. The blocks may be full or only partially full. The blocks look like $\ve{D}_k = [\ve{D_{ij}}]_k$
\end{itemize}

The inverse of a diagonal is simply: $d_{ii}^{-1} = 1/d_{ii}$, another diagonal matrix.

\textbf{Theorem}: The following are equivalent (see Math 54 or a textbook for proof):
%
\begin{enumerate}
\item $\ve{A}$ is regular ($\ve{A}^{-1}$ exists)
\item Rank($\ve{A}$) = n
\ifprintanswers
\item $\ve{A}\vec{x} = \vec{0}$ iff $\vec{x}=\vec{0}$
\else
\item
\fi
\item $\ve{A}\vec{x} = \vec{b}$ is uniquely solveable $\forall \vec{b}$
\item det($\ve{A}) \neq 0$
\end{enumerate}

%-------------------------------------------------------------
\subsection*{Minors, Cofactors, Determinants}
If the determinant is zero, then the matrix is singular, meaning we cannot invert it and numerical solutions are pretty much impossible.

\textbf{Properties of Determinants}
\begin{itemize}
\item det($\alpha\ve{A}$) = $\alpha^n$ det($\ve{A}$)

\ifprintanswers
\item det($\ve{A}^T$) = det($\ve{A}$)
\else
\item
\fi

\item det($\ve{A}^{-1}$) = 1/det($\ve{A}$)

\item det($\ve{AB}$) = det($\ve{A}$)det($\ve{B}$)

\item det($\ve{A}^k$) = [det($\ve{A}$)]$^k$

\item in general, det($\ve{A + B}$) $\neq$ det($\ve{A}$) + det($\ve{B}$)
\end{itemize}

Good illustration: \href{http://www.mathsisfun.com/algebra/matrix-inverse-minors-cofactors-adjugate.html}{http://www.mathsisfun.com/algebra/matrix-inverse-minors-cofactors-adjugate.html}

%If $\ve{A}$ is an $m \times n$ matrix and $k$ is an integer such that $0 < k \leq m$ and $k \leq n$ then a \textbf{$k \times k$ minor}, $M_{kk}$ of $\ve{A}$ is the determinant of the $k \times k$ matrix formed by deleting $m-k$ rows and $n-k$ columns.

For a square matrix, the \textbf{first order minor}, $M_{ij}$, just deletes the $i^{th}$ row and $j^{th}$ column and takes the determinant. E.g.
%
\begin{align}
    \ve{A} &= \begin{pmatrix}
        1 & 4 & 7 \\
        3 & 0 & 5 \\
        -1 & 9 & 11 \\
    \end{pmatrix}
    \qquad
    &M_{23} = \text{det}\begin{pmatrix}
       1 & 4 \\
       -1 & 9
    \end{pmatrix}
    = ((1 \times 9) - (4 \times -1)) = 13 \nonumber
\end{align}

\ifprintanswers
The corresponding $i,j$ \textbf{cofactor} of $\ve{A}$ is
%
\begin{equation}
C_{ij} = (-1)^{i+j} M_{ij} \nonumber
\end{equation}
\else
\vspace*{3em}
\fi
%
You can compute minors and cofactors for all of $\ve{A}$. The $n \times n$ matrix containing all of the cofactors is denoted $\ve{C}$ in this context.

Using these terms, the \underline{determinant} (which we use for lots of stuff) can be defined in terms of the Laplace expansion
%
\begin{align*}
\text{det}(\ve{A}) &= \sum_{j=1}^n a_{ij} C_{ij} \text{ for any i} \in \{1,...,n\} \\
%
&= \sum_{i=1}^n a_{ij} C_{ij} \text{ for any j} \in \{1,...,n\}
\end{align*}

For the above matrix, lets look at the Laplace expansion along the second column ($j = 2$; sum runs over $i$)

\begin{align}
\text{det}(\ve{A}) &= (-1)^{1+2} a_{12} M_{12} + (-1)^{2+2} a_{22} M_{22} + (-1)^{3+2} a_{32} M_{32} \nonumber \\
%
&= (-1)^{1+2} \cdot 4 \cdot \text{det}\begin{pmatrix}
        3 & 5 \\
        -1 & 11 \\ \end{pmatrix} + (-1)^{2+2} \cdot 0 \cdot \text{det}\begin{pmatrix}
        1 & 7 \\
        -1 & 11 \\ \end{pmatrix} + (-1)^{3+2} \cdot 9 \cdot \text{det}\begin{pmatrix}
        1 & 7 \\
        3 & 5 \\\end{pmatrix} \nonumber \\
%
&= -4 \cdot ((3 \cdot 11) - (5 \cdot -1)) + 0 -9 \cdot ((1 \cdot 5) - (7 \cdot 3)) = -8 \nonumber
\end{align}

\ifprintanswers
The inverse of $\ve{A}$ (which we often need) can be obtained from the determinant and cofactor matrix:
%
\begin{equation}
\ve{A}^{-1} = \frac{1}{\text{det}(\ve{A})}\ve{C}^T \nonumber
\end{equation}
\else
\vspace*{3em}
\fi
%
The benefit is that if we've used the cofactor method to get the determinant then we get the inverse for free.

\section*{Norms and Convergence}
We're going to look at direct and iterative methods to solve problems; we'll need these concepts to understand how the solution methods behave.

%-------------------------------------------------------------
\subsection*{Vector Norms}
Given $\vec{x}, \vec{y} \in \mathbb{R}^n$, a vector norm, denoted by $|| \cdot ||$, has the following properties:
%
\begin{enumerate}
\item $||\vec{x}|| > 0$; $||\vec{x}|| = 0$ iff $\vec{x} = 0$ (positive definite)
\ifprintanswers
\item $||\vec{x} + \vec{y}|| \leq ||\vec{x}|| + ||\vec{y}||$ (triangle inequality)
\else
\item
\fi
\item $||\alpha \vec{x}|| = |\alpha|\: ||\vec{x}||$ (homogeneous)
\end{enumerate}

The p-norm:
%
\begin{equation}
||\vec{x}||_p \equiv (|x_1|^p + |x_2|^p + \dots + |x_n|^p)^{1/p} \qquad p \geq 1 \nonumber
\end{equation}
%
\begin{itemize}
\item $||\vec{x}||_1 \equiv |x_1| + |x_2| + \dots + |x_n|$
\ifprintanswers
\item Euclidean norm (length) $||\vec{x}||_2 \equiv (|x_1|^2 + |x_2|^2 + \dots + |x_n|^2)^{1/2}$\\
\else
\item
\fi
\item $||\vec{x}||_{\infty} \equiv \displaystyle \max_{1 \leq i \leq n} |x_i|$
\end{itemize}

%-------------------------------------------------------------------
\subsection*{Inner Products}
Inner products are symmetric and bilinear form in $\mathbb{R}^n$:
%
\begin{itemize}
\item bi-linear (depends on two arguments) $\langle \vec{x}, \vec{y} \rangle$
\item symmetric $\langle \vec{x}, \vec{y} \rangle = \langle \vec{y}, \vec{x} \rangle$
\item linear $\langle \vec{x} + \alpha \vec{z}, \vec{y} \rangle = \langle \vec{x}, \vec{y} \rangle + \alpha \langle \vec{y}, \vec{z} \rangle$
\ifprintanswers
\item positive $\langle \vec{x}, \vec{x} \rangle$ $>$ 0 for $\vec{x} \neq 0$
\else
\item
\fi
\end{itemize}

The dot product is also known as the Euclidean inner product. One can define many inner products, but \textit{in this class assume we're using the Euclidean inner product} (dot product) unless otherwise specified.

\subsection*{Convergence}
We can use the concepts of norms and inner products to develop some relationships and talk about convergence. Note that there are two ways we think of convergence in numerical methods:
\begin{enumerate}
\item the \textbf{method}: the speed at which a numerical guess approaches the true solution with iteration count;
\ifprintanswers
\item the \textbf{discretization}: speed at which a numerical guess approaches the true solution with reduction in  grid size.
\else
\item
\fi
\end{enumerate}

%https://en.wikipedia.org/wiki/Rate_of_convergence
In numerical analysis, the speed at which a convergent sequence approaches its limit is what is commonly called the \textit{rate of convergence}. This concept is of practical importance in dealing with a sequence of successive approximations for an iterative method, as then typically fewer iterations are needed to yield a useful approximation if the rate of convergence is higher.

\ifprintanswers
We will first talk about this and we will use the \textit{Cauchy-Schwartz} inequality:
%
\begin{equation}
\langle \vec{x}, \vec{y} \rangle \leq ||\vec{x}||_2 ||\vec{y}||_2 \nonumber
\end{equation}
\else
\vspace{4em}
\fi
%
Two norms, denoted $|| \cdot ||_a$ and $|| \cdot ||_b$ here, are defined as being equivalent if there exists $C_1$ and $C_2$ such that $\forall \vec{x} \in \mathbb{R}^n$
%
\begin{equation}
C_1 ||\vec{x}||_a \leq ||\vec{x}||_b \leq C_2 ||\vec{x}||_a \qquad \text{where } C_1, C_2 = f(n) \nonumber
\end{equation}
%
This leads to the theorem that in $\mathbb{R}^n$ all norms are equivalent (offered without proof). E.g.:
\begin{align}
& ||\vec{x}||_{\infty} \leq ||\vec{x}||_2 \leq \sqrt[]{n} ||\vec{x}||_{\infty} \nonumber \\
%
& ||\vec{x}||_{\infty} \leq ||\vec{x}||_1 \leq n ||\vec{x}||_{\infty} \nonumber \\
%
& ||\vec{x}||_2 \leq ||\vec{x}||_1 \leq \sqrt[]{n} ||\vec{x}||_2 \nonumber
\end{align}

The inequalities above may not be `sharp' (A sharp inequality is when there could be no better inequality when making the comparison).

E.g., when comparing two real numbers/expressions, an inequality is sharp because we could not increase the left or decrease the right by a positive factor and still have it be true ($2 \leq 2$).

%\textbf{Example}: let's show the last item. We will use the Signum function, which extracts the sign of a real number
%\begin{equation}
%\text{sign}(x) \equiv \begin{cases}
%  -1 & \text{ if } x < 0 \nonumber \\
%  0  & \text{ if } x = 0 \nonumber \\
%  1  & \text{ if } x > 0 \nonumber
%\end{cases}
%\end{equation}
%%
%Let's define $v_i$ as sign$(x_i)$. Then
%%
%\begin{align}
%&< \vec{x}, \vec{v} > = \sum_i x_i v_i = \sum_i x_i \text{sign}(x_i) = \sum_i |x_i| = ||\vec{x}||_1 \nonumber \\
%%
%&\text{but by the Cauchy-Schwartz inequality,}\nonumber \\
%%
%&< \vec{x}, \vec{v} > \leq ||\vec{x}||_2 ||\vec{v}||_2 \nonumber \\
%%
%&\text{and we can give an upper bound of } ||\vec{v}||_2 \leq \sqrt{n} \text{(the right side of the inequality);} \nonumber \\
%%
%&\text{we can also say } ||\vec{x}||_2 \leq ||\vec{v}||_2 ||\vec{x}||_2 \leq \sqrt{n} ||\vec{x}||_2 \nonumber \\
%%
%&\text{thus } ||\vec{x}||_2 \leq ||\vec{x}||_1 \leq \sqrt{n} ||\vec{x}||_2 \nonumber
%\end{align}
%
%NEED $||\vec{x}||_2 \leq ||\vec{x}||_1$ !!!!

Norm equivalence is very important because if we can prove some property (usually convergence or an error bound) in \textbf{some} norm, we have effectively proven it in \textbf{all} norms.

\subsubsection*{Error}
For $\vec{x}$ (the real solution) and $\hat{x}$ (representing our numerical solution) $\in \mathbb{R}^n$ and some norm $p$ we define
%
\begin{itemize}
\ifprintanswers
\item Absolute error: $||\hat{x} - \vec{x}||_p$
\item Relative error: $\dfrac{||\hat{x} - \vec{x}||_p}{||\vec{x}||_p}$, where $\vec{x} \neq 0$
\else
\item
\item
\fi
\end{itemize}

\subsubsection*{Convergence}
Given a sequence $\lbrace \hat{x}^{(k)} \rbrace_{k=1,2,\dots,\infty}$ and some norm $p$, we say that $\lbrace \hat{x}^{(k)} \rbrace$ converges to $\vec{x}$ if
%
\begin{equation}
\lim_{k \rightarrow \infty} ||\hat{x}^{(k)} - \vec{x}||_p = 0 \nonumber
\end{equation}

%http://fourier.eng.hmc.edu/e176/lectures/NM/node3.html
%https://en.wikipedia.org/wiki/Rate_of_convergence
When we think of the \textbf{rate of convergence}, we can look at how quickly this limit is reached. We measure this as
\ifprintanswers
\[ \lim_{k \rightarrow \infty} \frac{||\hat{x}^{(k+1)} - \vec{x}||_p}{||\hat{x}^{(k)} - \vec{x}||_p^q} = \mu\]
\else
\\ \vspace*{2em}\\
\fi
Here $q \geq 1$ is called the \textit{order of convergence} and $\mu$ is the \textit{rate of convergences}. Also written as
\[||\hat{x}^{(k+1)} - \vec{x}||_p = \mu ||\hat{x}^{(k)} - \vec{x}||_p^q \rightarrow ||e_{k+1}||_p = \mu||e_{k}||_p^q\:.
\]
Consider
\begin{compactitem}
\item If $q=1$ and $\mu$ varies with iteration ($\mu = \mu_k$),  $||e_{k+1}||_p =\mu_k || e_k ||_p < || e_k ||_p$, then
  \begin{compactitem}
  \item if $\mu_k \rightarrow 1$ as $k \rightarrow \infty$, the convergence is sublinear
  \item if $\mu_k \rightarrow 0$ as $k \rightarrow \infty$, the convergence is superlinear
  \end{compactitem}
\item If $q=2$,  $|| e_{k+1} ||_p = \mu || e_k ||_p^2,\;\;(\mu>0)$, the convergence is quadratic.
\item If $q=3$,  $ || e_{k+1} ||_p = \mu || e_k ||_p^3,\;\;(\mu>0)$, the convergence is cubic.
\end{compactitem}

Similar concepts are used for \textbf{discretization methods}. The solution of the discretized problem converges to the solution of the continuous problem as the grid size goes to zero, and the speed of convergence is one of the factors of the efficiency of the method. The important parameter here for the convergence speed is not the iteration number $k$ but it depends on the number of grid points and grid spacing. In this case, the number of grid points $n$ in a discretization process is inversely proportional to the grid spacing.

We talked about this during integration.

In this case, a sequence $x_{n}$ is said to converge to $L$ with order $p$ if there exists a constant $C$ such that
\[ \|x_{n}-L\| < Cn^{-p} \quad \forall \:n\:.\]
This is written as $\|xn - L\| = O(n−p)$ using the ``big O" notation.

This is the relevant definition when discussing methods for numerical quadrature or the solution of ordinary differential equations. A practical method to calculate the rate of convergence for a discretization method is to implement the following formula:
\[ p\approx {\frac {\log {\frac {e_{new}}{e_{old}}}}{\log {\frac {h_{new}}{h_{old}}}}}\]

%--------------------------------------------------------------------------
\subsection*{Matrix Norms}
Given $\ve{A}, \ve{B} \in \mathbb{R}^{m \times n}$, a matrix norm, denoted by $|| \cdot ||$, has the following properties:
%
\begin{enumerate}
\ifprintanswers
\item $||\ve{A}|| > 0$ and $||\ve{A}|| = 0$ iff $\ve{A} = 0$ (positive definite)
\else
\item
\fi
\item $||\ve{A} + \ve{B}|| \leq ||\ve{A}|| + ||\ve{B}||$ (triangle inequality)
\item $||\alpha \ve{A}|| = |\alpha|\: ||\ve{A}||$ (homogeneous)
\end{enumerate}

E.g., the Frobenius norm
%
\begin{equation}
||\ve{A}||_F = \sqrt{ \sum_{i=1}^m \sum_{j=1}^n |a_{ij}|^2 } \nonumber
\end{equation}

\textbf{Definitions}:
\begin{itemize}
\item submultiplicative if $||\ve{A} \ve{B}|| \leq ||\ve{A}||\: ||\ve{B}||$

\item Not all norms are submultiplicative; \textit{we will only deal with those that are.}
\ifprintanswers
\item subordinate matrix norm for $\ve{A} \in \mathbb{R}^{m \times n}$ and $\vec{x} \in \mathbb{R}^n$:

$||\ve{A}|| \equiv \displaystyle \sup_{\vec{x} \neq \vec{0}} \dfrac{||\ve{A}\vec{x}||}{||\vec{x}||}$
\else
\item
\vspace*{3em}
\fi

\item \textbf{supremum}, or least upper bound, of a set $S$ of real numbers is denoted by sup$S$ and is defined to be the smallest real number that is greater than or equal to every number in $S$.

Consequently, the supremum is also referred to as the least upper bound (or LUB). If the supremum exists, it is unique, meaning that there will be only one supremum.
\end{itemize}

\textbf{examples:}
%
\begin{align}
||\ve{A}||_{1} &= \displaystyle \sup_{\vec{x} \neq \vec{0}} \frac{||\ve{A}\vec{x}||_{1}}{||\vec{x}||_{1}} =
\displaystyle \max_{1 \leq j \leq n} \sum_{i=1}^m |a_{ij}| \qquad \text{max absolute col sum} \nonumber \\
%
||\ve{A}||_{\infty} &= \displaystyle \sup_{\vec{x} \neq \vec{0}} \frac{||\ve{A}\vec{x}||_{\infty}}{||\vec{x}||_{\infty}} =
\displaystyle \max_{1 \leq i \leq m} \sum_{j=1}^n |a_{ij}| \qquad \text{max absolute row sum}\nonumber \\
%
||\ve{A}||_{2} &= \displaystyle \sup_{\vec{x} \neq \vec{0}} \frac{||\ve{A}\vec{x}||_{2}}{||\vec{x}||_{2}} = \text{ the sqrt of the max  eigenvalue of }\ve{A}^H\ve{A} \nonumber \\
&\text{is largest singular value for any matrix, or the spectral radius of a square matrix} \nonumber
\end{align}

For vectors, the infinity norm is the largest value. For matrices, it's the maximum absolute row sum\textemdash which is really the row that will most strongly change a vector, so that's analogous. For the $1$ matrix norm, the maximum absolute column sum corresponds to the vector component that will, overall, be changed the most. We'll talk about the $2$ matrix norm after we talk about spectral radius, which will require a quick review of eigenvalues.

The equivalence of norms we talked about with vectors also holds here. Some of the relationships are
%
\begin{align}
||\ve{A}||_{2} &\leq ||\ve{A}||_{F} \leq \sqrt{n}||\ve{A}||_{2} \nonumber \\
%
\frac{1}{\sqrt{n}}||\ve{A}||_{\infty} &\leq ||\ve{A}||_{2} \leq \sqrt{m}||\ve{A}||_{\infty} \nonumber \\
%
\frac{1}{\sqrt{m}}||\ve{A}||_{1} &\leq ||\ve{A}||_{2} \leq \sqrt{n}||\ve{A}||_{1} \nonumber
\end{align}

Again, as with vector norms, showing a property in some matrix norms implies that property in other matrix norms (which may be harder to compute). The inequalities above may not be `sharp'.

\section*{Eigenvalue Review}

Eigenvalues are a special set of scalars associated with a linear system of equations (a matrix equation) that are sometimes also known as characteristic roots.

Each eigenvalue is paired with a corresponding so-called eigenvector (or, in general, a corresponding right eigenvector and a corresponding left eigenvector; there is no analogous distinction between left and right for eigenvalues).

\ifprintanswers
The decomposition of a square matrix $\ve{A}$ into eigenvalues and eigenvectors is known as eigen decomposition, and the fact that \textbf{this decomposition is always possible as long as the matrix consisting of the eigenvectors of $\ve{A}$ is square} is known as the eigen decomposition theorem.
\else
\vspace*{4em}
\fi

Let $\ve{A} \in \mathbb{R}^{n \times n}$. If there is a vector $\vec{x} \in \mathbb{R}^{n}$ such that
%
\[\ve{A} \vec{x} = \lambda \vec{x}\]
%
for some scalar $\lambda$, then $\lambda$ is an eigenvalue of $\ve{A}$ with a corresponding (right) eigenvector $\vec{x}$. Note that the eigenvalue represents the ``stretching factor" in the direction of its associated eigenvector.

(\url{http://math.stackexchange.com/questions/54176/is-there-a-geometric-meaning-of-the-frobenius-norm})

\ifprintanswers
We find eigenvalues by re-writing the stated relationship as $(\ve{A} - \lambda \ve{I})\vec{x}=0$ and solving for the $\lambda$s that make this true. The $\lambda$s are then the eigenvalues and the $\vec{x}$s that go with them are the corresponding eigenvectors.
\else
\vspace*{4em}
\fi

A linear system of equations has nontrivial solutions iff the determinant vanishes (Cramer's rule; related to the theorem for finding out if a solution exits that we talked about above), so the solutions of this equation are given by
%
\[\det(\ve{A} - \lambda \ve{I})=0 \:.\]
%
This equation is known as the characteristic equation of $\ve{A}$, and the left-hand side is known as the characteristic polynomial.

\textbf{Example}
\begin{align}
    \ve{A} &= \begin{pmatrix}
        2 & 3 \\
        3 & -6
    \end{pmatrix}
    &\ve{A} - \lambda \ve{I} = \begin{pmatrix}
        2 - \lambda & 3 \\
        3 & -6 - \lambda
    \end{pmatrix} \nonumber \\
%
\det(\ve{A} - \lambda \ve{I}) &= (2 - \lambda)(-6 - \lambda) - (3)(3) \nonumber \\
%
&= -12 + 6\lambda -2\lambda + \lambda^2 -9 \nonumber \\
&= \lambda^2 + 4\lambda -21 = 0 \nonumber \\
0 &= (\lambda - 3)(\lambda + 7) \nonumber \\
&\boxed{\lambda = 3, -7} \nonumber
\end{align}

\begin{itemize}
\ifprintanswers
\item Eigenvalues may be real or complex.
\else
\item
\fi

\item Since an \nth degree polynomial has $n$ roots, $\ve{A}_{n \times n}$ is guaranteed to have $n$ real and/or complex eigenvalues, some of which may be repeated: $\lambda_1, \lambda_2, \dots, \lambda_n$.

This gives $\ve{A}\vec{u}_1 = \lambda_1 \vec{u}_1$, etc.

\item Left eigenvectors are found by reformulating the equation as $\vec{y}\ve{A} = \alpha \vec{y}$. In nuclear, we're used to seeing the right eigenvector formulation.
\end{itemize}

The spectrum of eigenvalues of a matrix $\ve{A}$ are defined formally as
\ifprintanswers
\[\sigma(\ve{A}) = [ \lambda \in \mathbb{C} : \det(\ve{A} - \lambda \ve{I})=0] \]
\else
\\ \vspace*{2em} \\
\fi
and an eigenvalue is
\[ \lambda \in \sigma(\ve{A})\:,\]
and
\begin{itemize}
\item $\sigma(\ve{A}) = \sigma(\ve{A}^T)$
\item $\cc{\sigma(\ve{A})} = \sigma(\ve{A}^H)$
\end{itemize}

%--------------------------------------------------------------------------
\subsection*{Aside about Singular Values}

\underline{Note:} Eigenvalues/vectors are only for square matrices. The analog for rectangular matrices is singular values. We are not going to focus on these, since we will be dealing with square matrices. However, here's a quick aside:

If a square matrix $\ve{A}$ has linearly-independent eigenvectors, it can be factored as $\ve{A} = \ve{P}\ve{D}\ve{P}^{-1}$. Here $\ve{D}$ is a diagonal matrix containing the eigenvalues of $\ve{A}$ and $\ve{P}$ contains the corresponding, linearly-independent eigenvectors of $\ve{A}$.

You cannot do this for a rectangular matrix. However, \textbf{any} matrix $\ve{A} \in \mathbb{C}^{m \times n}$ can be factored as $\ve{Q}\ve{\Sigma}\ve{V}^{-1}$, and this is called the singular value decomposition. $\ve{Q}$ and $\ve{V}$ are unitary, and $\ve{\Sigma}$ is diagonal, where the entries are the singular values of $\ve{A}$: $\sigma_1, \sigma_2, \dots, \sigma_n$ with $\sigma_1 \geq \cdots \geq \sigma_n$.

This comes from the notion that $\ve{A}^H\ve{A}$ is symmetric and can be orthogonally diagonalized (get linearly-independent eigenvectors). We can also think of this property as: the singular values of $\ve{A}$ are the square roots of the eigenvalues of $\ve{A}^H\ve{A}$.

To see the geometric meaning we can note that
\[\ve{A} = \ve{Q} \ve{\Sigma} \ve{V}^{-1} \rightarrow \ve{A}\ve{V} = \ve{Q} \ve{\Sigma} \:,\]
look at one column at a time and see
\[\ve{A}\vec{v}_i = \sigma_i \vec{q}_i \:, \qquad i = 1, \cdots, n \:.\]

Now you can see that these are like eigenvalues in terms geometric stretching. Finally, we note that $||\ve{A}||_F = \sqrt{\sigma_1^2 + \sigma_2^2 + \dots + \sigma_n^2}$, effectively meaning that the Frobenius norm measures the size of the distortion a matrix creates in a system. (see this reference for a good explanation: \url{http://www.math.iit.edu/$\sim$fass/477577\_Chapter\_2.pdf}).

%--------------------------------------------------------------------------
\subsection*{Spectral radius}

The spectral radius of $\ve{A} \in \mathbb{C}^{n \times n}$ is defined as
\[\rho(\ve{A}) \equiv \max \lbrace |\lambda|, \lambda \in \sigma(\ve{A}) \rbrace\]
With the properties:
%
\begin{itemize}
\ifprintanswers
\item $\rho(\ve{A})^k = \rho(\ve{A}^k)$
\item $\rho(\ve{A}) = \rho(\ve{A}^T)$
\item $\rho(\xi \ve{A}) = |\xi| \rho(\ve{A})$
\else
\item
\item
\item
\fi
\end{itemize}
%
Let's think about the meaning of the spectral radius. \textit{This is the largest change a matrix can induce on a vector while maintaining its direction.} That is what the matrix 2-norm means, which is analogous to the vector idea of length.

%\textbf{example}
%Show that the eigenvalues of a Hermitian matrix are real, noting that the eigenvalues of $\ve{A}$ satisfy
%\[\ve{A}\vec{x} = \lambda x \qquad \text{and } \vec{x} \neq \vec{0}\]
%and recall that $\ve{A} = \ve{A}^H = \ccm{A}^T$ defines a Hermitian matrix.
%%
%\begin{align}
%\cc{u}^T \ccm{A}^T &= \cc{\lambda} \cc{u}^T
%\qquad \text{The first step} \nonumber \\
%%
%\cc{u}^T \ve{A} &= \cc{\lambda} \cc{u}^T
%\qquad \text{apply Hermitian definition}\nonumber \\
%%
%\cc{u}^T \ve{A} \vec{u} &= \cc{\lambda} \cc{u}^T \vec{u}
%\qquad \text{right multiply by }\vec{u}\nonumber \\
%%
%\cc{u}^T \lambda \vec{u} &= \cc{\lambda} \cc{u}^T \vec{u}
%\qquad \text{apply eigenvalue definition }\nonumber \\
%%
%\lambda &= \cc{\lambda}
%\qquad \text{recognize that the }\vec{u}\text{s factor out}\nonumber
%\end{align}

 %--------------------------------------------------------------------------
\section*{Spectral radii are bounded}

(reference: \url{http://www.math.drexel.edu/~foucart/TeachingFiles/F12/M504Lect6.pdf})

For $\lambda \in \sigma(\ve{A})$
\ifprintanswers
\[|\lambda | \leq ||\ve{A}||\]
\[\rho(\ve{A}) \leq ||\ve{A}||\]
\else
\\ \vspace*{3em}
\fi

\underline{proof:}
\begin{enumerate}
\item The eigenvalues of $\ve{A}$ satisfy $\ve{A}\vec{x} = \lambda \vec{x}$, $\vec{x} \neq \vec{0}$.

\item Let the eigenvectors of $\ve{A}$ be collected in a matrix $\ve{X} = [\vec{x_1}, \vec{x_2}, \dots, \vec{x_n}]$, giving $\ve{AX} = \lambda \ve{X}$.

\item then
\begin{align}
|\lambda |\: ||\ve{X}|| &= ||\lambda \ve{X}||
\qquad \text{property of norms} \nonumber \\
%
                     &= ||\ve{A}\ve{X}||
                     \qquad \text{substitution} \nonumber \\
%
                     &\leq ||\ve{A}||\: ||\ve{X}||
                     \qquad \text{submultiplicative property} \nonumber \\
%
\therefore |\lambda | &\leq ||\ve{A}||
\qquad \text{cancellation}\nonumber \\
%
|\lambda| &\leq \rho(\ve{A}) \leq ||\ve{A}||
\qquad \text{definition of }\rho\text{ as max }|\lambda | \nonumber
\end{align}
\end{enumerate}

Note, we can use any submultiplicative matrix norm to bound from above the spectral radius (this will be important).

\end{document}


\documentclass[12pt, fullpage,letterpaper]{article}

\usepackage[margin=1in]{geometry}
\usepackage{url}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{xspace}
\usepackage{graphicx}
\usepackage{hyperref}

\newcommand{\semester}{Spring 2022}
\newcommand{\assignmentId}{1}
\newcommand{\releaseDate}{1 Feb, 2022}
\newcommand{\dueDate}{11:59pm, 18 Feb, 2022}

\newcommand{\bx}{{\bf x}}
\newcommand{\bw}{{\bf w}}

\title{CS 6190: Probabilistic Machine Learning \semester}
\author{Homework \assignmentId}
\date{Handed out: \releaseDate\\
  Due: \dueDate}

\begin{document}
\maketitle

\input{emacscomm}
\footnotesize
	\begin{itemize}
		\item You are welcome to talk to other members of the class about
		the homework. I am more concerned that you understand the
		underlying concepts. However, you should write down your own
		solution. Please keep the class collaboration policy in mind.

		\item Feel free discuss the homework with the instructor or the TAs.

		\item Your written solutions should be brief and clear. You need to
		show your work, not just the final answer, but you do \emph{not}
		need to write it in gory detail. Your assignment should be {\bf no
			more than 10 pages}. Every extra page will cost a point.

		\item Handwritten solutions will not be accepted.

		\item The homework is due by \textbf{midnight of the due date}. Please submit
		the homework on Canvas.
	\end{itemize}

\section*{Analytical problems [80 points + 30 bonus]}
\label{sec:q1}

\begin{enumerate}
\item~[8 points] A random vector, $\x = \left[\begin{array}{c}\x_1 \\  \x_2\end{array}\right]$ follows a multivariate Gaussian distribution,
\[
p(\x) = \N\big( \left[\begin{array}{c}\x_1 \\  \x_2\end{array}\right] |  \left[\begin{array}{c}\bmu_1 \\  \bmu_2\end{array}\right], \left[\begin{array}{cc} \bSigma_{11} & \bSigma_{12} \\  \bSigma_{21} & \bSigma_{22}\end{array}\right]\big).
\]
Show that the marginal distribution of $\x_1$ is $p(\x_1) = \N(\x_1| \bmu_1, \bSigma_{11})$.

\item~[\textbf{Bonus}][10 points] Given a Gaussian random vector, $\x \sim \N(\x|\bmu, \bSigma)$.  We have a linear transformation, $\y = \A\x + \b + \z$, where $\A$ and $\b$ are constants, $\z$ is another Gaussian random vector independent to $\x$, $p(\z) = \N(\z|\0, \bLambda)$. Show $\y$ follows Gaussian distribution as well, and derive its form. Hint: using characteristic function. You need to check the materials by yourself.

\item~[8 points] Show the differential entropy of the a multivariate Gaussian distribution $\N(\x|\bmu, \bSigma)$ is
\[
\mathrm{H}[\x] = \frac{1}{2}\log |\bSigma| + \frac{d}{2}(1 + \log 2\pi)
\]
where $d$ is the dimension of $\x$.
\item~[8 points] Derive the Kullback-Leibler divergence between two Gaussian distributions, $p(\x) = \N(\x|\bmu, \bSigma)$ and $q(\x) = \N(\x|\m, \Lambda)$, \ie $\mathrm{KL}(q || p)$.

\item~[8 points] Given a distribution in the exponential family,
\[
p(\x|\boldeta) = \frac{1}{Z(\boldeta)} h(\x)\exp\big(-\u(\x)^\top \boldeta\big).
\]
Show that
\[
\frac{\partial^2 \log Z(\boldeta)}{\partial \boldeta^2} = \mathrm{cov}(\u(\x)),
\]
where $\mathrm{cov}$ is the covariance matrix.

\item~[4 points] Is $\log Z(\boldeta)$ convex or nonconvex? Why?

\item~[8 points] Given two random variables $\x$ and $\y$, show that
\[
I(\x,\y) = H[\x] - H[\x|\y]
\]
where $I(\cdot, \cdot)$ is the mutual information and $H[\cdot]$ the entropy.

\item~[24 points] Convert the following distributions into the form of the exponential-family distribution. Please give the mapping from the expectation parameters to the natural parameters, and also represent the log normalizer as a function of the natural parameters.
\begin{itemize}
	\item Dirichlet distribution
	\item Gamma distribution
	\item Wishart distribution
\end{itemize}

\item~[6 points] Does student $t$ distribution (including both the scalar and vector cases) belong to the exponential family? Why?

\item~[6 points] Does the mixture of Gaussian distribution belong to the exponential family? Why?
\[
p(\x) = \frac{1}{2}\N(\x|\bmu, \bSigma) + \frac{1}{2}\N(\x|\m, \bLambda)
\]

\item~[\textbf{Bonus}][20 points] Given a distribution in the exponential family $p(\x|\boldeta)$, where $\boldeta$ are the natural parameters. As we discussed in the class, the distributions in the exponential family are often parameterized by their expectations, namely $\btheta = \EE\big(\u(\x)\big)$ where $\u(\x)$ are the sufficient statistics (recall Gaussian and Bernoulli distributions). Given an arbitrary distribution $p(\x|\balpha)$, the Fisher information matrix in terms of the distribution parameters $\balpha$ is defined as $\F(\balpha) = \EE_{p(\x|\balpha)}[- \frac{\partial^2 \log(p(\x|\balpha)) }{\partial \balpha^2}]$.
\begin{enumerate}
	\item~[5 points] Show that if we calculate the Fisher Information matrix in terms of the natural parameters, we have $\F(\boldeta) = \mathrm{cov}\big(\u(\x)\big)$.
	\item~[5 points] Show that $\frac{\partial \btheta}{\partial \boldeta} = \F(\boldeta)$.
	\item~[10 points] Show that the Fisher information matrix in terms of the expectation parameters is the inverse of that in terms of the natural parameters, $\F(\btheta) =\F^{-1}(\boldeta) $.
	\item~[5 points] Suppose we observed dataset $\Dcal$. Show that
	\[
	\frac{\partial \log p(\Dcal|\boldeta)}{\partial \boldeta} \F(\boldeta)^{-1} = \frac{\partial \log p(\Dcal|\btheta)}{\partial \btheta}
	\]
	and
	\[
	 \frac{\partial \log p(\Dcal|\btheta)}{\partial \btheta}\F(\btheta)^{-1} = \frac{\partial \log p(\Dcal|\boldeta)}{\partial \boldeta}.
	\]
	Note that I choose the orientation of the gradient vector to be consistent with Jacobian. So, in this case, the gradient vector is a row vector (rather than a column vector). If you want to use a column vector to represent the gradient, you can move the information matrix to the left. It does not influence the conclusion.
\end{enumerate}
\end{enumerate}

\section{Practice [20 points ]}
\begin{enumerate}
	\item~[5 Points] Look into the student t's distribution. Let us set the mean and precision to be $\mu = 0$ and $\lambda = 1$. Vary the degree of freedom $\nu = {0.1, 1, 10, 100, 10^6}$ and draw the density of the student t's distribution. Also, draw the density of the standard Gaussian distribution $\N(0,1)$. Please place all the density curves in one figure. Show the legend. What can you observe?

	\item~[5 points] Draw the density plots for Beta distributions: Beta(1,1), Beta(5, 5) and Beta (10, 10). Put the three density curves in one figure. What do you observe? Next draw the density plots for Beta(1, 2), Beta(5,6) and Beta(10, 11). Put the three density curves in another figure. What do you observe?

	\item~[10 points] Randomly draw 30 samples from a Gaussian distribution $\N(0, 2)$. Use the 30 samples as your observations to find the maximum likelihood estimation (MLE) for a Gaussian distribution and a student $t$ distribution. For both distributions, please use L-BFGS to optimize the parameters. For student $t$, you need to estimate the degree of the freedom as well. Draw a plot of the estimated the Gaussian distribution density, student $t$ density and the scatter data points. What do you observe, and why? Next, we inject three noises into the data: we append $\{8, 9, 10\}$ to the $30$ samples. Find the MLE for the Gaussian and student $t$ distribution again. Draw the density curves and scatter data points in another figure. What do you observe, and why?

\end{enumerate}
\end{document}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End: