\documentclass[notoc,notitlepage]{tufte-book}
% \nonstopmode
\usepackage{longtable,multirow,booktabs}
\usepackage{pgfplots}
\usepackage{classnotetitle}

\title{STAT330S18 - Mathematical Statistics}
\author{Johnson Ng}
\subtitle{Classnotes for Spring 2018}
\credentials{BMath (Hons), Pure Mathematics major, Actuarial Science Minor}
\institution{University of Waterloo}

\usepackage{soul}
\usepackage{listings}
\lstset{frame=tb,
  language=R,
  aboveskip=3mm,
  belowskip=3mm,
  frame=single,
  showstringspaces=false,
  columns=flexible,
  basicstyle={\small\ttfamily},
  numbers=none,
  numberstyle=\tiny\color{be-magenta},
  keywordstyle=\color{be-blue},
  commentstyle=\color{be-gray},
  stringstyle=\color{mauve},
  breaklines=true,
  breakatwhitespace=true,
  tabsize=3
}
\input{latex-classnotes-preamble.tex}

\setlength\extrarowheight{10pt}
\newcolumntype{e}{>{$}l<{$}}
\newcolumntype{w}{>{$}r<{$}}
\newcolumntype{m}{>{$}c<{$}}

\input{probnotation.tex}

\newcommand{\convd}{\overset{D}{\to}}
\newcommand{\convp}{\overset{P}{\to}}

\pgfmathdeclarefunction{gauss}{2}{%
  \pgfmathparse{1/(#2*sqrt(2*pi))*exp(-((x-#1)^2)/(2*#2^2))}%
}

\begin{document}
\input{latex-classnotes-header.tex}

\chapter*{Foreword}
  \label{chapter:foreword}

\begin{fullwidth}
  The proofs in this set of notes will be more rigourous compared to the expectations of the course (at least, for the course this term). If you are not the author and is interested in reading the notes, you may skip the proofs should you have little interest in them. The rigour is required almost exclusively for the author himself, for his own practice, and because he transferred his STAT230 course from a class that is clean of proofs.

  Also, many of the common mathematical notations will be heavily used both in the author's notes and proofs. The author cannot guarantee that his proofs are absolutely correct, but he tries, to the best of his abilities to minimize and assure of that. Should you be suspicious about the proofs, or should you notice errorneous ones, please do inform the author at \url{https://github.com/japorized/TeX_notes/issues}.

  You are also warned that the author is rather bummed with how the course is presented in the term that he is/was taking it, and so there may be sarcastic language (towards the lectures) mixed in his notes.
\end{fullwidth}

% chapter foreword (end)

\chapter{Lecture 1 May 1st 2018}
  \label{chapter:lecture_1_may_1st_2018}

\section{Introduction} % (fold)
\label{sec:introduction}

\begin{defn}[Sample Space]\label{defn:sample_space}\index{Sample Space}
  A \textbf{sample space}, \textbf{$S$} of a random experiment is the set of all possible outcomes of the experiment.
\end{defn}

\begin{eg}
  \label{eg:sample_space_eg}
  The following are some random experiments and their sample space.
  \begin{itemize}
    \item Flipping a coin\\
      $S = \{H, T\}$ where $H$ denotes head and $T$ tail.
    \item Rolling a 6-faced dice twice\\
      $S = \{(x, y) : x, y \in \mathbb{N}, \; 1 \leq x, y \leq 6 \}$
    \item Measuring a patient's height\\
      $S = R^+ = \{x \in \mathbb{R} : x \geq 0\}$
  \end{itemize}
\end{eg}

\begin{defn}[$\sigma$-field]\label{defn:sigma_field}\index{$\sigma$-field}
  Let $S$ be a sample space. The collection of sets $\mathscr{B} \subseteq \mathbb{P}(S)$\sidenote{The \hldefn{power set} of $S$, $\mathbb{P}(S)$, is defined as the set that contains all subsets of $S$.}, is called a $\sigma$-field (or \hldefn{$\sigma$-algebra}) on $S$ if:
  \begin{enumerate}
    \item $\emptyset \in \mathscr{B}$ and $S \in \mathscr{B}$;
    \item $\forall A \in \mathscr{B} \quad A^C \in \mathscr{B}$; \sidenote{We shall denote the compliment of a set by a superscript $C$ in this set of notes. The supplemental notes provided in the class uses an overhead bar, e.g. $\bar{A}$, while lecture notes will use $A^C$ and $A'$ interchangably.} and
    \item $\forall n \in \mathbb{N} \quad \forall \{A_j\}_{j = 1}^{n} \subseteq \mathscr{B} \quad \cup_{j=1}^{n} A_j \in \mathscr{B}$.
  \end{enumerate}
\end{defn}

\begin{defn}[Measurable Space]\label{defn:measurable_space}\index{Measurable Space}
  Given that $S$ is a non-empty set, and $\mathscr{B}$ is a $\sigma$-field, $(S, \mathscr{B})$ is a \textbf{measurable space}.\sidenote{A measurable space is a basic object in \hlnotea{measure theory}.}
\end{defn}

\begin{eg}
  \label{eg:sigma_field_eg}
  Consider $S = \{1, 2, 3, 4\}$. Check if $\mathscr{B} = \{\emptyset, \{1, 2, 3, 4\}, \{1, 2\}, \{3, 4\} \}$ is a $\sigma$-field on $S$.
  \begin{enumerate}
    \item It is clear that $\emptyset, S \in \mathscr{B}$.
    \item Note that $S^C = \emptyset$ and $\{1, 2\}^C = \{3, 4\}$.
    \item Note that the largest possible result of any countable union of the elements of $\mathscr{B}$ is $\{1, 2, 3, 4\}$, which is an element of $\mathscr{B}$.
  \end{enumerate}
\end{eg}

\newthought{Because} $(S, \mathscr{B})$ is a measurable space, we can define a measure on it.

\begin{defn}[Probability Measure]\label{defn:probability_measure}\index{Probability Measure}
  Suppose $S$ is a sample space of a random experiment. Let $\mathscr{B} = \{A_1, A_2, ...\} \subseteq \mathbb{P}(S)$ be the $\sigma$-field on $S$. The \hldefn{probability set function} (or \textbf{probability measure}), $P : \mathscr{B} \to [0, 1]$, is a function that satisfies the following:\sidenote{These conditions are also known as \hldefn{Kolmogorov Axioms}, or \hldefn{probability axioms}.}
  \begin{itemize}
    \item $\forall A \in \mathscr{B} \enspace P(A) \geq 0$;
    \item $P(S) = 1$;
    \item $\forall \{A_j\}_{j = 1}^{\infty} \subseteq \mathscr{B} \enspace \forall i \neq j \in \mathbb{N} \; A_i \cap A_j = \emptyset \implies$
      \begin{equation}\label{eq:probability_of_union_of_disjoint_sets}
        P \left( \bigcup_{j=1}^{\infty} A_j \right) = \sum_{j=1}^{\infty} P(A_j)
      \end{equation}
  \end{itemize}
  $(S, \mathscr{B}, P)$ is called a \hldefn{probability space}.
\end{defn}

\begin{eg}
  \label{eg:probability_measure}
  Consider flipping a coin where $S = \{H, T\}$. Let $P$ be defined as follows
  \begin{equation*}
    P(\{H\}) = \frac{1}{3} \quad P(\{T\}) = \frac{2}{3} \quad P(\emptyset) = 0 \quad  P(S) = 1
  \end{equation*}
  Conditions 1 and 2 of \cref{defn:probability_measure} are met. Notice that
  \begin{equation*}
    P(\{H\} \cup \{T\}) = P(S) = 1 \enspace \text{and} \enspace P(\{H\}) + P(\{T\}) = \frac{1}{3} + \frac{2}{3} = 1.
  \end{equation*}
  Hence condition 3 is also fulfilled.
\end{eg}

\begin{propo}[Properties of Probability Set Functions]\label{propo:properties_of_probability_set_functions}
  Let $P$ be a probability set function and $A, B$ be any set in $\mathscr{B}$. Prove the following:\sidenote{Many among these properties illustrate that the probability is indeed a \hlnotea{measure}.}
  \begin{enumerate}
    \item $P(A^C) = 1 - P(A)$
    \item $P(\emptyset) = 0$
    \item $P(A) \leq 1$
    \item $P(A \cap B^C) = P(A) - P(A \cap B)$
    \item $P(A \cup B) = P(A) + P(B) - P(A \cap B)$
    \item $A \subseteq B \implies P(A) \leq P(B)$
  \end{enumerate}
  \marginnote{\begin{ex}
    Prove that $A \subseteq B \iff B^C \subseteq A^C$.
  \end{ex}}
\end{propo}

\begin{proof}
  Let $S$ be the sample space for $P$.
  \begin{enumerate}
    \item Note that\\
      $A \in \mathscr{B} \implies A \in \mathbb{P}(S) \iff A \subseteq S$\\
      $A \in \mathscr{B} \iff A^C \in \mathscr{B} \implies A^C \subseteq S$.
      Also, since $A^C$ is the complement of $A$, it is clear that $S = A \cup A^C$.
      \begin{equation*}
        \therefore P(S) = 1 \iff P(A \cup A^C) = 1 \overset{1}{\iff} P(A) + P(A^C) = 1
      \end{equation*}
      where $1$ is by condition 3 in \cref{defn:probability_measure} since $A \cap A^C = \emptyset$ by definition of a complement of a set.

    \item Note that $S \cup \emptyset = S$ and $S \cap \emptyset = \emptyset$. Using a similar argument as above,
      \begin{equation*}
        1 = P(S) = P(S \cup \emptyset) = P(S) + P(\emptyset) \implies P(\emptyset) = 0
      \end{equation*}

    \item By 1 from above, $P(A) = 1 - P(A^C)$. Since $0 \leq P(A^C) \leq 1$, we have that $P(A)$ is at most $1$, as required.

    \item Note that $A = (A \cap B) \cup ( A \cap B^C )$. Clearly, $(A \cap B) \cap (A \cap B^C) = \emptyset$.\sidenote{This is an easy proof using the basic way of proving membership.} Hence by condition 3 in \cref{defn:probability_measure},
      \begin{equation*}
        P(A) = P(A \cap B) + P(A \cap B^C)
      \end{equation*}

    \item Consider $P(A \cup B) + P(A \cap B)$. By definition,
      \begin{equation*}
        A \cup B = (A \cap B^C) \cup (A \cap B) \cup (A^C \cap B)
      \end{equation*}
      where each of the sets in brackets are disjoint from each other\sidenote{Again, this is not hard to show}. By condition 3 of \cref{defn:probability_measure}, we would then have
      \begin{align*}
        &P(A \cup B) + P(A \cap B) \\
          &= P(A \cap B^C) + P(A \cap B) + P(A^C \cap B) + P(A \cap B) \\
          &= 2 P(A \cap B) + P(A) - P(A \cap B) + P(B) - P(A \cap B) \enspace \text{by 4} \\
          &= P(A) + P(B)
      \end{align*}

    \item Note that $B = B \cap S = B \cap (A^C \cup A) = (B \cap A^C) \cup A$. Clearly, $A \cap (B \cap A^C) \neq \emptyset$. By condition 3 in \cref{defn:probability_measure}, we thus have that
      \begin{equation*}\tag{$\dagger$}\label{eq:properties_prob_set_fn_6_1}
        P(B) = P(B \cap A^C) + P(A).
      \end{equation*}
      Suppose $A \subsetneq B$. Then $B \cap A^C \neq \emptyset$. I shall make the claim that $B \cap A^C \in \mathscr{B}$. Since $A \subseteq B$ we have that
      \begin{align*}
        a \in (B \cap A^C) &\iff a \in B \, \land \, a \in A^C \\
          &\iff a \in B \, \land \, a \notin A \\
          &\iff a \in (B \setminus A).
      \end{align*}
      But $B \setminus A$ is a subset of $B$ from the above steps\sidenote{This is rather obvious from the steps, since $\forall a \in (B \cap A^C)$, $a \in B$.}. Therefore, $(B \cap A^C) \subseteq B \in \mathscr{B}$ as required.

      With that done, by condition 1 in \cref{defn:probability_measure}, $P(B \cap A^C) \geq 0$. Hence from \cref{eq:properties_prob_set_fn_6_1}, we have that
      \begin{align*}
        P(B) &= P(B \cap A^C) + P(A) \\
          &\geq P(A)
      \end{align*}
      as required. \qed
  \end{enumerate}
\end{proof}
\begin{defn}[Conditional Probability]\label{defn:conditional_probability}\index{Conditional Probability}
  Suppose $S$ is a sample space of a random experiment, and $A, B \subseteq S$. The \hlnoteb{conditional probability of $A$ given $B$} is given by
  \begin{equation}\label{eq:conditional_probability}
    P(A | B) = \frac{P(A \cap B)}{P(B)} \quad \text{provided } P(B) > 0.
  \end{equation}
\end{defn}

\begin{defn}[Independent Events]\label{defn:independent_events}\index{Independent Events}
  Suppose $S$ is a sample space of a random experiment, and $A, B \subseteq S$. $A$ and $B$ are said to be \hlnoteb{independent of each other} if
  \begin{equation*}
    P(A \cap B) = P(A) P(B)
  \end{equation*}
\end{defn}

\begin{propo}[Boole's Inequality\index{Boole's Inequality}]\label{propo:boole_s_inequality}
  If $\{A_j\}_{j = 1}^{\infty}$ is a sequence of events, then
  \begin{equation*}
    P \left( \bigcup_{j = 1}^{\infty} A_j \right) \leq \sum_{j=1}^{\infty} P(A_j)
  \end{equation*}
\end{propo}

\begin{proof}
  \hlwarn{Proof shall be provided later}
\end{proof}

\begin{propo}[Bonferroni's Inequality\index{Bonferroni's Inequality}]\label{propo:bonferroni_s_inequality}
  If $\{A_j\}_{j = 1}^{k}$ is a set of events where $k \in \mathbb{N}$, then
  \begin{equation*}
    P \left( \bigcap_{j = 1}^{k} A_j \right) \geq 1 - \sum_{j=1}^{k} P(A^C_j)
  \end{equation*}
\end{propo}

\begin{proof}
  \hlwarn{Proof shall be provided later}
\end{proof}

\begin{propo}[Continuity Property\index{Continuity Property}]\label{propo:continuity_property}
  If $A_1 \subset A_2 \subset \hdots$ is a sequence where $A = \cup_{i = 1}^{n} A_i$, then
  \begin{equation*}
    \lim_{n \to \infty} P \left( \bigcup_{i = 1}^{n} A_i \right) = P(A)
  \end{equation*}
\end{propo}

\begin{proof}
  \hlwarn{Proof shall be provided later}
\end{proof}

% section introduction (end)

\section{Random Variable} % (fold)
\label{sec:random_variable}

\begin{defn}[Random Variable]\label{defn:random_variable}\index{Random Variable}
  In a given probability space $(S, \mathscr{B}, P)$, the function $X : S \to \mathbb{R}$ is called a \hlnoteb{random variable}\sidenote{We shall use rv as shorthand for random variable in this set of notes.} if
  \begin{equation}\label{eq:random_var_defn}
    P(X \leq x) = P \left( \{ \omega \in S \, : \, X(\omega) \leq x \} \right)
  \end{equation}
  is defined for all $x \in \mathbb{R}$\sidenote{$X \leq x$ is an abbreviation for $\{\omega \in S \, : \, X(\omega) \leq x \} \in \mathscr{B}$.}.
\end{defn}

\begin{eg}
  \label{eg:random_var_eg}
  In a coin flip experiment, we have that $S = \{H, T\}$ where $\mathbb{P}(S) = \{\emptyset, S, \{H\}, \{T\} \}$. Define $X$ : the number of heads in a flip, i.e.
  \begin{equation*}
    X(\{H\}) = 1 \text{ and } X(\{T\}) = 0
  \end{equation*}
  To prove why $X$ is a random variable given this definition, notice that
  \begin{align*}
    x < 0 &\implies P(X \leq x) = P(\{\omega \in S \, : \, X(\omega) < 0\}) = P(\emptyset) = 0 \\
    x \geq 1 &\implies P(X \leq x) = P(\{\omega \in S \, : \, X(\omega) \leq x\}) = P(\{H, T\}) \\
      & \qquad = P(\{H\}) + P(\{T\}) = 1 \text{ by Independence} \\
    0 \leq x < 1 &\implies P(X \leq x) = P(\{\omega \in S \, : \, X(\omega) \leq x\}) = P(T) \geq 0
  \end{align*}
  which shows that $P$ is deiined for all $x \in \mathbb{R}$. Hence $X$ is a random variable.
\end{eg}

\begin{defn}[Cumulative Distribution Function]\label{defn:cumulative_distribution_function}\index{Cumulative Distribution Function}
  The \hlnoteb{cumulative distribution function (c.d.f)} of a random variable $X$ is defined as
  \begin{equation*}
    \forall x \in \mathbb{R} \quad F(x) = P(X \leq x)
  \end{equation*}
\end{defn}

\begin{note}
  \newthought{Notice} that $F(x)$ is defined for \hlimpo{all} real numbers, and since it is a probability, we have $0 \leq F(x) \leq 1$.
\end{note}

\begin{propo}[Properties of the cdf\index{Properties of the cdf}]\label{propo:properties_of_the_cdf}
  \begin{enumerate}
    \item $\forall x_1 < x_2 \in \mathbb{R} \quad F(x_1) \leq F(x_2)$
    \item $\lim_{x \to -\infty} = 0 \, \land \, \lim_{x \to \infty} = 1$
    \item $\lim_{x \to a^+} F(x) = F(a)$ \sidenote{$F$ is a \hldefn{right-continuous} function.}
    \item $\forall a < b \in \mathbb{R} \quad P(a < X \leq b) = P(X \leq b) - P(X \leq a) = F(b) - F(a)$
    \item $P(X = b) = F(b) - \lim_{a \to b^-} F(a)$ \sidenote{This is also called \hlnotea{the magnitude of the jump}.}
  \end{enumerate}
\end{propo}

\begin{proof}
  \hlwarn{Proof shall be provided later}
\end{proof}

\begin{note}
  The definition and properties of the cdf hold for the rv $X$ regardless of whether $S$ is discrete (finite or countable) or not.
\end{note}

% section random_variable (end)

\section{Discrete Random Variable} % (fold)
\label{sec:discrete_random_variable}

\begin{defn}[Discrete Random Variable]\label{defn:discrete_random_variable}\index{Discrete Random Variable}
  An rv $X$ is a \hlnoteb{discrete random variable} when its image is finite or countably infinite, i.e. $X \in \{x_1, x_2, ...\}$. The function
  \begin{equation*}
    \forall x \in \mathbb{R} \quad f(x) := P(X = x) = F(x) - \lim_{\epsilon \to 0^+} F(x - \epsilon)
  \end{equation*}
  is its probability function, commonly known as the \hldefn{probability mass function} (pmf). The set $A := \{x : f(x) > 0\}$ is called the \hldefn{support set} of $X$, and
  \begin{equation}\label{eq:defn_discrete_rv_prob_sum}
    \sum_{x \in A} f(x) = \sum_{i=1}^{\infty} f(x_i) = 1.
  \end{equation}
\end{defn}

\begin{propo}[Properties of pmf]\label{propo:properties_of_pmf}\index{Properties of pmf}
  With the notation from \cref{defn:discrete_random_variable}, prove that
  \begin{enumerate}
    \item $\forall x \in \mathbb{R} \quad f(x) \geq 0$
    \item $\sum_{x \in A} f(x) = 1$
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item This result follows from the fact that $f$ is a pdf, a probability, i.e. $\forall x \in R$, $f(x) = 0$ is $x \notin S$ where $S$ is the sample space, and $0 \leq f(x) \leq 1$ if $x \in S$.
    \item Since $A = \{x : f(x) > 0\}$, we know that
      \begin{equation*}
         \sum_{x \in A} f(x) > 0.
      \end{equation*}
      If we consider all the elements of $A$, we have that the events $(X = x_i)$, for $x_i \in A$, constitutes the entire sample space. Therefore,
      \begin{equation*}
        \sum_{x \in A} f(x) = \sum_{x \in A} P(X = x) = P(S) = 1.
      \end{equation*}
  \end{enumerate}\qed
\end{proof}

\begin{ex}
  Consider an urn containing $r$ red marbles and $b$ black marbles. Find the pmf of the rv for the following:
  \begin{enumerate}
    \item $X =$ number of red balls in $n$ selections without replacement.
    \item $X =$ number of red balls in $n$ selections with replacement.
    \item $X =$ number of black balls selected before obtaining the first red ball if sampling is done with replacement.
    \item $X =$ number of black balls selected before obtaining the $k$th red ball if sampling is done with replacement.
  \end{enumerate}

  \begin{solution}
    \begin{enumerate}
      \item Let $d = \max\{n, r + b\}$. The desired pmf is therefore the pmf from the hypergeometric distribution
      \begin{equation*}
        \forall x \in \mathbb{Z}_{\leq r}^{+} \quad f(x) = \frac{\binom{r}{x} \binom{b}{d - x}}{\binom{r + b}{d}}.
      \end{equation*}
      \item $\forall x \in \mathbb{Z}^+ \quad f(x) = \binom{n}{x} \left( \frac{r}{r + b} \right)^x \left( \frac{b}{r + b} \right)^{n - x}$, which is the pmf of the binomial distribution.
      \item $\forall x \in \mathbb{Z}^{+} \quad f(x) = \left( \frac{b}{r + b} \right)^x \left( \frac{r}{r + b} \right)$
      \item $\forall x \in \mathbb{Z}^{+} \quad f(x) = \binom{x + k - 1}{k - 1} \left( \frac{b}{r + b} \right)^x \left( \frac{r}{r + b} \right)^k$
    \end{enumerate}
  \end{solution}
\end{ex}

\begin{eg}
  Consider the function
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{C \mu^x}{x!} & x \in \mathbb{Z}^+, \, \mu > 0 \\
      0                  & \text{otherwise}
    \end{cases}
  \end{equation*}
  Find $C$ such that $f(x)$ is a pmf for the rv $X$.

  \begin{solution}
    We have that\marginnote{This gives us that $\forall x \in \mathbb{Z}^+$, $f(x) = \frac{e^{- \mu} \mu^x}{x!}$, and this is, of course, the pmf of the \hlnotea{Poisson distribution}.}
    \begin{align*}
      1 &= \sum_{x \in \mathbb{Z}^+} \frac{C \mu^x}{x!} \\
        &= C \sum_{x \in \mathbb{Z}^+} \frac{\mu^x}{x!} \\
        &= C e^\mu
    \end{align*}
    Thus $C = e^{- \mu}$.
  \end{solution}
\end{eg}

\begin{ex}
  Prove that the pdf of $X \sim \Poi(\mu)$ sums to $1$ over all of its values.

  \begin{solution}
    \begin{align*}
      \sum_{x \in \mathbb{N}} \frac{\mu^x e^{- \mu}}{x!}
        &= e^{- \mu} \sum_{x \in \mathbb{N}} \frac{\mu^x}{x!} \\
        &= e^{- \mu} e^\mu \quad \because \sum_{x \in \mathbb{N}}^{\infty} \frac{k^x}{x!} = e^k \\
        &= 1
    \end{align*}
  \end{solution}
\end{ex}

\begin{ex}
  If $X$ is a random variable with pmf
  \begin{equation*}
    f(x) = \frac{- (1 - p)^x}{x \log p}, \enspace x = 1, 2, ... \; ; \; 0 < p < 1,
  \end{equation*}
  show that
  \begin{equation*}
    \sum_{x \in \mathbb{N}} f(x) = 1
  \end{equation*}

  \begin{solution}
    \begin{align*}
      \sum_{x \in \mathbb{N}} \frac{- (1 - p)^x}{x \log p}
        &= - \frac{1}{\log p} \sum_{x \in \mathbb{N}} \frac{(-1)^x (p - 1)^x}{x} \\
        &= - \frac{1}{\log p} \underbrace{ \left[ - (p - 1) + \frac{(p - 1)^2}{2} - \frac{(p - 1)^3}{3} + \hdots \right] }_{\text{Taylor expansion of } - \log p} \\
        &= 1
    \end{align*}
  \end{solution}
\end{ex}

% section discrete_random_variable (end)

% chapter lecture_1_may_1st_2018 (end)

\chapter{Lecture 2 May 03rd 2018}
  \label{chapter:lecture_2_may_03rd_2018}

\section{Continuous Random Variable} % (fold)
\label{sec:continuous_random_variable}

\begin{defn}[Continuous Random Variable]\label{defn:continuous_random_variable}\index{Continuous Random Variable}
  Suppose $X$ is an rv with cdf $F$. If $F$ is a continuous function for all $x \in \mathbb{R}$ and $F$ is differentiable except possibly at countably many points, then $X$ is a \hlnoteb{continuous rv}. The probability function, or more commonly known as the \hldefn{probability density function} (pdf), of $X$ is $f(x) = F'(x)$ wherever $F$ is differentiable on $x$ and $0$ otherwise.

  The set $A = \{x : f(x) > 0\}$ is called the \hlnoteb{support set} of $X$ and
  \begin{equation*}
    \int_{x \in A} f(x) \dif{x} = 1
  \end{equation*}
\end{defn}

\begin{propo}[Properties of pdf]\label{propo:properties_of_pdf}\index{Properties of pdf}
Let $X$ be a random variable and $f$ be its pdf.
  \begin{enumerate}
    \item $\forall x \in \mathbb{R} \quad f(x) \geq 0$
    \item $\int_{-\infty}^{\infty} f(x) \dif{x} = 1$
    \item $f(x) = \lim_{h \to 0} \frac{F(x + h) - F(x)}{h} = \lim_{h \to 0} \frac{P(x \leq X \leq x + h)}{h}$ (if the limit exists)
    \item $\forall x \in \mathbb{R} \quad F(x) = \int_{-\infty}^{x} f(t) \dif{t}$
    \item $P(a < X \leq b) = \int_{a}^{b} f(x) \dif{x} = F(b) - F(a)$
    \item $P(X = b) = F(b) - \lim_{a \to b^{-}} F(a) = F(b) - F(b) = 0$
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item The argument of this proof is similar to that provided in \cref{propo:properties_of_pmf}.
    \item Same as above, except that the support set can now have complete intervals.
    \item The first equation follows from the first principles of Calculus. The second equation follows by method of calculation using the cdf.
    \item $F(x) = P(X \leq x) = \int_{-\infty}^{x} f(t) \dif{t}$.
    \item This follows immediately from the above property.
    \item The first part of the equation is a way to interpret the above property. The limit equates to $F(b)$ since $F$ is continuous.
  \end{enumerate}\qed
\end{proof}

\begin{eg}
  Consider the function
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{\theta}{x^{\theta + 1}}   & x \geq 1 \\
      0                               & x < 1
    \end{cases}
  \end{equation*}
  For what values of $\theta$ is $f$ a pdf?

  \begin{solution}
    If $f$ is a pdf, then $\theta \geq 0$. In fact, $\theta \neq 0$; otherwise $f$ would be equivalently $0$ for all $x \in \mathbb{R}$, which would imply that $\int_\mathbb{R} f = 0$, which is impossible. It remains to check if $\theta > 0$ is a safe choice. Now
    \begin{equation*}
      \int_{1}^{\infty} \frac{\theta}{x^{\theta + 1}} \dif{x} = - \frac{1}{x^{\theta}} \at{1}{\infty} = 1
    \end{equation*}
    Note that the above integral is valid because $\frac{1}{x^{\theta + 1}} \leq \frac{1}{x}$. Therefore the choice of $\theta > 0$ is safe.
  \end{solution}
\end{eg}

% section continuous_random_variable (end)

\section{Examples of Discrete RVs}
\label{sec:examples_of_discrete_rvs}
% section Examples of Discrete RVs

\subsection{Binomial Distribution}
\label{sub:binomial_distribution}
\index{Binomial Distribution}
% subsection Binomial Distribution

\begin{defn}[Binomial RV]\label{defn:binomial_rv}
  Consider $X$ to be the number of successes in a sequence of $n$ experiments where
  \begin{enumerate}
    \item experiments are \hlnoteb{independent};
    \item the outcome of each experient is a \hlnoteb{binary} (e.g. success or failure); and
    \item has the \hlnoteb{probability of success, $p$} for each singular experiment.
  \end{enumerate}
  $X$ is called a \hlnotea{Binomial} rv, and we write $X \sim \Bin(n, p)$ and its pmf is
  \begin{equation*}
    P(X = x) = \begin{cases}
      \binom{n}{x} p^x (1 - p)^{n - x} & x = 0, 1, 2, ..., n \\
      0                                & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

% subsection Binomial Distribution (end)

\subsection{Geometric Distribution}
\label{sub:geometric_distribution}
\index{Geometric Distribution}
% subsection Geometric Distribution

\begin{defn}[Geometric RV]\label{defn:geometric_rv}
  Consider a sequence of independent success/failure (binary) experiments, each of which has a success probability of $p$. Let $X$ be the \hlnoteb{number of failures} before the \hlnoteb{first success} is reached. We call $X$ a \hlnotea{Geometric} rv, and we write $X \sim \Geo(p)$, and its pmf is
  \begin{equation*}
    P(X = x) = \begin{cases}
      (1 - p)^x p & x = 0, 1, 2, ..., n \\
      0           & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

\begin{note}
  Some authors would define the Geometric rv as:

  Let $X$ be the number of experiments until the first success.

  But that really is just a play of words.
\end{note}

% subsection Geometric Distribution (end)

\subsection{Poisson Distribution}
\label{sub:poisson_distribution}
\index{Poisson Distribution}
% subsection Poisson Distribution

\begin{defn}[Poisson RV]\label{defn:poisson_rv}
  Suppose $X$ is defined to be the number of occurrences of an event in a given time period. If the process on which the events occur satisfies the following:
  \begin{enumerate}
    \item The number of occurrences in non-overlapping intervals are independent of each other;
    \item The probability of the occurrence of an event in a short interval of length $h$ is proportional to $h$;
    \item For sufficiently short time periods of length $h$, the probability of $2$ or more events occurring in the interval is negligible, i.e. almost zero;
  \end{enumerate}
  then $X$ is a \hlnotea{Poisson} rv, and we write $X \sim \Poi(\lambda)$, with $\lambda > 0$, and the pmf is
  \begin{equation*}
    P(X = x) = \begin{cases}
      \frac{e^{- \lambda} \lambda^x}{x!} & x = 0, 1, ... \\
      0                                  & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

% subsection Poisson Distribution (end)

% section Examples of Discrete RVs (end)

\section{Examples of Continuous RVs}
\label{sec:examples_of_continuous_rvs}
% section Examples of Continuous RVs

\subsection{Normal/Gaussian Distribution}
\label{sub:normal_gaussian_distribution}
\index{Normal Distribution}\index{Gaussian Distribution}
% subsection Normal/Gaussian Distribution

\begin{defn}[Normal / Gaussian RV]\label{defn:normal_gaussian_rv}
  The \hlnoteb{Normal/Gaussian} Distribution is a continuous probability distribution that is symmetric about the mean, showing that data around the mean is more frequent than data far from the mean. If $X$ is a \hlnotea{Normal/Gaussian} rv, we write $X \sim \Nor(\mu, \sigma^2)$, and its pdf is
  \begin{equation*}
    f(x) = \frac{1}{\sqrt{2 \pi \sigma^2}} e^{- \frac{(x - \mu)^2}{2 \sigma^2}} \quad \text{for } x \in \mathbb{R}.
  \end{equation*}
\end{defn}

\begin{defn}[Standard Normal Distribution]\label{defn:standard_normal_distribution}
\index{Standard Normal Distribution}
  The \hlnoteb{Standard Normal Distribution} is the simplest case of a Normal Distribution. An rv $Z$ is called the \hlnotea{Standard Normal} rv if $\mu = 0$ and $\sigma = 1$. We write $Z \sim \Nor(0, 1)$ and its pdf is
  \begin{equation*}
    f(x) = \frac{1}{\sqrt{2 \pi}} e^{- \frac{x^2}{2}} \quad \text{for } x \in \mathbb{R}.
  \end{equation*}
\end{defn}

% subsection Normal/Gaussian Distribution (end)

\subsection{Uniform Distribution}
\label{sub:uniform_distribution}
\index{Uniform Distribution}
% subsection Uniform Distribution

\begin{defn}[Uniform RV]\label{defn:uniform_rv}
  If $X$ represents the result of drawing a real number from an interval $(a, b)$, with $a < b$, such that all numbers in between are equally likely to be chosen, then $X$ is called a \hlnotea{Uniform} rv, and we write $X \sim \Unif(a, b)$, and its pdf is
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{1}{b - a} & x \in (a, b) \\
      0               & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

% subsection Uniform Distribution (end)

\subsection{Exponential Distribution}
\label{sub:exponential_distribution}
\index{Exponential Distribution}
% subsection Exponential Distribution

\begin{defn}[Exponential RV]\label{defn:exponential_rv}
  Let $X$ show the time between two consecutive events in a \hlnoteb{Poisson process}, i.e. the 3 conditions in \nameref{defn:poisson_rv} are satisfied. Then $X$ is called an \hlnotea{Exponential} rv, and we write $X \sim \Exp(\theta)$, where $\theta > 0$, with its pdf
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{1}{\theta} e^{- \frac{x}{\theta}} & x > 0 \\
      0                                       & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

% subsection Exponential Distribution (end)

\subsection{Gamma Distribution}
\label{sub:gamma_distribution}
\index{Gamma Distribution}
% subsection Gamma Distribution

\begin{defn}[Gamma RV]\label{defn:gamma_rv}
  Let $X$ be the sum of $n$ independent \hlnotea{Exponential} rvs with some fixed $\theta$. Then $X$ is called a \hlnotea{Gamma} rv, in which we write $X \sim \Gamma(n, \theta)$, and its pdf is
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{x^{n - 1} e^{- \frac{x}{\theta}}}{\Gamma(n) \theta^n} & x > 0 \, \land \, \theta, n > 0 \\
      0                                                           & \text{otherwise}
    \end{cases}
  \end{equation*}
  where $\Gamma(n) = \int_{0}^{\infty} e^{-y} y^{n - 1} \dif{y} = (n - 1)!$, where the last equality is true when $n$ is an integer.
\end{defn}

\begin{note}
  The Gamma Distribution is usually used for when we are looking for the probability of the occurrence of the $n$-th event in the desired waiting time.
\end{note}

% subsection Gamma Distribution (end)

% section Examples of Continuous RVs (end)

\section{Functions of Random Variables}
\label{sec:functions_of_random_variables}
% section Functions of Random Variables

\newthought{Consider} the rv $X$ with pdf/pmf $f$ and cdf $F$. Given $Y = h(X)$ where $h$ is some real-valued function, we are interested in finding the pdf/pmf of $Y$.

The following are some possible scenarios:
\begin{enumerate}
  \item $X$ and $Y$ are both discrete;
  \item $X$ is continuous and $Y$ is discrete;
  \item $X$ and $Y$ are both continuous
\end{enumerate}
We may also define $Y = h(X)$ for a continuous rv $X$ such that $Y$ is \hlimpo{neither discrete nor continuous} (e.g. discrete for some values of $X$ while continuous for others).

\subsection{Discrete $X$ and Discrete $Y$}%
\label{sub:discrete_x_and_discrete_y}
% subsection discrete_x_and_discrete_y

If $X$ and $Y = h(X)$ are both discrete, we can derive $P(Y = y)$ by mapping values in $Y$ onto their corresponding value through $h$, i.e.
\begin{equation*}
  P(Y = y) = \sum_{\{x : h(x) = y\}} P(X = x)
\end{equation*}

\begin{ex}
  Let $X$ have the following probability function:
  \begin{equation*}
    f_X (x) = \begin{cases}
      \frac{e^{-1}}{x!} & x = 0, 1, 2, ... \\
      0                 & \text{otherwise}
    \end{cases}
  \end{equation*}
  Find the pmf of $Y = (X - 1)^2$.

  \begin{solution}
    Note that since
    \begin{equation*}
      \Dom X = \{0, 1, 2, 3, 4, ...\},
    \end{equation*}
    we have that
    \begin{equation*}
      \Dom Y = \{1, 0, 1, 4, 9, ...\}.
    \end{equation*}
    With that, note that
    \begin{align*}
      P(Y = 0) &= P(X = 1) = \frac{e^{-1}}{1!} \\
      P(Y = 1) &= P(X = 0 \text{ or } 2) = P(X = 0) + P(X = 2) \\
               &= \frac{e^{-1}}{0!} + \frac{e^{-1}}{2!}
                = e^{-1} \left( 1 + \frac{1}{2} \right) = \frac{3}{2} e^{-1} \\
      P(Y = 4) &= P(X = 3) = \frac{e^{-1}}{3!} \\
      P(Y = 9) &= P(X = 4) = \frac{e^{-1}}{4!}.
    \end{align*}
    Therefore, the pmf of $Y = (X - 1)^2$ is
    \begin{equation*}
      P(Y = y) = \begin{cases}
        e^{-1}                         & y = 0 \\
        \frac{3}{2} e^{-1}             & y = 1 \\
        \frac{e^{-1}}{(1 + \sqrt{y})!} & y = 4, 9, 16, ... \\
        0                              & \text{otherwise}
      \end{cases}
    \end{equation*}
  \end{solution}
\end{ex}

% subsection discrete_x_and_discrete_y (end)

\subsection{Continuous $X$ and Discrete $Y$}%
\label{sub:continuous_x_and_discrete_y}
% subsection continuous_x_and_discrete_y

If $X$ is continuous and $Y$ is discrete, we can use the method that we have used in the previous subsection, and replace $\Sigma$ by the integral sign $\int$, i.e. define $A := \{x : h(x) = y\}$ such that we have
\begin{equation*}
  P(Y = y) = \int_{A} f(x) \dif{x}
\end{equation*}

\begin{eg}[Example 2.9]\label{eg:cont_x_disc_y}
  Suppose $X$ is a random variable with the following probability function
  \begin{equation*}
    f_X(x) = \begin{cases}
      2e^{2x} & x > 0 \\
      0       & \text{otherwise}
    \end{cases}.
  \end{equation*}
  Suppose $Y = h(X)$ is defined as follows:
  \begin{equation*}
    Y = \begin{cases}
      1 & X < 1 \\
      2 & 1 \leq X \leq 2 \\
      3 & X > 2
    \end{cases}
  \end{equation*}
  Find the probability function of $Y$.

  \begin{solution}
    Note that $X \sim \Exp(\frac{1}{2})$. So it is clear that $X$ is a crv and since $Y = 1, 2$, or $3$, we have that $Y$ is discrete. Now
    \begin{align*}
      P(Y = 1) &= P(X < 1) = \int_{0}^{1} 2e^{-2x} \dif{x} \\
               &= -e^{-2x} \at{0}{1} = 1 - e^{-2} \\
      P(Y = 2) &= P(1 \leq X \leq 2) = \int_{1}^{2} 2e^{-2x} \dif{x} \\
               &= -e^{-2x} \at{1}{2} = e^{-2} - e^{-4} \\
      P(Y = 3) &= P(X > 2) = \int_{2}^{\infty} 2e^{-2x} \dif{x} \\
               &= -e^{-2x} \at{2}{\infty} = e^{-4}
    \end{align*}
    Thus the pmf is
    \begin{align*}
      P(Y = y) = \begin{cases}
        1 - e^{-2}      & Y = 1 \\
        e^{-2} - e^{-4} & Y = 2 \\
        e^{-4}          & Y = 3
      \end{cases}
    \end{align*}
  \end{solution}
\end{eg}

% subsection continuous_x_and_discrete_y (end)

\subsection{Continuous $X$ and Continuous $Y$}%
\label{sub:continuous_x_and_continuous_y}
% subsection continuous_x_and_continuous_y

If $X$ and $Y= h(X)$ are both continous, start with the definition of the cdf of $Y$, i.e.
\begin{equation*}
  F_Y(y) = P(Y \leq y) = P(h(X) \leq y)
\end{equation*}
solve the inequality for $X$, and then obtain the cdf of $Y$. We will then only need to differentiate the cdf wrt $y$ to get the pdf that we desire.

\begin{eg}[Example 2.10]
  Let $X$ have the following pdf:
  \begin{equation*}
    f_X (x) = \begin{cases}
      2e^{-2x} & x \geq 0 \\
      0        & \text{otherwise}
    \end{cases}
  \end{equation*}
  Find the pdf of $Y = \sqrt{X}$.

  \begin{solution}
    We have that the range of values where $f_Y (y) \leq 0$ is $y \geq 0$. Now
    \begin{align*}
      F_Y(y) &= P(Y \leq y) = P(\sqrt{X} \leq y) = P(X \leq y^2) \\
             &= \int_{0}^{y^2} 2e^{-2x} \dif{x} \\
             &= -e^{-2x} \at{0}{y^2} = 1 - e^{-2y^2}
    \end{align*}
    Therefore, the pdf of Y is
    \begin{equation*}
      f_Y(y) = \begin{cases}
        \frac{d}{dy} 1 - e^{-2y^2} = 4ye^{-2y^2} & y \leq 0 \\
        0 & \text{otherwise}
      \end{cases}.
    \end{equation*}
  \end{solution}
\end{eg}

% subsection continuous_x_and_continuous_y (end)

\subsection{A Formula for the Continuous Case}%
\label{sub:a_formula_for_the_continuous_case}
% subsection a_formula_for_the_continuous_case

\begin{thm}[One-to-One Transformation of a Random Variable]
\label{thm:one_to_one_transformation_of_a_random_variable}
  Suppose $X$ is a continuous random variable with pdf $f_X$ and support set $A = \{x : f_X(x) > 0\}$ and $Y = h(X)$ where $h$ is a real-valued function. Let $f_Y$ be the pdf of the rv $Y$ and let $B = \{y : f_Y(y) > 0\}$. If $h$ is a one-to-one function from $A$ to $B$ and if $h'$ is continuous, then
  \begin{equation*}
    f_Y(y) = f \big( h^{-1}(y) \big) \cdot \abs{ \frac{d}{dy}h^{-1}(y) }, \quad y \in B
  \end{equation*}
\end{thm}

\begin{proof}
  Note that since $h$ is one-to-one, it is monotonous. Suppose $h$ is increasing. Then $h^{-1}$ is also an increasing function. Note that the cdf of $Y$ is
  \begin{equation*}
    F_Y(y) = P(Y \leq y) = P(X \leq h^{-1}(y)) = F_X \big( h^{-1}(y) \big).
  \end{equation*}
  Then the cdf of $Y$ is
  \begin{align*}
    f_Y(y) = \frac{d}{dy} F_X \big( h^{-1}(y) \big) = f_X \big( h^{-1}(y) \big) \cdot \frac{d}{dy} h^{-1}(y)
  \end{align*}
  If $h$ is decreasing, then so is its inverse. Thus
  \begin{equation*}
    F_Y(y) = P(Y \leq y) =  P(X \geq h^{-1}(y)) = 1 - F_X( h^{-1}(y) )
  \end{equation*}
  Thus the cdf of $Y$ is
  \begin{equation*}
    f_Y(y) = \frac{d}{dy} ( 1 - F_X( h^{-1}(y) ) ) = - f_X( h^{-1} (y) ) \cdot \frac{d}{dy} h^{-1}(y).
  \end{equation*}
  Note that the pdf of $Y$ is indeed positive since $h^{-1}$ is decreasing.

  Combining the two, we have that
  \begin{equation*}
    f_Y(y) = f_X( h^{-1}(y) ) \cdot \abs{ \frac{d}{dy} h^{-1}(y) },
  \end{equation*}
  as required. \qed
\end{proof}

% subsection a_formula_for_the_continuous_case (end)

% section Functions of Random Variables (end)

% chapter lecture_2_may_03rd_2018 (end)

\chapter{Lecture 3 May 08th 2018}
\label{chp:lecture_3_may_08th_2018}
% chapter lecture_3_may_08th_2018

\section{Functions of Random Variables (Continued)}%
\label{sec:functions_of_random_variables_continued}
% section functions_of_random_variables_continued

\subsection{Special Cases}%
\label{sub:special_cases}
% subsection special_cases

\begin{eg}
  Recall \cref{eg:cont_x_disc_y}. Suppose $X$ is a rv with the following probability function
  \begin{equation*}
    f_X(x) = \begin{cases}
      2e^{-2x} & x > 0 \\
      0        & \text{otherwise}
    \end{cases}.
  \end{equation*}
  Define $Y = h(X)$ as follows:
  \begin{equation*}
    Y = \begin{cases}
      1 & X < 1 \\
      X & 1 \leq X \leq 2 \\
      3 & X > 2
    \end{cases}
  \end{equation*}
  Find the cdf of $Y$.

  \begin{solution}
    \hlwarn{Solution is given differently in the 2 sections. I am not happy with either solutions because some things don't add up. My opinion is that the definition of $Y$ is badly given, along with a badly phrased question. As a result, there are more ways than one to interpret an already confusing information, and thus we have ourselves one hell of a mess.}
  \end{solution}
\end{eg}

% subsection special_cases (end)

% section functions_of_random_variables_continued (end)

\section{Probability Integral Transformation}%
\label{sec:probability_integral_transformation}
% section probability_integral_transformation

\begin{thm}[Probability Integral Transformation]\index{Probability Integral Transformation}
\label{thm:probability_integral_transformation}
  If $X$ is a contiunous rv with cdf $F$, then $Y = F(X) \sim \Unif(0, 1)$. $Y = F(X)$ is called the \hlnoteb{probability integral transformation}.
\end{thm}

\begin{note}
  The distribution of $Y = F(X)$ can be proven.
\end{note}

\begin{proof}
  Let $X$ be a continuous rv and $Y = F(X)$. Since $F(X)$ is one-to-one and increasing (i.e. monotonous), there exists $F^{-1}(Y)$ that is a real-valued and increasing function. Then
  \begin{align*}
    F_Y(y) &= P(Y \leq y) = P( F_X(X) \leq y ) = P(X \leq F^{-1}(y) ) \\
           &= F( F^{-1}(y) ) = y
  \end{align*}
  Note that $F_Y(y) = y$ is the cdf of a $\Unif(0, 1)$ rv, i.e. the \hlnotea{standard uniform random variable}. Thus $Y \sim \Unif(0, 1)$.
\end{proof}

\begin{note}
  This theorem essentially states that any rv from a continuous distribution can be transformed into a standard uniform distribution.
\end{note}

\begin{eg}[Example 2.11]
  Suppose $X \sim \Exp(0 1)$. We know that $F_X(x) = 1 - e^{-10x}$ for all $x \in \mathbb{R}$a. By \nameref{thm:probability_integral_transformation}, we have that $Y = F_X(X) = 1 - e^{-10X} \sim \Unif(0, 1)$.
\end{eg}

  Note that the converse of \nameref{thm:probability_integral_transformation} is true:

\begin{thm}[Converse of Probability Integral Transformation]
\label{thm:converse_of_probability_integral_transformation}
  Suppose $X$ is a continuous rv with cdf $F$ such that $F^{-1}$ exists. If $U \sim \Unif(0, 1)$, we have that $Y = F^{-1}(U) \sim X$.
\end{thm}

\begin{proof}
  Note that
  \begin{align*}
    F_Y(y) &= P(Y \leq y) = P( F^{-1}(U) \leq y ) \\
           &= P(U \leq F_X(y)) = F_X(y).
  \end{align*}\qed
\end{proof}

\begin{eg}[Example 2.12]
  Suppose $X \sim \Unif(0, 1)$. Find a transformation $T$ such that $T(X) \sim \exp(\theta)$.

  \begin{solution}
    Let $Y = T(X) \sim \Exp(\theta)$. Note that
    \begin{equation*}
      F_Y(y) = 1 - e^{-\frac{y}{\theta}}, \quad y > 0
    \end{equation*}
    Observe that since
    \begin{equation*}
      x = 1 - e^{-\frac{y}{\theta}} \implies y = - \theta \ln (1 - x)
    \end{equation*}
    we have that
    \begin{equation*}
      F_Y^{-1}(X) = - \theta \ln (1 - X).
    \end{equation*}
    By \autoref{thm:converse_of_probability_integral_transformation}, we have that $T = F_Y^{-1}$.
  \end{solution}
\end{eg}

% section probability_integral_transformation (end)

\section{Location-Scale Families}%
\label{sec:location_scale_families}
% section location_scale_families

When we look into methods for constructing confidence intervals for an unknown parameter $\theta$. If the parameter $\theta$ is either a \textit{scale parameter} or \textit{location parameter}, then a confidence interval is easier to construct.

\begin{defn}[Location Parameter and Family]\index{Location Parameter}\index{Location Family}
\label{defn:location_parameter_and_family}
  Suppose $X$ is a continuous rv with pdf $f(x; \mu)$, where $\mu$ is a parameter of the distribution of $X$. Let $F_0(x) = F_X(x; \mu = 0)$, where $F_X$ is the cdf of $X$, and $f_0(x) = f(x; \mu = 0)$. The parameter $\mu$ is called a \hlnoteb{location paramter} of the distribution if
  \begin{equation*}
    F_X(x; \mu) = F_0( x - \mu ), \quad \mu \in \mathbb{R}
  \end{equation*}
  or equivalently,
  \begin{equation*}
    f(x; \mu) = f_0( x - \mu ), \quad \mu \in \mathbb{R}.
  \end{equation*}
  We say that $F$ belongs to a \hlnoteb{location family} of distributions.
\end{defn}

\begin{defn}[Scale Parameter and Family]\index{Scale Parameter}\index{Scale Family}
\label{defn:scale_parameter_and_family}
  Suppose $X$ is a continuous rv with pdf $f(x; \theta)$, where $\theta$ is a parameter of the distribution of $X$. Let $F_1(x) = F_X(x; \theta = 1)$, where $F_X$ is the cdf of $X$, and $f_1(x) = (x; \theta = 1)$. The parameter $\theta$ is called a \hlnoteb{scale parameter} of the distribution if
  \begin{equation*}
    F_X( x ; \theta ) = F_1( \frac{x}{\theta} ). \quad \theta > 0
  \end{equation*}
  or equivalently,
  \begin{equation*}
    f(x ; \theta) = \frac{1}{\theta} f_0( \frac{x}{\theta} ), \quad \theta > 0.
  \end{equation*}
  We say that $F$ belongs to a \hlnoteb{scale family} of distributions.
\end{defn}

\begin{defn}[Location-Scale Family]\index{Location-Scale Family}
\label{defn:location_scale_family}
  Suppose $X$ is an rv with cdf $F(x ; \mu, \sigma)$ where $\mu \in \mathbb{R}$ and $\sigma > 0$ are the parameters of the distribution. Let $Y = \frac{X - \mu}{\sigma}$. If the distribution of $Y$ does not depend on $\mu$ and/or $\sigma$, then $F$ is said to belong to a \hlnoteb{location-scale family} of distributions, with \hlnoteb{location parameter} $\mu$ and \hlnoteb{scale parameter} $\sigma$. In other words, $F$ belongs to a location-scale family of distributions if
  \begin{equation*}
    F(x; \mu, \theta) = F_0 \left( \frac{x - \mu}{\theta} \right),
  \end{equation*}
  where $F_0 (x) = F(x; \mu = 0, \theta = 1)$, or equivalently,
  \begin{equation*}
    f(x; \mu, \theta) = \frac{1}{\theta} f_0 \left( \frac{x - \mu}{\theta} \right),
  \end{equation*}
  where $f_0(x) = f(x; \mu = 0, \theta = 1)$.
\end{defn}

\begin{eg}[Example 2.13]
  Consider $X \sim \Gau(\mu, \sigma)$. Show that $F_X$ belongs to a location-scale family of distributions.

  We know that if $\mu = 0$ and $\sigma = 1$, then $Y = \frac{X - \mu}{\sigma} \sim \Gau(0, 1)$, and we know that $\Gau(0, 1)$ has no dependence on unknowns $\mu$ and $\sigma$. Therefore, $F_X$ belongs to the location-scale family of distributions, with location parameter $\mu$ and scale parameter $\sigma$.

  Another solution is to show that one of the equations in the definition is fulfilled. Observe that
  \begin{equation*}
    f_x(x) = \frac{1}{\sqrt{2 \pi \sigma^2}} e^{- \frac{(x - \mu)^2}{2 \sigma^2}}
  \end{equation*}
  So if we set $\mu = 0$ and $\sigma = 1$ to get $f_0$, we have that
  \begin{equation*}
    f_0(x) = \frac{1}{\sqrt{2 \pi}} e^{- \frac{x^2}{2}}.
  \end{equation*}
  Now, note that
  \begin{equation*}
    f(x) = \frac{1}{\sigma} \frac{1}{\sqrt{2 \pi}} e^{- \frac{1}{2} \left(\frac{(x - \mu)}{\sigma}\right)^2}.
  \end{equation*}
  Let $y = \frac{x - \mu}{\sigma}$, and we have ourselves
  \begin{equation*}
    f(x) = \frac{1}{\sigma} \frac{1}{\sqrt{2 \pi}} e^{- \frac{y^2}{2}} = \frac{1}{\sigma} f_0 (\frac{x - \mu}{\sigma})
  \end{equation*}
\end{eg}

\begin{eg}[Example 2.14]
  Consider $X \in \Gau(\mu, 2)$ where $\mu = E(X)$. Show that $\mu$ is a location parameter.

  We can use a similar approach as before and define $Y = X - \mu$ which follows $\Gau(0, 2)$. It is clear that we then have that $F_X$, the cdf of $X$, belongs to a location family of distributions.
\end{eg}

\begin{eg}[Example 2.15]\label{eg:3_3_3}
  Consider $X \sim \Exp(\theta)$. Show that $F_X$ belongs to a scale family of distributions and find the scale parameter.

  Note that
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{1}{\theta} e^{- \frac{x}{\theta}} & x > 0 \\
      0                                       & \text{otherwise}
    \end{cases}
  \end{equation*}
  Let $Y = \frac{X}{\theta}$. Then
  \begin{align*}
    F_Y (y) &= P(Y \leq y) = P(\frac{X}{\theta} \leq y) \\
            &= P(X \leq \theta y) = \int_{0}^{\theta y} \frac{1}{\theta} e^{-\frac{x}{\theta}} \dif{x} \\
            &= - e^{- \frac{x}{\theta}} \at{0}{\theta}{y} = 1 - e^{-y}
  \end{align*}
  and we have
  \begin{equation*}
    f_Y(y) = \begin{cases}
      e^{-y} & y > 0 \\
      0      & \text{otherwise}
    \end{cases}
  \end{equation*}
  Note that if we set $\sigma = 1$ to get $f_1$, we have
  \begin{equation*}
    f_1(x) = \begin{cases}
      e^{-x} & x > 0 \\
      0      & \text{otherwise}
    \end{cases}.
  \end{equation*}
  Therefore, $F_X$ belongs to a scale family of distributions.
\end{eg}

% section location_scale_families (end)

\section{Expectations}%
\label{sec:expectations}
% section expectations

\subsection{Expectations}%
\label{sub:expectations}
% subsection expectations

\begin{defn}[Expectation of A Discrete RV]
\label{defn:expectation_of_a_discrete_rv}
  If $X$ is a discrete rv with pmf $f$ and support set $A$, then the \hlnoteb{expectation} of $X$, or the \hldefn{expected value} of $X$ is defined by
  \begin{equation}\label{eq:expectation_discrete}
    E(X) = \sum_{x \in A} x f(x)
  \end{equation}
  provided that the sum converges absolutely, i.e.
  \begin{equation*}
    E(\abs{X}) = \sum_{x \in A} \abs{x} f(x) < \infty.
  \end{equation*}
  If $E(\abs{X})$ does not converge, then we say that $E(X)$ does not exist.
\end{defn}

\begin{defn}[Expectation of A Continuous RV]
\label{defn:expectation_of_a_continuous_rv}
  If $X$ is a continuous rv with pdf $f$ and support set $A$, then the \hlnoteb{expectation} of $X$, or the \hldefn{expected value} of $X$ is defined by
  \begin{equation}\label{eq:expectation_continuous}
    E(X) = \int_{x \in A} x f(x)
  \end{equation}
  provided that the integral converges absolutely, i.e.
  \begin{equation*}
    E(\abs{X}) = \int_{x \in A} \abs{x} f(x) < \infty.
  \end{equation*}
  If $E(\abs{X})$ does not converge, then we say that $E(X)$ does not exist.
\end{defn}

\begin{eg}[Example 2.16]
  Suppose $X \sim \Poi(\lambda)$. Calculate $E(X)$.

  \begin{solution}
    Note
    \begin{equation*}
      f(x) = \begin{cases}
        \frac{e^{- \lambda} \lambda^x}{x!}  & x = 0, 1, 2, ... \\
        0                                   & \text{otherwise}
      \end{cases}.
    \end{equation*}
    Then
    \begin{align*}
      E(X) &= \sum_{x = 0}^{\infty} x \frac{e^{-\lambda} \lambda^x}{x!} \\
           &= 0 + \sum_{x = 1}^{\infty} \frac{e^{-\lambda} \lambda^x}{(x - 1)!} \\
           &= e^{-\lambda} \lambda \sum_{x = 1}^{\infty} \frac{\lambda^{x - 1}}{(x - 1)!} \\
           &= e^{-\lambda} \lambda e^{\lambda} = \lambda
    \end{align*}
  \end{solution}
\end{eg}

\begin{eg}[Example 2.18]
  Suppose $X$ is an rv with
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{1}{x^2} & 1 < x < \infty \\
      0             & \text{otherwise}
    \end{cases}.
  \end{equation*}
  Calculate $E(X)$.

  \begin{solution}
    Observe that $x \cdot \frac{1}{x^2} = \frac{1}{x}$ and the antiderivative of $\frac{1}{x}$ is $\ln x$, which would need to be evaluated at $\ln \infty$. Thus, we should instead immediately check if the integral converges absolutely.
    \begin{align*}
      E(\abs{X}) &= \int_{1}^{\infty} \abs{x} \frac{1}{x^2} \dif{x} \\
                 &= \int_{1}^{\infty} \abs{x} \frac{1}{\abs{x} \abs{x}} \dif{x} \\
                 &= \int_{1}^{\infty} \frac{1}{\abs{x}} \dif{x} \\
                 &= \int_{1}^{\infty} \frac{1}{x} \dif{x},
    \end{align*}
    and we notice that the integral would not converge. Therefore, $E(X)$ does not exist.
  \end{solution}
\end{eg}

% subsection expectations (end)

% section expectations (end)

% chapter lecture_3_may_08th_2018 (end)

\chapter{Lecture 4 May 10th 2018}%
\label{chp:lecture_4_may_10th_2018}
% chapter lecture_4_may_10th_2018

\section{Expectations (Continued)}%
\label{sec:expectations_continued}
% section expectations_continued

\subsection{Expectations (Continued)}%
\label{sub:expectations_continued}
% subsection expectations_continued

\begin{thm}[Expectation from the cdf]
\label{thm:expectation_from_the_cdf}
  Suppose $X$ is a non-negative continuous rv with cdf $F$, and $E(X) < \infty$. Then
  \begin{equation}\label{eq:cont_cdf_to_expectation}
    E(X) = \int_{0}^{\infty} \left[ 1 - F(x) \right] \dif{x} = \int_{0}^{\infty} P(X \geq x) \dif{x}
  \end{equation}
  If $X$ is a discrete rv with cdf $F$, and $E(X) < \infty$, then
  \begin{equation}\label{eq:discrete_cdf_to_expectation}
    E(X) = \sum_{x = 0}^{\infty} \left[ 1 - F(x) \right] = \sum_{x = 0}^{\infty} P(X \geq x)
  \end{equation}
\end{thm}

\begin{proof}
  Note that for a continuous rv $X$, we have
  \begin{equation*}
    1 - F(x) = P(X \geq x) = \int_{x}^{\infty} f(t) \dif{t}
  \end{equation*}
  Therefore,
  \begin{equation*}
    \int_{0}^{\infty} \left[ 1 - F(x) \right] \dif{x} = \int_{0}^{\infty} \int_{x}^{\infty} f(t) \dif{t} \dif{x}.
  \end{equation*}
  Since $1 - F(x)$ is a finite value, so is $\int_{0}^{\infty} f(t) \dif{t}$, and thus we can apply \hlnotea{Fubini's Theorem}\sidenote{Condition for Fubini's Theorem to hold is that the integrand of the double integral must be absolutely convergent. See \href{https://en.wikipedia.org/wiki/Fubini\%27s_theorem}{Wikipedia}.}:
  \begin{equation*}
    \int_{0}^{\infty} \left[ 1 - F(x) \right] \dif{x} = \int_{0}^{\infty} \int_{x}^{\infty} f(t) \dif{t} \dif{x} = \int_{0}^{\infty} \int_{0}^{t} f(t) \dif{x} \dif{t}
  \end{equation*}
  Note that the limits of the integral utilizes the following figure:
	\begin{center}
		\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$t$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$x$};
				\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(3,0)--(3,3)--(0,0);
        \draw[line width=0pt,dashed](3,3)--(0,0) node[midway,left] [label=left:\textcolor{light}{$t = x$}] {};
		\end{tikzpicture}
	\end{center}
  \marginnote{Work on the discrete case as an exercise.
    \begin{ex}
      For a non-negative discrete rv $X$ with cdf $F$ and $E(X) < \infty$, prove that
      \begin{equation*}
        E(X) = \sum_{x = 0}^{\infty} [ 1 - F(x) ]
      \end{equation*}
    \end{ex}
  }
  With that, note that
  \begin{equation*}
    \int_{0}^{t} f(t) \dif{x} = x f(t) \at{0}{t} = tf(t)
  \end{equation*}
  Since $t$ is just a dummy variable, we can indeed let $t = x$, and thus we have
  \begin{equation*}
    \int_{0}^{\infty} \left[ 1 - F(x) \right] \dif{x} = \int_{0}^{\infty} xf(x) \dif{x} = E(X)
  \end{equation*}
  as required.

  \qed
\end{proof}

\begin{eg}[Example 2.20]
  Suppose $X \sim \Exp(\theta)$. Use \cref{thm:expectation_from_the_cdf} to calculate $E(X)$.

  \begin{solution}
    Note that $X$ is a non-negative rv. The cdf of $X \im \Exp(\theta)$ is
    \begin{equation*}
      F_X (x) = 1 - e^{-\frac{x}{\theta}}.
    \end{equation*}
    Then
    \begin{align*}
      E(X) &= \int_{0}^{\infty} 1 - F_X(x) \dif{x} = \int_{0}^{\infty} e^{-\frac{x}{\theta}} \dif{x} \\
           &= -\theta e^{-\frac{x}{\theta}} \at{0}{\infty} = \theta
    \end{align*}\qed
  \end{solution}
\end{eg}

\begin{thm}[Expected Value of a Function of X]
\label{thm:expected_value_of_a_function_of_x}
  Suppose $h(x)$ is a real-valued function.

  If $X$ is a discrete rv with pmf $f$ and support set $A$, then
  \begin{equation}\label{eq:discrete_function_to_expectation}
    E[ h(x) ] = \sum_{x \in A} h(x) f(x)
  \end{equation}
  provided that the sum converges absolutely.

  If $X$ is a continuous rv with pdf $f$, then
  \begin{equation}\label{eq:continous_function_to_expectation}
    E[ h(x) ] = \int_{-\infty}^{\infty} h(x) f(x) \dif{x}
  \end{equation}
  provided that the integral converges absolutely.
\end{thm}

The proof is, unfortunately, not trivial. One would have to look into Lesbesgue integrals (or at the very least, Riemann-Stieltjes integrals) in order to prove this statement. This ``theorem'' is also called \hlnotea{The Law of the Unconscious Statistician} [\href{https://en.wikipedia.org/wiki/Law_of_the_unconscious_statistician}{Reference - Wikipedia}]\index{Law of the Unconscious Statistician}. An idea of the proof is given on \href{https://math.stackexchange.com/questions/1277800/expected-value-of-a-function-of-a-random-variable}{Math SE}.

\begin{eg}
  Suppose $X \sim \Unif(0, \theta)$. Calculate $E(X^2)$.

  \begin{solution}
    \begin{equation*}
      E(X^2) = \int_{0}^{\theta} \frac{x^2}{\theta} \dif{x} = \frac{1}{\theta} \frac{x^3}{3} \at{x = 0}{\theta} = \frac{\theta^2}{3}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{ex}
  Find the pdf of $Y = X^2$ and find $E(Y)$ by evaluating $\int_{-\infty}^{\infty} y f_Y(y) \dif{y}$
\end{ex}

\begin{thm}[Linearity of Expectation]
\index{Linearity - Expectation}
\label{thm:linearity_of_expectation}
  Suppose $X$ is an rv with pf $f$. Let $a_i, b_i \in \mathbb{R}$, for $i = 1, ..., n$, be constants, and $g_i (x)$, for $i = 1, ..., n$, are real-valued functions. Then
\marginnote{This theorem essentially states that the expectation is a linear operator.}
  \begin{equation}\label{eq:linearity_of_expectation}
    E \left[ \sum_{i = 1}^{n} \left( a_i g_i(X) + b_i \right) \right] = \sum_{i = 1}^{n} \left( a_i E[ g_i(X) ] + b_i \right)
  \end{equation}
  provided that $E[g_i(X)] < \infty$ for $i = 1, ..., n$.
\end{thm}

\begin{proof}
  Suppose $X$ is a discrete rv with support set $A$. Then
  \begin{align*}
    E \left[ \sum_{i = 1}^{n} \left( a_i g_i(X) + b_i \right) \right]
      &= \sum_{x \in A} \left[ \sum_{i = 1}^{n} ( a_i g_i(x) + b_i ) \right] f(x) \quad \because \cref{thm:expected_value_of_a_function_of_x} \\
      &= \sum_{x \in A} \sum_{i = 1}^{n} \left[ a_i g_i(x) f(x) + b_i f(x) \right] \\
      &= \sum_{i = 1}^{n} \sum_{x \in A} \left[ a_i g_i(x) f(x) + b_i f(x) \right] \qquad (*) \\
      &= \sum_{i = 1}^{n} \left[ a_i \sum_{x \in A} g_i(x) f(x) + b_i \sum_{x \in A} f(x) \right] \\
      &= \sum_{i = 1}^{n} (a_i E[g_i(X)] + b_i)
  \end{align*}
  where note that $(*)$ is valid because $a_i, b_i$ are constants, and $g_i(x), f(x)$ are finite real-valued functions.
\end{proof}

\begin{note}
  In general, $E(g(X)) \neq g(E(X))$ unless if $g$ is a linear function. For example, for $a, b \in \mathbb{R}$, we have
  \begin{equation*}
    E(aX + b) = aE(X) + b
  \end{equation*}
\end{note}

% subsection expectations_continued (end)

\subsection{Moments and Variance}%
\label{sub:moments_and_variance}
% subsection moments_and_variance

Since these concepts were introduced in STAT230 and were given little treatment in the lecture, we shall only cover over them briefly.

\begin{defn}[Variance]\index{Variance}
\label{defn:variance}
  The expectation tof the squared deviation of an rv from its mean is called the \hlnoteb{variance}, i.e. for an rv $X$ with mean $\mu = E(X)$,
  \begin{equation*}
    \sigma^2 = \Var(X) = E[ (X - \mu)^2 ] = E(X^2) - E(X)^2
  \end{equation*}
\end{defn}

\begin{defn}[Moments]\index{Moments}
\label{defn:moments}
  Let $X$ be an rv with mean $\mu$.

  The $k$\textsuperscript{th} \hlnoteb{moment about the origin} is defined as:
  \begin{equation*}
    E(X^k)
  \end{equation*}

  The $k$\textsuperscript{th} \hlnoteb{moment about the mean} is defined as:
  \begin{equation*}
    E[ (X - \mu)^k ]
  \end{equation*}

  The $k$\textsuperscript{th} \hldefn{factorial moment} is defined as:
  \begin{equation*}
    E[ X^{(k)} ] = E [ X (X - 1) \hdots (X - k + 1) ] = E \left[ \frac{X!}{(X - k)!} \right]
  \end{equation*}
\end{defn}

\begin{thm}[Variance of a Linear Function]
\label{thm:variance_of_a_linear_function}
  Suppose $X$ is an rv with pf $f$ and $a, b \in \mathbb{R}$. Then
  \begin{equation*}
    \Var(aX + b) = a^2 \Var(X)
  \end{equation*}
\end{thm}

\begin{proof}
  Observe that
  \begin{align*}
    \Var(aX + b) &= E[ (aX + b)^2 ] - E(aX + b)^2 \\
                 &= E[ a^2 X^2 + 2abX + b^2 ] - (aE(X) + b)^2 \\
                 &= a^2 E(X^2) + 2abE(X) + b^2 - (a^2 E(X)^2 + 2abE(X) + b^2) \\
                 &= a^2 E(X^2) - a^2 E(X)^2 = a^2 \Var(X)
  \end{align*}\qed
\end{proof}

\begin{eg}[Example 2.22 (course notes - 2.6.10 (1))]
  If $X \sim \Poi(\theta)$, then $E[ X^{(k)} ] = \theta^k$ for $k = 1, 2, ...$.

  \begin{solution}
    Note
    \begin{equation*}
      f_X (x) = \begin{cases}
        \frac{e^{-\theta} \theta^x}{x!} & x = 0, 1, 2, ... \\
        0                               & \text{otherwise}
      \end{cases}
    \end{equation*}
    So
    \marginnote{Note that it is not necessarily true that
    \begin{equation*}
      x (x - 1)\hdots(x - k + 1) = \frac{x!}{(x - k)!}
    \end{equation*}
    for $0 \leq x \leq k - 1$. And so we can only say that the equality is true for $x \geq k$, and hence we have the approach that we use in $(*)$.
    }
    \begin{align*}
      E[ X^{(k)} ]
        &= E( X(X-1)(X-2)\hdots(X-k+1) ) \\
        &= \sum_{x = 0}^{\infty} x(x-1)(x-2)\hdots(x-k+1) \frac{e^{-\theta} \theta^x}{x!} \\
        &= 0 + \sum_{x = k}^{\infty} x(x-1)(x-2)\hdots(x-k+1) \frac{e^{-\theta} \theta^x}{x!} \qquad (*) \\
        &= \sum_{x = k}^{\infty} \frac{x!}{(x - k)!} \frac{e^{-\theta} \theta^x}{x!} \quad \because x(x-1)\hdots(x-k+1)=\frac{x!}{(x - k)!}\\
        &= e^{-\theta} \theta^k \sum_{x = k}^{\infty} \frac{\theta^{x - k}}{(x - k)!} \\
        &= e^{-\theta} \theta^k \sum_{y = 0}^{\infty} \frac{\theta^y}{y!} \qquad \text{let } y = x - k \\
        &= e^{-\theta} \theta^k e^{\theta} = \theta^k
    \end{align*}
    where for $(*)$ we have that $\sum_{x = 0}^{k - 1} x(x-1)\hdots(x-k+1) A = 0$ for any $A \in \mathbb{R}$.
  \end{solution}
\end{eg}

% subsection moments_and_variance (end)

% section expectations_continued (end)

\section{Inequalities}%
\label{sec:inequalities}
% section inequalities

\subsection{Markov/Chebyshev Style Inequalities}%
\label{sub:markov_chebyshev_style_inequalities}
% subsection markov_chebyshev_style_inequalities

\begin{thm}[Markov's Inequality]
\index{Markov's Inequality}
\label{thm:markov_s_inequality}
  If $X$ is a non-negative rv and $a > 0$, then the probability that $X$ is no less than $a$ is no greater than the expectation of $X$ divided by $a$, i.e.
  \begin{equation}\label{eq:markov_s_inequality}
    P(X \geq a) \leq \frac{E(X)}{a}
  \end{equation}
\end{thm}

\begin{proof}
  We shall prove for the discrete case. Suppose $X$ is a non-negative discrete rv with pf $f$. Let $A \subset S$, where $S$ is the sample space, such that \\\noindent$A = \{w \in S : X(w) \geq a\}$.
  \marginnote{
  \begin{ex}
    Prove Markov's Inequality for a continuous rv.
  \end{ex}
  }
  \begin{align*}
    E(X) &= \sum_{x \in S} x f(x) \\
         &= \sum_{x \in A} xf(x) + \sum_{x \notin A} xf(x) \\
         &\geq \sum_{x \in A} xf(x) \quad \because \sum_{x \notin A} xf(x) \geq 0  \\
         &\geq \sum_{x \in A} af(x) \\
         &= a \sum_{x \in A} f(x) = a \cdot P(A) \\
         &= a \cdot P(\{w \in S : X(w) \geq a\}) = a P(X \geq a).
  \end{align*}\qed
\end{proof}

\begin{thm}[Markov's Inequality 2]
\index{Markov's Inequality 2}
\label{thm:markov_s_inequality_2}
  If $X$ is a non-negative rv and $a, k > 0$, then the probability that $X$ is no less than $a$ is no greater than the expectation of $X$ divided by $a$, i.e.
  \begin{equation}\label{eq:markov_s_inequality_2}
    P(\abs{ X } \geq a) \leq \frac{E(\abs{ X }^k)}{a^k}
  \end{equation}
\end{thm}

\begin{proof}
  We shall, again, prove for the discrete case. Suppose $X$ is a non-negative discrete rv with pf $f$. $A := \{w \in S : \abs{X(w)} \geq a\} \subseteq S$. Then
  \marginnote{\hlnotec{Question:} Can we write
    \begin{equation*}
      P(\{w \in S : \abs{X(w)} \geq a\}) = P(\abs{X} \geq a)?
    \end{equation*}
    \begin{ex}
      Prove for the continuous case.
    \end{ex}
  }
  \begin{align*}
    E(\abs{X}^k) &= \sum_{x \in S} \abs{x}^k f(x) \\
                 &= \sum_{x \in A} \abs{x}^k f(x) + \sum_{x \notin A} \abs{x}^k f(x) \\
                 &\geq \sum_{x \in A} \abs{x}^k f(x) \geq \sum_{x \in A} af(x) \\
                 &= a^k P(A) = a^k P(\abs{X} \geq a).
  \end{align*}\qed
\end{proof}

\begin{thm}[Chebyshev's Inequality]
\index{Chebyshev's Inequality}
\label{thm:chebyshev_s_inequality}
  Suppose $X$ is an rv with finite mean $\mu$ and finite variance $\sigma^2$. Then for any $k > 0$,
  \begin{equation}\label{eq:chebyshev_s_inequality}
    P(\abs{X - \mu} \geq k \sigma) \leq \frac{1}{k^2}
  \end{equation}
\end{thm}

\begin{proof}
  By \cref{thm:markov_s_inequality_2},
  \begin{equation*}
    P(\abs{X - \mu} \geq k \sigma) \leq \frac{E(\abs{X - \mu}^2)}{(k\sigma)^2} = \frac{1}{k^2}
  \end{equation*} since $E(\abs{X - \mu}^2) = \Var(X) = \sigma^2$.\qed
\end{proof}

\begin{eg}[Example 2.23]
  A post office handles, on average, $10000$ letters a day. What can be said about the probability that it will handle at least $15000$ letters tomorrow?

  \begin{solution}
    $X :=$ number of letters handled in a day. Note that by its definition, $X$ is a non-negative discrete rv. Then, using \cref{thm:markov_s_inequality}, since $E(X) = 10000$
    \begin{equation*}
      P(X \geq 15000) \leq \frac{10000}{15000} = \frac{2}{3}.
    \end{equation*}
    Thus, we know that there is less than two-third of chance that the post office will handle more than $15000$ tomorrow.
  \end{solution}
\end{eg}

% subsection markov_chebyshev_style_inequalities (end)

% section inequalities (end)

% chapter lecture_4_may_10th_2018 (end)

\chapter{Lecture 5 May 15th 2018}
\label{chp:lecture_5_may_15th_2018}
% chapter Lecture 5 May 15th 2018

\section{Inequalities (Continued)}%
\label{sec:inequalities_continued}
% section inequalities_continued

\subsection{Markov/Chebyshev Style Inequalities (Continued)}%
\label{sub:markov_chebyshev_style_inequalities_continued}
% subsection markov_chebyshev_style_inequalities_continued

\begin{eg}[Example 2.24]
  A post office handles $10000$ letters per day with a variance of $2000$ letters. What can be said about the probability that this post office handles between $8000$ and $12000$ letters tomorrow? What about the probability that more than $15000$ letters come in (use \cref{thm:chebyshev_s_inequality})?

  \begin{enumerate}
    \item \textit{Probability that this post office handles between $8000$ and $12000$ letters tomorrow:}
      \begin{align*}
        &P(8000 < X < 12000) \\
          &= P(-2000 < X - 10000 < 2000) \\
          &= P(\abs{X - 10000} < 2000) = 1 - P(\abs{X - 10000} \geq 2000) \\
          &\geq 1 - \frac{1}{(\sqrt{2000})^2} \quad \because \cref{thm:chebyshev_s_inequality} \, \land \, k = \frac{2000}{\sigma} = \sqrt{2000} \\
          &= \frac{1999}{2000}
      \end{align*}

    \item \textit{Probability that more than 15000 letters come in:}
      \begin{align*}
        P(X > 15000) &= P(X - 10000 > 15000 - 10000) \\
          &= P(X - 10000 > 5000) \\
          &\leq P(X - 10000 > 5000) + P(X - 10000 < 5000) \\
          &\leq P(\abs{X - 10000} > 5000) \\
          &\leq \frac{1}{\left( \frac{5000}{\sqrt{2000}} \right)^2} = \frac{2000}{5000^2}
      \end{align*}
  \end{enumerate}
\end{eg}

% subsection markov_chebyshev_style_inequalities_continued (end)

% section inequalities_continued (end)

\section{Moment Generating Function}
\label{sec:moment_generating_function}
% section Moment Generating Function

Moment generating functions are important because they uniquely define the distribution of an rv.

\begin{defn}[Moment Generating Function]\label{defn:moment_generating_function}
\index{Moment Generating Function}
  If $X$ is an rv, then $M_X (t) = E(e^{tx})$ is called the \hlnoteb{moment generating function} (mgf) of $X$ provided this expectation exists for all $t \in (-h , h)$ for some $h > 0$.
\end{defn}

\begin{note}
  When determining the mgf of an rv, the values of $t$ for which the expectation exists must always be stated. The range of $t$ where the expectation is defined is ``essentially'' the \hlnotea{radius of convergence}.
\end{note}

\begin{ex}[Example 2.25 (2.9.2 (1) of the course notes)]
  Find the mgf of $X \sim \Gamma(\alpha, \beta)$. Make sure you specify the domain on which the mgf is defined.

  \begin{solution}
    Note that the pdf of the Gamma distribution is:
    \begin{align*}
      f(x) &= \begin{cases}
        \frac{1}{\beta^\alpha \Gamma(\alpha)} x^{\alpha - 1} e^{- \frac{X}{\beta} } & x > 0 \\
        0 & \text{otherwise}
      \end{cases}
    \end{align*}
    Therefore
    \begin{align*}
      M_X (t) &= E(e^{tx})
        = \int_{0}^{\infty}e^{tx} \frac{1}{\beta^\alpha} x^{\alpha - 1} e^{- \frac{x}{\beta}} \dif{x} \\
        &= \frac{1}{\beta^\alpha} \int_{0}^{\infty}  \frac{1}{\Gamma( \alpha )} x^{\alpha} e^{-x \left(\frac{1}{\beta} - t \right)} \dif{x} \\
        &= \frac{\left( \frac{\beta}{1 - t\beta} \right)^\alpha}{\beta^\alpha} \underbrace{ \int_{0}^{\infty} \frac{1}{\left( \frac{\beta}{1 - t \beta} \right)^\alpha \Gamma(\alpha)} x^{\alpha - 1} e^{-\frac{x}{\frac{\beta}{1 - t \beta}}} \dif{x} }_{\text{sum over all values for pdf of } \Gamma(\alpha, \frac{\beta}{1 - t \beta} = 1)} \quad \text{for } \frac{1}{\beta} - t > 0 \\
        &= (1 - t \beta)^{- \alpha} \qquad \text{for } t < \frac{1}{\beta}
    \end{align*}
  \end{solution}
\end{ex}

\begin{defn}[Indicator Function]\index{Indicator Function}
\label{defn:indicator_function}
  The function $\mathbb{1}_A$ is called the \hlnoteb{indicator function} of the set $A$, i.e.
  \begin{equation}\label{eq:indicator_function}
    \mathbb{1}_A = \begin{cases}
      1 & \text{if } A \text{ occurs } \\
      0 & \text{if } A^C \text{ occurs }
    \end{cases}
  \end{equation}
\end{defn}

\begin{eg}
  The pdf
  \begin{equation*}
    f(x) = \begin{cases}
      \frac{1}{\theta} & 0 \leq x \leq \theta \\
      0                & \text{otherwise}
    \end{cases}
  \end{equation*}
  can be represented as
  \begin{equation*}
    f(x) = \frac{1}{\theta} \mathbb{1}_{\{0 \leq x \leq \theta \}}
  \end{equation*}
\end{eg}

\begin{eg}[Example 2.26]
  Find the mgf of $X \sim \Poi(\lambda)$. Make sure you specify the domain on which the mgf is defined.

  \begin{solution}
    Note that the pmf of $X$ is
    \begin{equation*}
      f_X(x) = \frac{e^{-\lambda} \lambda^x}{x!} \mathbb{1}_{\{0, 1, 2, ...\}}
    \end{equation*}
    The mgf is thus
    \begin{align*}
      M_X(t) &= E(e^{tX}) = \sum_{x = 0}^{\infty} e^{tx} \frac{e^{-\lambda} \lambda^x}{x!} \\
             &= e^{-\lambda} \sum_{x = 0}^{\infty} \frac{(e^t \lambda)^x}{x!} = e^{-\lambda} e^{e^{t} \lambda} \\
             &= e^{\lambda( e^t - 1 )} \qquad \forall t \in \mathbb{R}
    \end{align*}
  \end{solution}
\end{eg}

\begin{propo}[Properties of the MGF]
\label{propo:properties_of_the_mgf}
  Suppose $X$ is an rv. Then
  \begin{enumerate}
    \item $M_X(0) = 1$
    \item Suppose the derivatives $M_X^{(k)}(t)$, for $k = 1, 2, ...$, exists for $t \in (-h, h)$ for some $h > 0$, then the \hlnotea{Maclaurin Series}\sidenote{The Maclaurin series is the Taylor expansion around $0$.} of $M_X(t)$ is
      \begin{equation*}
        M_X(t) = \sum_{k = 0}^{\infty} \frac{M_X^{(k)} (t) \at{t = 0}{}}{k!} t^k
      \end{equation*}
    \item If the mgf exists, then the $k$\textsuperscript{th} moment of $X$ is:
      \begin{equation*}
        E(X^k) = \frac{d^k M_X(t)}{dt^k} \at{t = 0}{}
      \end{equation*}
    \item Putting 2 and 3 together, we have
      \begin{equation*}
        M_X(t) = \sum_{k = 0}^{\infty} \frac{E(X^k)}{k!} t^k
      \end{equation*}
  \end{enumerate}
  The final item shows why $M_X(t)$ is called the \hlnoteb{moment generating function}.
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item $M_X(t) \at{t = 0}{} = E(e^{tX}) \at{t = 0}{} = E(e^{0}) = 1$
    \item This is simply a result of using the Maclaurin series.
    \item Note that
      \begin{align*}
        E(e^{tX})
          &= E \left[ 1 + tX + \frac{1}{2} (tX)^2 + \frac{1}{3!} (tX)^3 + \hdots \right] \\
          &= 1 + tE(X) + \frac{t^2}{2} E(X^2) + \frac{t^3}{3!} E(X^3) + \hdots.
      \end{align*}
      So
      \begin{equation*}
        \frac{d^k}{dt^k} E(e^{tX}) \at{t = 0}{}
          = \frac{k!}{k!} E(X^k) + \underbrace{ \frac{k! \cdot t}{(k + 1)!} E(X^{k + 1}) + \hdots }_{= 0 \text{ when } t = 0} \at{t = 0}{} = E(X^k)
      \end{equation*}
  \end{enumerate}\qed
\end{proof}

\begin{eg}[Example 2.27]
  A discrete random variable $X$ has the pmf
  \begin{equation*}
    f(x) = \left( \frac{1}{2} \right)^{x + 1} \mathbb{1}_{\{0, 1, 2, ...\}}
  \end{equation*}
  Derive the mgf of $X$ and use it calculate its mean and variance.

  \begin{align*}
    M_X(t) &= \sum_{x = 0}^{\infty} e^{tx} \left( \frac{1}{2} \right)^{x + 1} \\
           &= \frac{1}{2} \cdot \sum_{x = 0}^{\infty} \left( \frac{e^t}{2} \right)^x \\
           &= \frac{1}{2} \cdot \frac{1}{1 - \frac{e^t}{2}} \quad \text{for } \abs{\frac{e^t}{2}} < 1 \text{ or } t < \ln 2 \\
           &= \frac{1}{2 - e^t}
  \end{align*}
  To get the first two moments,
  \begin{align*}
    E(X) &= \frac{d}{dt} M_X(t) \at{t = 0}{} \\
      &= \frac{e^t}{( 2 - e^t )^2} \at{t = 0} = 1 \\
    E(X^2) &= \frac{d^2}{dt^2} M_X(t) \at{t = 0}{} \\
      &= \frac{e^t}{(2 - e^t)^2} + \frac{2e^t}{(2 - e^t)^3} \at{t = 0}{} \\
      &= 1 + 2 = 3
  \end{align*}
  Thus we have that the expected value and variance are
  \begin{align*}
    E(X) &= 1 \\
    \Var(X) &= E(X^2) - E(X)^2 = 3 - 1 = 2
  \end{align*}
  respectively.
\end{eg}

\subsection{MGF of a Linear Transformation}
\label{sub:mgf_of_a_linear_transformation}
% subsection MGF of a Linear Transformation

\begin{thm}[MGF of a Linear Transformation]
\label{thm:mgf_of_a_linear_transformation}
  Suppose the rv $X$ has an mgf $M_X(t)$ defined for $t \in (-h, h)$ for some $h > 0$. Let $Y = aX + b$, where $a, b \in \mathbb{R}$ and $a \neq 0$. Then the mgf of $Y$ is
  \begin{equation}\label{eq:mgf_of_a_linear_transformation}
    M_Y(t) = e^{bt} M_X(at), \quad \abs{t} \leq \frac{h}{\abs{a}}.
  \end{equation}
\end{thm}

\begin{proof}
  Observe that
  \begin{align*}
    M_Y(t) = E(e^{tY}) = E(e^{t(aX + b)}) = E(e^{atX}e^{tb}) = e^{bt} M_X(at).
  \end{align*}
  The range of $t$ is
  \begin{equation*}
    \abs{at} < h \overset{a \neq 0}{\iff} \abs{t} < \frac{h}{\abs{a}}
  \end{equation*}
\end{proof}

\begin{eg}[Example 2.28]
  Consider $X \sim \Unif(\theta_1, \theta_2)$. Find the mgf of $Y = 5X + 3$.

  \begin{solution}
    Note that
    \begin{align*}
      M_X(t) &= \int_{\theta_1}^{\theta_2} \frac{e^{tx}}{\theta_2 - \theta_1} \dif{x} \\
        &= \begin{cases}
          \frac{e^{tx}}{t ( \theta_2 - \theta_1 )} \at{\theta_1}{\theta_2} & t \neq 0 \\
          1 & t = 0
        \end{cases} \\
        &= \begin{cases}
          \frac{e^{t \theta_2} - e^{t \theta_1}}{t (\theta_2 - \theta_1)} & t \neq 0 \\
          1 & t = 0
        \end{cases}
    \end{align*}
    Thus by \cref{thm:mgf_of_a_linear_transformation},
    \begin{equation*}
      M_Y(t) = e^{3t} M_X(5t) = \begin{cases}
        e^{3t} \frac{e^{5t \theta_2} - e^{5t \theta_1}}{5t (\theta_2 - \theta_1)} & t \neq 0 \\
        1 & t = 0
      \end{cases}
    \end{equation*}
  \end{solution}
\end{eg}

% subsection MGF of a Linear Transformation (end)

\subsection{Uniqueness of the MGF}
\label{sub:uniqueness_of_the_mgf}
% subsection Uniqueness of the MGF

\begin{thm}[Uniqueness of the MGF]
\label{thm:uniqueness_of_the_mgf}
  Suppose the rv $X$ has mgf $M_X(t)$ and the rv $Y$ has mgf $M_Y(t)$. Suppose also that $M_X(y) = M_Y(t)$ for all $t \in (-h, h)$ for some $h > 0$. Then $X$ and $Y$ have the same distribution, that is, $\forall s \in \mathbb{R}$,
  \begin{equation*}
    P(X \leq s) = F_X(s) = F_Y(s) = P(Y \leq s)
  \end{equation*}
\end{thm}

\begin{proof}
  The proof of this theorem is not trivial. See \href{https://math.stackexchange.com/q/2388038}{this comment} on Math SE for information. It appears that the 2nd bullet point points to a material that I might be able to understand. If I can find that material, and understand it, I may change this proof section to become my own notes.
\end{proof}

\begin{eg}[Example 2.29]
  Suppose $X \sim \Unif(0, 1)$. Define $Y = -2 \log X$, and use the mgf method to show that $Y \sim \chi_2^2$.\\
\noindent ( Hint: Find mgf of $\chi_2$ and show that $Y$ has the same mgf )

  \begin{solution}
    Let $Z = \chi_2^2$. The pdf of $Z$ is therefore
    \begin{equation*}
      f_Z (z) = \frac{1}{2} e^{- \frac{z}{2}} \mathbb{1}_{\{z > 0\}}.
    \end{equation*}
    Then
    \begin{align*}
      M_Z(t) = E(e^{tZ}) &= \int_{0}^{\infty} e^{tz} \frac{1}{2} e^{- \frac{z}{2}} \dif{z} \\
        &= \frac{1}{2} \int_{0}^{\infty} e^{(t - \frac{1}{2}) z} \dif{z} \\
        &= \begin{cases}
          \frac{1}{2} \frac{1}{t - \frac{1}{2}} e^{(t - \frac{1}{2}) z} \at{z = 0}{\infty} & t \neq \frac{1}{2} \\
          \infty & t = \frac{1}{2}
        \end{cases} \\
        &= \frac{1}{2t - 1} \qquad t \neq \frac{1}{2}
    \end{align*}
  \end{solution}
\end{eg}

% subsection Uniqueness of the MGF (end)

% section Moment Generating Function (end)

% chapter Lecture 5 May 15th 2018 (end)

\chapter{Lecture 6 May 17th 2018}%
\label{chp:lecture_6_may_17th_2018}
% chapter lecture_6_may_17th_2018

\section{Joint Distributions}%
\label{sec:joint_distributions}
% section joint_distributions

\subsection{Introduction to Joint Distributions}%
\label{sub:introduction_to_joint_distributions}
% subsection introduction_to_joint_distributions

\begin{note}[Motivation]
  Most studies collect information for multiple variables per subject rather than just one variable. Because these variables may interfere/interact with each other and hence give us results that may not be fully reliant on a single variable, it is in our interest to study the interaction of these variables.

  To start off with the basics, we will first look at the bivariate case of a joint distribution.
\end{note}

% subsection introduction_to_joint_distributions (end)

\subsection{Joint and Marginal CDFs}%
\label{sub:joint_and_marginal_cdfs}
% subsection joint_and_marginal_cdfs

\begin{defn}[Joint CDF]\index{Joint CDF}
\label{defn:joint_cdf}
  Suppose $X$ and $Y$ are rvs defined on a sample space $S$. The \hlnoteb{joint cdf} of $X$ and $Y$ is given by
  \begin{equation*}
    \forall (x, y) \in \mathbb{R}^2 \qquad F(x, y) = P(X \leq x, Y \leq y).
  \end{equation*}
\end{defn}

\begin{note}
  \begin{itemize}
    \item Depending on whether $X$ and $Y$ are both discrete or both continuous, we can derive the joint pmf or joint pdf of $(X, Y)$, respectively.
    \item \cref{defn:joint_cdf} only concerns two variables (a bivariate case), but we can certainly extend the idea to a $k$-dimensional joint cdf for the rvs $X_1, X_2, ..., X_k$ as $\forall (x_1, x_2, ..., x_k) \in \mathbb{R}^k$,
    \begin{equation*}
      F(x_1, x_2, ..., x_k) = P(X_1 \leq x_1, X_2 \leq x_2, ..., X_k \leq x_k).
    \end{equation*}
  \end{itemize}
\end{note}

\begin{propo}[Properties of Joint CDF]
\label{propo:properties_of_joint_cdf}
  Suppose $X, Y$ are rvs, either both continuous or discrete, and has a joint cdf $F$. Then
  \begin{enumerate}
    \item $F$ is non-decreasing in $x$ for fixed $y$.
    \item $F$ is non-decreasing in $y$ for fixed $x$.
    \item $\lim_{x \to - \infty} F(x, y) = 0$ and $\lim_{y \to -infty} F(x, y) = 0$.
    \item $\lim_{(x, y) \to (-\infty, -\infty)} F(x, y) = 0$ and $\lim_{(x, y) \to (\infty, \infty)} F(x, y) = 1$
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item Suppose not, i.e. that we have instead that $F$ is decreasing for $x$. Then for $x_1 < x_2 \in \mathbb{R}$, we would have
      \begin{align*}
        &F(x_1, y) > F(x_2, y) \\
        &\implies P(X \leq x_1, Y \leq y) > P(X \leq x_2, Y \leq y)
      \end{align*}
      In other words,
      \begin{align*}
        &P(\{ (w, v) : (w, v) \in S, \; X(w) \leq x_1 , \, Y(v) \leq y\}) \\
        &> P(\{ (w, v) : (w, v) \in S, \; X(w) \leq x_2 , \, Y(v) \leq y\})
      \end{align*}
      However, note that for fixed $y$, since $x_1 < x_2$, we must have that
      \begin{align*}
        &\{(w, v) \in S : X(w) \leq x_1 , Y(v) \leq y \} \\
        &\subseteq \{(w, v) \in S : X(w) \leq x_2, Y(v) \leq y \}.
      \end{align*}
      By \cref{propo:properties_of_probability_set_functions}, we have that
      \begin{align*}
        &P(\{ (w, v) : (w, v) \in S, \; X(w) \leq x_1 , \, Y(v) \leq y\}) \\
        &\leq P(\{ (w, v) : (w, v) \in S, \; X(w) \leq x_2 , \, Y(v) \leq y\}).
      \end{align*}
      This is clearly a contradiction.
    \item The proof for this statement is similar to the above.
    \item Note that
      \begin{align*}
        \lim_{x \to -\infty} F(x, y) &= \lim_{x \to - \infty} P(X \leq x, Y \leq y) \\
          &= P(X \leq -\infty, Y \leq y) \\
          &= P([ X \leq -\infty ] \cap [ Y \leq y ]) \\
          &= P( \emptyset \cup [Y \leq y] ) = P(\emptyset) = 0
      \end{align*}
      The proof for the case where $y \to -\infty$ is similar.
    \item This is simply a consequence of 3.
  \end{enumerate}
\end{proof}

\begin{note}
    We say that $F$ is a joint cdf if it satisfies all the conditions in \cref{propo:properties_of_joint_cdf}.\sidenote{Many literature actually claims this, and it does look like it will be assumed so for this class.}
\end{note}

\begin{eg}[Example 3.1]\label{eg:eg_3_1}
  Consider the following joint cdf of two rvs $(X_1, X_2)$:
  \begin{equation*}
    F(x_1, x_2) = \begin{cases}
      0    & x_1 < 0 \, \lor \, x_2 < 0 \\
      0.49 & 0 \leq x_1 < 1 \, \land \, 0 \leq x_2 < 1 \\
      0.7  & 0 \leq x_1 < 1 \, \land \, x_2 > 1 \\
      0.7  & x_1 \geq 1 \, \land \, 0 \leq x_2 < 1 \\
      1    & x_1 \geq 1 \, \land \, x_2 \geq 1
    \end{cases}
  \end{equation*}
  Flipping an unfair coin with $P(\{H\}) = 0.3$ twice independently, we define for $i = 1, 2$
  \begin{equation*}
    X_i = \begin{cases}
      1 & \text{if the } i \text{\textsuperscript{th} flip is heads} \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
  The joint cdf of $(X_1, X_2)$ is the given $F$ above. Verify that under this experiment, $F$ is indeed a cdf.

  \begin{solution}
    Note that conditions 3 and 4 of \cref{propo:properties_of_joint_cdf} are automatically satisfied by the definition of $F$.

    \hlwarn{incomplete example}
  \end{solution}
\end{eg}

\begin{defn}[Marginal CDF]\index{Marginal CDF}
\label{defn:marginal_cdf}
  For the rvs $X, Y$ with joint cdf $F$, the \hlnoteb{marginal cdf} of $X$ is
  \marginnote{Note that the marginal cdf is defined for both discrete and continuous cases.}
  \begin{equation*}
    F_X(x) = P(X \leq x) = \lim_{y \to \infty} F(x, y) = F(x, \infty) \quad \forall x \in \mathbb{R}
  \end{equation*}
  and the \hlnoteb{marginal cdf} of $Y$ is
  \begin{equation*}
    F_Y(y) = P(Y \leq y) = \lim_{x \to \infty} F(x, y) = F(\infty, y) \quad \forall y \in \mathbb{R}
  \end{equation*}
\end{defn}

\begin{eg}
  Based on \cref{eg:eg_3_1}, derive $F_{X_i}(x_i)$ for $i = 1, 2$.

  \begin{solution}
    \begin{align*}
      F_{X_1} (x_1) &= \lim_{x_2 \to \infty} F(x_1, x_2) \\
          &= \begin{cases}
            0   & x_1 < 0 \\
            0.7 & 0 \leq x_1 < 1 \\
            1   & x_1 \geq 1
          \end{cases}
    \end{align*}
    The solution for $F_{X_2}(x_2)$ is similar.
  \end{solution}
\end{eg}

% subsection joint_and_marginal_cdfs (end)

\subsection{Joint Discrete RVs}%
\label{sub:joint_discrete_rvs}
% subsection joint_discrete_rvs

\begin{defn}[Joint Discrete RV]\index{Joint Discrete Random Variables}
\label{defn:joint_discrete_rv}
  Suppose $X$ amd $Y$ are rvs defined on a sample space $S$. If $S$ is discrete then $X$ and $Y$ are discrete rvs. The \hlnoteb{joint pmf}\index{Joint PMF} of $X$ and $Y$ is given by
  \begin{equation*}
    \forall (x, y) \in \mathbb{R}^2 \quad f(x, y) = P(X = x, Y = y).
  \end{equation*}
  The set $A = \{(x, y) : f(x, y) > 0\}$ is called the \hldefn{support set} of $(X, Y)$.
\end{defn}

\begin{propo}[Properties of Joint PMF]
\label{propo:properties_of_joint_pmf}
  Suppose $X, Y$ are discrete rvs with joint pmf $f$ and support set $A$. Then
  \begin{enumerate}
    \item $\forall (x, y) \in \mathbb{R}^2 \qquad f(x, y) \geq 0$
    \item $\underset{(x, y) \in A}{\sum \enspace \sum} f(x, y) = 1$
    \item $\forall R \subset \mathbb{R}^2$,
      \begin{equation*}
        P[ (X, Y) \in R ] = \underset{(x, y) \in R}{\sum \enspace \sum} f(x, y)
      \end{equation*}
  \end{enumerate}
\end{propo}

The proof is analogous to the univariate case as seen in \cref{propo:properties_of_pmf}

\begin{eg}[Example 3.2]\label{eg:eg_3_2}
  Consider the following joint pmf where the numbers inside the table show $P(X = x, Y = y)$. Find $c$. Then, calculate $P(X + Y \leq 2)$.
  \[
  \begin{tabular}{c | c | c | c}
          & x = -2 & x = 0 & x = 2 \\
    \hline
    y = 0 & 0.05   & 0.1   & 0.15 \\
    \hline
    y = 1 & 0.07   & 0.11  & c \\
    \hline
    y = 2 & 0.02   & 0.25  & 0.05 \\
  \end{tabular}
  \]

  \begin{solution}
    Since the sum of all the probabilities must be $1$, thus
    \begin{equation*}
      c = 1 - 0.05 - 0.07 - 0.02 - \hdots - 0.15 - 0.05 = 0.2.
    \end{equation*}
    Notice that the only cases where $X + Y > 2$ is when
    \begin{itemize}
      \item $X = 2, Y = 1$; and
      \item $X = 2, Y = 2$.
    \end{itemize}
    Thus
    \begin{align*}
      P(X + Y \leq 2) &= 1 - P(X = 2, Y = 1) - P(X = 2, Y = 2) \\
        &= 1 - 0.2 - 0.05 = 0.75
    \end{align*}
  \end{solution}
\end{eg}

\begin{eg}[Example 3.3]
  A small college has $90$ male and $30$ female professors. An ad hoc committee of $5$ is selected at random to write the vision and mission of the college. Let $X$ and $Y$ be the number of men and women in this committee, respectively. Derive the joint distribution of $(X, Y)$.

  \begin{solution}
    Observe that the support set of this distribution is
    \begin{equation*}
      A = \{(x, y) : x + y = 5, x, y = 0, 1, 2, 3, 4, 5 \}.
    \end{equation*}
    We have that the distribution is
    \begin{equation*}
      P(X = x, Y = y) = \begin{cases}
        \frac{\binom{90}{x} \binom{30}{y}}{\binom{120}{5}} & \substack{x, y = 0, \; 1, \; 2, \; 3, \; 4, \; 5 \\ x + y = 5} \\
        0             & \text{otherwise}
      \end{cases}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{defn}[Marginal Distribution - Discrete Case]\index{Marginal Distribution}
\label{defn:marginal_distribution_discrete_case}
  Suppose $X$ and $Y$ are discrete rvs with joint pf $f$. Then the \hlnoteb{marginal pf} of $X$ is
  \begin{equation*}
    \forall x \in \mathbb{R}^2 \quad f_X (x) = P(X = x) = \sum_{y \in \mathbb{R}} f(x, y),
  \end{equation*}
  and the \hlnoteb{marginal pf} of $Y$ is
  \begin{equation*}
    \forall y \in \mathbb{R}^2 \quad f_Y (y) = P(Y = Y) = \sum_{x \in \mathbb{R}} f(x, y).
  \end{equation*}
\end{defn}

\begin{eg}[Example 3.4]
  Consider the joint pmf from \cref{eg:eg_3_2}. Find the marginal distributions, i.e. marginal pmfs of $X$ and $Y$.
  \[
  \begin{tabular}{c | c | c | c}
          & x = -2 & x = 0 & x = 2 \\
    \hline
    y = 0 & 0.05   & 0.1   & 0.15 \\
    \hline
    y = 1 & 0.07   & 0.11  & 0.2 \\
    \hline
    y = 2 & 0.02   & 0.25  & 0.05
  \end{tabular}
  \]

  \begin{solution}
    Using the definition, we have that
    \begin{equation*}
      f_X(x) = \sum_{y \in \mathbb{R}} f(x, y) = \begin{cases}
        0.14 & x = -2 \\
        0.46 & x = 0 \\
        0.40 & x = 2
      \end{cases}
    \end{equation*}
    and
    \begin{equation*}
      f_Y(y) = \sum_{x \in \mathbb{R}} f(x, y) = \begin{cases}
        0.3  & y = 0 \\
        0.38 & y = 1 \\
        0.32 & y = 2
      \end{cases}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{eg}[Example 3.5]
  Suppose that a penny and a nickel are each tossed $10$ times so that every pair of sequences of tosses ($n$ tosses in each sequence) is equally likely to occur. Let $X$ be the number of heads obtained with the penny, and $Y$ be the number of heads obtained with the nickel. It can be shown that (show it!) the joint pmf of $X$ and $Y$ is as follows.
  \begin{equation*}
    P(X = x, Y = y) = \begin{cases}
      \binom{10}{x} \binom{10}{y} \left( \frac{1}{2} \right)^{20} & x, y = 0, ..., 10 \\
      0                                                           & \text{otherwise}
    \end{cases}
  \end{equation*}

  \begin{solution}
    Note that the support set of $X$ and $Y$ are the same, i.e.
    \begin{equation*}
      A_X = A_Y = \{ 0, 1, ..., 10 \}.
    \end{equation*}
    We may assume that the penny and the nickel are fair coins, i.e. if we let $p_x$ and $p_y$ be the probability of getting a head for a penny and nickel, respectively, then $p_x = p_y = \frac{1}{2}$. Since there are $10$ ways to get $x$ heads with the penny, and similarly so for the nickel, we have that
    \begin{align*}
      P(X = x, Y = y) &= \begin{cases}
        \binom{10}{x} \binom{10}{y} \left( \frac{1}{2} \right)^{10} \left( \frac{1}{2} \right)^{10} & x, y = 0, 1, ..., 10 \\
        0            & \text{otherwise}
      \end{cases} \\
      &= \begin{cases}
        \binom{10}{x} \binom{10}{y} \left( \frac{1}{2} \right)^{20} & x, y = 0, 1, ... , 10 \\
        0   & \text{otherwise}
      \end{cases}
    \end{align*}
    as required.
  \end{solution}
\end{eg}

\begin{note}
  It is interesting to observe that the two rvs in the last example have seemingly no relationship with one another in terms of the experiment conducted, since they do not affect each other. This leads us to introducing the next concept.
\end{note}

% subsection joint_discrete_rvs (end)

\subsection{Independence of Discrete RVs}%
\label{sub:independence_of_discrete_rvs}
% subsection independence

\begin{defn}[Independence of Discrete RVs]\index{Independence}
\label{defn:independence_of_discrete_rvs}
  Two rvs $X$ and $Y$ with joint cdf $F$ are said to be \hlnoteb{independent} if and only if
  \begin{equation*}
    \forall x, y \in \mathbb{R} \quad F(x, y) = F_X(x) F_Y(y)
  \end{equation*}
\end{defn}

\begin{thm}[Independence by PF]
\label{thm:independence_by_pf}
  \marginnote{I am not certain as to why this is presented as a theorem that repeats the definition. As so, the prove for the 2nd equation will not be shown.}
  Suppose $X$ and $Y$ are rvs with joint cdf $F$, joint pf $f$, marginal cdf $F_X$ and $F_Y$ respectively, and marginal pf $f_X$ and $f_Y$ respectively. Also, suppose that $A_X = \{x : f_X (x) > 0\}$ is the support set of $X$ and $A_Y = \{y : f_Y(y) > 0\}$ is the support set of $Y$. Then $X$ and $Y$ are independent rvs if and only if either
  \begin{equation*}
    \forall (x, y) \in A_X \times A_Y \quad f(x, y) = f_X(x) f_Y(y)
  \end{equation*}
  holds, or
  \begin{equation*}
    \forall x, y \in \mathbb{R} \quad F(x, y) = F_X(x) F_Y(y)
  \end{equation*}
\end{thm}

\begin{proof}
  The $(\implies)$ direction is simply a result of \hlnotea{Clairaut's Theorem}\sidenote{Work needs to be done to show that our statement actually satisfies the condition for Clairaut's Theorem to apply. Clairaut's Theorem states that:

  \begin{thm*}[Clairaut's Theorem]
  \index{Clairaut's Theorem}
  \label{thm*:clairaut_s_theorem}
    If $(x_0, y_0)$ is a point in the domain of a function $f$ with
    \begin{itemize}
      \item $f$ is defined on all points in an open disk centered at $(x_0, y_0)$;
      \item the first partial derivatives, $f_{xy}$ and $f_{yx}$ are all continuous for all points in the open disk.
    \end{itemize}
    Then $f_{xy}(x_0, y_0) = f_{yx}(x_0, y_0)$.
  \end{thm*}
  }. While the $(\impliedby)$ direction is a direct result of applying double integrals. \qed
\end{proof}

\begin{eg}[Example 3.6]
  Suppose $X$ and $Y$ are discrete rvs with joint pf
  \begin{equation*}
    f(x, y) = \frac{\theta^{x + y} e^{- 2 \theta}}{x! y!} \mathbb{1}_{\{x, y = 0, 1, ...\}}.
  \end{equation*}
  Are $X$ and $Y$ independent of each other?

  \begin{solution}
    Note that we may write $f$ as
    \begin{equation*}
      f(x, y) = \left( \frac{\theta^x e^{-\theta}}{x!} \cdot \frac{\theta^y e^{- \theta}}{y!} \right) \mathbb{1}_{\{x, y = 0, 1, ...\}}
    \end{equation*}
    and so this suggests that we can indeed break down $f$ into two parts, each only affected by $x$ and $y$ respectively, ``indenpdent'' of each other. Indeed, since
    \begin{align*}
      f_X(x) &= \sum_{y = 0}^{\infty} \frac{\theta^{x + y} e^{- \theta}}{x! y!} \mathbb{1}_{\{x,y = 0, 1, ... \}} \\
        &= \sum_{y = 0}^{\infty} \left( \frac{\theta^x e^{-\theta}}{x!} \cdot \frac{\theta^y e^{- \theta}}{y!} \right) \mathbb{1}_{\{x = 0, 1, ...\}} \\
        &= \frac{\theta^x e^{-\theta}}{x!} \underbrace{ \sum_{y = 0}^{\infty} \frac{\theta^y e^{-\theta}}{y!} }_{\text{sum of pmf of } \Poi(\theta) = 1} \\
        &= \frac{\theta^x e^{-\theta}}{x!}
    \end{align*}
    Similarly, we can obtain
    \begin{equation*}
      f_Y(y) = \frac{\theta^y e^{-\theta}}{y!}
    \end{equation*}
    Multiplying $f_X(x)$ and $f_Y(y)$ together, we indeed get back to the original joint pmf.
  \end{solution}
\end{eg}

% subsection independence (end)

% section joint_distributions (end)

% chapter lecture_6_may_17th_2018 (end)

\chapter{Lecture 7 May 24th 2018}%
\label{chp:lecture_7_may_24th_2018}
% chapter lecture_7_may_24th_2018

\section{Joint Distributions (Continued)}%
\label{sec:joint_distributions_continued}
% section joint_distributions_continued

\subsection{Independence of Discrete RVs (Continued)}%
\label{sub:independence_of_discrete_rvs_continued}
% subsection independence_continued

\begin{eg}[Example 3.7]
  Consider the joint pmf below from \cref{eg:eg_3_2}. Are $X$ and $Y$ independent? Prove or disprove.
  \[
  \begin{tabular}{c | c | c | c | c}
             & x = -2 & x = 0 & x = 2 & P(Y = y) \\
    \hline
    y = 0    & 0.05   & 0.1   & 0.15  & 0.3 \\
    \hline
    y = 1    & 0.07   & 0.11  & 0.2   & 0.38 \\
    \hline
    y = 2    & 0.02   & 0.25  & 0.05  & 0.32 \\
    \hline
    P(X = x) & 0.14   & 0.46  & 0.4
  \end{tabular}
  \]

  \begin{solution}
    Note that
    \begin{gather*}
      P(X = -2, Y = 0) = 0.5 \text{ but }\\
      P(X = -2) P(Y = 0) = 0.14 \cdot 0.3 = 0.042 \neq 0.5.
    \end{gather*}
    Thus $X$ and $Y$ are not independent.
  \end{solution}
\end{eg}

% subsection independence_continued (end)

\subsection{Joint Continuous RVs}%
\label{sub:joint_continuous_rvs}
% subsection joint_continuous_rvs

\begin{defn}[Joint Continuous RVs]\index{Joint Continuous Random Variables}
\label{defn:joint_continuous_rvs}
  Two random variables $X$ and $Y$ are said to be \hlnoteb{jointly continuous} if there exists a function $f(x, y)$ such that the joint cdf of $X$ and $Y$ can be written as
  \begin{equation*}
    \forall (x, y) \in \mathbb{R}^2 \quad F(x,y) = \int_{-\infty}^{x} \int_{-\infty}^{y} f(t_1, t_2) d_{t_2} d_{t_1}.
  \end{equation*}
  The function $f$ is called the \hldefn{joint density function} of $X$ and $Y$. It follows from the above defintiion that when the second partial derivative exists, we have
  \begin{equation*}
    f(x, y) = \frac{\partial^2}{\partial x \partial y} F(x, y)
  \end{equation*}
  The set $\{(x, y) : f(x, y) > 0 \}$ is called the \hldefn{support set} of $(X, Y)$.
\end{defn}

\begin{note}[Convention]
  Define $f(x, y) = 0$ when $\frac{\partial^2}{\partial x \partial y} F(x, y)$ does not exist.
\end{note}

\begin{eg}[Example 3.8]
  Suppose $X$ and $Y$ have joint pdf $f(x, y) = \mathbb{1}_{\{0 < x, y < 1\}} = \mathbb{1}_{\{0 < x < 1, \, 0 < y < 1 \}}$. Calculate the joint cdf of $X$ and $Y$.

  \begin{solution}
    \begin{equation*}
      F(x, y) = \begin{cases}
        0         & x \leq 0, \, \lor \, y \leq 0 \\
        \int_{0}^{x} \int_{0}^{y} 1 \dif{s} \dif{t} = xy & 0 < x < 1 \, \land \, 0 < y < 1 \\
        \int_{0}^{1} \int_{0}^{y} 1 \dif{s} \dif{t} = y  & x \geq 1 \, \land \, 0 < y < 1 \\
        \int_{0}^{x} \int_{0}^{1} 1 \dif{s} \dif{t} = x  & 0 < x < 1 \, \land \, y \geq 1 \\
        \int_{0}^{1} \int_{0}^{1} 1 \dif{s} \dif{t} = 1  & x \geq 1 \, \land \, y \geq 1
      \end{cases}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{propo}[Properties of Joint PDF]
\label{propo:properties_of_joint_pdf}
  \begin{enumerate}
    \item $\forall (x, y) \in \mathbb{R}^2 \quad f(x, y) \geq 0$
    \item $\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} f(x, y) \dif{x} \dif{y} = 1$
    \item $\forall B \subset \mathbb{R}^2$,
      \begin{equation*}
        P[ (X, Y) \in B ] = \underset{(x, y) \in B}{\int \int} f(x, y) \dif{x} \dif{y}
      \end{equation*}
  \end{enumerate}
\end{propo}

\begin{proof}

\end{proof}

\begin{eg}[Example 3.9]
  Suppose that $f(x, y) = Kxy \cdot \mathbb{1}_{\{0 < x \, y < 1 \}}$ for some constant $K > 0$. Find $K$ so that $f$ is a valid joint pdf. If $X$ and $Y$ have the joint density $f$, calculate $P(X > Y)$.

  \begin{solution}
    Note that
    \begin{equation*}
      1 = \int_{0}^{1} \int_{0}^{1} Kxy \dif{x} \dif{y} = \frac{K}{4}.
    \end{equation*}
    Thus $K = 4$. To solve the next part, observe that for $X > Y$, we have the diagram to the right to show the support set of the joint distribution.
    \marginnote{
	  \begin{center}
	  	\begin{tikzpicture}
          \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
          \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
	  			\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(2.5,0)--(2.5,2.5)--(0,0);
          \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway,left] [label=left:\textcolor{light}{$y = x$}] {};
          \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
          \node [label={270:1}] at (2.5,0) {};
	  	\end{tikzpicture}
	  \end{center}
    }
    The shaded region is the support set. We then have
    \begin{align*}
      P(X > Y) &= \int_{0}^{1} \int_{0}^{x} 4xy \dif{y} \dif{x} = \int_{0}^{1} 2xy^2 \at{0}{x} \dif{x} \\
        &= \int_{0}^{1} 2x^3 \dif{x} = \frac{1}{2} x^3 \at{0}{1} = \frac{1}{2}
    \end{align*}
  \end{solution}
\end{eg}

\begin{eg}[Example 3.10]
  Suppose that
  \begin{equation*}
    f(x, y) = \begin{cases}
      Cxy & 0 < x, y < 1, \, x + y < 1 \\
      0   & \text{otherwise}
    \end{cases}
  \end{equation*}
  Find $C$ so that $f(x, y)$ is a valid joint probability density function, and calculate $P(Y^2 < X)$.
\end{eg}

\begin{solution}
  Note that the diagram on the right shows the support set of $(X, Y)$.\marginnote{
  \begin{center}
  	\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
  			\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(2.5,0)--(0,2.5)--(0,0);
        \draw[line width=0pt,dashed](0,2.5)--(2.5,0);
        \node [label=right:\textcolor{light}{$y = 1 - x$}] at (1.25,1.25) {};
        \node [label={180:1}] at (0,2.5) {};
        \node [label={270:1}] at (2.5,0) {};
  	\end{tikzpicture}
  \end{center}
  } To find $C$,
  \begin{align*}
    1 &= \int_{0}^{1} \int_{0}^{1 - x} Cxy \dif{y} \dif{x} = \int_{0}^{1} \frac{C}{2}xy^2 \at{0}{1 - x} \dif{x} \\
      &= C \int_{0}^{1} \frac{1}{2} x (x^2 - 2x + 1) \dif{x} = C \int_{0}^{1} \frac{1}{2} ( x^3 - 2x^2 + x ) \dif{x} \\
      &= C \left( \frac{1}{8} x^4 - \frac{1}{3} x^3 + \frac{1}{4} x^2 \right) \at{0}{1} = C \left( \frac{3}{24} - \frac{8}{24} + \frac{6}{24} \right) = \frac{C}{24}.
  \end{align*}
  And so $C = 24$.
  \pagebreak

  To calculate $P(Y^2 < X)$, note the diagram to the right. Then
  \marginnote{
  \begin{center}
  	\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
  			\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25,rotate=-22](1.5450849718747371205,0) +(180:1.5450849718747371205) arc (180:108:1.5450849718747371205) -- (1.43,0.57);
  			\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](1.5450849718747371205,0) -- (2.5,0) -- (1.5450849718747371205,0.9549150281252628795) -- (1.5450849718747371205,0);
        \draw[line width=0pt,dashed](0,2.5)--(2.5,0);
        \node [label=right:\textcolor{light}{$y = 1 - x$}] at (1.25,1.25) {};
        \draw[line width=0pt,rotate=-22,dashed] (1.5450849718747371205,0) +(180:1.5450849718747371205) arc (180:108:1.5450849718747371205);
        \node [label={180:1}] at (0,2.5) {};
        \node [label={270:1}] at (2.5,0) {};
        \draw[line width=0pt,dashed] (1.5450849718747371205,0) -- (1.5450849718747371205,0.9549150281252628795);
        \node [label={270:$\frac{3 + \sqrt{5}}{2}$}] at (1.5450849718747371205,0) {};
        \draw[line width=0pt,dashed] (0,0.9549150281252628795) -- (1.5450849718747371205,0.9549150281252628795);
        \node [label={180:$\frac{-1 + \sqrt{5}}{2}$}] at (0,0.9549150281252628795) {};
  	\end{tikzpicture}
  \end{center}
  Solve for $y = 1 - x$ and $y^2 = x$ to get the intersection.
  }
  \begin{align*}
    P(Y^2 < X) &= \int_{0}^{\frac{3 + \sqrt{5}}{2}} \int_{0}^{\sqrt{x}} 24xy \dif{y} \dif{x} + \int_{\frac{ 3 + \sqrt{5} }{2}}^{1} \int_{0}^{1 - x} 24xy \dif{y} \dif{x} \\
      &= \int_{0}^{\frac{3 + \sqrt{5}}{2}} 12xy^2 \at{0}{\sqrt{x}} \dif{x} + \int_{\frac{3 + \sqrt{5}}{2}}^{1} 12xy^2 \at{0}{1 - x} \dif{x} \\
      &= 4x^3\at{0}{\frac{3 + \sqrt{5}}{2}} + \int_{\frac{3 + \sqrt{5}}{2}}^{1} 24 (x^3 - 2x^2 + x) \dif{x} \\
      &= 4 \left( \frac{3 + \sqrt{5}}{2} \right)^3 + 24 \left[ \frac{1}{4}x^4 - \frac{2}{3}x^3 + \frac{1}{2}x^2 \right] \at{\frac{3 + \sqrt{5}}{2}}{1} = \hdots
  \end{align*}
  We shall not proceed to get the final solution since it is a messy process and the result is not important.
\end{solution}
% subsection joint_continuous_rvs (end)

\subsection{Marginal Distribution (Continuous)}%
\label{sub:marginal_distribution_continuous}
% subsection marginal_distribution_continuous

\begin{defn}[Marginal PDF]\index{Marginal Probability Density Function}
\label{defn:marginal_pdf}
  Suppose $X$ and $Y$ are continuous rvs with joint pdf $f$. Then the marginal pdf of $X$ is given by
  \begin{equation*}
    \forall x \in \mathbb{R} \quad f_X(x) = \int_{-\infty}^{\infty} f \dif{y},
  \end{equation*}
  and the marginal pdf of $Y$ is
  \begin{equation*}
    \forall y \in \mathbb{R} \quad f_Y(y) = \int_{-\infty}^{\infty} f \dif{x}.
  \end{equation*}
\end{defn}

\begin{eg}[Example 3.11]\label{eg:3_11}
  Suppose $X$ and $Y$ have joint pdf $f(x, y) = K(x + y) \mathbb{1}_{0 \leq x < y \leq 1}$ for some constant $K$. Find $K$. Then, calculate the marginal density of $X$.
  \pagebreak

  \begin{solution}
  A diagram showing the region of the support set is on the right.
  \marginnote{
	\begin{center}
		\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
				\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(0,2.5)--(2.5,2.5)--(0,0);
        \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway,left] [label=left:\textcolor{light}{$y = x$}] {};
        \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
        \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
		\end{tikzpicture}
	\end{center}
  }

  To get $K$,
  \begin{align*}
    1 &= \int_{0}^{1} \int_{x}^{1} K(x + y) \dif{y} \dif{x} = \int_{0}^{1} \left( Kxy + \frac{1}{2}Ky^2 \right) \at{x}{1} \dif{x} \\
      &= \int_{0}^{1} Kx + \frac{K}{2} - Kx^2 - \frac{1}{2}Kx^2 \dif{x} \\
      &= \frac{K}{2} \left( x^2 + x - x^3 \right) \at{0}{1} = \frac{K}{2}
  \end{align*}
  Thus $K = 2$.

  To get the marginal density of $X$, note that our joint pdf is now the following:
  \begin{equation*}
    f(x, y) = 2(x + y) \mathbb{1}_{\{0 \leq x < y \leq 1\}}
  \end{equation*}
  Thus
  \begin{equation*}
    \int_{x}^{1} 2(x + y) \dif{y} = 2xy + y^2 \at{x}{1} = 2x + 1 - 3x^2
  \end{equation*}
  And hence
  \begin{equation*}
    f_X(x) = \begin{cases}
      -3x^2 + 2x + 1 & 0 \leq x \leq 1 \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
  \end{solution}
\end{eg}

% subsection marginal_distribution_continuous (end)

\subsection{Independence of Continuous RVs}%
\label{sub:independence_of_continuous_rvs}
% subsection independence_of_continuous_rvs

\begin{defn}[Independence of Continuous RVs]\index{Independence}
\label{defn:independence_of_continuous_rvs}
  Two random variables $X$ and $Y$ with joint cdf $F$ and joint pdf $f$ are independent iff
  \begin{equation*}
    \forall x, y \in \mathbb{R} \quad F(x, y) = F_X(x) F_Y(y)
  \end{equation*}
  or\sidenote{It's really an "AND"}
  \begin{equation*}
    \forall x, y \in \mathbb{R} \quad f(x, y) = f_X(x) f_Y(y).
  \end{equation*}
\end{defn}

\begin{note}
  A necessary, but insufficient, condition for $X$ and $Y$ to be independent is that
  \begin{equation*}
    \supp(X, Y) = \supp(X) \times \supp(Y)
  \end{equation*}
\end{note}

\begin{eg}[Example 3.12]
  Are random variables $X$ and $Y$ introduced in \cref{eg:3_11} independent? Explain.

  \begin{solution}
    Recall that the pdf was given as
    \begin{equation*}
      f(x, y) = 2(x + y) \mathbb{1}_{\{1 \leq x < y \leq 1\}}.
    \end{equation*}
    We derived the marginal pdf of $X$ in the earlier example:
    \begin{equation*}
      f_X (x) = (-3x^2 + 2x + 1) \mathbb{1}_{\{0 \leq x \leq 1\}}.
    \end{equation*}
    To get the marginal pdf of $Y$, note
    \begin{equation*}
      \int_{0}^{y} 2(x + y) \dif{x} = x^2 + 2xy \at{0}{y} = 3y^2.
    \end{equation*}
    Thus
    \begin{equation*}
      f_Y(y) = \begin{cases}
        3y^2 & 0 \leq y \leq 1 \\
        0 & \text{otherwise.}
      \end{cases}
    \end{equation*}
    Note that
    \begin{equation*}
      f_X(x) f_Y(y) = -9x^2y^2 + 6xy^2 + 3y^2 \quad 0 \leq x < y \leq 1
    \end{equation*}
    which is not equal to $f$. Thus, $X$ and $Y$ are not independent.
  \end{solution}
\end{eg}

% subsection independence_of_continuous_rvs (end)

% section joint_distributions_continued (end)

% chapter lecture_7_may_24th_2018 (end)

\chapter{Lecture 8 May 29th 2018}%
\label{chp:lecture_8_may_29th_2018}
% chapter lecture_8_may_29th_2018

\section{Joint Distributions (Continued 2)}%
\label{sec:joint_distributions_continued_2}
% section joint_distributions_continued_2

\subsection{Independence of Continuous RVs (Continued)}%
\label{sub:independence_of_continuous_rvs_continued}
% subsection independence_of_continuous_rvs_continued

\begin{eg}[Example 3.12 (3.4.8 course note)]
  Suppose $X$ and $Y$ are continuous with joint pdf
  \begin{equation*}
    f(x, y) = \frac{3}{2} y (1 - x^2) \mathbb{1}_{\{-1 \leq x \leq 1 \}} \mathbb{1}_{\{0 \leq y \leq 1 \}}
  \end{equation*}
  Are $X$ and $Y$ independent?

  \begin{solution}
    The marginal pdf of $X$ is
    \begin{align*}
      f_X(x) &= \int_{0}^{1} \frac{3}{2}y (1 - x^2) \dif{y} = \frac{3}{4}y^2 (1 - x^2) \at{0}{1} \\
        &= \begin{cases}
          \frac{3}{4} ( 1 - x^2 ) & -1 \leq x \leq 1 \\
          0 & \text{otherwise}
        \end{cases}
    \end{align*}
    The marginal pdf of $Y$ is
    \begin{align*}
      f_Y(x) &= \int_{-1}^{1} \frac{3}{2} y (1 - x^2) \dif{x} = \frac{3}{2} y \left(x - \frac{1}{3}x^3 \right) \at{-1}{1} \\
      &= \begin{cases}
        2y & 0 \leq y \leq 1 \\
        0 & \text{otherwise.}
      \end{cases}
    \end{align*}
    Clearly, we have
    \begin{equation*}
      f_X(x) f_Y(y) = \frac{3}{2} y (1 - x^2) = f(x, y) \quad -1 \leq x \leq 1, \, 0 \leq y \leq 1.
    \end{equation*}
    Thus $X$ and $Y$ are independent.
  \end{solution}
\end{eg}

\begin{thm}[Factorization Theorem for Independence]
\index{Factorization Theorem}
\label{thm:factorization_theorem_for_independence}
  Suppose $X$ and $Y$ are rvs with joint pf $f$, and marginal pf $f_X$ and $f_Y$, respectively. Suppose also that
  \begin{align*}
    A &= \{(x, y) : f(x, y) > 0\} \text{ is the support set of } (X, Y) \\
    A_X &= \{x : f_X(x) > 0 \} \text{ is the support set of } X \text{, and} \\
    A_Y &= \{y : f_Y(y) > 0 \} \text{ is the support set of } Y
  \end{align*}
  Then $X$ and $Y$ are independent rvs iff $A = A_X \times A_Y$ and there exist non-negative functions $g$ and $h$ such that
  \begin{equation*}
    f(x, y) = g(x) h(y)
  \end{equation*}
  for all $(x, y) \in A_X \times A_Y$.
\end{thm}

\begin{proof}
  The $\implies$ direction is straightforward: Since $X$ and $Y$ are independent, we have that $f = f_X f_Y$, and so clearly, $A = A_X \times A_Y$ and so $\forall (x, y) \in A = A_x \times A_Y$, we have that $f_X$ and $f_Y$ are non-negative.

  For the $\impliedby$ direction, note that
  \begin{align*}
    f_Y(y) &= \int_{x \in A_X} g(x)h(y) \dif{x} = h(y) \int_{x \in A_X} g(x) \dif{x} \\
    f_X(x) &= \int_{y \in A_Y} g(x)h(y) \dif{y} = g(x) \int_{y \in A_Y} h(x) \dif{y}.
  \end{align*}
  Thus,
  \begin{align*}
    f_X(x) f_Y(y) &= g(x)h(y) \int_{x \in A_X} g(x) \dif{x} \int_{y \in A_Y} h(y) \dif{y} \\
      &= g(x) h(x) \int_{x \in A_X} \int_{y \in A_Y} g(x)h(y) \dif{y} \dif{x} = g(x) h(y)
  \end{align*}
  where line 2 is by \hlnotea{linearity of integration}. Thus $f(x, y) = f_X(x) f_Y(y)$. Thus $X$ and $Y$ are independent. \qed
\end{proof}

\begin{note}
  \begin{enumerate}
    \item If \cref{thm:factorization_theorem_for_independence} holds, then $f_X$ will be proportional to $g$ and $f_Y$ will be proportional to $h$. Clearly so, since
      \begin{gather*}
        g(x) \cdot h(y) = f_X(x) f_Y(y) \\
        g(x) \propto f_X(x) \, \land \, h(y) \propto f_Y(y)
      \end{gather*}

    \item The definitions and theorems can be easily extended to the random vector $(X_1, X_2, ..., X_n)$. Indeed, if we apply mathematical induction on the proof above, we will be able to get our desired result.\sidenote{I wonder if this statement is equivalent to the \href{https://en.wikipedia.org/wiki/Sufficient_statistic\#Fisher\%E2\%80\%93Neyman_factorization_theorem}{Fisher-Neyman Factorization Theorem}.}
  \end{enumerate}
\end{note}

% subsection independence_of_continuous_rvs_continued (end)

\subsection{Conditional Distributions}%
\label{sub:conditional_distributions}
% subsection conditional_distributions

\begin{defn}[Conditional Distributions]\index{Conditional Distributions}\index{Conditional PF}
\label{defn:conditional_distributions}
  Suppose $X$ and $Y$ are rvs with joint pf $f$, and marginal pfs $f_X$ and $f_Y$, respectively. Suppose also that $A = \{(x, y) : f(x, y) > 0 \}$. The \hlnoteb{conditional pf} of $X$ given $Y = y$ is given by
  \begin{equation*}
    f_X(x | y) = \frac{f(x, y)}{f_Y(y)}
  \end{equation*}
  for $(x, y) \in A$ provided that $f_Y(y) \neq 0$. The \hlnoteb{conditional pf} of $Y$ given $X = x$ is given by
  \begin{equation*}
    f_Y(y | x) = \frac{f(x, y)}{f_X(x)}
  \end{equation*}
  for $(x, y) \in A$ provided that $f_X(x) \neq 0$.
\end{defn}

\begin{remark}
  If $X$ and $Y$ are discrete rvs then
  \begin{equation*}
    f_X(x | y) = P(X = x | Y = y) = \frac{P(X = x, Y = y)}{P(Y = y)} = \frac{f(x, y)}{f_Y(y)}
  \end{equation*}
  and
  \begin{equation*}
    \sum_{x} f_X(x | y) = \sum_{x} \frac{f(x, y)}{f_Y(y)} = \frac{1}{f_Y(y)} \sum_{x} f(x, y) = \frac{f_Y(y)}{f_Y(y)} = 1,
  \end{equation*}
  and similarly so for $f_Y(y | x)$. Similarly, if $X$ and $Y$ are both continuous rvs, then
  \begin{equation*}
    \int_{-\infty}^{\infty} f_X(x | y) \dif{x} = \int_{-\infty}^{\infty} \frac{f(x, y)}{f_Y(y)} \dif{x} = \frac{1}{f_Y(y)} \int_{-\infty}^{\infty} f(x, y) \dif{x} = \frac{f_Y(y)}{f_Y(y)} = 1
  \end{equation*}

  Now consider if $X$ is a continuous rv such that $f_X(x) \neq P(X = x)$ and $P(X = x) = 0$ for all $x$. Then to justify the definition of the conditional pdf of $Y$ given $X = x$, when $X$ and $Y$ are both continuous rvs, we consider $P(Y \leq y | X = x)$ using the limit approach:
  \begin{align*}
    P(Y \leq y | X = x) &= \lim_{h \to 0} P( Y \leq y | x \leq X \leq x + h ) \\
      &= \lim_{h \to 0} \frac{\int_{x}^{x + h} \int_{-\infty}^{y} f(u, v) \dif{v} \dif{u} }{ \int_{x}^{x + h} f_X(u) \dif{u} } \\
      &= \lim_{h \to 0} \frac{\frac{d}{dh} \int_{x}^{x + h} \int_{-\infty}^{y} f(u, v) \dif{v} \dif{u}}{ \frac{d}{dh} \int_{x}^{x + h} f_X(u) \dif{u} } \quad \text{by } \hlnotea{L'H\^opital's Rule} \\
      &= \lim_{h \to 0} \frac{\int_{-\infty}^{y} \frac{d}{dh} \int_{x}^{x + h} f(u, v) \dif{u} \dif{v} }{\frac{d}{dh} \int_{x}^{x + h} f_X(u) \dif{u} } \quad (1) \\
      &= \lim_{h \to 0} \frac{\int_{-\infty}^{y} f(x + h, v) \dif{v}}{f_X(x + h)} \quad (2) \\
      &= \frac{\int_{-\infty}^{y} f(x, v) \dif{v} }{f_X(x)}
  \end{align*}
  where $(1)$ is by assuming that the integrands are all convergent so that we may interchange the integral signs and the differential operator, and $(2)$ by the \hlnotea{Fundamental Theorem of Calculus}. If we differentiate the last line with respect to $y$, by the Fundamental Theorem of Calculus, we have
  \begin{equation*}
    \frac{d}{dy} P(Y \leq y | X = x) = \frac{f(x, y)}{f_X(x)}
  \end{equation*}
  which justifies the using of our definition
  \begin{equation*}
    f_Y(y | x) = \frac{f(x, y)}{f_X(x)}.
  \end{equation*}
\end{remark}

\begin{eg}\label{eg:basic_conditional_probability}
  A fair coin is flipped 10 times independently.
  \begin{enumerate}
    \item What is the distribution of $Y$, the number of heads in 10 flips?
    \item Suppose the first 4 flips have all landed on tails. What is the distribution of $Y$ given this information?
  \end{enumerate}

  \begin{solution}
    \begin{enumerate}
      \item Clearly, we know that $Y \sim \Bin \left(10, \frac{1}{2}\right)$.
      \item Since each flip is independent of each other and the first four flips have already been determined, the range of values for $Y$ changes from $\{0,...,10\}$ to $\{0, ..., 6\}$. Since the experiment is still essentially the same, we have that
      \begin{equation*}
        Y \, | \, \text{ first 4 flips are tails} \sim \Bin \left( 6, \frac{1}{2} \right).
      \end{equation*}
    \end{enumerate}
  \end{solution}
\end{eg}

\begin{eg}[Example 3.13]
  Consider the experiment carried out in \cref{eg:basic_conditional_probability}. Let
  \begin{align*}
    X &:= \text{ number of heads in the first } 4 \text{ flips } \\
    Y &:= \text{ number of heads in } 10 \text{ flips }
  \end{align*}
  Derive the conditional distribution of $Y$ given that the first 4 flips landed on heads, i.e. derive the distribution for $Y | X = 4$.

  \begin{solution}
    Let $W$ be the number of heads in the last 6 flips. Then $W$ has the same distribution as in part 2 of our earlier example. Also, $X \sim \Bin\left(4, \frac{1}{2}\right)$ Clearly, $Y = X + W$. We proceed to derive the joint pf of $X$ and $Y$:
    \begin{align*}
      P(X = x, \, Y = y) &= P(X = x, \, X + W = y) = P(X = x, \, W = y - x) \\
        &= P(X = x) P(W = y - x) \quad \text{by }\hlnoteb{Independence} \\
        &= \binom{4}{x} \left( \frac{1}{2} \right)^x \left( \frac{1}{2} \right)^{4 - x} \cdot \binom{6}{y - x} \left( \frac{1}{2} \right)^{y - x} \left( \frac{1}{2} \right)^{6 - y + x} \\
        &= \binom{4}{x} \binom{6}{y - x} \left( \frac{1}{2} \right)^{10}
    \end{align*}
    Then
    \begin{align*}
      P(Y | X = 4) &= \frac{P(X = 4, \, Y = y)}{P(X = 4)} \\
        &= \frac{\binom{4}{4} \binom{6}{y - 4} \left( \frac{1}{2} \right)^{10}}{\binom{4}{4} \left( \frac{1}{2} \right)^4 \left( \frac{1}{2} \right)^{0}} \\
        &= \binom{6}{y - 4} \left( \frac{1}{2} \right)^{6} \quad y \in \{4, 5, ..., 10\}.
    \end{align*}
    We may also re-label the conditional distribution to have
    \begin{equation*}
      \binom{6}{y^*} \left( \frac{1}{2} \right)^{6} \quad y^* \in \{0, ..., 6\}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{eg}[Example 3.14]
  From \cref{eg:3_11}, we had that
  \begin{equation*}
    f(x, y) = 2 ( x + y ) \mathbb{1}_{\P0 \leq x < y \leq 1\}}
  \end{equation*}
  and the marginal density of $X$ is
  \begin{equation*}
    f_X(x) = (2x - 3x^2 + 1) \mathbb{1}_{\{ 0 \leq x < 1 \}}.
  \end{equation*}
  Derive the conditional distribution of $Y | X = \frac{1}{2}$.

  \begin{solution}
    Observe that
    \begin{align*}
      f(y | X = \frac{1}{2}) = \frac{f \left( \frac{1}{2}, \, y \right)}{f_X \left( \frac{1}{2} \right)} = \frac{2 \left( \frac{1}{2} + y \right)}{2 \left( \frac{1}{2} \right) - 3 \left( \frac{1}{2} \right)^2 + 1} = \frac{8}{13} ( 1 + 2y )
    \end{align*}
    for $\frac{1}{2} < y \leq 1$.
  \end{solution}
\end{eg}

\begin{propo}[Properties of Conditional Distributions]
\label{propo:properties_of_conditional_distributions}
  Let $X$ and $Y$ be rvs. If both $X$ and $Y$ are discrete, then
\marginnote{
\begin{ex}
  Prove \cref{propo:properties_of_conditional_distributions}.
\end{ex}
}
  \begin{itemize}
    \item $\sum_{x} f(x | y) = 1$;
    \item $F(x | y) = \sum_{\{w : w \leq x\}} f(w | y)$; and
    \item $f(x | y) = F(x | y) - F(x^- | y)$.
  \end{itemize}
  If $X$ and $Y$ are both continuous, then
  \begin{itemize}
    \item $\int_{x} f(x | y) \dif{x} = 1$;
    \item $F(x | y) = \int_{-\infty}^{x} f(t | y) \dif{t}$; and
    \item $f(x | y) = \frac{\partial}{\partial x} F(x | y)$
  \end{itemize}
\end{propo}

\begin{thm}[Product Rule]
\index{Product Rule}
\label{thm:product_rule}
  Suppose $X$ and $Y$ are rvs with joint pf $f$, marginal pfs $f_X(x)$ and $f_Y(y)$ respectively, and conditional pfs $f_X(x | y)$ and $f_Y(y | x)$ respectively. Then
  \begin{equation*}
    f(x, y) = f_X(x | y) f_Y(y) = f_Y(y | x) f_X(x).
  \end{equation*}
\end{thm}

\begin{proof}
  Notice once and for all that by rearranging the definition of conditional distribution
  \begin{equation*}
    f_X(x | y) f_Y(y) = f(x, y) = f_Y(y | x) f_X(x)
  \end{equation*}\qed
\end{proof}

\begin{propo}[Independence from Conditionality]
\label{propo:independence_from_conditionality}
  Suppose $X$ and $Y$ are rvs with marginal pfs $f_X(x)$ and $f_Y(y)$ respectively, and conditional pfs $f_X(x | y)$ and $f_Y(y | x)$ respectively. Let $A_X = \{x : f_X(x) > 0\}$ and $A_Y = \{ y : f_Y(y) > 0 \}$. $X$ and $Y$ are independent rvs iff either of the following holds:
  \begin{equation*}
    \forall x \in A_X \quad f_X(x | y) = f_X(x)
  \end{equation*}
  or
  \begin{equation*}
    \forall y \in A_Y \quad f_Y(y | x) = f_Y(y).
  \end{equation*}
\end{propo}

\begin{proof}
  Suppose that $X$ and $Y$ are independent rvs. Then
  \begin{equation*}
    f(x, y) = f_X(x) f_Y(y).
  \end{equation*}
  Then
  \begin{align*}
    f_X(x | y) &= \frac{f(x, y)}{f_Y(y)} = \frac{f_X(x) f_Y(y)}{f_Y(y)} = f_X(x) \\
    f_Y(y | x) &= \frac{f(x, y)}{f_X(x)} = \frac{f_X(x) f_Y(y)}{f_Y(y)} = f_Y(y).
  \end{align*}
  for $x \in A_X$ and $y \in A_Y$.

  Suppose WLOG that $f_X(x | y) = f_X(x)$. Thus by \cref{thm:product_rule}
  \begin{equation*}
    f(x, y) = f_X(x | y) f_Y(y) = f_X(x) f_Y(y) \text{ for } x \in A_X, \, y \in A_Y.
  \end{equation*}
  \qed
\end{proof}

\begin{eg}[Example 3.15]
  In a game of chance, a random number is generated from $P \sim \BetaDist( \alpha, \beta )$. Given $P = p$, a coin with $P( \{H\} ) = p$ is flipped independently $n$ times, where the player is rewarded the same amount of dollars as the number of heads in $n$. Calculate the probability that a random player earns at least \$1 in this game.

  \begin{solution}
    Let $X$ be the number of heads that appear in $n$ flips, which equates to the total amount of \$1 earned. Then
    \begin{equation*}
      X \, | P = p \sim \Bin(n, p)
    \end{equation*}
    However, note that
    \begin{equation*}
      P (X \geq 1) = P(\text{earn at least \$1}) = 1 - P(\text{earn nothing}) = 1 - P(X = 0)
    \end{equation*}
    To get $P(X = x)$, we need to do the following: note that the support set of $P$ is from $0$ to $1$, then
    \marginnote{
    \begin{defn}[Beta Distribution]\index{Beta Distribution}
    \label{defn:beta_distribution}
    If $X \sim \BetaDist(\alpha, \beta)$, then
      \begin{equation*}
        f(x) = \frac{x^{\alpha - 1} (1 - x)^{\beta - 1}}{B(\alpha, \beta)}
      \end{equation*}
      where
      \begin{equation*}
        B(\alpha, \beta) = \frac{\Gamma(\alpha) \Gamma(\beta)}{\Gamma(\alpha + \beta)}
      \end{equation*}
    \end{defn}
    }
    \begin{align*}
    Pr(X = x) &= \int_{0}^{1} Pr(X = x, P = p) \dif{p} \\
           &= \int_{0}^{1} Pr(X = x | P = p) Pr(P = p) \dif{p} \qquad \text{by } \cref{thm:product_rule} \\
           &= \int_{0}^{1} \binom{n}{x} p^x (1 - p)^{n - x} \frac{p^{\alpha - 1} (1 - x)^{\beta - 1} }{B(\alpha, \beta)} \dif{p} \\
           &= \binom{n}{x} \frac{1}{B(\alpha, \beta)} \int_{0}^{1} p^{\alpha + x  -1} (1 - p)^{\beta + n - x - 1} \dif{p} \\
           &= \binom{n}{x} \frac{B(\alpha + x , \beta + n - x)}{B(\alpha, \beta)} \int_{0}^{1} \underbrace{ \frac{p^{\alpha + x - 1} (1 - p)^{\beta + n - x - 1}}{B (\alpha + x, \beta + n - x)} }_{\text{pdf of } \BetaDist(\alpha + x, \beta + n - x)} \dif{p} \\
           &= \binom{n}{x} \frac{B( \alpha + x, \beta + n - x )}{B( \alpha, \beta )}
    \end{align*}
    Therefore,
    \begin{align*}
      P(X \geq 1) &= 1 - P(X = 0) = 1 - \binom{n}{0} \frac{\Gamma(\alpha)\Gamma(\beta + n)}{\Gamma( \alpha + \beta + n )} \frac{\Gamma( \alpha + \beta )}{\Gamma(\alpha) \Gamma(\beta)} \\
        &= 1 - \frac{\Gamma(\alpha + \beta) \Gamma(\beta + n)}{\Gamma( \alpha + \beta + n ) \Gamma(\beta)}
    \end{align*}
  \end{solution}
\end{eg}

% subsection conditional_distributions (end)

\subsection{Joint Expectations}%
\label{sub:joint_expectations}
% subsection joint_expectations

\begin{defn}[Joint Expectation]\index{Joint Expectation}
\label{defn:joint_expectation}
  Suppose $h(x, y)$ is a real-valued function. If $X$ and $Y$ are discrete rvs with joint pf $f$ and support $A$, then
  \begin{equation*}
    E[ h(x, y) ] = \underset{(x, y) \in A}{\sum\sum} h(x, y) f(x, y).
  \end{equation*}
  provided that the joint sum converges absolutely.

  If $X$ and $Y$ are continuous rvs with joint pf $f$, then
  \begin{equation*}
    E[ h(x, y) ] = \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} h(x, y) f(x, y) \dif{x} \dif{y}
  \end{equation*}
  provided that the joint integral converges absolutely.

  This is also known as the \hldefn{Law of the Unconscious Statistician} for two rvs.
\end{defn}

\begin{eg}[Example 3.16]
  Consider $X$ and $Y$ with the following joint probability distribution. Calculate $E(XY)$.
  \[
  \begin{tabular}{c | c | c | c}
         & x = -2 & x = 0 & x = 2 \\
  \hline
  y = 0  & 0.05   & 0.1   & 0.15 \\
  \hline
  y = 1  & 0.07   & 0.1   & 0.2 \\
  \hline
  y = 2  & 0.02   & 0.25  & 0.05
  \end{tabular}
  \]

  \begin{solution}
    \begin{align*}
      E(XY) &= \sum_{x} \sum_{y} xy f(x, y) \\
        &= -2 (1) (0.07) - 2(2)(0.02) + 2(1)(0.2) + 2(2)(0.05) \\
        &= 0.38
    \end{align*}
  \end{solution}
\end{eg}

% subsection joint_expectations (end)

% section joint_distributions_continued_2 (end)

% chapter lecture_8_may_29th_2018 (end)

\chapter{Lecture 9 May 31st 2018}%
\label{chp:lecture_9_may_31st_2018}
% chapter lecture_9_may_31st_2018

\section{Joint Distributions (Continued 3)}%
\label{sec:joint_distributions_continued_3}
% section joint_distributions_continued_3

\subsection{Joint Expectations (Continued)}%
\label{sub:joint_expectations_continued}
% subsection joint_expectations_continued

\begin{thm}[Linearity of Expectation in Bivariate Case]\index{Linearity - Expectation}
\label{thm:linearity_of_expectation_in_bivariate_case}
  Suppose $X$ and $Y$ are two rvs with joint pf $f$, $a_i, \, b_i$, for $i = 1, ..., n$, are constants, and $g_i(x, y)$, for $i = 1, ..., n$, are real-valued functions. Then
  \begin{equation*}
    E\left[ \sum_{i=1}^{n} \left( a_i g_i (X, Y) + b_i \right) \right] = \sum_{i = 1}^{n} ( a_i E[ g_i(X, Y) ] ) + \sum_{i=1}^{n} b_i
  \end{equation*}
  provided that $E[g_i(X, Y)]$ is finite for $i = 1, ..., n$.
\end{thm}

\begin{proof}
  This is simply an extension of \cref{thm:linearity_of_expectation}.
\end{proof}

\begin{thm}[Implication of Independence on Joint Expectation]
\label{thm:implication_of_independence_on_joint_expectation}
  If $X$ and $Y$ are independent rvs with joint pf $f$, and $g(x)$ and $h(y)$ are real valued funcctions, then
  \begin{equation*}
    E[ g(X) h(Y) ] = E[ g(X) ] E[ h(Y) ]
  \end{equation*}
\end{thm}

\begin{proof}
  We shall prove for the discrete case and leave the continuous case for future exercises.\marginnote{
  \begin{ex}
    Prove \cref{thm:implication_of_independence_on_joint_expectation} for the continuous case.
  \end{ex}
  }

  Observe that
  \begin{align*}
    E[ g(X) h(Y) ] &= \sum_{x} \sum_{y} g(x) h(y) f(x, y) \quad \because \cref{defn:joint_expectation} \\
      &= \sum_{x} \sum_{y} g(x) h(y) f_X(x) f_Y(y) \\
      &= \sum_{x} g(x) f_X(x) \sum_{y} h(y) f_Y(y) \\
      &= E[ g(X) ] E[ h(Y) ]
  \end{align*}
  where $f_X(x)$ and $f_Y(y)$ are the marginal pfs of $X$ and $Y$ respectively.\qed
\end{proof}

We may repeatedly apply the above proof for $n$ rvs through induction and get the following result.

\begin{thm}[Generalized Implication of Independence on Joint Expectation]
\label{thm:generalized_implication_of_independence_on_joint_expectation}
  If $X_1, X_2, ..., X_n$, for some $n \in \mathbb{N}$, are independent rvs and $h_1, h_2, ..., h_n$ are real valued functions, then
  \begin{equation*}
    E \left[ \prod_{i = 1}^{n} h_i (X_i) \right] = \prod_{i = 1}^{n} E[ h_i(X_i) ].
  \end{equation*}
\end{thm}

% subsection joint_expectations_continued (end)

\subsection{Covariance}%
\label{sub:covariance}
% subsection covariance

\newthought{Independence} of two rvs $X$ and $Y$ implies that knowledge of the value of $X$ does not provide any information whatsoever about the distribution of $Y$. Essentially, we can say that there is no ``relationship'' between $X$ and $Y$. In statistics, \hlnoteb{linear relationships} are often the subject of interest. The strength of a linear relationship is related to \hlnoteb{covariance} and measured by the \hlnoteb{correlation coefficient}, usually denoted by $\rho$.

It can be shown that when $X$ and $Y$ have no linear relationship iff their covariance is $0$.

On a related thought, does covariance relate to independence? If so, how?

\begin{defn}[Covariance]\index{Covariance}
\label{defn:covariance}
  The \hlnoteb{covariance} of rvs $X$ and $Y$ is given by
  \begin{equation}\label{eq:covariance}
    \Cov( X, Y ) = E[ (X - \mu_X) (Y - \mu_Y) ] = E(XY) - \mu_X \mu_Y.
  \end{equation}
  where $\mu_X = E[X]$ and $\mu_Y = E[Y]$.

  If $\Cov(X, Y) = 0$, then $X$ and $Y$ are called \hldefn{uncorrelated} rvs.
\end{defn}

\begin{note}
  Note that the 2nd and 3rd term are equivalent in \cref{eq:covariance} since
  \begin{align*}
    E[ (X - \mu_X) (Y - \mu_Y) ]
      &= E[ XY - \mu_X Y - \mu_Y X + \mu_X \mu_Y ] \\
      &= E[ XY ] - \mu_X E[Y] - \mu_Y E[X] + \mu_X \mu_Y \because \cref{thm:linearity_of_expectation_in_bivariate_case} \\
      &= E[ XY ] - \mu_X \mu_Y - \mu_X \mu_Y + \mu_X \mu_Y = E[ XY ] - \mu_X \mu_Y
  \end{align*}

  From here, it is easy to see from \cref{thm:implication_of_independence_on_joint_expectation}, that since $E[XY] = E[X] E[Y] = \mu_X \mu_Y$, we have that the independence of $X$ from $Y$ will imply that $\Cov(X, Y) = 0$.

  However, the converse of the above is \hlimpo{not true}.
\end{note}

\begin{eg}
  Source: \href{https://stats.stackexchange.com/questions/12842/covariance-and-independence\#12844}{Stats SE}

  Let $X$ be an rv that it is $ - 1$ or $1$ with probability $0.5$. Then let $Y$ be an rv such that $Y=0$ if $X=-1$, and $Y$ is randomly $-1$ or $1$ with probability $0.5$ if $X=1$.

  Clearly $X$ and $Y$ are highly dependent (since knowing $Y$ allows me to perfectly know $X$). They both have zero mean:
  \begin{align*}
    E[X] &= -1 \left(\frac{1}{2}\right) + 1 \left(\frac{1}{2}\right) = 0 \\
    E[Y] &= -1 \left(\frac{1}{2}\right) + 1 \left(\frac{1}{2}\right) = 0
  \end{align*}
  and
  \begin{align*}
    E[ XY ] &= (-1) \cdot 0 P(X = -1, Y = 0) + 1 (-1) \cdot P(X = 1, Y = -1) \\
      &\qquad + 1 (1) P(X = 1, Y = 1) \\
      &= - \frac{1}{4} + \frac{1}{4} = 0
  \end{align*}
  Thus $\Cov(X, Y) = 0$

  Or more generally, take any distribution $P(X)$ and any $P(Y | X)$ such that $P(Y = a | X)=P(Y=-a | X)$ for all $X$ (i.e., a joint distribution that is symmetric around the $x$ axis), and you will always have zero covariance. But you will have non-independence whenever $P(Y | X) \neq P(Y)$, i.e., the conditionals are not all equal to the marginal, and vice versa for symmetry around the y axis.
\end{eg}

\begin{note}
  \begin{itemize}
    \item If $\Cov(X, Y) = 0$, then $X$ and $Y$ are called \hlnoteb{uncorrelated} rvs.
    \item By definition, $\Cov(X, X) = \Var(X)$, since
      \begin{equation*}
        \Cov(X, X) = E[ (X - \mu_X)^2 ] = \Var(X)
      \end{equation*}
  \end{itemize}
\end{note}

\begin{eg}[Example 3.17]
  Consider the rvs $X$ and $Y$ with the joint pdf
  \begin{equation*}
    f(x, y) = \begin{cases}
      2 & 0 \leq x \leq y \leq 1 \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
  Calculate $\Cov(X, Y)$.

  \begin{solution}
  Observe the diagram of the support set of $X$ and $Y$ to our right.
  \marginnote{
	\begin{center}
		\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
				\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(0,2.5)--(2.5,2.5)--(0,0);
        \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway,left] [label=left:\textcolor{light}{$y = x$}] {};
        \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
        \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
		\end{tikzpicture}
	\end{center}
  }

  Then we can calculate
  \begin{align*}
    E[ XY ]  & = \int_{0}^{1} \int_{x}^{1} 2xy \dif{y} \dif{x} = \int_{0}^{1} xy^2 \at{x}{1} \dif{x} \\
             & = \int_{0}^{1} x - x^3 \dif{x} = \frac{1}{2} - \frac{1}{4} = \frac{1}{4} \\
    f_X(x)   & = \int_{x}^{1} 2 \dif{y} = \begin{cases}
                  2 - 2x & 0 \leq x \leq 1 \\
                  0      & \text{otherwise}
                \end{cases} \\
    E[X]     & = \int_{0}^{1} x (2 - 2x) \dif{x} = 1 - \frac{2}{3} = \frac{1}{3}\\
    f_Y(y)   & = \int_{0}^{y} 2 \dif{x} = \begin{cases}
                  2y & 0 \leq y \leq 1 \\
                  0  & \text{otherwise}
                \end{cases} \\
    E[Y]     & = \int_{0}^{1} 2y^2 \dif{y} = \frac{2}{3}
  \end{align*}
  Thus
  \begin{equation*}
    \Cov(X, Y) = \frac{1}{4} - \frac{1}{3} \left( \frac{2}{3} \right) = \frac{1}{36}
  \end{equation*}
  We observe that the covariance is positive. This implies a \hlnoteb{positive linear relationship}. However, we cannot tell from this value the strength of the relationship between $X$ and $Y$.
  \end{solution}
\end{eg}

\begin{eg}[Example 3.18]
  Consider rvs $X$ and $Y$ with the joint pf
  \begin{equation*}
    f(x, y) = \frac{xy}{7} \mathbb{1}_{\{y = 1, 2\}} \mathbb{1}_{\{x = 0, ..., y\}}.
  \end{equation*}
  Calculate $\Cov(X, Y)$.

  \begin{solution}
    The following table captures all the probabilities that can be found from the given pf

    \[
    \begin{tabular}{c | c | c | c}
            & x = 0 & x = 1       & x = 2 \\
      \hline
      y = 1 & 0     & $\frac{1}{7}$ & 0 \\
      \hline
      y = 2 & 0     & $\frac{2}{7}$ & $\frac{4}{7}$
    \end{tabular}
    \]

    Observe that we thus have
    \begin{align*}
      f_X(x) &= \begin{cases}
        \frac{3}{7} & x = 1 \\
        \frac{4}{7} & x = 2
      \end{cases} \\
      E[X] &= \frac{3}{7} + 2 \left( \frac{4}{7} \right) = \frac{11}{7} \\
      f_Y(y) &= \begin{cases}
        \frac{1}{7} & y = 1 \\
        \frac{6}{7} & y = 2
      \end{cases} \\
      E[Y] &= \frac{1}{7} + 2 \left( \frac{6}{7} \right) = \frac{13}{7}
    \end{align*}
    Also,
    \begin{equation*}
      E[XY] = \frac{1}{7} + 2 \left( \frac{2}{7} \right) + 4 \left( \frac{4}{7} \right) = 3
    \end{equation*}
    Therefore,
    \begin{equation*}
      \Cov(X, Y) = E[XY] - E[X]E[Y] = 3 - \frac{11}{7} \frac{13}{7} = \frac{4}{49}
    \end{equation*}
  \end{solution}
\end{eg}

\begin{thm}[Variance of Linear Combinations]
\label{thm:variance_of_linear_combinations}
  Suppose $X$ and $Y$ are rvs and $a, b, c$ are real constants. Then
  \begin{equation*}
    \Var(aX + bY + c) = a^2 \Var(X) + b^2 \Var(Y) + 2ab \Cov (X, Y)
  \end{equation*}
\end{thm}

\begin{proof}
  Let $E[ aX + bY + c ] = \mu$. Observe that
  \begin{align*}
    &\Var[ aX + bY + c ] \\
    &= E \left[ [ ( aX + bY + c ) - \mu ]^2 \right] \\
    &= E \left[ (aX + bY + c)^2 - 2\mu (aX + bY + c) + \mu^2 \right] \\
    &= E \left[ a^2X^2 + abXY + acX + abXY + b^2 Y^2 + bcY + c^2 \right. \\
    & \qquad \left. - 2\mu(aX + bY+c) + \mu^2 \right] \\
    &= a^2 E[X^2] + 2ab E[XY] + ac E[X] + b^2 E[Y^2] + bc E[Y] + c^2 - \mu^2
  \end{align*}
  Note that
  \begin{align*}
    \mu^2 &= E \left[ aX + bY + c \right]^2 \\
      &= \left( aE[X] + bE[Y] + c \right)^2 \\
      &= a^2 E[X]^2 + b^2 E[Y]^2 + 2abE[X]E[Y] + acE[X] + bcE[Y] + c^2
  \end{align*}
  Therefore, we have that
  \begin{align*}
    &\Var[aX + bY + c] \\
    &= a^2 E[X^2] + a^2 E[X]^2 + b^2 E[Y^2] - b^2 E[Y]^2 + 2ab E[XY] - 2ab E[X]E[Y] \\
    &= a^2 \left( E[X^2] - E[X]^2 \right) + b^2 \left( E[Y^2] - E[Y]^2 \right) + 2ab \left( E[XY] - E[X]E[Y] \right) \\
    &= a^2 \Var(X) + b^2 \Var{Y} + 2ab \Cov(X, Y)
  \end{align*}
  as required. \qed
\end{proof}

By applying \cref{thm:variance_of_linear_combinations} repeatedly, we have the following generalized theorem.

\begin{thm}[Generalized Variance of Linear Combinations]
\label{thm:generalized_variance_of_linear_combinations}
  Suppose $X_1, X_2, ..., X_n$ are rvs with $\Var(X_i) = \sigma_i^2$, and $a_1, a_2, ..., a_n$ are real constants. Then
  \begin{equation*}
    \Var \left( \sum_{i=1}^{n} a_i X_i \right) = \sum_{i = 1}^{n} a_i^2 \sigma_i^2 + 2 \sum_{i=1}^{n-1} \sum^{n}_{j = i + 1} a_i a_j \Cov(X_i, X_j)
  \end{equation*}
\end{thm}

Note that the prove the above, we have to also use the fact that
\begin{equation*}
  \Cov(X_i, X_j) = \Cov(X_j, X_i)
\end{equation*}

\begin{note}
  Note that in \cref{thm:generalized_variance_of_linear_combinations}, if the rvs are independent rvs, then $\Cov(X_i, X_j) = 0$ for $i \neq j$, thus wiping off the 2nd term in the equation, leaving us with
  \begin{equation*}
    \Var \left( \sum_{i=1}^{n} a_i X_i \right) = \sum_{i=1}^{n} a_i^2 \sigma_i^2
  \end{equation*}
\end{note}

\begin{eg}[Example 3.19]
  To build a ship engine piece, suppose two pole-shaped components $A$ and $B$ are attached at one end to each other to make one long pole-shaped component $C$. Suppose the length of part $A$ is an rv with a mean of $3$ inches and a variance of $0.25$ inch$^2$. Similarly, the length of component $B$ is an rv with a mean of $25$ inches and a variance of $0.5$ inch$^2$.

  Find the mean and the variance of the length of part $C$ if
  \begin{enumerate}
    \item the lengths of $A$ and $B$ are independent;
    \item the covariance between lengths of $A$ and $B$ is $-0.3$ inch$^2$.
  \end{enumerate}
\end{eg}

\begin{solution}
  Note that we are given that $C = A + B$
  \begin{enumerate}
    \item We have that
      \begin{equation*}
        E(C) = E(A + B) = E(A) + E(B) = 3 + 25 = 28.
      \end{equation*}
      For variance, since $A$ and $B$ are independent, $\Cov(A, B) = 0$, thus
      \begin{align*}
        \Var(C) &= \Var(A + B) = \Var(A) + \Var(B) + 2 \Cov(A, B) \\
          &= 0.25 + 0.5 + 0 = 0.75
      \end{align*}

    \item Since $C$ is a linear equation, the covariance does not affect the expectation and thus we still have
    \begin{equation*}
      E(C) = 28.
    \end{equation*}
    Now, given that $\Cov(A, B) = -0.3$, we have
    \begin{equation*}
      \Var(C) = \Var(A) + \Var(B) + 2 \Cov(A, B) = 0.75 - 0.6 = 0.15.
    \end{equation*}
  \end{enumerate}
\end{solution}

% subsection covariance (end)

\subsection{Correlation}%
\label{sub:correlation}
% subsection correlation

The \hlnoteb{covariance} is a real number which depends on the units of measurement of $X$ and $Y$. The information part of a covariance is its \hlimpo{sign}, unless if it is used as the context.

To use the covariance as the context, and to quantitatively measure the strength of a linear relationship, which we have discussed and desired before, we use the \hlnoteb{correlation coefficient}.

\begin{defn}[Correlation Coefficient]\index{Correlation Coefficient}
\label{defn:correlation_coefficient}
  The \hlnoteb{correlation coefficient} of rvs $X$ and $Y$ is given by
  \marginnote{
  \noindent Note that this is the definition of the \hldefn{Pearson Correlation Coefficient}. There are other correlation coefficients but we will be using only Pearson, at it seems.
  }
  \begin{equation*}
    \rho(X, Y) = \frac{\Cov(X, Y)}{\sigma_X \sigma_Y}
  \end{equation*}
  where $\sigma_X = \sqrt{\Var(X)}$ and $\sigma_Y = \sqrt{\Var(Y)}$.
\end{defn}

\begin{propo}[Properties of the Correlation Coefficient]
\label{propo:properties_of_the_correlation_coefficient}
  Let $X$ and $Y$ be rvs, and $\rho(X, Y)$ the correlation coefficient of $X$ and $Y$. Then
  \begin{enumerate}
    \item $\abs{\rho(X, Y)} \leq 1$;
    \item ( \hlnoteb{perfect positive linear relationship}  ) \\
      $\rho(X, Y) = 1 \iff Y = aX + b$ for some $a > 0$;
    \item ( \hlnoteb{perfect inverse linear relationship} ) \\
      $\rho(X, Y) = -1 \iff Y = aX + b$ for some $a < 0$.
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item This is somewhat beyond the scope of what we can cover now but we shall use this result presented on \href{https://en.wikipedia.org/wiki/Covariance\#Relationship_to_inner_products}{Wikipedia}:
      \begin{equation*}
        \abs{ \Cov(X, Y) } \leq \sqrt{ \Var(X) \Var(Y) }.
      \end{equation*}
      Then given the formula of $\rho(X, Y)$, the proof is complete:
      \begin{equation*}
        \abs{ \rho(X, Y) } = \abs{ \frac{\Cov(X, Y)}{\sqrt{ \Var(X) \Var(Y) }} } \leq \frac{\sqrt{ \Var(X) \Var(Y) }}{\sqrt{ \Var(X) \Var(Y) }} = 1
      \end{equation*}
  \end{enumerate}
  Proving 2 and 3 is currently outside of my abilities. Refer to this \href{https://math.stackexchange.com/questions/478137/proving-y-ax-b-given-correlation-coefficient-rhox-y-1#478147}{Math SE} Q\&A for a hint on how to prove this statement.
\end{proof}

\begin{eg}[Example 3.20]
  Consider rvs $X$ and $Y$ with the joint pdf
  \begin{equation*}
    f(x, y) = \begin{cases}
      2 & 0 \leq x \leq y \leq 1 \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
  Calculate $\rho(X, Y)$.
\end{eg}

\begin{solution}
    The diagram to the right is an illustration of the region of support for $X$ and $Y$.
    \marginnote{
		\begin{tikzpicture}
        \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
        \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
				\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(0,2.5)--(2.5,2.5)--(0,0);
        \draw[line width=0pt](2.5,2.5)--(0,0) node[midway,left] [label=left:\textcolor{light}{$y = x$}] {};
        \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
        \draw[line width=0pt](2.5,2.5)--(0,2.5) node[left] {1};
		\end{tikzpicture}
    }
    We now calculate the following values:
    \begin{align*}
      E(XY)  &= \int_{0}^{1} \int_{x}^{1} 2xy \dif{y} \dif{x} = \int_{0}^{1} x - x^3 \dif{x} = \frac{1}{4} \\
      f_X(x) &= \int_{x}^{1} 2 \dif{y} = 2 - 2x \quad 0 \leq x \leq 1 \\
      f_Y(y) &= \int_{0}^{y} 2 \dif{x} = 2y \quad 0 \leq y \leq 1 \\
      E(X)   &= \int_{0}^{1} 2x - 2x^2 \dif{x} = \left( x^2 - \frac{2}{3}x^3 \right)\at{0}{1} = \frac{1}{3} \\
      E(X^2) &= \int_{0}^{1} 2x^2 - 2x^3 \dif{x} = \left( \frac{2}{3}x^3 - \frac{1}{2}x^4 \right)\at{0}{1} = \frac{1}{6} \\
      E(Y)   &= \int_{0}^{1} 2y^2 \dif{y} = \frac{2}{3} \\
      E(Y^2) &= \int_{0}^{1} 2y^3 \dif{y} = \frac{1}{2} \\
      \Var(X)&= E(X^2) - E(X)^2 = \frac{1}{6} - \left( \frac{1}{3} \right)^2 = \frac{1}{18} \\
      \Var(Y)&= E(Y^2) - E(Y)^2 = \frac{1}{2} - \left( \frac{2}{3} \right)^2 = \frac{1}{18}
    \end{align*}
    Therefore we have that
    \begin{gather*}
      \Cov(X, Y) = E(XY) - E(X)E(Y) = \frac{1}{4} - \frac{2}{9} = \frac{1}{36} \\
      \sqrt{\Var(X) \Var(Y)} = \frac{1}{18}
    \end{gather*}
    and so
    \begin{equation*}
      \rho(X, Y) = \frac{\Cov(X, Y)}{\sqrt{\Var(X) \Var(Y)}} = \frac{\frac{1}{36}}{\frac{1}{18}} = \frac{1}{2}
    \end{equation*}
\end{solution}

% subsection correlation (end)

\subsection{Conditional Expectation}%
\label{sub:conditional_expectation}
% subsection conditional_expectation

\begin{defn}[Conditional Expectation]\index{Conditional Expectation}
\label{defn:conditional_expectation}
  Let $G$ be a real-valued function. The \hlnoteb{conditional expectation} of $g(Y)$ given $X = x$, denoted as $g(Y) \, | \, \left( X = x \right)$ is given by
  \begin{equation*}
    E \left[ g(Y) \, | \, x \right] = \sum_{y} g(y) f_Y( y \, | \, x )
  \end{equation*}
  if $Y | (X = x)$ is a discrete rv and
  \begin{equation*}
    E \left[ g(Y) \, | \, x \right] = \int_{y} g(y) f_Y( y \, | \, x )
  \end{equation*}
  if $Y | (X = x)$ is a continuous rv. This definition only holds provided that the sum and the integral converges absolutely. The conditional expectation of $h(X)$ given $Y = y$, where $h$ is a real-valued function, is defined in a similar manner.

  We also call $E[ Y \, | \, X = x ]$ the \hldefn{conditional mean}, which may be denoted as $E( Y \, | \, x )$, and $\Var(Y \, | \, X = x)$ the \hldefn{conditional variance}, which may be denoted as $\Var(Y \, | \, x)$.
\end{defn}

\begin{note}
  Note that there is also the notation $E(Y \, | \, X)$, which is an rv, and hence different from $E(Y \, | \, x)$.
\end{note}

\begin{eg}[Example 3.21]
  Consider $f(x, y) = 8xy \mathbb{1}_{\{0 < x < y < 1\}}$. Calculate the conditional mean and the conditional variance of $X \, | \, \left( Y = \frac{1}{2} \right)$.
\end{eg}

\begin{solution}
  The diagram to the right illustrates the region of support for $X$ and $Y$.
  \marginnote{
  \begin{tikzpicture}
      \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
      \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
  		\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25](0,0)--(0,2.5)--(2.5,2.5)--(0,0);
      \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway,left] [label=left:\textcolor{light}{$y = x$}] {};
      \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
      \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
  \end{tikzpicture}
  }
  To derive the conditional distribution, we first need $f_Y(y)$.
  \begin{equation*}
    f_Y(y) = \int_{0}^{y} 8xy \dif{x} = 4x^2y \at{0}{y} = 4y^3
  \end{equation*}
  Thus
  \begin{equation*}
    f_X \left(X \, | \, Y = \frac{1}{2}\right) = \frac{f\left(x, \frac{1}{2}\right)}{f_Y\left(\frac{1}{2}\right)} = \frac{4x}{\frac{1}{2}} = 8x \quad 0 < x < \frac{1}{2}
  \end{equation*}
  Therefore the conditional mean is
  \begin{equation*}
    E\left[ X  \, | \, \frac{1}{2} \right] = \int_{0}^{\frac{1}{2}} 8x^2 \dif{x} = \frac{8}{3} \cdot \frac{1}{2^3} = \frac{1}{3},
  \end{equation*}
  the second conditional moment is
  \begin{equation*}
    E \left[ X^2 \, | \, \frac{1}{2} \right] = \int_{0}^{\frac{1}{2}} 8x^3 \dif{x} = 2 \cdot \frac{1}{2^4} = \frac{1}{8},
  \end{equation*}
  and so the conditional variance is
  \begin{equation*}
    \Var \left( X^2 \, | \, \frac{1}{2} \right) = E\left[ X^2 \, | \, \frac{1}{2} \right] + E\left[ X \, | \, \frac{1}{2} \right]^2 = \frac{1}{8} - \frac{1}{9} = \frac{1}{72}
  \end{equation*}
\end{solution}

% subsection conditional_expectation (end)

% section joint_distributions_continued_3 (end)

% chapter lecture_9_may_31st_2018 (end)

\chapter{Lecture 10 Jun 05th 2018}%
\label{chp:lecture_10_jun_05th_2018}
% chapter lecture_10_jun_05th_2018

\section{Joint Distribution (Continued 4)}%
\label{sec:joint_distribution_continued_4}
% section joint_distribution_continued_4

\subsection{Conditional Expectation (Continued)}%
\label{sub:conditional_expectation_continued}
% subsection conditional_expectation_continued

\begin{eg}[Example 3.22]
  Given the joint distribution below, calculate $\Var(X \, | \, Y = 1)$ and compare it to $\Var(X)$.
  \[
  \begin{tabular}{c | c | c | c || c}
            & x = -2 & x = 0 & x = 2 & P(Y = y) \\
  \hline
  y = 0     & 0.05   & 0.1   & 0.15  & 0.3 \\
  \hline
  y = 1     & 0.07   & 0.11  & 0.2   & 0.38 \\
  \hline
  y = 2     & 0.02   & 0.25  & 0.05  & 0.32 \\
  \hline
  \hline
  P(X = x)  & 0.14   & 0.46  & 0.4   &
  \end{tabular}
  \]
\end{eg}

\begin{solution}
  Note that
  \begin{align*}
    \Var(X) &= E(X^2) - E(X)^2 \\
      &= 4 \cdot 0.14 + 4 \cdot 0.4 - ( -2 \cdot 0.14 + 2 \cdot 0.4 )^2 = 1.8896
  \end{align*}
  To get $\Var(X \, | \, Y = 1)$, we first need
  \begin{equation*}
    f(X \, | \, Y = 1) = \frac{P(X = x, Y = 1)}{P(Y = 1)} = \begin{cases}
      \frac{0.07}{0.38} = 0.1842 & x = -2 \\
      \frac{0.11}{0.38} = 0.2895 & x = 0 \\
      \frac{0.2}{0.38} = 0.5263 & x = 2
    \end{cases}
  \end{equation*}
  Thus
  \begin{align*}
    \Var(X \, | \, Y = 1) &= E[ X^2 | Y = 1 ] - E[ X \, | \, Y = 1 ]^2 \\
      &= \frac{14}{19} + \frac{40}{19} - ( \frac{13}{19} )^2 = \frac{857}{361} = 2.3740
  \end{align*}
\end{solution}

\begin{propo}[Independence on Conditional Expectation]
\label{propo:independence_on_conditional_expectation}
  If $X$ and $Y$ are independent rvs then $E[ g(Y) \, | \, x ] = E [ g(Y) ]$ and $E[ h(X) \, | \, y ] = E[ h(X) ]$.
\end{propo}

\begin{proof}
  We shall prove one of the above for the other will follow a similar argument. Also, we shall prove the continuous case and leave the discrete case as an exercise.\marginnote{ \begin{ex}
    Prove the discrete case for \cref{propo:independence_on_conditional_expectation}.
  \end{ex} }

  Observe that
  \begin{align*}
    E [ g(Y) \, | \, X = x ] &= \int_{y} g(y) \frac{f(x, y)}{f_X (x)} \dif{y} \\
      &= \int_{y} g(y) \frac{f_X(x) f_Y(y)}{f_X(x)} \dif{y} \quad \because \text{ independence } \\
      &= \int_{y} g(y) f_Y(y) \dif{y} = E[g(Y)]
  \end{align*}\qed
\end{proof}

\begin{thm}[Law of Total Expectation]
\index{Law of Total Expectation}
\label{thm:law_of_total_expectation}
  Suppose $X$ and $Y$ are rvs, then
  \begin{equation*}
    E\left( E[ g(Y) \, | \, X ] \right) = E[ g(Y) ]
  \end{equation*}
  If $g$ is the identity function, we have $E( E[Y \, | \, X] ) = E(Y)$.
\end{thm}

\begin{proof}
  We shall prove for the discrete case and leave the continuous case as an exercise.\marginnote{ \begin{ex}
    Prove the continuous case for \cref{thm:law_of_total_expectation}.
  \end{ex} }
  Observe that
  \begin{align*}
    E[ g(Y) \, | \, X ] &= \sum_{y} \left[ g(y) \cdot P(Y = y \, | \, X) \right] \\
    E[E[ g(Y) \, | \, X ]] &= \sum_{x} \left[ \sum_{y} \left[ g(y) \cdot P(Y = y \, | \, X) \right] \right] P(X = x) \\
      &= \sum_{x} \sum_{y} g(y) \cdot P(X = x, Y = y) \\
      &= \sum_{y} g(y) \sum_{x} P(X = x, Y = y) \\
      &= \sum_{y} \left[ g(y) \cdot P(Y = y) \right] = E[ g(Y) ]
  \end{align*}\qed
\end{proof}

\begin{eg}[Example 3.23 - A Classical Example]
\marginnote{This is a very classical example to illustrate the power of the \hlnoteb{Law of Total Expectation}.}
A man is lost in a mine, and 3 paths are in front of him. If he takes path 1, after 3 hours, he will be back at his current place. If he takes path 2, the time to get out of the mine (in hours) follows an $\Exp(1)$ distriburion. If he takes the 3rd path, he will be back to his current place after 2 hours. Suppose that the man cannot recognize which path he took every time he comes back to the original spot (after going through either path 1 or 3), and so he randomly chooses a path every time he comes back to this original spot. What is the expected time that he will take to get out of the mine?
\end{eg}

$ $ \\

\begin{solution}
  Let $X$ an rv that represents the path number, i.e. $X = 1, 2$ or $3$, and let $Y$ represent the total time that the man takes to exit the mine. We are given that
  \begin{equation*}
    P(X = 1) = P(X = 2) = P(X = 3) = \frac{1}{3}.
  \end{equation*}
  We are also given that
  \begin{align*}
    E[ Y \, | \, X = 1 ] &= 3 + E[Y] \\
    E[ Y \, | \, X = 2 ] &= 1 \quad \because Y \, | \, (X = 2) \sim \Exp(1) \\
    E[ Y \, | \, X = 3 ] &= 2 + E[Y].
  \end{align*}
  Therefore, to get the expected time, by \cref{thm:law_of_total_expectation},
  \begin{align*}
    E[Y] &= E[ E[ Y \, | \, X ] ] \\
      &= \frac{1}{3} \cdot E[ Y \, | \, X = 1 ] + \frac{1}{3} \cdot E[ Y \, | \, X = 2 ] + \frac{1}{3} \cdot E[ Y \, | \, X = 3 ] \\
      &= \frac{1}{3} \cdot ( 3 + E[Y] ) + \frac{1}{3} \cdot 1 + \frac{1}{3} (2 + E[Y]) \\
      &= 2 + \frac{2}{3} E[Y]
  \end{align*}
  and hence
  \begin{equation*}
    E[Y] = 6
  \end{equation*}
\end{solution}

\begin{thm}[Law of Total Variance]
\index{Law of Total Variance}
\label{thm:law_of_total_variance}
  Suppose $X$ and $Y$ are rvs. Then
  \begin{equation*}
    \Var(Y) = E[ \Var(Y \, | \, X) ] + \Var[ E(Y \, | \, X) ]
  \end{equation*}
\end{thm}

\begin{proof}
  Note that
  \begin{align*}
    \Var(Y | X) &= E( Y^2 | X ) - E( Y | X )^2 \\
    E[ \Var(Y | X) ] &= E[ E( Y^2 | X ) - E( Y|X )^2 ] \\
      &= E[Y^2] - E [ E(Y|X)^2 ] \\
      &= E[Y^2] - \left[ \Var( E(Y|X) ) + E \left( E(Y|X) \right)^2 \right] \\
      &= \Var(Y) - \Var( E(Y | X) )
  \end{align*}
  By rearranging the above, we get
  \begin{equation*}
    \Var(Y) = E[ \Var(Y|X) ] + \Var[ E(Y|X) ]
  \end{equation*}\qed
\end{proof}

\begin{eg}[Example 3.24 (Course Note 3.7.11)]
  Suppose $P \sim \Unif(0, 0.1)$ and $Y \, | \, P = p \sim \Bin(10, p)$. Find $E(Y)$ and $\Var(Y)$.
\end{eg}

\begin{solution}
  \begin{align*}
    E[Y] &= E[ E[Y|P] ] = E[ 10P ] = 10 E[P] \\
      &= 10 \cdot \int_{0}^{0.1} \frac{p}{0.1} \dif{p} = \frac{10}{0.2} p^2 \at{0}{0.1} = 0.05 \\
    \text{Note: }& E[P^2] = \int_{0}^{0.1} \frac{p^2}{0.1} \dif{p} = \frac{p^3}{0.3}\at{0}{0.1} = \frac{0.001}{0.3} = \frac{1}{300} \\
    \Var(Y) &= E[ \Var(V|P) ] + \Var[ E[Y|P] ] \\
      &= E[ 10P(1 - P) ] + \Var[ 10P ] \\
      &= 10E[P] - 10E[P^2] + 100 \Var(P) \\
      &= 0.05 - \frac{1}{30} + 100 \left[ E[P^2] - E[P]^2 \right] \\
      &= \frac{1}{60} + 100 \left[ \frac{1}{300} - 0.05^2 \right] = \frac{1}{60} + 100 \left[ \frac{1}{300} - \frac{1}{400} \right] \\
      &= \frac{1}{60} + \frac{1}{12} = \frac{1}{10}
  \end{align*}
\end{solution}

% subsection conditional_expectation_continued (end)

\subsection{Joint Moment Generating Functions}%
\label{sub:joint_moment_generating_functions}
% subsection joint_moment_generating_functions

\begin{defn}[Joint Moment Generating Functions]\index{Joint Moment Generating Functions}
\label{defn:joint_moment_generating_functions}
  The \hlnoteb{joint moment generating function} of two rvs $X$ and $Y$ is defined as
  \begin{equation*}
    M( t_1, t_2 ) = E\left( e^{t_1 X + t_2 Y} \right)
  \end{equation*}
  if the expectation exists for all $t_1 \in (-h_1, h_1)$ and $t_2 \in (-h_2, h_2)$ for some $h_1, h_2 > 0$.

  More generally, if $X_1, X_2, ..., X_n$ are rvs, then
  \begin{equation*}
    M( t_1, t_2, ..., t_n ) = E \left[ \exp\left( \sum_{i=1}^{n} t_i X_i \right) \right]
  \end{equation*}
  is called the \hlnoteb{joint mgf} of $X_1, X_2, ..., X_n$ is the expectation exists for all $t_i \in (-h_i, h_i)$ for some $h_i > 0$, where $i = 1, ..., n$.
\end{defn}

\begin{defn}[Joint Moments and Marginal MGF]\index{Joint Moments}\index{Marginal MGF}
\label{defn:joint_moments_and_marginal_mgf}
  Given the joint mgf $M(t_1, t_2)$, we can calculate the joint moments. In particular,
  \begin{equation*}
    E\left( X^j Y^k \right) = \frac{\partial^{j + k}}{\partial t_i^j \partial t_2^k} M(t_1, t_2) \at{( t_1, t_2 ) = (0, 0)}{}
  \end{equation*}
  If $M(t_1, t_2)$ exists for all $t_1 \in (-h_1, h_1)$ and $t_2 \in (-h_2, h_2)$ for some $h_1, h_2 > 0$, then the mdf of $X$ is given by
  \begin{equation*}
    M_X(t) = E \left( e^{tX} \right) = M(t, 0) \quad t \in (-h_1, h_1)
  \end{equation*}
  and the mgf of $Y$ is given by
  \begin{equation*}
    M_Y(t) = E \left( e^{tY} \right) = M(0, t) \quad t \in (-h_2, h_2).
  \end{equation*}
\end{defn}

\begin{eg}[Example 3.25]\label{eg:3_25}
  Given the joint distribution below, calculate the joint mgf $M(t_1, t_2)$, the first joint moment, $E[XY]$, from the joint mgf, and the marginal mgf of $X$ and that of $Y$.
  \[
  \begin{tabular}{c | c | c}
          & x = -1 & x = 1 \\
    \hline
    y = 1 & 0.5    & 0.3 \\
    \hline
    y = 2 & 0.1    & 0.1
  \end{tabular}
  \]
\end{eg}

\begin{solution}
  Since all probabilities are provided,
  \begin{align*}
    M(t_1, t_2) &= E\left( e^{t_1 X + t_2 Y} \right) = \sum_{x} \sum_{y} e^{t_1 x + t_2 y} P(X = x, Y = y) \\
      &= 0.5 e^{-t_1 + t_2} + 0.3 e^{t_1 + t_2} + 0.1 e^{-t_1 + 2t_2} + 0.1 e^{t_1 + 2t_2} \\
    E(XY) &= \frac{\partial^2}{\partial t_1 \partial t_2} M(t_1, t_2) \at{t_1 = 0, \, t_2 = 0}{} \\
      &= \frac{\partial}{\partial t_1} \left[ 0.5 e^{-t_1 + t_2} + 0.3 e^{t_1 + t_2} + 0.2 e^{-t_1 + 2t_2} + 0.2e^{t_1 + 2t_2} \right] \at{t_1 = 0, t_2 = 0}{} \\
      &= -0.5 e^{-t_1 + t_2} + 0.3e^{t_1 + t_2} - 0.2 e^{-t_1 + 2t_2} + 0.2e^{t_1 + 2t_2} \at{t_1 = 0, t_2 = 0}{} \\
      &= -0.2 \\
    M_X(t_1) &= M(t_1, 0) = 0.5e^{-t_1} + 0.3e^{t_1} + 0.1 e^{-t_1} + 0.1^{t_1} \\
      &= 0.6e^{-t_1} + 0.4 e^{t_1} \\
    M_Y(t_2) &= M(0, t_2) = 0.5e^{t_2} + 0.3e^{t_2} + 0.1 e^{2t_2} + 0.1 e^{2t_2} \\
      &= 0.8 e^{t_2} + 0.2 e^{2t_2}
  \end{align*}
\end{solution}

\begin{propo}[Independence on Joint MGF]
\label{propo:independence_on_joint_mgf}
  Suppose $X$ and $Y$ are rvs with joint mgf $M(t_1, t_2)$ which exists $\forall t_1 \in (-h_1, h_1), \, t_2 \in (-h_2, h_2)$, for some $h_1, h_2 > 0$. Then $X$ and $Y$ are independent rvs iff
  \begin{equation*}
    \forall t_1 \in (-h_1, h_1), \, t_2 \in (-h_2, h_2) \quad M(t_1, t_2) = M_X(t_1) M_Y(t_2)
  \end{equation*}
  where $M_X(t_1) = M(t_1, 0)$ and $M_Y(t_2) = M(0, t_2)$.
\end{propo}

\begin{proof}
  \hlwarn{to be proven later}
\end{proof}

\begin{eg}[Example 3.26 (Course Note 3.8.5)]
  Suppose $X$ and $Y$ are continous rvs with joint pdf
  \begin{equation*}
    f(x, y) = e^{-y} \quad 0 < x < y < \infty
  \end{equation*}
  Find the joint mdf of $X$ and $Y$. Are $X$ and $Y$ independent rvs? What is the marginal mgf of $X$ and $Y$?
\end{eg}

\begin{solution}
  \begin{align*}
    M(t_1, t_2) &= E[ e^{t_1 X + t_2 Y} ] = \int_{0}^{\infty} \int_{0}^{y} e^{t_1 x + t_2 y} e^{-y} \dif{x} \dif{y} \\
      &= \int_{0}^{\infty} \frac{1}{t_1}e^{t_1 x + t_2 y - y} \at{0}{y} \dif{y} \\
      &= \int_{0}^{\infty} \frac{1}{t_1} \left[e^{y(t_2 - 1)} - e^{y( t_1 + t_2 - 1 )}\right] \dif{y} \\
      &= \frac{1}{t_1} \left[ \frac{1}{t_2 - 1}e^{y(t_2 - 1)} - \frac{1}{t_1 + t_2 - 1)} e^{y(t_1 + t_2 - 1)} \right] \at{0}{\infty} \\
      &= \frac{1}{t_1} \left[ \frac{t_1}{(t_2 - 1) (t_1 + t_2 - 1)} \right] \\
      &= \frac{1}{(t_2 - 1)(t_1 + t_2 - 1)} \quad t_2 < 1 \, \land \, t_1 + t_2 < 1 \\
    M_X(t_1) &= M(t_1, 0) = \frac{1}{t_1 - 1} \quad t_1 < 1 \\
    M_Y(t_2) &= M(0, t_2) = \frac{1}{(t_2 - 1)^2} \quad t_2 < 1
  \end{align*}
  Observe that
  \begin{equation*}
    M_X(t_1) M_Y(t_2) = \frac{1}{(t_1 - 1)(t_2 - 1)^2} \neq M(t_1, t_2)
  \end{equation*}
  and so by \cref{propo:independence_on_joint_mgf}, $X$ and $Y$ are not independent.
\end{solution}

\begin{eg}[Example 3.27]
  Investigate the independence of $X$ and $Y$ in \cref{eg:3_25} using the mgf method.
\end{eg}

\begin{solution}
  We had that
  \begin{align*}
    M_X(t_1) &= 0.6^{-t_1} + 0.4e^{t_1} \quad t_1 \in \mathbb{R} \\
    M_Y(t_2) &= 0.8e^{t_2} + 0.2 e^{2t_2} \quad t_2 \in \mathbb{R}.
  \end{align*}
  Since
  \begin{equation*}
    M_X \left( \frac{1}{2} \right) M_Y\left( \frac{1}{2} \right) \neq M\left( \frac{1}{2}, \frac{1}{2} \right)
  \end{equation*}
  we have that $X$ and $Y$ are not independent.
\end{solution}

% subsection joint_moment_generating_functions (end)

% chapter lecture_10_jun_05th_2018 (end)

\chapter{Lecture 11 Jun 07th 2018}%
\label{chp:lecture_11_jun_07th_2018}
% chapter lecture_11_jun_07th_2018

\subsection{Working with Multivariate Cases}%
\label{sub:working_with_multivariate_cases}
% subsection working_with_multivariate_cases

Almost everything that has been introduced above can be extended to cases where we have more than just $2$ rvs. For example:

\begin{defn}[k-variate CDF]\index{k-variate CDF}
\label{defn:k_variate_cdf}
  The \hlnoteb{k-variate CDF}, $k > 2$, rvs $X_1, ..., X_k$ is defined as
  \begin{equation*}
    F(x_1, ..., x_k) = P(X_1 \leq x_1, X_2 \leq x_2, ..., X_k \leq x_k).
  \end{equation*}
  In the continuous case, we may write
  \begin{equation*}
    f(x_1, ..., x_k) = \frac{\partial^k}{\partial x_1 \hdots \partial x_k} F(x_1, ..., x_k).
  \end{equation*}
\end{defn}

\begin{defn}[k-variate Support Set]\index{k-variate Support Set}
\label{defn:k_variate_support_set}
  The support set of the distribution for $X_1, X_2, ..., X_k$ is
  \begin{equation*}
    \{(x_1, ..., x_k) : f(x_1, ..., x_k) > 0 \}
  \end{equation*}
\end{defn}

We also have the following:

\begin{propo}[Law of Total Probability - Multivariate]
\label{propo:law_of_total_probability_multivariate}
  If $X_1, ..., X_k$ are continuous rvs, then
  \begin{equation*}
    \int_{x_1} \hdots \int_{x_k} f(x_1, ..., x_k) \dif{x_1} \hdots \dif{x_k} = 1.
  \end{equation*}
  Should they by discrete, then
  \begin{equation*}
    \sum_{x_1} \hdots \sum_{x_k} f(x_1, ..., x_k) = 1
  \end{equation*}
\end{propo}

\begin{defn}[k-Variate Marginal Distribution]
\label{defn:k_variate_marginal_distribution}
  To get the marginal distribution of a subset of $m$ variables from $X_1, ..., X_k$ ($1 \leq m \leq k$), we will sum or integrate over the other ones if they are discrete or continuous, respectively. For example,
  \begin{equation*}
    f(x_1, x_2, x_3) = \int_{x_4} \hdots \int_{x_k} f(x_1, ..., x_k) \dif{x_4} \hdots \dif{x_k}
  \end{equation*}
\end{defn}

\begin{defn}[k-Variate Joint MGF]\index{k-Variate Joint MGF}
\label{defn:k_variate_joint_mgf}
  The joint mgf of $X_1, ..., X_k$ is defined as
  \begin{equation*}
    M(t_1, t_2, ..., t_k) = E \left( e^{t_1 X_1 + \hdots + t_k X_k} \right)
  \end{equation*}
\end{defn}

\begin{propo}[Independence for Multivariate Cases]
\label{propo:independence_for_multivariate_cases}
  If $X_1, ..., X_k$ are independent, then
  \begin{gather*}
    f(x_1, ..., x_k) = \prod_{i=1}^{k} f_{X_i}(x_i) \qquad F(x_1, ..., x_k) = \prod_{i=1}^{k} F_{X_i}(x_i) \\
    M(t_1, ..., t_k) = \prod_{i=1}^{k} M_{X_i}(t_i)
  \end{gather*}
\end{propo}

\newthought{There are} many different examples of multivariate distributions. We shall discuss two:
\begin{itemize}
  \item Multinomial Distribution
  \item Multivariate Normal Distribution
\end{itemize}

The \hlnoteb{multinomial distribution} is an extension of the binomial distribution to cases where there are more categories than two results. For a multinomial distribution, we have that
\begin{itemize}
  \item the experiment involves $n$ trials, each with $k$ categories
  \item the outcome of trials are independent of each other
  \item the probability of each category, $p_i$, remains the same across $n$ trials
  \item $X = (X_1, ..., X_k) \sim \Mult(n, p_1, ..., p_k)$ counts the number of elements in each category among the $n$ trials.
\end{itemize}

\begin{defn}[Mutlinomial Distribution]\index{Mutlinomial Distribution}
\label{defn:mutlinomial_distribution}
  Suppose $X_1, ..., X_k$ are discrete rvs with joint pf
  \begin{equation*}
    f(x_1, ..., x_k) = \frac{n!}{x_1! x_2! \hdots x_k! x_{k + 1}!} p_1^{x_1} p_2^{x_2} \hdots P_k^{x_k} p_{k + 1}^{x_{k + 1}}
  \end{equation*}
  where $x_i = 0, ..., n,$, $x_{k + 1} = n - \sum_{i=1}^{k} x_i$, $0 < p_i < 1$, $p_{k + 1} = 1 - \sum_{i=1}^{k} p_i$, for $i = 1, ..., k + 1$.

  Under these conditions, $(X_1, ..., X_k)$ is said to have a multinomial distribution, and we write $(X_1, ..., X_k) \sim \Mult(n, p_1, ..., p_k)$.
\end{defn}

\begin{note}
  Observe that $\Bin(n, p) = \Mult(n, p, p)$.
\end{note}

\begin{propo}[Properties of Multinomial Distribution]
\label{propo:properties_of_multinomial_distribution}
  Suppose $(X_1, ..., X_k) \sim \Mult(n, p_1, ..., p_k)$, then
  \begin{enumerate}
    \item $\forall (t_1, ..., t_k) \in \mathbb{R}^k$, the random vector $(X_1, ..., X_k)$ has joint mgf
      \begin{equation*}
        M(t_1, ..., t_k) = E \left( e^{t_1 X_1 + \hdots t_k X_k} \right) = (p_1 e^{t_1} + \hdots + p_k e^{t_k} + p_{k + 1})^n
      \end{equation*}

    \item Any subset of $X_1, ..., X_{k + 1}$ also has a multinomial distribution. In particular, $X_i \sim \Bin(n, p_i), \, i = 1, ..., k + 1$.
    \item If $T = X_i + X_j$, for $i \neq j$, then $T \sim \Bin(n, p_i + p_j)$
    \item $\Cov(X_i, X_j) = -n p_i p_j$, for $i \neq j$
    \item The conditional distribution of any subset of $(X_1, ..., X_{k + 1})$ given the rest of the coordinates is a multinomial distribution. In particular, the conditional pf of $X_i$ given $X_j = x_j$, $i \neq j$, is
      \begin{equation*}
        X_i \, | \, X_j = x_j \sim \Bin\left( n - x_j, \frac{p_i}{1 - p_j} \right)
      \end{equation*}

    \item The conditional distribution of $X_i$ given $T = X_i + X_j = t$, for $i \neq j$, is
      \begin{equation*}
        X_i \, | \, X_i + X_j = t \sim \Bin \left( t, \frac{p_i}{p_i + p_j} \right)
      \end{equation*}
  \end{enumerate}
\end{propo}

\newthought{We shall} look at the bivariate normal distribution so that it is easier to be explained. The same idea can be extended to a multivariate Normal distribution.

\begin{defn}[Bivariate Normal Distribution]\index{Bivariate Normal Distribution}
\label{defn:bivariate_normal_distribution}
    Let $X_1$ and $X_2$ be rvs with joint pdf
    \begin{align*}
      f(x_1, x_2)
        &= \frac{1}{2 \pi \abs{\Sigma}^{\frac{1}{2}}} \exp \left\{ - \frac{1}{2} ( x - \mu )^T \Sigma^{-1} (x - \mu) \right\} \\
        &= \frac{1}{2 \pi \sigma_1 \sigma_2 \sqrt{ 1 - \rho^2 }} \exp \left\{ - \frac{1}{2( 1 - \rho^2 )} \left[ \frac{(x_1 - \mu_1)^2}{\sigma_1^2} \right. \right. \\
        &\qquad \left. \left. + \frac{(x_2 - \mu_2)^2}{\sigma_2^2} - \frac{2 \rho (x_1 - \mu_1)(x_2 - \mu_2)}{\sigma_1 \sigma_2} \right] \right\}
    \end{align*}
    where $(x_1, x_2) \in \mathbb{R}^2$ and
    \begin{equation*}
      x = \begin{pmatrix} x_1 \\ x_2 \end{pmatrix}, \mu = \begin{pmatrix} \mu_1 \\ \mu_2 \end{pmatrix}, \Sigma = \begin{bmatrix}
        \sigma_1^2 & \rho \sigma_1 \sigma_2 \\
        \rho \sigma_1 \sigma_2 & \sigma_2^2
      \end{bmatrix}
    \end{equation*}
    where $\Sigma$ is a nonsingular matrix. Then $X = (X_1, X_2)^T$ is said to have a bivariate normal distribution, and we write $X \sim \BVN(\mu, \Sigma)$.
\end{defn}

\begin{propo}[Properties of Bivariate Normal Distribution]
\label{propo:properties_of_bivariate_normal_distribution}
  Suppose $X \sim \BVN(\mu, \Sigma)$, where
  \begin{equation*}
    \mu = \begin{pmatrix}
      \mu_1 \\ \mu_2
    \end{pmatrix},
    \Sigma = \begin{bmatrix}
      \sigma_1^2 & \rho \sigma_1 \sigma_2 \\
      \rho \sigma_1 \sigma_2 & \sigma_2^2
    \end{bmatrix}
  \end{equation*}
  \begin{enumerate}
    \item $X$ has a joint mgf of
      \begin{equation*}
        M(t_1, t_2) = E[ \exp\left( t^T X \right) ] = E\left( e^{t_1 X_1 + t_2 X_2} \right) = \exp\left( \mu^T t + \frac{1}{2} t^T \Sigma t \right)
      \end{equation*}
      $\forall (t_1, t_2) \in \mathbb{R}^2$.

    \item $X_1 \sim N(\mu_1, \sigma_1^2)$ and $X_2 \sim N(\mu_2, \sigma_2^2)$
    \item $\Cov(X_1, X_2) = \rho \sigma_1 \sigma_2$ and $\Corr(X_1, X_2) = \rho$ where $\abs{rho} \leq 1$
    \item $X_1$ and $X_2$ are independent rvs iff $\rho = 0$
    \item $c = (c_1, c_2)^T$ is a nonzero vector of constants $\implies$
      \begin{equation*}
        c^T X = \sum_{i=1}^{2} c_i X_i \sim N \left( c^T \mu, c^T \Sigma c \right).
      \end{equation*}
    \item If $A$ is a $2 \times 2$ nonsingular matrix and $b$ is a $2 \times 1$ vector, then $Y = AX + b \sim \BVN\left( A\mu + b, A \Sigma A^T \right)$.
    \item
      \begin{gather*}
        X_2 \, | \, X_1 = x_1 \sim N \left( \mu_2 + \rho \frac{\sigma_2}{\sigma_1}( x_1 - \mu_1 ), \sigma_2^2 (1 - \rho^2) \right) \\
        X_1 \, | \, X_2 = x_2 \sim N \left( \mu_1 + \rho \frac{\sigma_1}{\sigma_2}( x_2 - \mu_2 ), \sigma_1^2 (1 - \rho^2) \right)
      \end{gather*}

    \item $(X - \mu)^T \Sigma^{-1} (X - \mu) \sim \chi^2 (2)$
  \end{enumerate}
\end{propo}

\begin{defn}[Random Sample (IID)]\index{Random Sample}
\label{defn:random_sample_iid}
  Rvs $X_1, ..., X_2$ are said to form a \hlnoteb{simple random sample} or are said to be \hlnoteb{independent and identically distributed} (IID) if $X_1, ..., X_n$ are independent, and $f_{X_i} = f_{X_j}$, $\forall i \neq j$.
\end{defn}

\begin{eg}[Example 3.28]
  Let $X_1, ..., X_{10}$ be a random sample of standard normal distribution. Let $Y_1$ denote the number of these variables that are between $-1$ and $1$, let $Y_2$ denote the number that have absolute value between $1$ and $2$, and let $Y_3$ denote the number that have absolute value larger than $2$. Calculate:
  \begin{enumerate}
    \item $P(Y_1 \leq 2)$
    \item $E[ Y_2 \, | \, Y_1 = 5 ]$
  \end{enumerate}
\end{eg}

% subsection working_with_multivariate_cases (end)

% section joint_distribution_continued_4 (end)

% chapter lecture_11_jun_07th_2018 (end)

\chapter{Lecture 12 Jun 12th 2018}%
\label{chp:lecture_12_jun_12th_2018}
% chapter lecture_12_jun_12th_2018

\section{Functions of Random Variables}%
\label{sec:functions_of_random_variables}
% section functions_of_random_variables

\subsection{Transformation of Two or More Random Variables}%
\label{sub:transformation_of_two_or_more_random_variables}
% subsection transformation_of_two_or_more_random_variables

In earlier lectures we discussed about basic transformations from one random variable to another, for example, from a continuous rv $X$ to $Y = g(X)$. In particular, we methods were presented:

\begin{itemize}
  \item \newthought{The direct method}, i.e. $P(Y \leq y) = P( g(X) \leq y )$, and taking the derivative of $P(Y \leq y)$ with respect to $y$.
  \item \newthought{Using the mgf of $Y$}, and then translate it as the mgf of $X$.
\end{itemize}

\begin{note}[Recall]
  In \cref{sub:a_formula_for_the_continuous_case}, we used the following idea to obtain the result that we desire: for rvs $X$ and $Y = g(X)$ where $g$ is some continuous and injective function
  \begin{align*}
    P(Y \leq y) &= P( g(X) \leq y ) = P(X \leq g^{-1}(y)) = F_X( g^{-1}(y) ) \\
    f_Y(y) &= \frac{d}{dy} F_X( g^{-1}(y) ) = ( g^{-1}(y) )' f_X( g^{-1}(y) )
  \end{align*}
\end{note}

In this chapter, we will now study the case where we have more than one rv involved. In particular, let $X$ and $Y$ be two continuous rvs with joint pdf $f(x, y)$. Our questions are:
\begin{enumerate}
  \item What is the distribution of $U = h_1( X, Y )$?
  \item What is the joint distribution of $U = h_1(X, Y)$ and $V = h_2 (X, Y)$?
\end{enumerate}

To answer the first question, we can actually still employ the direct method:

\begin{eg}[Example 4.1 (Course Notes 4.1.1)]\label{eg:4_1}
  Suppose $X$ and $Y$ are continuous rvs with joint pdf
  \begin{equation*}
    f(x, y) = 3y \mathbb{1}_{0 \leq x \leq y \leq 1}
  \end{equation*}
  Find the pdf of $T = XY$.
\end{eg}

\begin{solution}
  First, note that\sidenote{I wonder if the last step is actually valid. The support of $X$ definitely includes $0$, so the division would not make sense with $\frac{t}{X}$. We can, however, still make sense of the event in the 2nd term, which we would have $P(0 \leq t)$. Should we be concerned about $X = 0$, or can we neglect that single point given that $X$ is a continuous rv? \\
  $ $ \\}
  \begin{equation*}
    P(T \leq t) = P( XY \leq t ) = P\left(Y \leq \frac{t}{X}\right)
  \end{equation*}
  The diagram to the right shows us the support of the joint probability.\marginnote{
  \begin{tikzpicture}
      \draw[->] (-0.5, 0) -- (4, 0) node[right] {$x$};
      \draw[->] (0, -0.5) -- (0, 4) node[above] {$y$};
  		\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25] (0,0) -- (0,2.5) -- (2.5,2.5) -- (0,0);
      \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway] [label=left:\textcolor{light}{$y = x$}] {};
      \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
      \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
      \draw[domain=0.25:4,dashed] plot ({\x},{1/\x}) node[above] {$y = \frac{t}{x} \quad 0 < t < 1$};
      \draw[domain=1.75:4,dashed] plot ({\x},{7/\x}) node[above] {$y = \frac{t}{x} \quad t > 1$};
      \node at (0.25, 0.75) {A};
      \node at (1, 2) {B};
  \end{tikzpicture}
  }
  We observe that if $t \leq 0$, then $P( T \leq t ) = 0$, and if $t \geq 1$, then $P( T \leq t ) = 1$. Now if $0 < t < 1$, the region that we are looking for is the shaded region with the label A, and so we consider
  \begin{align*}
    P( T \leq t ) &= 1 - P(B) = 1 - \underset{B}{\int \int} f(x, y) \dif{x} \dif{y} \\
                  &\overset{(1)}{=} 1 - \int_{\sqrt{t}}^{1} \int_{\frac{t}{y}}^{\sqrt{t}} f(x, y) \dif{x} \dif{y} - \int_{\sqrt{t}}^{1} \int_{\sqrt{t}}^{y} f(x, y) \dif{x} \dif{y} \\
                  &\overset{(2)}{=} 1 - \int_{\sqrt{t}}^{1} \int_{\frac{t}{y}}^{y} f(x, y) \dif{x} \dif{y} \\
                  &= 1 - \int_{\sqrt{t}}^{1} \int_{\frac{t}{y}}^{y} 3y \dif{x} \dif{y} \\
                  &= 1 - \int_{\sqrt{t}}^{1} 3y \left( y - \frac{t}{y} \right) \dif{y} \quad \because \text{\hlnotea{FTC}} \\
                  &= 1 - \frac{\sqrt{t}}{1} \left( 3y^2 - 3t \right) \dif{y} \\
                  &= 1 - \left[ y^3 - 3ty \right]\at{\sqrt{t}}{1} = 1 - ( 1 - 3t - \sqrt{t}^3 + 3t \sqrt{t} ) \\
                  &= 3t - 2t\sqrt{t} \text{ for } 0 < t < 1
  \end{align*}
  where for step $(1)$, we broke B into two parts, in particular at $x = \sqrt{t}$ where $y = x$ and $y = \frac{t}{x}$ coincide, and step $(2)$ is true by linearity of integration. With that, i.e. with the CDF of $T$, we can then obtain
  \begin{equation*}
    f_T(t) = \begin{cases}
      3 - 3\sqrt{t} & 0 < t < 1 \\
      0             & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{solution}

\begin{eg}[Example 4.2 (Course Note 4.1.2)]
  Using the info in \cref{eg:4_1}, find the pdf of $T = \frac{X}{Y}$.
\end{eg}

\begin{solution}
  The diagram to the right shows the support of $X, Y$ and the function $t = \frac{x}{y}$.\marginnote{
  \begin{tikzpicture}
      \draw[->] (-0.5, 0) -- (3, 0) node[right] {$x$};
      \draw[->] (0, -0.5) -- (0, 3) node[above] {$y$};
  		\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25] (0,0) -- (0,2.5) -- (2.5,2.5) -- (0,0);
      \draw[line width=0pt,dashed](2.5,2.5)--(0,0) node[midway] [label=left:\textcolor{light}{$y = x$}] {};
      \draw[line width=0pt,dashed](2.5,2.5)--(2.5,0) node[below] {1};
      \draw[line width=0pt,dashed](2.5,2.5)--(0,2.5) node[left] {1};
      \draw[domain=-0.5:0.5,dashed] plot ({\x},{-\x});
      \draw[domain=-0.5:3,dashed] plot ({\x},{0.5*\x}) node[right] {$y = \frac{1}{t} x \quad t > 1$};
      \draw[domain=-0.25:1.5,dashed] plot ({\x},{2*\x}) node[right] {$y = \frac{1}{t}x \quad 0 < t < 1$};
      \node at (0.5,2) {A};
      \node at (1.5,2) {B};
  \end{tikzpicture}
  }
  We observe that if $t = 0$, then $x = 0$, and we would have the line on the axis, and so $P(T \leq t) = 0$. If $t < 0$, we would have $y = mx$ where $m = \frac{1}{t} < 0$, which, regardless of what $t < 0$ is, will not interact with the support of $X$ and $Y$. So for $t < 0$, $P(T \leq t) = 0$. Now if $t > 0$, we have
  \begin{equation*}
    P(T \leq t) = P\left(\frac{X}{Y} \leq t\right) = P\left( Y \geq \frac{1}{t}X \right).
  \end{equation*}
  Consider the case where $t \geq 1$, we have that the event would still cover the entire support set of $X$ and $Y$, and so $P (T \leq t) = 1$ for $t \geq 1$. With that, the only remaining case is when $0 < t < 1$. In this case,
  \begin{equation*}
    P( T \leq t ) = \int_{0}^{1} \int_{0}^{ty} 3y \dif{x} \dif{y} = \int_{0}^{1} 3ty^2 \dif{y} = t
  \end{equation*}
  and so the pdf of $T$ is
  \begin{equation*}
    f_T(t) = \begin{cases}
      1 & 0 < t < 1 \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{solution}

\begin{eg}[Example 4.3 (Course Note 4.1.3) - Order Statistics]\label{eg:4_3}
  Suppose $X_1, ..., X_n$ are IID samples, each from a continuous distribution, and with pdf $f$ and cdf $F$. Find the pdf of
  \begin{enumerate}
    \item $T = \min( X_1, ..., X_n ) = X_{(1)}$
    \item $Y = \max( X_1, ..., X_n ) = X_{(n)}$
  \end{enumerate}
\end{eg}

\begin{solution}
  \begin{enumerate}
    \item For $T$, we have that its cdf is\sidenote{The use the \hlnotea{Law of Total Probability} here so that we can use the following argument: ``the \textit{smallest} rv is larger than $t$, and so rest of the rvs must be the same.''}
      \begin{align*}
        P(T \leq t) &= 1 - P(T > t) = 1 - P( \min(X_1, ..., X_n) > t ) \\
                    &= 1 - P( X_1 > t, X_2 > t, ..., X_n > t ) \\
                    &= 1 - \prod_{i=1}^{n} P(X_i > t) \qquad \because \text{independence} \\
                    &= 1 - \prod_{i=1}^{n} P(X_1 > t) \qquad \because \text{identical distribution} \\
                    &= 1 - P(X_1 > t)^n = 1 - [ 1 - F_{X_1}(t) ]^n
      \end{align*}
      and so its pdf is
      \begin{align*}
        f_T(t) &= - \frac{d}{dt} [ 1 - F_{X_1}(t) ]^n = -n ( -F_{X_1}(t) )' [ 1 - F_{X_1}(t) ]^{n - 1} \\
               &= n f_{X_1}(t) [ 1 - F_{X_1}(t) ]^{n - 1}.
      \end{align*}
      Since $T$ relies entirely on $X_1$ (due to IID), and since we did not have to condition on the values of $t$, we have that
      \begin{equation*}
        \supp(T) = \supp(X_1) = \supp(X_i) \quad \text{ for } i = 1, ..., n.
      \end{equation*}

    \item For $Y$, we have that its cdf is\sidenote{This time, we do not have to employ the \hlnotea{Law of Total Probability}, because we simply have that ``the \textit{largest} rv is smaller than $t$, and so must the rest of the rvs.''}
      \begin{align*}
        P(Y \leq t) &= P( \max(X_1, ..., X_n) \leq y) = P(X_1 \leq y, ..., X_n \leq y) \\
                    &= \prod_{i=1}^{n} P(X_i \leq y) \quad \because \text{independence} \\
                    &= \prod_{i=1}^{n} P(X_1 \leq y) \quad \because \text{identical distribution} \\
                    &= P(X_1 \leq t)^n = F_{X_1}(t)^n
      \end{align*}
      and therefore its pdf is
      \begin{align*}
        f_Y(y) &= \frac{d}{dy} F_{X_1}(y)^n = n f_{X_1}(y) F_{X_1}(y)^{n - 1}.
      \end{align*}
  \end{enumerate}
\end{solution}

\begin{ex}
  From \cref{eg:4_3}, find the joint distribution of $X_{(1)}$ and $X_{(n)}$.
\end{ex}

% subsection transformation_of_two_or_more_random_variables (end)

\subsection{One-to-One Bivariate Transformations}%
\label{sub:one_to_one_bivariate_transformations}
% subsection one_to_one_bivariate_transformations

\begin{defn}[One-to-One Bivariate Transformation]\index{One-to-One Transformation}
\label{defn:one_to_one_bivariate_transformation}
Let $X$ and $Y$ be rvs, and $R_{XY} = \supp[(X, Y)] \in \mathbb{R}^2$. We define
  \begin{gather*}
    U = h_1(X, Y) \quad V = h_2(X, Y) \\
    S : R_{XY} \to \mathbb{R}^2 \text{ by } (x, y) \mapsto ( h_1(x, y), h_2(x, y )
  \end{gather*}
  The mapping $S$ is called a \hlnoteb{one-to-one mapping} if and only if \sidenote{There is nothing magnificent about this definition, since this is simply the definition of a one-to-one function.} $\forall (u, v) \in R_{UV}$, $\exists! (x, y) \in R_{XY}$, $\exists w_1, w_2$ that are functions such that
  \begin{equation*}
    x = w_1(u, v) \quad y = w_2(u, v)
  \end{equation*}
  i.e. $\exists S^{-1} : R_{UV} \to R_{XY}$ such that $(u, v) \mapsto (x, y)$. The \hldefn{Jacobian} of the transformation $S^{-1}$ is
  \begin{equation*}
    \frac{\partial(x, y)}{\partial (u, v)} = \begin{vmatrix}
      \frac{\partial x}{\partial u} & \frac{\partial x}{\partial v} \\
      \frac{\partial y}{\partial u} & \frac{\partial y}{\partial v}
    \end{vmatrix} = \abs{ \frac{\partial (u, v)}{\partial (x, y)} }^{-1}
  \end{equation*}
  where $\frac{\partial (u, v)}{\partial (x, y)}$ is the Jacobian of the transformation $S$.
\end{defn}

\begin{thm}[One-to-One Bivariate Transformations]
\index{One-to-One Bivariate Transformations}
\label{thm:one_to_one_bivariate_transformations}
  \marginnote{This is a generalization of \cref{thm:one_to_one_transformation_of_a_random_variable}.} Let $X$ and $Y$ be continuous rvs with joint pdf $f_{XY}$ and let $R_{XY} = \{( x, y ) : f(x, y) > 0\}$ be the support set of $(X, Y)$, and $R_{UV}$ be the support set of $(U, V)$. Suppose the transformation $S: R_{XY} \to R_{UV}$ defined by
  \begin{equation*}
    U = h_1(X, Y) \quad V = h_2(X, Y)
  \end{equation*}
  is a one-to-one transformation, with inverse transformation
  \begin{equation*}
    X = w_1(U, V) \quad Y = w_2(U, V).
  \end{equation*}
  Then $g(u, v)$, the joint pdf of $U$ and $V$, is given by
  \begin{equation*}
    \forall (u, v) \in R_{UV} \quad g(u, v) = f(w_1(u,v), w_2(u,v)) \abs{ \frac{\partial (x, y)}{\partial (u, v)} }.
  \end{equation*}
\end{thm}

\begin{proof}
  Let the inverse transformation be labelled as $S^{-1} : R_{UV} \supset B \to A \subset R_{XY}$. Then
  \begin{align*}
    \underset{B}{\int\int} g(u, v) \dif{v} \dif{u}
      &= P[ (U, V) \in B ] = P[ (X, Y) \in A ] \\
      &= \underset{A}{\int\int} f(x, y) \dif{x} \dif{y} \\
      &= \underset{B}{\int\int} f( w_1 (u, v), w_2 (u, v) ) \abs{ \frac{\partial (x, y)}{\partial (u, v)} } \dif{u} \dif{v}
  \end{align*}
  where the last step is by the \href{http://mathworld.wolfram.com/ChangeofVariablesTheorem.html}{Change of Variables Theorem}. And so by comparing integrands, we have
  \begin{equation*}
    \forall (u, v) \in R_{UV} \quad g(u, v) = f(w_1(u,v), w_2(u,v)) \abs{ \frac{\partial (x, y)}{\partial (u, v)} }
  \end{equation*}
  as required. \qed
\end{proof}

\begin{propo}[Properties of the Jacobian]\index{Jacobian}
\label{propo:properties_of_the_jacobian}
Given the setup in \cref{defn:one_to_one_bivariate_transformation}, we have that \begin{enumerate}
    \item if $S$ is a linear transformation, i.e. $\exists a_1, b_1, c_1, a_2, b_2, c_2 \in \mathbb{R}$ such that $u(x, y) = a_1 x + b_1 y + c_1$ and $v(x, y) = a_2 x + b_2 y + c_2$, then the Jacobian is a constant;
    \item if $S$ is a one-to-one transformation, then $\abs{\frac{\partial(x, y)}{\partial (u, v)}} \neq 0$
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item We have
      \begin{gather*}
        \frac{\partial u}{\partial x} = a_1 \qquad \frac{\partial u}{\partial y} = b_1 \\
        \frac{\partial v}{\partial x} = a_2 \qquad \frac{\partial v}{\partial y} = b_2
      \end{gather*}
      and so
      \begin{equation*}
        \abs{ J } = \frac{\partial (u, v)}{\partial (x, y)} = \begin{vmatrix}
            a_1 & b_1 \\
            a_2 & b_2
        \end{vmatrix} = a_1 b_2 - a_2 b_1
      \end{equation*}
      which is a constant, as required.

    \item \hlwarn{I have no idea how to prove this.}
  \end{enumerate}
\end{proof}

\begin{eg}[Example 4.4 (Course Notes 4.2.4)]
  Suppose $X \sim \Gam(a, 1)$ and $Y \sim \Gam(b, 1)$ independently. Find the joint pdf of $U = X + Y$ and $V = \frac{X}{X + Y}$. Show that $U \sim \Gam(a + b, 1)$ and $V \sim \BetaDist(a, b)$, independently. Find $E(V)$.
\end{eg}

\begin{solution}
  Given $U = X + Y$ and $V = \frac{X}{X + Y}$, rearranging variables, we have
  \begin{equation*}
    X = UV \text{ and } Y = U ( 1 - V )
  \end{equation*}
  In order for $U$ to have a Gamma distribution, we need $U$ to be non-negative, which we do since both $X$ and $Y$ have Gamma distributions.
  Note that the transformation is indeed one to one, since $\forall (u_1, v_1), (u_2, v_2) \in R_{UV}$ with
  \begin{gather*}
    u_1 = x_1 + y_1 \quad v_1 = \frac{x_1}{x_1 + y_1} \\
    u_2 = x_2 + y_2 \quad v_2 = \frac{x_2}{x_2 + y_2},
  \end{gather*}
  we have that, if we let $\phi$ denote the transformation,
  \begin{gather*}
    \phi(u_1, v_1) = \phi(u_2, v_2) \\
    \implies \left( x_1 + y_1 , \frac{x_1}{x_1 + y_1} \right) = \left( x_2 + y_2, \frac{x_2}{x_2 + y_2} \right)
  \end{gather*}
  which then
  \begin{align}
    x_1 + y_1 &= x_2 + y_2 \label{eq:eg_4_4_1} \\
    \frac{x_1}{x_2 + y_2} &= \frac{x_2}{x_2 + y_2} \nonumber \\
    \overset{\cref{eq:eg_4_4_1}}{\implies} x_1 &= x_2 \nonumber \\
    \overset{\cref{eq:eg_4_4_1}}{\implies} y_1 &= y_2. \nonumber
  \end{align}
  We shall now get the Jacobian so that we may use \cref{thm:one_to_one_bivariate_transformations}, so that we may consequently get the distributions for $U$ and $V$.
  \begin{gather*}
    J = \begin{vmatrix}
      \frac{\partial x}{\partial u} & \frac{\partial x}{\partial v} \\
      \frac{\partial y}{\partial u} & \frac{\partial y}{\partial v}
    \end{vmatrix}
    = \begin{vmatrix}
      v     & u \\
      1 - v & -u
    \end{vmatrix}
    = -vu - u(1 - v) = -u \\
    \abs{J} = u
  \end{gather*}
  By \cref{thm:one_to_one_bivariate_transformations}, and since $X$ and $Y$ are independent, we have
  \begin{align*}
    f_{U, V}(u, v) &= f_{X, Y}(x, y) \cdot \abs{J} = f_X(x) f_Y(y) \cdot \abs{J} \\
                   &= \frac{x^{a - 1} e^{-x}}{\Gamma(a)} \frac{y^{b - 1} e^{-y}}{\Gamma(b)} \cdot \abs{J} \\
                   &= \frac{e^{-a} e^{-b}}{\Gamma(a) \Gamma(b)} (uv)^{a - 1} u^{b - 1} ( 1 - v )^{b - 1} u \\
                   &= \underbrace{ \frac{u^{a + b - 1} e^{-(a + b)}}{\Gamma(a + b)} }_{\text{pdf of } \Gam(a + b, 1)} \cdot \underbrace{ \frac{\Gamma(a + b)}{\Gamma(a) \Gamma(b)} v^{a - 1} (1 - v)^{b - 1} }_{\text{pdf of } \BetaDist(a, b)}
  \end{align*}
  We have already shown that $U$ is a non-negative rv, and so $U \sim \Gam(a + b, 1)$ as required. Note that for $V$, we have
  $X + Y > X > 0 \implies 1 > \frac{X}{X + Y} > 0 \quad \because X + Y \neq 0$ and so $0 < V < 1$. Therefore $V \sim \BetaDist(a, b)$.

  With that, we can look for $E[V]$.
  \begin{align*}
    E[V] &= \int_{0}^{1} v \frac{\Gamma(a + b)}{\Gamma(a) \Gamma(b)} v^{a - 1} (1 - v)^{b - 1} \dif{v} \\
         &= \frac{\Gamma(a + b) \Gamma(a + 1)}{\Gamma(a) \Gamma(a + b + 1)} \int_{0}^{1} \underbrace{ \frac{\Gamma(a + b + 1)}{\Gamma(a + 1) \Gamma(b)} v^a (1 - v)^{b - 1} }_{\text{pdf of } \BetaDist(a + 1, b)} \dif{v} \\
         &= \frac{a \Gamma(a) \Gamma(a + b)}{(a + b) \Gamma(a) \Gamma(a + b)} = \frac{a}{a + b}
  \end{align*}
\end{solution}

% subsection one_to_one_bivariate_transformations (end)

% section functions_of_random_variables (end)

% chapter lecture_12_jun_12th_2018 (end)

\chapter{Lecture 13 Jun 14th 2018}%
\label{chp:lecture_13_jun_14th_2018}
% chapter lecture_13_jun_14th_2018

\section{Functions of Random Variables (Continued)}%
\label{sec:functions_of_random_variables_continued}
% section functions_of_random_variables_continued

\subsection{One to One Bivariate Transformations (Continued)}%
\label{sub:one_to_one_bivariate_transformations_continued}
% subsection one_to_one_bivariate_transformations_continued

\begin{eg}[Example 4.5 (Course Notes 4.2.8)]
  Suppose $X$ and $Y$ are continuous rvs with joint pdf
  \begin{equation*}
    f(x, y) = e^{-x - y} \mathbb{1}_{\{0 < x, y < \infty\}}.
  \end{equation*}
  Let $U = X + Y$ and $V = X - Y$. Find the joint pdf of $U$ and $V$. (Note: Be sure to specify the support of $(U, V)$.) Then, find the marginal pdf of $U$ and $V$ respectively.
\end{eg}

\begin{solution}
  Note that because $0 < x, y < \infty$,
  \begin{gather*}
    u = x + y \implies 0 < u < \infty \\
    v = x - y \implies -\infty < v < \infty
  \end{gather*}
  The Jacobian is
  \begin{equation*}
    \abs{ \frac{\partial (u, v)}{\partial (x, y)} } = \abs{ \begin{vmatrix}
        1 & 1 \\
        1 & -1
    \end{vmatrix} } = \abs{-2} = 2
  \end{equation*}
  and so
  \begin{equation*}
    \abs{ \frac{\partial(x, y)}{\partial (u, v)} } = \abs{ \frac{\partial (u, v)}{\partial (x, y)} }^{-1} = \frac{1}{2}.
  \end{equation*}
  Note that
  \begin{equation*}
    X = \frac{U - V}{2} \quad Y = \frac{U - V}{2}.
  \end{equation*}
  and so
  \begin{gather*}
    x = \frac{u + v}{2} > 0 \implies u > -v \\
    y = \frac{u - v}{2} > 0 \implies u > v.
  \end{gather*}
  The diagram to the right shows the support of $U, V$.\marginnote{
  \begin{tikzpicture}
      \draw[->] (-0.5, 0) -- (3, 0) node[right] {$u$};
      \draw[->] (0, -3) -- (0, 3) node[above] {$v$};
  		\draw[line width=0pt,draw=none,fill=light,fill opacity=0.25] (0,0) -- (3,-3) -- (3,3) -- (0,0);
      \draw[domain=-0.5:3] plot ({\x},{\x}) node[right] [label=right:\textcolor{light}{$v = u$}] {};
      \draw[domain=-0.5:3] plot ({\x},{-\x}) node[right] [label=right:{$v = -u$}] {};
  \end{tikzpicture}
  }
  With that,
  \begin{align*}
    g(u, v) &= f( \frac{u + v}{2}, \frac{u - v}{2} ) \cdot \frac{1}{2} \\
            &= \frac{1}{2} e^{- \left( \frac{u + v}{2} \right) - \left( \frac{u - v}{2} \right)} = \frac{1}{2} e^{-u}.
  \end{align*}
  With that, the marginal pdf of $V$ is\sidenote{$V$ is also called the \href{http://atomic.phys.uni-sofia.bg/local/nist-e-handbook/e-handbook/eda/section3/eda366c.htm}{Double Exponential Distribution}.}
  \begin{align*}
    v < 0 \implies f_V(v) &= \int_{-v}^{\infty} \frac{1}{2} e^{-u} \dif{u} = \frac{1}{2} e^v \\
    v \geq 0 \implies f_V(v) &= \int_{v}^{\infty} \frac{1}{2} e^{-v} \dif{u} = \frac{1}{2} e^{-v}.
  \end{align*}
  For the marignal pdf of $U$, we have
  \begin{align*}
    f_U(u) &= \int_{-u}^{u} \frac{1}{2}e^{-u} \dif{v} = ue^{-u} \quad \text{ for } 0 < u \leq \infty.
  \end{align*}
\end{solution}

% subsection one_to_one_bivariate_transformations_continued (end)

\subsection{Moment Generating Function Method}%
\label{sub:moment_generating_function_method}
% subsection moment_generating_function_method

This method is particularly useful in finding distributions of sums of independent rvs.

\begin{thm}[Sums of MGF]
\label{thm:sums_of_mgf}
Suppose $X_1, ..., X_n$ are independent rvs and $X_i$ has mgf $M_i(t)$ which exists for $t \in (-h, h)$ for some $H > 0$. The mgf of $Y = \sum_{i=1}^{n} X_i$ is given by
\begin{equation*}
  M_Y(t) = \prod_{i=1}^{n} M_i(t)
\end{equation*}
for $t \in (-h, h)$.
\end{thm}

\begin{proof}
  Observe that
  \begin{align*}
    M_Y(t) &= E\left[ e^{tY} \right] = E\left[ e^{t \sum_{i=1}^{n} X_i} \right] = E\left[ \prod_{i=1}^{n} e^{tX_i} \right] \\
           &= \prod_{i=1}^{n} E\left[ e^{tX_i} \right] = \prod_{i=1}^{n} M_i(t)
  \end{align*}
  and $t \in (-h, h)$ is preserved. \qed
\end{proof}

\begin{note}
  \begin{enumerate}
    \item If $X_i$'s are IID rvs each with mgf $M(t)$ then $Y$ has mgf
      \begin{equation*}
        M_Y(t) = [ M(t) ]^n \quad \text{ for } t \in (-h, h).
      \end{equation*}

    \item Used in conjunction with the Uniqueness Theorem for mgfs, this theorem can be used to find the distribution of $Y$.
  \end{enumerate}
\end{note}

\begin{ex}
  Show the following results:
  \begin{enumerate}
    \item If $X \sim \Gam(\alpha, \beta)$, where $\alpha \in \mathbb{N}$, then $\frac{2X}{\beta} \sim \chi^2 (2 \alpha)$.
    \item If $X_i \sim \Gam(\alpha_i, \beta), \, i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \Gam\left( \sum_{i=1}^{n} \alpha_i, \, \beta \right).
      \end{equation*}
    \item If $X_i \sim \Gam(1, \beta) = \Exp(\beta)$, $i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i  \sim \Gam(n, \, \beta).
      \end{equation*}
    \item If $X_i \sim \Gam\left( \frac{k_i}{2}, 2 \right) = \chi^2 ( k_i )$, $i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \chi^2 \left( \sum_{i=1}^{n} k_i \right).
      \end{equation*}
    \item If $X_i \sim \Nor( \mu, \, \sigma^2 )$, $i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} \left( \frac{X_i - \mu}{\sigma} \right)^2 \sim \chi^2(n).
      \end{equation*}
    \item If $X_i \sim \Poi(\mu_i)$, $i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \Poi\left( \sum_{i=1}^{n} \mu_i \right).
      \end{equation*}
    \item If $X_i \sim \Bin(n_i, p)$, $i = 1, ..., n$ independently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \Bin\left( \sum_{i=1}^{n} n_i, p \right).
      \end{equation*}
    \item If $X_i \sim \NB(k_i, p)$, $i = 1, ..., n$ indepdently, then
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \NB\left( \sum_{i=1}^{n} k_i, p \right).
      \end{equation*}
  \end{enumerate}
\end{ex}

\begin{solution}
  \begin{enumerate}
    \item Using the mgf method,
      \begin{align*}
        M_Y(t) &= E\left[ e^{tY} \right] = E\left[ e^{t \cdot \frac{2x}{\beta}} \right] = \int_{0}^{\infty} e^{ \frac{2tx}{\beta} } \frac{1}{\beta^\alpha \Gamma(a)} x^{\alpha - 1} e^{- \frac{x}{\beta}} \dif{x} \\
               &= \frac{1}{\beta^\alpha} \int_{0}^{\infty} \frac{1}{\Gamma(a)} x^{\alpha - 1} e^{ - \frac{x ( 1 - 2t )}{\beta} } \dif{x} \\
               &\overset{(*)}{=} \frac{1}{\beta^\alpha} \left( \frac{\beta}{1 - 2t} \right)^\alpha \int_{0}^{\infty} \underbrace{ \frac{1}{\left( \frac{\beta}{1 - 2t} \right) \Gamma(\alpha)} x^{\alpha - 1} e^{ - \frac{x}{\left( \frac{\beta}{1 - 2t} \right)} } }_{\text{pdf of } \Gam\left(\alpha, \frac{\beta}{1 - 2t}\right)} \dif{x} \\
               &= \left( \frac{1}{1 - 2t} \right)^\alpha = ( 1 - 2t )^\alpha
      \end{align*}
      where in $(*)$ we note that $1 - 2t > 0$ and so $t > \frac{1}{2}$. Observe that the mgf of $Y$ is the mgf of $\chi^2 ( 2 \alpha )$ and so by the \cref{thm:uniqueness_of_the_mgf}, $Y = \frac{2X}{\beta} \sim \chi^2 (2 \alpha)$.
    \item Using the mgf method, let $Y = \sum_{i=1}^{n} X_i$
      \begin{align*}
        M_Y(t) &= E\left[ e^{tY} \right] = E\left[ \prod_{i=1}^{n} e^{tX_i} \right] = \prod_{i=1}^{n} E[ e^{tX_i} ] \\
               &= \prod_{i=1}^{n} M_{X_i}(t) = \prod_{i=1}^{n} ( 1 - \beta t )^{-\alpha_1} = (1 - \beta t)^{- \sum_{i=1}^{n} \alpha_1}
      \end{align*}
      and we observe that the last term is the mgf of $\Gam\left( \sum_{i=1}^{n} \alpha_i, \beta \right)$ and so the result follows.
    \item From $(2)$ is $\alpha_i = 1$ for $i = 1, ..., n$, then $\sum_{i=1}^{n} \alpha_1 = \sum_{i=1}^{n} 1 = n$. The result follows.
    \item By $(2)$, we have $\sum_{i=1}^{n} X_i \sim \Gam\left( \sum_{i=1}^{n} \frac{k_1}{2}, 2 \right)$. Then by $(1)$, we have that
      \begin{equation*}
        \sum_{i=1}^{n} X_i \sim \chi^2\left( \sum_{i=1}^{n} k_i \right)
      \end{equation*}
      as required.
    \item We know that $Y_i = \frac{X_i - \mu}{\sigma} \sim Z(0, 1)$. Let $W_i = Y_i^2$. Then
      \begin{align*}
        M_{W_i}(t) &= E\left[ e^{tY_i} \right] = \int_{-\infty}^{\infty} \frac{1}{\sqrt{2 \pi}} e^{-\frac{y^2}{2}} e^{ty^2} \dif{y} \\
                   &= \int_{-\infty}^{\infty} \frac{1}{\sqrt{2 \pi}} \exp\left\{-\frac{y^2}{2 ( 1 - 2t )^{-1}} \right\} \dif{y} \\
                   &= (1 - 2t)^{-\frac{1}{2}} \int_{-\infty}^{\infty} \underbrace{ \frac{1}{\sqrt{2 \pi} (1 - 2t)^{-\frac{1}{2}}} \exp \left\{ \frac{y^2}{2 ( 1 - 2t )^{-1}} \right\} }_{\text{pdf of } \Nor\left(0, (1 - 2t)^{-\frac{1}{2}}\right)} \dif{y} \\
                   &= (1 - 2t)^{-\frac{1}{2}},
      \end{align*}
      which is the mgf of $\chi^2 (1)$. Let $W = \sum_{i=1}^{n} W_i$. Then
      \begin{equation*}
        M_W(t) = \prod_{i=1}^{n} (1 - 2t)^{-\frac{1}{2}} = (1 - 2t)^{-\frac{n}{2}}
      \end{equation*}
      which is the mgf of $\chi^2 ( n )$.
    \item Let $Y = \sum_{i=1}^{n} X_i$. Then
      \begin{equation*}
        M_Y(t) = \prod_{i=1}^{n} e^{\mu_i (e^t - 1)} = \exp\left[ (e^t - 1) \sum_{i=1}^{n} \mu_i \right]
      \end{equation*}
      is the mgf of $\Poi\left( \sum_{i=1}^{n} \mu_i \right)$.
    \item Let $Y = \sum_{i=1}^{n} X_i$. Then
      \begin{equation*}
        M_Y(t) = \prod_{i=1}^{n} ( pe^t + q )^{n_i} = ( p e^t + q )^{\sum_{i=1}^{n} n_i}
      \end{equation*}
      is the mgf of $\Bin\left( \sum_{i=1}^{n} n_i, p \right)$.
    \item Again, a similar approach: let $Y = \sum_{i=1}^{n} X_i$. Then
      \begin{align*}
        M_Y(t) = \prod_{i=1}^{n} \left( \frac{1 - p}{1 - pe^t} \right)^{k_i} = \left( \frac{1 - p}{1 - pe^t} \right)^{\sum_{i=1}^{n} k_i}
      \end{align*}
      is the mgf of $\NB\left(\sum_{i=1}^{n} k_i, p\right)$.
  \end{enumerate}
\end{solution}

% subsection moment_generating_function_method (end)

% section functions_of_random_variables_continued (end)

% chapter lecture_13_jun_14th_2018 (end)

\chapter{Lecture 14 Jun 19th 2018}%
\label{chp:lecture_14_jun_19th_2018}
% chapter lecture_14_jun_19th_2018

\section{Functions of Random Variables (Continued 2)}%
\label{sec:functions_of_random_variables_continued_2}
% section functions_of_random_variables_continued_2

\subsection{Moment Generating Function Method (Continued)}%
\label{sub:moment_generating_function_method_continued}
% subsection moment_generating_function_method_continued

\begin{thm}[Gaussian Distribution]\index{Gaussian Distribution}
\label{thm:gaussian_distribution}
If $X_i \sim \Nor( \mu_i, \sigma_i^2 )$, with $i = 1, ..., n$ independently, then
\begin{equation*}
  \sum_{i=1}^{n} a_i X_i \sim \Nor\left( \sum_{i=1}^{n} a_i \mu_i, \sum_{i=1}^{n} a_i^2 \sigma_i^2 \right).
\end{equation*}
\end{thm}

\begin{proof}
  Let $Y_i = a_i X_i$. Then by \cref{thm:mgf_of_a_linear_transformation}, we have
  \begin{equation*}
    M_{Y_i}(t) = M_{X_i}(a_i t) = \exp\left[ \mu_i a_i t + \frac{\sigma_i^2 a_i^2 t^2}{2} \right],
  \end{equation*}
  which implies that $Y_i \sim \Nor( a_i \mu_i, a_i^2 \sigma_i^2 )$ by \cref{thm:uniqueness_of_the_mgf}. Then let $Y = \sum_{i=1}^{n} Y_i$. Then
  \begin{equation*}
    M_Y(t) = \prod_{i=1}^{n} \exp\left[ \mu_i a_i t + \frac{\sigma_i^2 a_i^2 t_2}{2} \right] = \exp\left[ t \sum_{i=1}^{n} a_i \mu_i + \frac{t^2}{2} \sum_{i=1}^{n} a_i^2 \sigma_i^2 \right],
  \end{equation*}
  which implies that $Y \sim \Nor\left( \sum_{i=1}^{n} a_i \mu_i , \sum_{i=1}^{n} a_i^2 \sigma_i^2 \right)$.\qed
\end{proof}

\begin{thm}[Properties of the Gaussian Distribution]
\label{thm:properties_of_the_gaussian_distribution}
  Asumme $X_1, ..., X_n \sim \Nor( \mu, \sigma_2 )$, independently, where $\bar{X} = \frac{1}{n} \sum_{i=1}^{n} X_i$, and $S^2 = \frac{\sum_{i=1}^{n} ( X_i - \bar{X} )^2}{n - 1}$. Then
  \begin{enumerate}
    \item $\bar{X} \sim \Nor\left( \mu, \frac{\sigma^2}{n} \right)$.
    \item (\hldefn{Cochran's Theorem}) $\frac{( n - 1 ) S^2}{\sigma^2} = \frac{\sum_{i=1}^{n} (X_i - \bar{X})^2}{\sigma^2} \sim \chi^2 ( n - 1 )$.
    \item (\hldefn{t-test}) $\frac{\bar{X} - \mu}{S / \sqrt{n}} \sim t(n - 1)$.
  \end{enumerate}
\end{thm}

\begin{proof}
  \begin{enumerate}
    \item Let $Y_i = \frac{X_i}{n}$. \cref{thm:gaussian_distribution} implies that $Y_i \sim \Nor\left( \frac{\mu}{n} , \frac{\sigma^2}{n^2} \right)$. Let $Y = \sum_{i=1}^{n} Y_i$. Then \cref{thm:gaussian_distribution} implies that $Y \sim \Nor\left( \sum_{i=1}^{n} \frac{\mu}{n}, \sum_{i=1}^{n} \frac{\sigma^2}{n^2} \right) = \Nor\left( \mu, \frac{\sigma^2}{n} \right)$.\qed
    \item The proof of Cochran's Theorem is beyond the scope of this course, for it uses knowledge from linear algebra and involving Fourier Transforms. [\href{https://en.wikipedia.org/wiki/Cochran\%27s_theorem}{Reference - Wikipedia}]
    \item The proof of this statement is non-trivial: [\href{https://en.wikipedia.org/wiki/Student\%27s\_t\-distribution}{Reference - Wikipedia}] [\href{https://math.stackexchange.com/questions/474733/derivation-of-the-density-function-of-student-t-distribution-from-this-big-integ}{Reference - Math SE}]
 \end{enumerate}
\end{proof}

\begin{thm}[F Distribution]
\index{F Distribution}
\label{thm:f_distribution}
Suppose $X_1, ..., X_n$ is a random sample from the $\Nor(\mu_1, \sigma_1^2)$ distribution and independently $Y_1, ..., Y_m$ is a random sample from the $\Nor(\mu_2, \sigma_2^2)$ distribution. Let
\begin{equation*}
  S_1^2 = \frac{1}{n - 1} \sum_{i=1}^{n} (X_i - \bar{X})^2 \text{ and } S_2^2 = \frac{1}{m - 1} \sum_{i=1}^{m} (Y_i - \bar{Y})^2.
\end{equation*}
Then
\begin{equation*}
  \frac{S_1^2 / \sigma_1^2}{S_2^2 / \sigma_2^2} \sim F( n - 1, m - 1 ).
\end{equation*}
\end{thm}

\begin{proof}
  This is, once again, an incomplete proof. Rigor shall be appended where it is due or they shall be stated with reference, or if it is at a level that I cannot understand and that I cannot find a reference, I shall state that as well.

  Note that the definition of the F distribution is as follows:
  \begin{equation*}
    F_{\gamma_1, \gamma_2} = \frac{\chi^2( \gamma_1 ) / \gamma_1}{\chi^2( \gamma_2 ) / \gamma_2}
  \end{equation*}
  where $\chi^2(\gamma_1)$ and $\chi^2(\gamma_2)$ are independent.

  Now we are given that $X_1, ..., X_n$ and $Y_1, ..., Y_m$ are independent of one another. It can likely be shown, using induction, that the following two are independent of one another:
  \begin{gather*}
    \sum_{i=1}^{n} \left( \frac{X_i - \bar{X}}{\sigma_1} \right)^2 \sim \chi^2( n - 1 ) \\
    \sum_{j=1}^{m} \left( \frac{Y_j - \bar{Y}}{\sigma_2} \right)^2 \sim \chi^2( m - 1 ).
  \end{gather*}
  Then
  \begin{equation*}
    \frac{\sum\limits_{i=1}^{n} \left( \frac{X_1 - \bar{X}}{\sigma_1} \right)^2}{\sum\limits_{j=1}^{m} \left( \frac{Y_j - \bar{Y}}{\sigma_2} \right)^2}
    = \frac{( n - 1 ) S_1^2 / \sigma_1^2}{( m - 1 ) S_2^2 / \sigma_2^2}.
  \end{equation*}
  With that, we have that
  \begin{equation*}
    \frac{S_1^2 / \sigma_1^2}{S_2^2 / \sigma_2^2} = \frac{\frac{(n-1)S_1^2}{\sigma_1^2} / ( n - 1 )}{\frac{(m-1)S_2^2}{\sigma_2^2} / ( m - 1 )}.
  \end{equation*}
  The result follows.
\end{proof}

% subsection moment_generating_function_method_continued (end)

% section functions_of_random_variables_continued_2 (end)

% chapter lecture_14_jun_19th_2018 (end)

\chapter{Lecture 15 Jun 21st 2018}%
\label{chp:lecture_15_jun_21st_2018}
% chapter lecture_15_jun_21st_2018

\section{Limiting or Asymptotic Distributions}%
\label{sec:limiting_or_asymptotic_distributions}
% section limiting_or_asymptotic_distributions

\subsection{Convergence in Distribution}%
\label{sub:convergence_in_distribution}
% subsection convergence_in_distribution

\begin{defn}[Convergence in Distribution]\index{Convergence in Distribution}\index{Limiting Distribution}\index{Asymptotic Distribution}
\label{defn:convergence_in_distribution}
  Let $X_1, X-2, ..., X_n, ...$ be a sequence of rvs such that $X_i$ has cdf $F_i$, for $i \in \mathbb{N}$. Let $X$ be an rv with cdf $F$. We say that $X_i$ \hlnoteb{converges in distribution} to $X$ and we write
  \begin{equation*}
    X_n \overset{D}{\to} X
  \end{equation*}
  if
  \begin{equation*}
    \lim_{i \to \infty} F_i(x) = F(x)
  \end{equation*}
  at all points $x$ at which $F$ is continuous. We call $F$ the \hlnoteb{limiting} or \hlnoteb{asymptotic distribution} of $X_i$.
\end{defn}

The following theorem, that we shall not prove, will be useful for the remainder of the notes.

\begin{thm}[Taylor Series with Lagrange's Remainder]
\index{Taylor Series with Lagrange's Remainder}
\label{thm:taylor_series_with_lagrange_s_remainder}
  Suppose that $f:[a, b] \to \mathbb{R}$ is infinitely differentiable, and $x \in [a, b]$. Then $\forall x \in [a, b]$ and $\forall k \in \mathbb{N}$,
  \begin{equation*}
    f(x) = \sum_{i=0}^{k} \frac{f^{(i)}(c)(x - c)^i}{i!} + \frac{f^{(k + 1)}( \zeta_X )(x - c)^{k + 1}}{(k + 1)!}
  \end{equation*}
  for some $\zeta_X \in [c, x]$.
\end{thm}

The proof for the above involves the use of the \hlnotea{Mean Value Theorem for Integrals}.

\begin{thm}[Generalized Limit Definition of $e$]
\label{thm:generalized_limit_definition_of_e_}
  If $b, c \in \mathbb{R}$ are constants and $\lim\limits_{n \to \infty} \psi(n) = 0$, then
  \begin{equation*}
    \lim_{n \to \infty} \left[ 1 + \frac{b}{n} + \frac{\psi(n)}{n} \right]^{cn} = e^{bc}.
  \end{equation*}
\end{thm}

\begin{proof}
  The above equation can be rewritten as
  \begin{equation*}
    \lim_{n \to \infty} e^{cn \log \left( 1 + \frac{b}{n} + \frac{\psi(n)}{n} \right)} = e^{bc}
  \end{equation*}
  and so it suffices to prove that
  \begin{equation*}
    \lim_{n \to \infty} cn \log\left( 1 + \frac{b}{n} + \frac{\psi(n)}{n} \right) = bc
  \end{equation*}
  Note that the Taylor Expansion with Lagrange's Remainders (\cref{thm:taylor_series_with_lagrange_s_remainder}) of $log(1 + x)$, where we pick $c = 0$ and $k = 1$ for convenience,
  \begin{equation*}
    \log(1 + x) = \frac{\log(1) x^0}{0!} + \frac{\left( \frac{1}{1 - 0} \right) x^1}{1!} + \frac{- \frac{1}{(1 + \zeta_x)^2} x^2}{2!} = x - \frac{x^2}{2 ( 1 + \zeta_x )}
  \end{equation*}
  where $\zeta \in [0, x]$. Then
  \begin{equation*}
    cn \log \left[ 1 + \frac{b}{n} + \frac{\psi(n)}{n} \right] = cb + c \psi(n) - \frac{c (b + \psi(n))^2}{2n(1 + \zeta)^2}
  \end{equation*}
  where $\zeta \in \left[0, \frac{b + \psi(n)}{n}\right]$. Note that by \hlnotea{L'H\^opital's Rule}, we have that
  \begin{equation*}
    \lim_{n \to \infty} \frac{b + \psi(n)}{n} = \lim_{n \to \infty} \frac{\psi'(n)}{1} = 0
  \end{equation*}
  and so the possible highest value for $\zeta$ goes to $0$ as $n \to \infty$. Then
  \begin{equation*}
    \lim_{n \to \infty} \frac{c( b + \psi(n) )^2}{2n ( 1 + \zeta )^2} = 0.
  \end{equation*}
  As a result, we have
  \begin{equation*}
    \lim_{n \to \infty} cn \log\left( 1 + \frac{b}{n} + \frac{\psi(n)}{n} \right) = bc
  \end{equation*}
  as required.\qed
\end{proof}

\begin{crly}[Limit definition of $e$]
\label{crly:limit_definition_of_e_}
  If $b, c \in \mathbb{R}$ are constants, then
  \begin{equation*}
    \lim_{n \to \infty} \left( 1 + \frac{b}{n} \right)^{cn} = e^{bc}
  \end{equation*}
\end{crly}

\begin{eg}[Example 5.1 (Course Notes 5.1.4)]
  Let $X_i \sim \Exp(1)$, where $i \in \mathbb{N}$, independently so. Consider the sequence of rvs $Y_1, Y_2, ..., Y_n, ...$, where $Y_n = \max( X_1, ..., X_n ) - \log n$. Find the limiting distribution of $Y_n$.
\end{eg}

\begin{solution}
  Firstly, observe that to find the support set of $Y_n$, note
  \begin{gather*}
    0 < x_1, ..., x_n < \infty \\
    0 < \max( x_1, ..., x_n ) < \infty \\
    -\log n < \max( x_1, ..., x_n ) - \log n < \infty
  \end{gather*}
  so
  \begin{equation*}
    \supp(Y) = (- \log n, \infty)
  \end{equation*}
  Now the pf of $Y_n$ is
  \begin{align*}
    F_n(y) &= P(Y_n \leq y) = P( \max(X_1, ..., X_n) - \log n \leq y ) \\
           &= P( \max(X_1, ..., X_n) \leq y + \log n ) \\
           &= \prod_{i=1}^{n} P(X_1 \leq y + \log n) \quad \because X_1, ..., X_n \text{ are IID } \\
           &= \prod_{i=1}^{n} \left[ 1 - e^{- y - \log n} \right] = \prod_{i=1}^{n} \left[ 1 - \frac{e^-y}{n} \right] = \left[ 1 - \frac{e^-y}{n} \right]^n.
  \end{align*}
  Thus the limiting distribution of $Y_i$ is
  \begin{equation*}
    \lim_{n \to \infty} F_n(y) = \lim_{n \to \infty} \left[ 1 - \frac{e^-y}{n} \right]^n = e^{-e^{-y}} \text{ for } y \in ( -\log n, \infty ).
  \end{equation*}
\end{solution}

\begin{eg}[Example 5.2 (Course Notes 5.1.5)]
  Let $X_i \sim \Unif(0, \theta)$, $i \in \mathbb{N}$, independently so. Consider the sequence of random variables $Y_1, Y_2, ..., Y_n, ...$, where $Y_n = \max(X_1, ..., X_n)$. Find the limiting distribution of $Y_n$.
\end{eg}

\begin{solution}
  Clearly, support of $Y_n = ( 0, \theta )$. Note that
  \begin{equation*}
    f_{X_i}(x) = \frac{1}{\theta} \mathbb{1}_{0 < x < \theta}.
  \end{equation*}
  Now since $X_i$ are IID,
  \begin{equation*}
    F_n (y) = P(Y_n \leq y) = \prod_{i=1}^{n} P( X_1 \leq y ) = \prod_{i=1}^{n} \frac{y}{\theta} = \left( \frac{y}{\theta} \right)^n \quad y \in ( 0, \theta )
  \end{equation*}
  Then the limiting distribution is
  \begin{equation*}
    \lim_{n \to \infty} F_n(y) = \lim_{n \to \infty} \left( \frac{y}{\theta} \right)^n = \begin{cases}
      0 & y < \theta \\
      1 & y \geq \theta
    \end{cases}
  \end{equation*}
  Define $Y$ to have a distribution such that $P(Y = \theta) = 1$. Then
  \begin{equation*}
    Y_n \convd Y.
  \end{equation*}
\end{solution}

\begin{defn}[Degenerate Distribution]\index{Degenerate Distribution}
\label{defn:degenerate_distribution}
  A function $F$ is the cdf of a degenerate distribution at value $y = c$ if
  \begin{equation*}
    F(y) = \begin{cases}
      0 & y < c \\
      1 & y \geq c
    \end{cases}.
  \end{equation*}
  In other words, $F$ is the CDF of a discrete distribution where
  \begin{equation*}
    P(Y = y) = \begin{cases}
      1 & y = c \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
\end{defn}

We have that the earlier example gives us that the limiting distribution is a degenerate distribution.

% subsection convergence_in_distribution (end)

\subsection{Convergence in Probability}%
\label{sub:convergence_in_probability}
% subsection convergence_in_probability

\begin{defn}[Convergence in Probability]\index{Convergence in Probability}
\label{defn:convergence_in_probability}
  A sequence of rvs $X_1, X_2,..., X_n, ...$ \hlnoteb{converges in probability} to an rv $X$ if, for every $\epsilon > 0$,
  \begin{equation*}
    \lim_{n \to \infty} P ( \abs{X_n - X} \geq \epsilon ) = 0
  \end{equation*}
  or equivalently
  \begin{equation*}
    \lim_{n \to \infty} P(\abs{X_n - X} < \epsilon) = 1
  \end{equation*}
  We write
  \begin{equation*}
    X_n \convp X.
  \end{equation*}
\end{defn}

\begin{eg}[Example 5.3]
  Consider $X \sim \Bernoulli(0.3)$. Define the sequence $X_n = \left(1 + \frac{1}{n}\right) X$, $n \in \mathbb{N}$. Show that $X_n \convp X$.
\end{eg}

\begin{solution}
  Note that
  \begin{equation*}
    P(X = x) = \begin{cases}
      0.3 & x = 1 \\
      0.7 & x = 0
    \end{cases}
  \end{equation*}
  Since $X_n = \left(1 + \frac{1}{n}\right) X$, we have
  \begin{equation*}
    X_1 = 2X, \, X_2 = \frac{3}{2} X, \, X_3 = \frac{4}{3} X, \, ... \,.
  \end{equation*}
  Note that $\abs{X_n - X} = \abs{\left(1 - \frac{1}{n}\right) X - X} = \abs{\frac{1}{n}X} = \frac{1}{n} X$, so
  \begin{align*}
    P(\abs{X_n - X} < \epsilon) &= P \left( \frac{1}{n} X < \epsilon \right) = P( X < n \epsilon ) \\
                                &= \begin{cases}
    P( X = 0 ) = 0.7 & n < \frac{1}{\epsilon} \\
    P( X = 1 ) + P( X = 0 ) = 1 & n \geq \frac{1}{\epsilon}
                                \end{cases}.
  \end{align*}
  Therefore
  \begin{equation*}
    \lim_{n \to \infty} P( \abs{X_n - X} < \epsilon ) = 1
  \end{equation*}
  and so
  \begin{equation*}
    X_n \convp X.
  \end{equation*}
\end{solution}

We shall look into the following proposition in the next lecture.

\begin{propononum}
  Suppose $\forall n \in \mathbb{N}$, $\{ X_n \}$ is a sequence of rvs. Then
  \begin{equation*}
    X_n \convp X \implies X_n \convd X
  \end{equation*}
  where $X$ is an rv.
\end{propononum}

% subsection convergence_in_probability (end)

% section limiting_or_asymptotic_distributions (end)

% chapter lecture_15_jun_21st_2018 (end)

\chapter{Lecture 16 Jun 26th 2018}%
\label{chp:lecture_16_jun_26th_2018}
% chapter lecture_16_jun_26th_2018

\section{Limiting or Asymptotic Distributions (Continued)}%
\label{sec:limiting_or_asymptotic_distributions_continued}
% section limiting_or_asymptotic_distributions_continued

\subsection{Convegence in Probability (Continued)}%
\label{sub:convegence_in_probability_continued}
% subsection convegence_in_probability_continued

Before proving the proposition introduced in last class, we require the following lemma.

\begin{lemma}
\label{lemma:cdf_bounded_abv}
  Let $X$, $Y$ be rvs, $a \in \mathbb{R}$, and $\epsilon > 0$. Then
  \begin{equation*}
    P( Y \leq a ) \leq P( X \leq a + \epsilon ) + P ( \abs{Y - X} > \epsilon )
  \end{equation*}
\end{lemma}

\begin{proof}
  Note that
  \begin{align}
    P(Y \leq a) &= P(Y \leq a, X \leq a + \epsilon) + P( Y \leq a, X > a + \epsilon ) \label{eq:lemma_cdf_bounded_abv_1} \\
                &\leq P(X \leq a + \epsilon) + P( Y - X \leq a - X , a - X < - \epsilon ) \label{eq:lemma_cdf_bounded_abv_2} \\
                &\leq P(X \leq a + \epsilon) + P( Y - X < - \epsilon ) \label{eq:lemma_cdf_bounded_abv_3} \\
                &\leq P(X \leq a + \epsilon) + P( Y - X < - \epsilon ) + P( Y - X > \epsilon ) \label{eq:lemma_cdf_bounded_abv_4} \\
                &= P(X \leq a + \epsilon) + P( \abs{Y - X} > \epsilon ) \nonumber
  \end{align}
  where \cref{eq:lemma_cdf_bounded_abv_1} is by \hlnotea{Law of Total Probability}, \cref{eq:lemma_cdf_bounded_abv_2} and \cref{eq:lemma_cdf_bounded_abv_3} are by the fact that for non-empty sets $A$ and $B$, $P(A \cap B) \leq P(A)$, and \cref{eq:lemma_cdf_bounded_abv_4} is because $P(Y - X > \epsilon) \geq 0$.\qed
\end{proof}

\begin{propo}[Convergence in Probability Implies Convergence in Distribution]
\label{propo:convergence_in_probability_implies_convergence_in_distribution}
\marginnote{This only states for the case of scalar random variables.}
  Suppose $\forall n \in \mathbb{N}$, $\{ X_n \}$ is a sequence of rvs. Then
  \begin{equation*}
    X_n \convp X \implies X_n \convd X
  \end{equation*}
  where $X$ is an rv.
\end{propo}

\begin{proof}
  By \cref{lemma:cdf_bounded_abv}, we have that\sidenote{We choose to use \cref{lemma:cdf_bounded_abv} to have $P(X_n \leq a)$ in two inequalities. Note that our goal is to show that
  \begin{equation*}
    \lim_{n \to \infty} P(X_n \leq a) = P(X \leq a).
  \end{equation*}
We already know that $\epsilon \to 0$, so we should use that to our advantage in the case of $X$. Also, in hindsight, it is clear why we choose this method as the \hlnotea{Squeeze Theorem} is suitable to help us reach our conclusion.}
  \begin{gather*}
    P(X_n \leq a) \leq P(X \leq a + \epsilon) + P(\abs{X_n - X} > \epsilon) \\
    P(X \leq a - \epsilon) \leq P(X_n \leq a) + P(\abs{X_n - X} > \epsilon)
  \end{gather*}
  Then
  \begin{align*}
    &P(X \leq a - \epsilon) - P(\abs{X_n - X} > \epsilon) \\
    &\leq P(X_n \leq a) \\
    &\leq P(X \leq a + \epsilon) + P(\abs{X_n - X} > \epsilon)
  \end{align*}
  As $n \to \infty$, by assumption that $X_n \convp X$, we have that $P(\abs{X_n - X} > \epsilon) \to 0$, and so
  \begin{equation*}
    P(X \leq a - \epsilon) \leq \lim_{n \to \infty} P(X_n \leq a) \leq P(X \leq a + \epsilon)
  \end{equation*}
  Now as $\epsilon \to 0^+$, note that we must have the cdf of $X$ be continuous on $a$ by assumption, and so
  \begin{equation*}
    P(X \leq a) \leq \lim_{n \to \infty} P(X_n \leq a) \leq P(X \leq a)
  \end{equation*}
  and so by the \hlnotea{Squeeze Theorem},
  \begin{equation*}
    \lim_{n \to \infty} P(X_n \leq a) = P(X \leq a)
  \end{equation*}
  which then
  \begin{equation*}
    X_n \convd X
  \end{equation*}
  as required.\qed
\end{proof}

\begin{defn}[Convergence in Probability to a Constant]
\label{defn:convergence_in_probability_to_a_constant}
  A sequence of rvs $\{X_i\}_{i \in \mathbb{N}}$ converges in probability to a constant $b \in \mathbb{R}$ if $\forall \epsilon > 0$,
  \begin{equation*}
    \lim_{n \to \infty} P( \abs{X_n - b} \geq \epsilon ) = 0
  \end{equation*}
  or equivalently
  \begin{equation*}
    \lim_{n \to \infty} P( \abs{X_n - b} < \epsilon ) = 1.
  \end{equation*}
  We write
  \begin{equation*}
    X_n \convp b.
  \end{equation*}
\end{defn}

\begin{eg}[Example 5.4 - \hldefn{Weak Law of Large Numbers}]\label{eg:weak_law_of_large_numbers}\label{eg:bernoulli_s_theorem}\marginnote{The Weak Law of Large Numbers is also known as \hldefn{Bernoulli's Theorem}.}
  Let $X_1, ..., X_n, ...$ be a sequence of IID rvs, each having mean $\mu$ and variance $\sigma^2$. Let
  \begin{equation*}
    \bar{X}_n = \frac{1}{n} \sum_{i=1}^{n} X_i.
  \end{equation*}
  Then $\bar{X}_n \convp \mu$.
\end{eg}

\begin{solution}
  Observe that
  \begin{equation*}
    E\left[\bar{X}_n\right] = E\left[ \frac{1}{n} \sum_{i=1}^{n} X_i \right] = \frac{1}{n} \sum_{i=1}^{n} E[X_i] = \frac{n \mu}{n} = \mu,
  \end{equation*}
  Also, we have that
  \begin{equation*}
    \Var\left( \bar{X}_n \right) = \Var\left( \frac{1}{n} \sum_{i=1}^{n} X_i \right) = \frac{1}{n^2} \sum_{i=1}^{n} \Var(X_i) = \frac{1}{n^2} \cdot n \sigma^2 = \frac{\sigma^2}{2}.
  \end{equation*}
  Then by \cref{thm:markov_s_inequality_2}, we have that $\forall \epsilon > 0$,
  \begin{equation*}
    P(\abs{\bar{X}_n - \mu} \geq \epsilon) \leq \frac{\Var(\bar{X}_n)}{\epsilon^2} = \frac{\sigma^2}{n \epsilon^2}.
  \end{equation*}
  Therefore, we have
  \begin{equation*}
    0 \leq P(\abs{\bar{X}_n - \mu} \geq \epsilon) \leq \min\left( 1, \frac{\sigma^2}{n \epsilon^2} \right)
  \end{equation*}
  As $n \to \infty$, by \hlnotea{Squeeze Theorem}, we have that
  \begin{equation*}
    \lim_{n \to \infty} P(\abs{\bar{X}_n - \mu} \geq \epsilon) = 0
  \end{equation*}
  i.e. $\bar{X}_n \convp \mu$.
\end{solution}

Note that the converse of \cref{propo:convergence_in_probability_implies_convergence_in_distribution} is not generally true. \hlwarn{(Example required)} However, with an additional condition, the converse becomes true.

\begin{propo}[Partial Converse of \cref{propo:convergence_in_probability_implies_convergence_in_distribution}]
\label{propo:partial_converse_of_convp_implies_convd}
  Suppose $\forall n \in \mathbb{N}$, $\{X_n\}$ is a sequence of rvs, each with cdf $F_n$. If
  \begin{equation*}
    \lim_{n \to \infty} F_n(x) = \lim_{n \to \infty} P( X_n \leq x ) = \begin{cases}
      0 & x < c \\
      1 & x > c
    \end{cases},
  \end{equation*}
  then
  \begin{equation*}
    X_n \convp c.
  \end{equation*}
\end{propo}

\begin{proof}
  Note that for $\epsilon > 0$, we have
  \begin{align*}
    P( \abs{X_n - c} > \epsilon ) &= P( X_n - c < -\epsilon ) + P( X_n - c > \epsilon ) \\
                                  &= P( X_n < c - \epsilon ) + P( X_n > c + \epsilon ) \\
                                  &= 1 - P( X_n < c + \epsilon ) + P ( X_n < c - \epsilon )
  \end{align*}
  By assumption, we have that
  \begin{gather*}
    \lim_{n \to \infty} P( X_n \leq c + \epsilon ) = 1 \\
    \lim_{n \to \infty} P( X_n \leq c - \epsilon ) = 0
  \end{gather*}
  So
  \begin{equation*}
    \lim_{n \to \infty} P(\abs{X_n - c} > \epsilon) = 1 - 1 - 0 = 0
  \end{equation*}
  and so by definition, we have $X_n \convp c$.\qed
\end{proof}

\begin{defn}[Double Parameter Exponential Distribution]\index{Double Parameter Exponential Distribution}
\label{defn:double_parameter_exponential_distribution}
We say that an rv $X \sim \Exp(\lambda, \theta)$ when $X$ has pdf
\begin{equation*}
  f(x) = e^{- \frac{x - \theta}{\lambda}} \text{ for } x \in (\theta, \infty)
\end{equation*}
where $\lambda$ is the scale parameter, and $\theta$ the location parameter.
\end{defn}

\begin{eg}[Example 5.5 (Course Notes 5.2.5)]
  Let $X_i \sim \Exp(1, \theta)$, $i = 1, 2, ...$, independently. Consider the sequence of rvs $Y_1, Y_2, ...$ where $Y_n = \min(X_1, X_2, ..., X_n)$, $n = 1, 2, ...$. Show that $Y_n \convp \theta$.
\end{eg}

\begin{solution}
  Since we want to show that $Y_n \convp \theta$, and $\theta$ is a constant, we can use \cref{propo:partial_converse_of_convp_implies_convd}.
  Thus we need to show that
  \begin{equation*}
    \lim_{n \to \infty} F_{Y_n} (y) = \lim_{n \to \infty} P ( Y_n \leq y ) = \begin{cases}
      0 & y < \theta \\
      1 & y > \theta
    \end{cases}
  \end{equation*}
  Since $Y_n$ is defined as the minimum of $n$ of the first $X_i$ rvs, we need to use the \hlnotea{Law of Total Probability} in order to be able to make sense of the order statistics, i.e.
  \begin{equation*}
    P( Y_n \leq y ) = 1 - P( Y_n > y )
  \end{equation*}
  Now
  \begin{align*}
    P( Y_n > y ) &= \prod_{i=1}^{n} P(X_i > y) = \prod_{i=1}^{n} \left[ \int_{y}^{\infty} e^{- ( x - \theta )} \dif{x} \right] \\
                 &= \prod_{i=1}^{n} \left[ e^{- (y - \theta)} \right] = e^{ -n (y - \theta) }
  \end{align*}
  Thus
  \begin{align*}
    \lim_{n \to \infty} P (Y_n \leq y) &= \begin{cases}
      1 - \lim\limits_{n \to \infty} P( Y_n > y ) & y > \theta \\
      0 & y \leq \theta
    \end{cases} \\
    &= \begin{cases}
       1 - \lim\limits_{n \to \infty} e^{-n (y - \theta)} = 1 & y > \theta \\
       0 & y \leq \theta
    \end{cases}
  \end{align*}
  since if $y < \theta$
  \begin{equation*}
    P( Y_n \leq y ) = P( \min( X_1, X_2, ..., X_n ) \leq y ) = 0.
  \end{equation*}
  Note that if $y = \theta$, then
  \begin{equation*}
    e^{-n ( y - \theta )} = 1.
  \end{equation*}
  The proof is complete with \cref{propo:partial_converse_of_convp_implies_convd}.
\end{solution}

% subsection convegence_in_probability_continued (end)

\subsection{Limit Theorems}%
\label{sub:limit_theorems}
% subsection limit_theorems

\begin{propo}[Convergence in Distribution and MGF]
\label{propo:convergence_in_distribution_and_mgf}
Let $X_1, X_2, ..., X_n, ...$ be a sequence of rvs such that $X_n$ has mgf $M_n(t)$. Let $X$ be an rv with mgf $M(t)$.
\begin{equation*}
  X_n \convd X \iff \left[ \exists h > 0 \; \forall t \in (-h, h) \quad \lim_{n \to \infty} M_n(t) = M(t) \right]
\end{equation*}
\end{propo}

\begin{proof}
  Note that
  \begin{equation*}
    \lim_{n \to \infty} M_n(t) = \lim_{n \to \infty} \int\limits_{\supp(X_n)} e^{tx} \frac{d}{dx} F_{X_n} (x) \dif{x}
  \end{equation*}
  and
  \begin{equation*}
    M(t) = \int\limits_{\supp(X)} e^{tx} \frac{d}{dx} F_X(x) \dif{x}
  \end{equation*}
  The result follows assuming that the integral converges\sidenote{This allows us to ``swap'' the limit, the integral sign, and the differential operator.}.\qed
\end{proof}

\begin{eg}
  Consider the sequence $Y_1, Y_2, ...$, where $Y_i \sim \Bin\left(n , \frac{\mu}{n}\right)$, for $i = 1, 2, ...$. Find the limiting distribution of $Y_n$.
\end{eg}

\begin{solution}[Example 5.6]
  Note that $Y_n \sim \Bin\left(n, \frac{\mu}{n}\right) \implies$
  \begin{equation*}
    M_{Y_n}(t) = \left( \frac{\mu}{n} e^t + 1 - \frac{\mu}{n}\right)^n = \left( 1 + \frac{\mu \left( e^t - 1 \right)}{n} \right)^n.
  \end{equation*}
  So
  \begin{equation*}
    \lim_{n \to \infty} M_{Y_n}(t) = \lim_{n \to \infty} \left( 1 + \frac{\mu \left( e^t - 1 \right)}{n} \right)^n = e^{\mu \left(e^t - 1\right)}
  \end{equation*}
  which is the mgf of a Poisson Distribution. Thus by \cref{propo:convergence_in_distribution_and_mgf}, we have that
  \begin{equation*}
    Y_n \convd Y \sim \Poi(\mu).
  \end{equation*}
\end{solution}

\begin{eg}[Example 5.7]
  Let $Y_1, Y_2, ... \sim \Gau(\mu, \sigma)$. Then
  \begin{equation*}
    \frac{\bar{Y}_n - \mu}{\sigma / \sqrt{n}} \sim \Gau(0, 1)
  \end{equation*}
  where $\bar{Y}_n = \frac{1}{n} \sum_{i=1}^{n} Y_i$.
\end{eg}

\begin{solution}
  Note that by \cref{thm:mgf_of_a_linear_transformation}, we have that
  \begin{equation*}
  M_{\bar{Y}_n}(t) = \prod_{i=1}^{n} M_{Y_i}\left(\frac{t}{n}\right) = \prod_{i=1}^{n} e^{\frac{\mu t}{n} + \frac{(\sigma t)^2}{2 n^2}} = e^{ \mu t + \frac{\sigma^2 t^2}{n} }.
  \end{equation*}
  which is the mgf of $\Gau\left(\mu, \frac{\sigma}{\sqrt{n}}\right)$. Let $X_n = \frac{\bar{Y}_n - \mu}{\sigma / \sqrt{n}}$.

  Then again, by \cref{thm:mgf_of_a_linear_transformation}, we have
  \begin{align*}
    M_{X_n}(t) &= e^{- \frac{\mu t}{\sigma / \sqrt{n}}} M_{\bar{Y}_n}(t) \left( \frac{t}{\sigma / \sqrt{n}} \right) \\
    &= e^{- \frac{\mu t}{\sigma / \sqrt{n}}} e^{ \frac{\mu t}{\sigma / \sqrt{n}} + \frac{\sigma^2 / n}{2} \left( \frac{t}{\sigma / \sqrt{n}} \right)^2 } = e^{\frac{t^2}{2}}
  \end{align*}
  which is the mgf of $\Gau(0, 1)$.
\end{solution}

\begin{eg}[Example 5.8]\label{eg:5_8}
  Let $Y_n \sim \Gam( n, 2 )$. Find the limiting distribution of
  \begin{equation*}
    W_N = \frac{1}{2 \sqrt{n}} ( Y_n - 2n ).
  \end{equation*}
\end{eg}

\begin{solution}
  By \cref{thm:mgf_of_a_linear_transformation}, we have
  \begin{align*}
    M_{W_n}(t) &= e^{ - t \sqrt{n} } M_{ Y_n } (t) \left( \frac{t}{2 \sqrt{n}} \right) = e^{- t \sqrt{n}} \left( 1 - 2 \left( \frac{t}{2 \sqrt{n}} \right) \right)^{-n} \\
               &= \left[ e^{\frac{t}{\sqrt{n}}} \left( 1 - \frac{t}{\sqrt{n}} \right) \right]^{-n} = \left[ \left(1 - \frac{t}{\sqrt{n}}\right) \sum_{k=0}^{\infty} \frac{(t / \sqrt{n})^k}{k!} \right]^{-n} \\
               &= \left[ \left( 1 - \frac{t}{\sqrt{n}} \right) \left( 1 + \frac{t}{\sqrt{n}} + \frac{t^2}{2n} + \frac{t^3}{3 \sqrt{n^3}} + \hdots \right) \right]^{-n} \\
               &= \left( 1 - \frac{t}{\sqrt{n}} + \frac{t}{\sqrt{n}} - \frac{t^2}{n} + \frac{t^2}{2 n} - \frac{t^3}{2 \sqrt{n^3}} + \frac{t^3}{3 \sqrt{n^3}} - \frac{t^4}{3 n^2} + \hdots \right)^{-n} \\
               &= \left( 1 - \frac{t^2}{2n} - \smash{ \underbrace{ \frac{t^3}{6 \sqrt{n^3}} - \frac{t^4}{3n^2} + \hdots }_{\to 0 \text{ as } n \to \infty} } \right)^{-n}
  \end{align*}

  \noindent So we have, by \cref{thm:generalized_limit_definition_of_e_},
  \begin{equation*}
    \lim_{n \to \infty} M_{W_n}(t) = \lim_{n \to \infty} ( 1 - \frac{t^2 / 2}{n} - \frac{t^3}{6 \sqrt{n^3}} - \frac{t^4}{3 n^2} + \hdots )^{-n} = e^{-(- \frac{t^2}{2})} = e^{\frac{t^2}{2}},
  \end{equation*}
  which is the mgf of $\Gau(0, 1)$. So we know that
  \begin{equation*}
    W_n \convd W \sim \Gau(0, 1)
  \end{equation*}
  for some limiting distribution $W$ that has a $\Gau(0, 1)$ distribution.
\end{solution}

\begin{thm}[Central Limit Theorem]
\index{Central Limit Theorem}
\label{thm:central_limit_theorem}
Suppose $X_1, X_2,...$ is a sequence of IID rvs with $E(X_i) = \mu$ and $\Var(X_i) = \sigma^2 < \infty$, for $i = 1, 2, ...$. Then\marginnote{\faLightbulbO \; We actually also need the condition that all moments of $\frac{X_i - \mu}{\sigma}$ exists and is bounded. }
\begin{equation*}
  \frac{\bar{X}_n - \mu}{\sigma / \sqrt{n}} = \frac{\sqrt{n} ( \bar{X}_n - \mu )}{\sigma} \convd Z \sim \Nor(0, 1)
\end{equation*}
where $\bar{X}_n = \frac{1}{n} \sum_{i=1}^{n} X_i$.
\end{thm}

\begin{proof}
  Since the first and second moment exist, the mgf must be well-defined, and is at least of class $C^2$ \sidenote{You should remember this from your calculus courses. Otherwise, \href{https://en.wikipedia.org/wiki/Differentiable_function\#Differentiability_classes}{[Reference - Wiki]}.}. We will use \cref{propo:convergence_in_distribution_and_mgf} and\\ \noindent \cref{thm:taylor_series_with_lagrange_s_remainder} in this proof. Firstly note that
  \begin{equation*}
    E\left[ \sum_{i=1}^{n} X_i \right] = n \mu \text{ and } \Var\left( \sum_{i=1}^{n} X_i \right) = n \sigma^2
  \end{equation*}
  Let $\Sigma X_n = \sum_{i=1}^{n} X_i$ and $Z = \frac{\sqrt{n} ( \bar{X}_n - \mu )}{\sigma}$. Note that
  \begin{equation*}
    Z = \frac{\sqrt{n} ( \bar{X}_n - \mu )}{\sigma} = \frac{n \bar{X}_n - \mu n}{\sigma \sqrt{n}} = \frac{\Sigma X_n - n \mu}{\sigma \sqrt{n}} = \sum_{i=1}^{n} \frac{X_i - \mu}{\sigma \sqrt{n}}.
  \end{equation*}
  Let $Y_i = \frac{X_i - \mu}{\sigma}$ so that
  \begin{equation*}
    Z = \sum^{n}_{i=1} \frac{1}{\sqrt{n}} Y_i.
  \end{equation*}
  Now the mgf of $Z$, by \cref{thm:mgf_of_a_linear_transformation} and \cref{propo:independence_on_joint_mgf}, is
  \begin{equation}\label{eq:clt_eq1}
    M_Z(t) = \prod_{i=1}^{n} M_{Y_i}\left(\frac{t}{\sqrt{n}}\right) = M_{Y_i}\left(\frac{t}{\sqrt{n}}\right)^n.
  \end{equation}
  Note that
  \begin{gather*}
    E(Y_i) = E\left( \frac{X_i - \mu}{\sigma} \right) = \frac{1}{\sigma} E(X) - \frac{\mu}{\sigma} = 0 \\
    \Var(Y_i) = \Var\left( \frac{X_i - \mu}{\sigma} \right) = \frac{1}{\sigma^2} \Var(X_i) = 1.
  \end{gather*}
  So by \cref{thm:taylor_series_with_lagrange_s_remainder} up to the second derivative, we have
  \begin{equation*}
    M_{Y_i}\left(\frac{t}{\sqrt{n}}\right) = 1 + \frac{t^2}{2n} + \frac{M_{Y_i}^{(3)} ( \zeta ) \left( \frac{t}{\sqrt{n}} \right)^3}{3!}
  \end{equation*}
  where $\zeta \in \left[0, \frac{t}{\sqrt{n}}\right]$. Note that the last term in the above equation goes to $0$ faster than the second term as $n \to \infty$, since we would have $n^{\frac{3}{2}}$ in the denominator, which is larger than $n$ in the second term. With that, continuing with \cref{eq:clt_eq1} and now taking the limit as $n \to \infty$, we have
  \begin{equation*}
    \lim_{n \to \infty} M_Z (t) = \lim_{n \to \infty} \left[ 1 + \frac{t^2}{2n} + \phi(n) \right]^n
  \end{equation*}
  where
  \begin{equation*}
    \phi(n) = \frac{M_{Y_i}^{(3)} ( \zeta ) \left( \frac{t}{\sqrt{n}} \right)^3}{3!}.
  \end{equation*}
  Then by \cref{thm:generalized_limit_definition_of_e_}, we have
  \begin{equation*}
    \lim_{n \to \infty} M_Z(t) = e^{\frac{t^2}{2}}
  \end{equation*}
  which is the mgf of $\Nor(0, 1)$. This completes the proof.\qed
\end{proof}

\begin{eg}[Example 5.9]
  Revisit \cref{eg:5_8} using the \hyperref[thm:central_limit_theorem]{\faCoffee \; Central Limit Theorem}. Given
  \begin{equation*}
    W_n = \frac{Y - 2n}{2 \sqrt{n}}
  \end{equation*}
  where $Y \sim \Gam( n, 2 )$, find sequences $a_n$ and $b_n$ such that
  \begin{equation*}
    a_n ( Y - b_n ) \convd Z \sim G(0, 1).
  \end{equation*}
\end{eg}

\begin{solution}
  \marginnote{This involves the knowledge that the sum of $n$ exponential distributions with mean $\mu$ results in a gamma distribution with parameter $n$ and $\mu$. Since this has never been proven in this set of notes, it shall be proven here in this solution.} Let $X_1, X_2, ...$ be a sequence of IID rvs with distribution $\Exp(2)$. Note that we then have $E(X_i) = 2$ and $\Var(X_i) = 4$. Let $Y_n = \sum_{i=1}^{n} X_i$. Note that
  \begin{equation*}
    M_{Y_n}(t) = [ M_{X_i}(t) ]^n = (1 - 2t)^{-n}
  \end{equation*}
  which is the mgf of $\Gam(n, 2)$. By CLT, we have that
  \begin{equation*}
    \frac{\frac{1}{n} Y_n - 2}{2 / \sqrt{n}} \convd \Gau(0, 1).
  \end{equation*}
  Note that
  \begin{equation*}
    \frac{\frac{1}{n}Y_n - 2}{2 / \sqrt{n}} = \frac{\sqrt{n}}{2 n} ( Y_n - 2n )
  \end{equation*}
  and so
  \begin{equation*}
    a_n = \frac{1}{2}\sqrt{n} \text{ and } b_n = 2n
  \end{equation*}
\end{solution}

\begin{eg}[Example 5.10 (Course Notes 5.3.6)]
  Suppose $Y_n \sim \chi^2(n)$, $n = 1, 2, ...$. Consider the sequence of rvs $Z_1, Z_2,...$ where $Z_n = (Y_n - n) / \sqrt{2n}$. Show that
  \begin{equation*}
    Z_n = \frac{Y_n - n}{\sqrt{2n}} \convd Z \sim G(0, 1).
  \end{equation*}
\end{eg}

\begin{solution}
  \marginnote{Once again our class pulls the card of ``not explaining or showing stuff that they are supposed to''. We will be using the fact that a sum of $n$ Chi-Squared Distirbutions, each with degree of freedom k, will result in a gamma distribution with parameter $\frac{nk}{2}$ and $2$. This will be proven in this exercise. } Let $X_1, X_2, ... \sim \chi^2(1)$ be IID rvs, and so $E[X_i] = 1$ and $\Var(X_i) = 2$ for $1 \leq i \leq n$, and let $Y_n = \sum_{i=1}^{n} X_i$. Note that
  \begin{equation*}
    M_{Y_n}(t) = [ M_{X_i}(t) ]^n = (1 - 2t)^{-\frac{n}{2}}
  \end{equation*}
  is the mgf of $\chi^2 (n)$. By CLT, we have that
  \begin{equation*}
    \frac{\frac{1}{n} \sum_{i=1}^{n} X_i - 1}{\sqrt{2} / \sqrt{2}} \convd Z \sim \Gau(0, 1).
  \end{equation*}
  Note that
  \begin{equation*}
    \frac{\frac{1}{n} \sum_{i=1}^{n} X_i - 1}{\sqrt{2} / \sqrt{n}} = \frac{Y_n - n}{\sqrt{2} n / \sqrt{n}} = \frac{Y_n - n}{\sqrt{2n}}.
  \end{equation*}
  This completes our example.
\end{solution}

\begin{eg}[Example 5.11 (Course Notes 5.3.7)]
  Suppose $Y_n \sim \Bin(n, p)$, $n = 1, 2, ...$. Consider the sequence of rvs $Z_1, Z_2, ...$, where $Z_n = \frac{Y_n - np}{\sqrt{np(1 - p)}}$. Show that
  \begin{equation*}
    Z_n = \frac{Y_n - np}{\sqrt{np(1 - p)}} \convd Z \sim \Nor(0, 1).
  \end{equation*}
\end{eg}

\begin{solution}
  Let $X_1, X_2, ...$ be a sequence of IID Bernoulli trials with success probability $0 \leq p \leq 1$. Note that for $i \in \mathbb{N}$, we have that $E[X_i] = p$ and $\Var(X_i) = p(1 - p)$. Now let $Y_n = \sum_{i=1}^{n} X_i$. We note that this satisfies our assumption, i.e. $Y_n \sim \Bin(n, p)$ since
  \begin{equation*}
    M_{Y_n}(t) = [ M_{X_1} ]^n = (pe^t + 1 - p)^n
  \end{equation*}
  is the mgf of $\Bin(n, p)$. By CLT, we have that
  \begin{equation*}
    \frac{\frac{1}{n} \sum_{i=1}^{n} X_i - p}{\sqrt{ p(1 - p) } / \sqrt{n}} \convd Z \sim \Nor(0, 1).
  \end{equation*}
  Note that
  \begin{equation*}
    \frac{\frac{1}{n} \sum_{i=1}^{n} X_i - p}{\sqrt{p (1 - p) / \sqrt{n}}} = \frac{Y_n - np}{\sqrt{np(1 - p)}}
  \end{equation*}
  and so this completes our proof.
\end{solution}

\begin{propo}[Other Limit Theorems]
\label{propo:other_limit_theorems}
  \begin{enumerate}
    \item $X_n \convp a \, \land \, g$ continuous at $x = a \implies g(X_n) \convp g(a)$.
    \item $X_n \convp a \, \land \, Y_n \convp b \, \land \, g$ continous at $(x, y) = (a, b) \implies g(X_n, Y_n) \convp g(a, b)$.
    \item (\hldefn{Slutsky's Theorem}) $X_n \convd X \, \land \, Y_n \convp b \, \land \, g$ continous at $(x, y) = (x, b)$ for all $x \in \supp( X ) \implies g(X_n, Y_n) \convd g(X, b)$.
    \item (\hldefn{Continuous Mapping Theorem}) $X_n \convd X \, \land \, g$ continous at all $x \in \supp(X) \implies g(X_n) \convd g(X)$.
  \end{enumerate}
\end{propo}

\begin{proof}
  \begin{enumerate}
    \item Since $g$ is continuous at $a$, we have that
      \begin{equation*}
        \forall \epsilon > 0 \; \exists \delta > 0 \; \forall x \in \supp(X) \; ( \abs{x -a} < \delta \implies \abs{g(x) - g(a)} < \epsilon )
      \end{equation*}
      It follows that
      \begin{equation*}
        P[ \abs{ g(X_n ) - g(a)} < \epsilon ] \geq P[ \abs{X_n - a} < \delta ]
      \end{equation*}
      since $P(B) \geq P(A)$ whenever $A \subset B$ \sidenote{\st{While I do not have a rigorous proof of this for our case here, it is a sensible result seeing that $\delta$ depends on $\epsilon$.} My hunch was right: thinking about the two probability measures using the definition of a random variable, we see that the $w \in S$ that works for $\abs{X_n - a} < \delta$ will definitely work for $\abs{g(X_n) - g(a)} < \epsilon$, but the converse is not necessarily true. Therefore, the events covered in the left term is a larger set that contains the event on the right.}. Since $X_n \convp a$, we have that for any $\delta > 0$, we have
      \begin{equation*}
        1 \geq \lim_{n \to \infty} P[ \abs{g(X_n) - g(a)} < \epsilon ] \geq \lim_{n \to \infty} P[ \abs{X_n - a} < \delta ] = 1.
      \end{equation*}
      Thus
      \begin{equation*}
        \lim_{n \to \infty} P[ \abs{g(X_n) - g(a)} < \epsilon ] = 1
      \end{equation*}
      and so
      \begin{equation*}
        g(X_n) \convp g(a)
      \end{equation*}
      as required.

    \item This is simply a more general case than $(1)$.\qed

    \item See this \href{https://en.wikipedia.org/wiki/Slutsky\'s_theorem}{Wikipedia article} for a proof. Requires knowledge of measure theory (in particular, convergence of measures).

    \item See this \href{https://en.wikipedia.org/wiki/Continuous_mapping_theorem}{Wikipedia article} for a proof. Requires knowledge of measure theory (in particular, convergence of measures).
  \end{enumerate}
\end{proof}

\begin{eg}
  If $X_n \convp 10$ and $Y_n \convp 2$, then $X_n Y_n \convp 20$ since $g(x, y) = xy$ is continuous at $(10, 2)$.

  \noindent If $Z_n \convd Z \sim G(0, 1)$ and $a_n \convp a$ where $a$ is a constant, then $a_n Z_n \convd aZ \sim (0, a)$ since $g(a, z) = az$ is continous at $(a, z)$ for all $z \in \supp(Z)$..
\end{eg}

\begin{eg}[Example 5.12 (Course Notes 5.3.10)]
  If $X_n \convp a > 0$, $Y_n \convp b \neq 0$ and $Z_n \convd Z \sim \Gau(0, 1)$, the find the limiting distributions of each of the following:
  \begin{enumerate}
    \item $\sqrt{X_n}$
    \item $X_n + Y_n$
    \item $X_n Z_n$
    \item $\frac{1}{Z_n}$
  \end{enumerate}
\end{eg}

\begin{solution}
  \begin{enumerate}
    \item Since $a > 0$, we have the function $g(x) = \sqrt{x}$ is continuous on $a$, and so by \cref{propo:other_limit_theorems}, we have that $\sqrt{X_n} \convp \sqrt{a}$.
    \item Since the function $g(x, y) = x + y$ is continous on the real number 2-tuple $(a, b)$, we have that $X_n + Y_n \convp a + b$.
    \item Since the function $g(x, z) = xz$ is continuous on the real number 2-tuple $(x, z)$ for all $x \in \supp(X_n)$ and $z \in \supp(Z_n)$, by \hlnotea{Slutsky's Theorem}, we have $X_n Z_n \convd a Z \sim \Gau(0, a)$.
    \item Note that the function $g(z) = \frac{1}{z}$ is continuous for all $z \in \supp(X)$ except at $z = 0$. But since $Z \sim \Gau(0, 1)$ is a continuous distribution, at a single point $z = 0$, $P(Z = 0) = 0$, and in a distribution it is of negligible value\sidenote{This is a painful thing to write down for me...}. Therefore, we have that
      \begin{equation*}
        \frac{1}{Z_n} \convd \frac{1}{Z}
      \end{equation*}
  \end{enumerate}
\end{solution}

\begin{eg}[Example 5.13 (Course Notes 5.3.11)]\label{eg:involved_eg_for_limit_theorems}
  Suppose $X_1, X_2, ... \sim \Poi(\mu)$ are IID rvs. Define $Z_n = \frac{\sqrt{n} ( \bar{X}_n - \mu )}{\sqrt{\bar{X}_n}}$. Find the limiting distribution of $Z_n$.
\end{eg}

\begin{solution}
  Firstly, note that the parameter for a Poisson distribution is positive. Note that by CLT, we have
  \begin{equation*}
    \frac{\bar{X}_n - \mu}{\sqrt{\mu} / \sqrt{n}} \sim Z \sim \Gau(0, 1)
  \end{equation*}
  Note that since $\sqrt{m}$ is a constant, we have that $\sqrt{m} \convp \sqrt{m}$ is trivially true. Thus by our limit theorems, we have
  \begin{equation*}
    \sqrt{n} ( \bar{X}_n - \mu ) \sim \sqrt{m} Z \sim \Gau(0, \sqrt{m}).
  \end{equation*}
  Now by the \hyperref[eg:weak_law_of_large_numbers]{Weak Law of Large Numbers}, we have
  \begin{equation*}
    \bar{X}_n \convp \mu.
  \end{equation*}
  Since a function $g(x) = \sqrt{x}$ is continuous for $x = \mu > 0$, we have that
  \begin{equation*}
    \sqrt{\bar{ X }}_n \convp \sqrt{m}.
  \end{equation*}
  Then by the \hlnoteb{Continuous Mapping Theorem}, we have that
  \begin{equation*}
    Z_n = \frac{\sqrt{n} ( \bar{X}_n - \mu )}{\sqrt{\bar{X}_n}} \sim \frac{\sqrt{\mu} Z}{\sqrt{\mu}} \sim \Gau(0, 1).
  \end{equation*}
\end{solution}

\begin{thm}[Generalized $\delta$-Method]
\label{thm:generalized_delta_method}
  Let $X_1, X_2, ...$ be a sequence of rvs such that
  \begin{equation*}
    n^b (X_n - a) \convd X
  \end{equation*}
  for some $b > 0$. Suppose the function $g(x)$ is differentiable at $a$ and $g'(a) \neq 0$. Then
  \begin{equation*}
    n^b [ g(X_n) - g(a) ] \convd g'(a) X.
  \end{equation*}
\end{thm}

\begin{proof}
  Using the \hlnotea{Mean Value Theorem}, we have that
  \begin{equation}\label{eq:delta_method_eq1}
    g(X_n) = g(a) + g'(c) ( X_n - a )
  \end{equation}
  for some $c$ in between $X_n$ and $a$. Since $X_n \convp a$ \sidenote{\hlwarn{ I have no idea why... }}, and, WLOG, $X_n < c < a$, $c \convp a$. Then by the \hlnoteb{Continuous Mapping Theorem}, we have $g(c) \convp g(a)$.

  \noindent Now by rearranging \cref{eq:delta_method_eq1} and multiplying both sides by $n^b$, we get
  \begin{equation*}
    n^b [ g(X_n) - g(a) ] = g'(c) n^b [ X_n - a ] \convd g'(a) X
  \end{equation*}
  by \hlnoteb{Slutsky's Theorem}.\qed
\end{proof}

\begin{crly}[$\delta$-Method]
\index{$\delta$-Method}
\label{crly:_delta_method}
  Let $X_1, X_2, ...$ be a sequence of IID rvs with mean $\mu$ and variance $\sigma^2$. Suppose the function $g(x)$ is differentiable at $\mu$ and $g'(\mu) \neq 0$. Then
  \begin{equation*}
    \sqrt{n} [ g(\bar{X}_n) - g(\mu) ] \convd Z \sim N\left( 0, [ g'(\mu) ]^2 \sigma^2 \right)
  \end{equation*}
\end{crly}

\begin{proof}
  Note that by the same working as in \cref{eg:involved_eg_for_limit_theorems} for $\sqrt{n} ( \bar{X}_n - \mu )$, we have that\sidenote{This is actually stated as an assumption on the $\delta$-Method Wikipedia article - [\href{https://en.wikipedia.org/wiki/Delta_method}{Reference}].}
  \begin{equation*}
    \sqrt{n} ( \bar{X}_n - \mu ) \convd Y \sim \Nor(0, \sigma^2).
  \end{equation*}
  Then by \cref{thm:generalized_delta_method}, we have that
  \begin{equation*}
    \sqrt{n} ( g(\bar{X}_n) - g(\mu) ) \convd g'(\mu) Y \sim \Nor\left( 0, [g'(\mu)]^2 \sigma^2 \right)
  \end{equation*}
  as required.\qed
\end{proof}

\begin{eg}[Example 5.14]
  Let $X_1, X_2, ... \sim \Geo(p)$ is a sequence of IID rvs, each with support $\supp(X_i) = \{0, 1, 2, ...\}$. Derive the limiting distribution of
  \begin{equation*}
    W_n = \sqrt{n} \left( \frac{1}{\bar{X}_n} - \frac{p}{1 - p} \right)
  \end{equation*}
\end{eg}

\begin{solution}
  Note that by CLT, we have that
  \begin{equation*}
    \frac{\bar{X}_n - \frac{1 - p}{p}}{\sqrt{\frac{1 - p}{p^2}} / \sqrt{n}} \convd Z \sim \Nor(0, 1)
  \end{equation*}
  Then using \cref{propo:other_limit_theorems}, we have
  \begin{equation*}
    \sqrt{n} \left( \bar{X}_n - \frac{1 - p}{p} \right) \convd \sqrt{ \frac{1 - p}{p^2} } Z \sim N\left(0, \frac{1 - p}{p^2}\right)
  \end{equation*}
  \hlwarn{We took} $g(x) = \frac{1}{x}$ and the given solution in class somehow circumvents the fact that $x = 0$ is a case.
\end{solution}

\begin{eg}[Example 5.15]
  Suppose that $X_1, X_2,... \sim \Exp(\theta)$ is a sequence of IID rvs. Find constants $a_n$ and $b_n$ such that
  \begin{equation*}
    W_n = b_n (\bar{X}_n^2 - a_n)
  \end{equation*}
  has a non-denegerate limiting distribution.
\end{eg}

\begin{solution}
  By CLT, we have
  \begin{equation*}
    \frac{\bar{X}_n - \theta}{\theta / \sqrt{n}} \convd Z \sim \Nor(0, 1)
  \end{equation*}
  which then by \cref{propo:other_limit_theorems}, we have
  \begin{equation*}
    \sqrt{n} ( \bar{X}_n - \theta ) \convd \theta Z \sim N(0, \theta^2).
  \end{equation*}
  Let $g(x) = x^2$. Then $g$ is continous on any $x \in \supp(X_n)$ and on $\theta$. Then using \cref{crly:_delta_method}, we have
  \begin{equation*}
    \sqrt{n} ( \bar{X}_n^2 - \theta^2 ) \sim \theta^2 Z \sim \Nor(0, 4 \theta^2).
  \end{equation*}
  So we just need to pick $b_n = \sqrt{n}$ and $a_n = \theta^2$.
\end{solution}

% subsection limit_theorems (end)

% section limiting_or_asymptotic_distributions_continued (end)

% chapter lecture_16_jun_26th_2018 (end)

\chapter{Lecture 17 Jun 28th 2018}%
\label{chp:lecture_17_jun_28th_2018}
% chapter lecture_17_jun_28th_2018

\section{Estimation}%
\label{sec:estimation}
% section estimation

\newthought{Suppose} $X_1, ..., X_n \sim f(x; \theta)$ is an IID sequence of rvs, where $f(x; \theta)$ is the pf of the $X_i$'s. The joint distribution of $X_1, ..., X_n$ is
\begin{equation*}
  \prod_{i=1}^{n} f(x_i; \theta)
\end{equation*}
where the unknown parameter $\theta$ can either be a scalar in $\Omega$, where $\Omega$ is the parameter space or the set of possible values of $\theta$, or a vector, i.e. $\theta = \begin{pmatrix} \theta_1 & \theta_2 & \hdots & \theta_n \end{pmatrix}^T$.

We are interested in making inferences about the unknown parameter $\theta$, i.e. we want to find \hlnoteb{estimators} (point and interval) of $\theta$ and we want to test our hypothesis about $\theta$.

Before proceeding the the rest of this chapter (actual chapter, ahem...), we require the following definitions.

\begin{defn}[Statistic]\index{Statistic}
\label{defn:statistic}
  A \hlnoteb{statistic}, $T = T(X) = T(X_1, ..., X_n)$, is a function of the data which does not depend on any unknown parameter(s).
\end{defn}

\begin{eg}
  Suppose $X_1, ..., X_n$ is a random sample from a distribution with $E[X_i] = \mu$ and $\Var(X_i) = \sigma^2$, where $\mu$ and $\sigma^2$ are unknown. The sample mean $\bar{X} = \frac{1}{n} \sum_{i=1}^{n} X_i$ and the sample variance $S^2 = \frac{1}{n} \sum_{i=1}^{n} (X_i - \bar{X})^2$ are statistics, while $\frac{\bar{X} - \mu}{\sigma / \sqrt{n}}$ is not a statistic.
\end{eg}

\begin{defn}[Estimators and Estimates]\index{Estimator}\index{Estimate}
\label{defn:estimator}
A statistic $T = T(X) = T(X_1, ..., X_n)$ that is used to estimate $\tau(\theta)$, a function of $\theta$, is called an \hlnoteb{estimator} of $\tau(\theta)$, and an observed value of the statistic $t = t(x) = t(x_1, ..., x_n)$ is called an \hlnoteb{estimate} of $\tau(\theta)$.
\end{defn}

\begin{eg}
  Suppose $X_1, ..., X_n$ are IID rvs with $E[X_i] = \mu$. The rv $\bar{X}$ is an estimator of $\mu$. For a given set of observations, $x_1, ..., x_n$, the number $\bar{x} = \frac{1}{n} \sum_{i=1}^{n} x_i$ is an estimate of $\mu$.
\end{eg}

There are certain ``properties'' of an estimator that we look for, in particular, for an estimator, $\tilde{\theta}$, of an unknown parameter $\theta$, we want that the estimator
\begin{itemize}
  \item is not biased, aka an \hldefn{unbiased estimator}, i.e. $E[\tilde{\theta}]$;
  \item has small variance;
  \item is consistent, i.e. $\tilde{\theta} \convp \theta$.
\end{itemize}

\subsection{Maximum Likelihood Estimation}%
\label{sub:maximum_likelihood_estimation}
% subsection maximum_likelihood_estimation

\newthought{Suppose} $X$ is a discrete rv with pf $P(X = x; \theta) = f(x; \theta), \, \theta \in \Omega$ where the scalar parameter $\theta$ is unknown. Suppose $x$ is an observed value of the rv $X$. Then the probability of observing this value is
\begin{equation*}
  P(X = x; \theta) = f(x; \theta).
\end{equation*}
With the observed value of $x$ substituted into $f(x; \theta)$, we have a function of the parameter $\theta$, referred to as the \hlnoteb{likelihood function}, and denoted $L(\theta)$. In the absense of any other information, it seems logical (temptingly so) that we should estimate the parameter $\theta$ using a value that is ``the most compatible'' with the data. E.g., we might choose the value of $\theta$ of which it maximizes the probability of the observed data, or equivalently, the value of $\theta$ which maximizes the likelihood function $L(\theta)$.

But first, let us formally state the definition of a likelihood function.

\begin{defn}[Likelihood function]\index{Likelihood function}
\label{defn:likelihood_function}
  Suppose $X$ is an rv with pf $f(x; \theta)$, where $\theta \in \Omega$ is a scalar. If $x$ is the observed data, then the \hlnoteb{likelihood function} for $\theta$ based on $x$ is
  \begin{equation*}
    L(\theta) = P(X = x; \theta) = f(x; \theta) \quad \text{ for } \theta \in \Omega.
  \end{equation*}

  Similarly so, suppose $X_1, ..., X_n$ is a random sample from a distribution, each with pf $f(x; \theta)$, and let $x_1, ..., x_n$ be the observed data. Then the \hlnoteb{likelihood function} for $\theta$ based on $x_1, ..., x_n$ is
  \begin{equation*}
    L(\theta) = P(X_1 = x_1, ..., X_n = x_n ; \theta) = \prod_{i=1}^{n} f(x_i; \theta) \quad \text{ for } \theta \in \Omega.
  \end{equation*}
\end{defn}

\begin{note}
  As discussed earlier, a natural estimate of $\theta$ is the value which maximizes the probability of the observed sample, and we denote this notion as
  \begin{equation*}
    \hat{\theta} = \argmax_{\theta \in \Omega} L(\theta)
  \end{equation*}
  and call $\hat{\theta}$ the \hldefn{maximum likelihood estimate} (ML estimate). In practice, it is often convenient to work, instead, with the natural logarithm of the likelihood function, in which we call the \hldefn{Log-likelihood}:
  \begin{equation*}
    \ell(\theta) = \ln L(\theta).
  \end{equation*}
  Note that the maximum likelihood estimate for both the likelihood function and the log-likelihood are the same since $\log$ is a strictly increasing function.
\end{note}

\begin{eg}[Example 6.1]\label{eg:6_1}
  Consider flipping a coin repeatedly, where for $i \in \mathbb{N}$
  \begin{equation*}
    X_i = \begin{cases}
      1 & \text{if the } i \text{\textsuperscript{th} flip lands on heads} \\
      0 & \text{otherwise}
    \end{cases}
  \end{equation*}
  Based on $4$ independent flips, we are given that $X_1, X_2, X_3, X_4 \sim \Bernoulli(p)$ are IID rvs. The sample has been observed as $x_1 = 1, \, x_2 = 1 \, x_3 = 0, \, x_4 = 1$. Write the probability of this sample as a function of $p$. Then, compute the likelihood function and find the ML estimate.
\end{eg}

\begin{solution}
  Since the $X_i$'s are IID rvs, we have
  \begin{align*}
    f(x_1, x_2, x_3, x_4 ; p) &= \prod_{i=1}^{4} P(X_i = x_i ; p) \\
                              &= \prod_{i=1}^{4} p^{x_i} (1 - p)^{1 - x_i} = p^{\sum_{i=1}^{4} x_i} (1 - p)^{\sum_{i=1}^{4} ( 1 - x_i )}.
  \end{align*}
  Note that the likelihood function for $p$ under the observed values is therefore
  \begin{equation*}
    L(p) = p^3 (1 - p).
  \end{equation*}
  To find the ML estimate, we do
  \begin{align}
    0 = \frac{dL(p)}{dp} = 3p^2 - 4p^3 = p^2( 3 - 4p )
  \end{align}
  and we observe that $p = 0$ or $\frac{3}{4}$. Clearly, $p = \frac{3}{4}$ maximizes the likelihood function.
\end{solution}

\begin{note}
  \newthought{We have been} talking almost solely about the discrete case, so what about the continuous case? We have that $P(X = x) = 0$, and so we consider a small neighbourhood of radius $\delta > 0$. Then for a small $\delta > 0$ around any point $x \in \supp(X)$, we have that
  \begin{equation}\label{eq:ml_estimate_for_continuous_case_eq1}
    P( x - \delta < X < x + \delta; \theta ) = \int_{x - \delta}^{x + \delta} f(t; \theta) \dif{t} \approx 2 \delta \cdot f(x; \theta).
  \end{equation}
  And so for an observed value $x$, since $\delta$ is fixed in \cref{eq:ml_estimate_for_continuous_case_eq1}, we have that the value of $\theta$ that maximizes $f(t; \theta)$ also maximizes $2 \delta \cdot f(t ; \theta)$.
\end{note}

\begin{warning}
  We shall clarify the following notations: we use
  \begin{itemize}
    \item $\tilde{\theta}$ as the estimator, which is an rv; and
    \item $\hat{\theta}$ as an estimate, which is a fixed value.
  \end{itemize}
\end{warning}

\begin{eg}[Example 6.2 (Course Notes 6.2.4)]\label{eg:6_2}
  Recall the coin flip example in \cref{eg:6_1}. Suppose $X_1, ..., X_n \sim \Bernoulli(p)$ is a sequence of IID rvs. Calculate the ML estimate of $p$.
\end{eg}

\begin{solution}
  Since $X_1, ..., X_n$ are IID, let $\vec{x} = ( x_1 \; x_2 \; \hdots \; x_n )$, then the joint distribution is
  \begin{align*}
    f(\vec{x}; p) &= \prod_{i=1}^{n} f(x_i; p) \\
                  &= \prod_{i=1}^{n} p^{x_i} ( 1 - p )^{1 - x_i} = p^{ \sum\limits_{i=1}^{n} x_i }(1 - p)^{\sum\limits_{i=1}^{n} (1 - x_i)}
  \end{align*}
  To get an ML estimator, we shall use the log-likelihood:
  \begin{equation*}
    \ell(p) = \left( \sum_{i=1}^{n} x_i \right) \ln p + \left( n - \sum_{i=1}^{n} x_i \right) \ln (1 - p)
  \end{equation*}
  and so to get the ML estimator\marginnote{Note that when looking for an \hlnoteb{ML estimate}, we should also check for the case when $\frac{\partial^2}{\partial p^2} \ell(p) \at{p = \hat{p}}{} < 0$ to ensure maximality (instead of minimality).}
  \begin{align*}
    0 = \frac{\partial}{\partial p} \ell(p) \at{p = \hat{p}}{} &= \left( \sum_{i=1}^{n} x_i \right) \left( \frac{1}{\hat{ p }} \right) - \left( n - \sum_{i=1}^{n} x_i \right) \left( \frac{1}{1 - \hat{ p }} \right) \\
                                                               &= \frac{\Sigma x_i}{\hat{p}} - \frac{n - \Sigma x_i}{1 - \hat{p}} = \frac{(1 - \hat{p}) \Sigma x_i - n\hat{p} + \hat{p} \Sigma x_i}{\hat{p} (1 - \hat{p})} \\
                                                               &= \frac{\Sigma x_i - n\hat{p}}{\hat{p}(1 - \hat{p})}
  \end{align*}
  where we represent $\sum_{i=1}^{n} x_i$ by $\Sigma x_i$ for sanity. Thus we have that\marginnote{Note that the \hlnoteb{ML estimator} in this case would be represented as
  \begin{equation*}
    \tilde{p} = \bar{X}.
  \end{equation*}}
  \begin{equation*}
    n \hat{p} = \sum_{i=1}^{n} x_i \implies \hat{p} = \frac{1}{n} \sum_{i=1}^{n} x_i = \bar{x}
  \end{equation*}
\end{solution}

\begin{eg}[Example 6.3 (Course Notes 6.2.5)]\label{eg:6_3}
  Suppose we have the data $\vec{x} = ( x_1 \; x_2 \; \hdots \; x_n )$ for the sequence of IID rvs $X_1, X_2, ..., X_n \sim \Poi(\theta)$. Find the likelihood function, log-likelihood, the ML estimate $\hat{\theta}$, and the ML estimator $\tilde{\theta}$.
\end{eg}

\begin{solution}
  Since each of the $X_i$'s are IID, we have that their joint pmf, and in particular, their likelihood function, is
  \begin{equation*}
    L(\theta) = \prod_{i=1}^{n} \frac{e^{- \theta } \theta^x}{x!} = e^{-n \theta} \theta^{ \sum\limits_{i=1}^{n} x_i } \left( \prod\limits_{i=1}^{n} x! \right)^{-1}.
  \end{equation*}
  Then the log-likelihood is
  \begin{equation*}
    \ell(\theta) = - n \theta + \left( \sum_{i=1}^{n} x_i \right) \log \theta - \sum_{i=1}^{n} \log ( x_i ! ).
  \end{equation*}
  To get the ML estimate, note that
  \begin{equation*}
    0 = \frac{\partial}{\partial \theta} \ell(\theta) \at{\theta = \hat{\theta}}{} = -n + \frac{1}{\hat{ \theta }} \sum_{i=1}^{n} x_i
  \end{equation*}
  and so
  \begin{equation*}
    \hat{\theta} = \frac{1}{n} \sum^{n}_{i=1} x_i = \bar{x}.
  \end{equation*}
  Consequently, we have that the ML estimator is
  \begin{equation*}
    \tilde{\theta} = \bar{X}.
  \end{equation*}
\end{solution}

% subsection maximum_likelihood_estimation (end)

% section estimation (end)

% chapter lecture_17_jun_28th_2018 (end)

\chapter{Lecture 18 Jul 3rd 2018}%
\label{chp:lecture_18_jul_3rd_2018}
% chapter lecture_18_jul_3rd_2018

\section{Estimation (Continued)}%
\label{sec:estimation_continued}
% section estimation_continued

\subsection{Maximum Likelihood Estimation (Continued)}%
\label{sub:maximum_likelihood_estimation_continued}
% subsection maximum_likelihood_estimation_continued

\begin{defn}[Score Function]\index{Score Function}
\label{defn:score_function}
  The \hlnoteb{score function} is defined as
  \begin{equation*}
    S(\theta) = S(\theta; x) = \frac{d}{d\theta}\ell(\theta) = \frac{d}{d\theta} \ln L(\theta) \quad \theta \in \Omega.
  \end{equation*}
\end{defn}

\begin{note}
  Notice that to find the ML estimate, we usually set the score function to zero and solve for $S(\theta) = 0$.
\end{note}

\begin{defn}[Information Function]\index{Information Function}
\label{defn:information_function}
  The \hlnoteb{information function} is defined as
  \begin{equation*}
    I(\theta) = I(\theta ; x) = - \frac{d^2}{d\theta^2} \ell(\theta) = - \frac{d^2}{d\theta^2} \ln L(\theta) \quad \theta \in \Omega
  \end{equation*}
  If $\hat{\theta}$ is the ML estimate of $\theta$, then $l(\hat{\theta})$ is called the \hldefn{observed information}.
\end{defn}

\begin{eg}\label{eg:6_2_sub1}
  Going back to our example in \cref{eg:6_2}, we had that the likelihood function was
  \begin{equation*}
    L(p) = p^{\sum\limits_{i=1}^{n} x_i} (1 - p)^{n - \sum\limits_{i=1}^{n} x_i}
  \end{equation*}
  the log-likelihood was
  \begin{equation*}
    \ell(p) = \left( \sum_{i=1}^{n} x_i \right) \ln p + \left( n - \sum_{i=1}^{n} x_i \right) \ln (1 - p).
  \end{equation*}
  So the score function is
  \begin{equation*}
    S(p) = \frac{d}{dp} \ell(p) = \frac{1}{p} \sum_{i=1}^{n} x_i - \frac{1}{1 - p} \left( n - \sum_{i=1}^{n} x_i \right)
  \end{equation*}
  Consequently, the information function is
  \begin{align*}
    I(p) &= - \frac{d^2}{dp^2} \ell(p) = - \frac{d}{dp} S(p) \\
         &= \frac{1}{p^2} \sum_{i=1}^{n} x_i + \frac{1}{(1 - p)^2} \left( n - \sum_{i=1}^{n} x_i \right)
  \end{align*}
\end{eg}

\begin{note}
  Recall from the calculus knowledge of the 2nd derivative being the ``curvature'' of the original function, and so $I(\theta)$ is a function about the curvature of the log-likelihood. In particular, we have that $I(\hat{\theta})$ tells us about the convacity of the log-likelihood function at the ML estimate.

  Note that the information function is a function that has two variables: the unknown parameter $\theta$, and the data $\vec{X} = (X_1 \; ... \; X_n)$.

  In a later lecture, we shall see how the observed information $I(\hat{\theta})$ can be used to construct approximate confidence intervals for the unknown parameter $\theta$.
\end{note}

\begin{defn}[Fisher Information]\index{Fisher Information}
\label{defn:fisher_information}
  If $\theta$ is a scalar, then the \hlnoteb{expected information}, or $\hlnoteb{Fisher information}$ (function) is given by
  \begin{equation*}
    J(\theta) = E[ I(\theta ; \vec{X}) ] = E\left[ - \frac{\partial^2}{\partial \theta^2} \ell(\theta; \vec{X}) \right] \quad \theta \in \Omega
  \end{equation*}
\end{defn}

\begin{note}
  Just to take away the layers of definitions and compare the Fisher information with our pf for the rv(s), if $X_1, ..., X_n$ is a random sample (i.e. IID rvs), each with pf $f(x; \theta)$, then
  \begin{equation*}
    J(\theta) = E\left[ - \frac{\partial^2}{\partial \theta^2} \ell(\theta; \vec{X}) \right] = nE\left[ - \frac{\partial^2}{\partial \theta^2} \ln f(\vec{X}; \theta) \right]
  \end{equation*}
  where $\vec{X} = ( X_1 \; ... \; X_n )$.
\end{note}

\begin{eg}[Example 6.4 (Course Notes 6.2.10)]
  Suppose $X_1, ..., X_n \sim \Bernoulli(p)$ is a sequence of IID rvs. We have showed in \cref{eg:6_2} that the ML estimator of $p$ is $\tilde{p} = \bar{X}$. Calculate the Fisher information and compare it with the variance of the ML estimator of $p$.
\end{eg}

\begin{solution}
  Recall that from \cref{eg:6_2_sub1}, we had
  \begin{align*}
    \ell(p) &= \left( \sum_{i=1}^{n} x_i \right) \ln p + \left( n - \sum_{i=1}^{n} x_i \right) \ln ( 1 - p ) \\
    S(p) &= \frac{1}{p} \sum_{i=1}^{n} x_i - \frac{1}{1 - p} \left( n - \sum_{i=1}^{n} x_i \right) \\
    I(p) &= \frac{1}{p^2} \sum_{i=1}^{n} x_i + \frac{1}{(1 - p)^2} \left( n - \sum_{i=1}^{n} x_i \right),
  \end{align*}
  So the Fisher information is
  \begin{align*}
    E[ I(p) ] &= E\left[ \frac{1}{p^2} \sum_{i=1}^{n} X_i + \frac{1}{(1 - p)^2} \left( n - \sum_{i=1}^{n} X_i \right) \right] \\
              &= \frac{1}{p^2} \sum_{i=1}^{n} E[X_i] + \frac{1}{(1 - p)^2} \left( n - \sum_{i=1}^{n} E[X_i] \right) \\
              &= \frac{1}{p^2} ( np ) + \frac{1}{(1 - p)^2} ( n - np ) \\
              &= \frac{n}{p} + \frac{n}{1 - p} = \frac{n}{p ( 1 - p )}.
  \end{align*}
  On the other hand, note that the variance of the ML estimator is
  \begin{equation*}
    \Var(\tilde{p}) = \Var(\bar{X}) = \frac{1}{n} \Var(X_i) = \frac{p( 1 - p )}{n}.
  \end{equation*}
\end{solution}

\begin{eg}[Example 6.5 (Course Notes 6.2.10)]
  Suppose $X_1, ..., X_n \sim \Poi(\theta)$ is a sequence of IID rvs. We showed in \cref{eg:6_3} the ML estimator of $\theta$ is $\tilde{\theta} = \bar{X}$. Calculate the Fisher information and compare it with the variance of the ML estimator of $\theta$.
\end{eg}

\begin{solution}
  We had the that log-likelihood is
  \begin{equation*}
    \ell(\theta) = -n \theta + \left( \sum_{i=1}^{n} x_i \right) \log \theta - \sum_{i=1}^{n} \log (x_i!).
  \end{equation*}
  So the score function is
  \begin{equation*}
    S(\theta) = -n + \frac{1}{\theta} \sum_{i=1}^{n} x_i,
  \end{equation*}
  and so the information function is
  \begin{equation*}
    I(\theta; x) = \frac{1}{\theta^2} \sum_{i=1}^{n} x_i.
  \end{equation*}
  Then the Fisher information is
  \begin{equation*}
    E[ I(\theta; \vec{X}) ] = \frac{n}{\theta^2} E[X_i] = \frac{n}{\theta}.
  \end{equation*}
  The variance of the ML estimator of $\theta$ is
  \begin{equation*}
    \Var(\tilde{\theta}) = \frac{1}{n} \Var(X_i) = \frac{\theta}{n}.
  \end{equation*}
\end{solution}

\begin{note}
  We observe from the above two examples that
  \begin{equation*}
    J(\theta) = \frac{1}{\Var{\tilde{\theta}}}
  \end{equation*}
  where $\tilde{\theta} = \bar{X}$. It is tempting to verify if this is the case in general, but there does not seem to be reliable sources that points to this case (at least, online searches have yet to yield me results). The lecture claims that this is only true for certain families of distributions.
\end{note}

\begin{eg}[Example 6.6 (Course Notes 6.2.12)]\label{eg:6_6}
  Suppose $X_1, ..., X_n$ is a random sample, each from a distribution with pdf
  \begin{equation*}
    f(x; \theta) = \theta x^{\theta - 1} \quad 0 \leq x \leq 1, \enspace \theta > 0.
  \end{equation*}
  Find the score function, the ML estimator of $\theta$, the information function and the observed information.
\end{eg}

\begin{solution}
  Since $X_1, ..., X_n$ are IID, we have
  \begin{equation*}
    L(\theta) = \prod_{i=1}^{n} \theta x_i^{\theta - 1} = \theta^n \left( \prod_{i=1}^{n} x_i \right)^{\theta - 1}.
  \end{equation*}
  So the log-likelihood is
  \begin{equation*}
    \ell(\theta) = n \ln \theta + ( \theta - 1 ) \ln \left( \prod_{i=1}^{n} x_i \right) = n \ln \theta + ( \theta - 1 ) \sum_{i=1}^{n} \ln x_i.
  \end{equation*}
  The score function is therefore
  \begin{equation*}
    S(\theta) = \frac{n}{\theta} + \sum_{i=1}^{n} \ln x_i,
  \end{equation*}
  and so the ML estimator of $\theta$ is
  \begin{equation*}
    \tilde{\theta} = - \frac{n}{\sum\limits_{i=1}^{n} \ln x_i}.
  \end{equation*}
  From the score function, the information function is
  \begin{equation*}
    I(\theta) = \frac{n}{\theta^2},
  \end{equation*}
  and so the observed information is
  \begin{equation*}
    I(\hat{\theta}) = \frac{n}{\left( - \frac{n}{\sum_{i=1}^{n} \ln x_i} \right)^2} = \frac{1}{n} \left( \sum_{i=1}^{n} \ln x_i \right)^2
  \end{equation*}
\end{solution}

\begin{note}
  In the above example, note that the Fisher information is
  \begin{equation*}
    J(\theta) = E[I(\theta; \vec{X})] = \frac{n}{\theta^2}.
  \end{equation*}
\end{note}

\begin{eg}[Example 6.7 (Course Notes 6.2.13)]\label{eg:eg_where_derivatives_for_mle_fail}
  Suppose $X_1, ..., X_n \sim \Unif(0, \theta)$ is a random sample. Find the ML estimator of $\theta$.
\end{eg}

\begin{solution}
  Note that $f_{X_i}(x_i; \theta) = \frac{1}{\theta}$ for $0 \leq x_i \leq \theta$. So the likelihood function is
  \begin{align*}
    L(\theta) &= \prod_{i=1}^{n} \frac{1}{\theta} \mathbb{1}_{\{0 \leq x_i \leq \theta\}} \\
              &= \theta^{-n} \mathbb{1}_{\{ 0 \leq x_1 \leq \theta, \, 0 \leq x_2 \leq \theta, \, ..., \, 0 \leq x_n \leq \theta\}} \\
              &= \theta^{-n} \mathbb{1}_{\{0 \leq x_{(n)} \leq \theta\}}
  \end{align*}
  where we use the order statistics notation to simplify the equation. Notice that if we take the derivative of the likelihood function from this point and try to get the ML estimate/estimator, we would run into the following equation:
  \begin{equation*}
    - \frac{n}{\theta^{n + 1}} = 0
  \end{equation*}
  in which we would have trouble getting the MLE.
\end{solution}

\begin{eg}[Example 6.8]
  Suppose $X_1, ..., X_n$ is a random sample, each from the $\Unif(\theta, \theta + 1)$ distribution. Find  the ML estimator of $\theta$.
\end{eg}

\begin{solution}
  This time, we have that the pdf for each $X_i$ is
  \begin{equation*}
    f_{X_i}(x_i ; \theta) = \mathbb{1}_{\{\theta \leq x_i \leq \theta + 1\}}.
  \end{equation*}
  Then the likelihood function is\marginnote{
  \begin{tikzpicture}
      \draw[->] (-0.5, 0) -- (3, 0) node[right] {$\theta$};
      \draw[->] (0, -0.5) -- (0, 2) node[above] {$L(\theta)$};
  		\draw (1,1) -- (2,1);
      \draw[dashed] (1,1) -- (1, 0) node[below] {$x_{(n)} - 1$};
      \draw[dashed] (2,1) -- (2, 0) node[below] {$x_{(1)}$};
  \end{tikzpicture}
  }
  \begin{align*}
    L(\theta) &= \prod_{i=1}^{n} f(x_i; \theta) = \mathbb{1}_{\{\theta \leq x_1 \leq \theta + 1, \, ..., \, \theta \leq x_n \leq \theta + 1\}} \\
              &= \mathbb{1}_{\{\theta \leq x_{(1)}\}} \mathbb{1}_{\{ x_{(n)} \leq \theta + 1 \} } \\
              &= \mathbb{1}_{\{\theta \leq x_{(1)}\}} \mathbb{1}_{\{ x_{(n)} - 1 \leq \theta \} } = \mathbb{1}_{\{x_{(n)} - 1 \leq \theta \leq x_{(1)} \}}
  \end{align*}
  where we, once again, use the order statistics notation. Consequently, we have the graph of $\theta$ versus $\L(\theta)$ on the right.

  Thus we observe that the MLE is not unique, since it can be any number between $x_{(n)} - 1$ and $x_{(1)}$.
\end{solution}

\begin{thm}[Invariance Property of the MLE]
\index{Invariance Property of MLE}
\label{thm:invariance_property_of_the_mle}
  Suppose $\tau = h(\theta)$ is an injective function of $\theta$. Suppose also that $\hat{\theta}$ is the ML estimator of $\theta$. Then $\hat{\tau} = h(\theta)$ is the ML estimator of $\tau$.
\end{thm}

\begin{warning}
  We are short on certain tools to actually prove this theorem.
\end{warning}

\begin{eg}[Example 6.9]
  Suppose $X_1, ..., X_n \sim f(x; \theta) = \theta x^{\theta - 1} \mathbb{1}_{\{0 < x < 1\}}$, where $\theta > 0$. Find the MLE of the median of the distribution.
\end{eg}

\begin{solution}
  Recall from \cref{eg:6_6} that we had
  \begin{equation*}
    \hat{\theta} = - \frac{n}{\sum\limits_{i=1}^{n} \ln x_i}.
  \end{equation*}
  Let $m$ be the median. The goal is to use \cref{thm:invariance_property_of_the_mle}. \hlwarn{Something feels off...}
\end{solution}

% section estimation_continued (end)

% chapter lecture_18_jul_3rd_2018 (end)

\chapter{Lecture 19 July 5th 2018}%
\label{chp:lecture_19_july_5th_2018}
% chapter lecture_19_july_5th_2018

\section{Estimation (Continued 2)}%
\label{sec:estimation_continued_2}
% section estimation_continued_2

\subsection{Maximum Likelihood Estimation (Continued 2)}%
\label{sub:maximum_likelihood_estimation_continued_2}
% subsection maximum_likelihood_estimation_continued_2

\begin{defn}[Relative Likelihood]\index{Relative Likelihood}
\label{defn:relative_likelihood}
  Suppose $X_1, ..., X_n \sim f(x; \theta)$ is an IID sequence of rvs where the likelihood function is $L(\theta)$ and the MLE of $\theta$ is $\hat{\theta}$. The \hlnoteb{relative likelihood} function is defined by
  \begin{equation*}
    R(\theta) = R(\theta; x) = \frac{L(\theta)}{L(\hat{\theta})}, \quad \theta \in \Omega
  \end{equation*}
\end{defn}

\begin{note}
  Note that since $L(\hat{\theta})$ is maximum since $\hat{\theta}$ is the maximum likelihood estimate, we have that $L(\theta) \leq L(\hat{\theta})$, and so\sidenote{Note that the likelihood function $L(\theta) = f(x; \theta)$ is nonnegative, since the cdf of a distribution is nondecreasing, and so the pf must be nonnegative.}
  \begin{equation*}
    0 \leq R(\theta) = \frac{L(\theta)}{L(\hat{\theta})} \leq 1.
  \end{equation*}
\end{note}

\begin{note}
  Given the data $\theta = (x_1 \; ... \; x_n)$, we say that
  \begin{itemize}
    \item if $R(\theta)$ is small (usual threshold is $0.1$), then we say that $\theta$ is \hldefn{implausible};
    \item if $R(\theta)$ is large (usual threshold is $0.5$), then we say that $\theta$ is \hldefn{plausible}.
  \end{itemize}
\end{note}

\begin{defn}[Likelihood Region \& Likelihood Interval]\index{Likelihood Region}\index{Likelihood Interval}
\label{defn:likelihood_region_n_likelihood_interval}
  The set of $\theta$ values for which $R(\theta) \geq p$ is called a $100p \%$ \hlnoteb{likelihood region} for $\theta$. If the region is an interval of real values, then it is called a $100p\%$ \hlnoteb{likelihood interval} (LI) for $\theta$.
\end{defn}

\begin{note}
  Using the definition, we can extend on the previous note:
  \begin{itemize}
    \item Values inside the $10\%$ LI are referred to as \hlnoteb{plausible} and values outside of this interval as \hlnoteb{implausible};
    \item Values inside a $50\%$ LI are very plausible; and
    \item Values outside a $1\%$ LI are very implausible in light of the data.
  \end{itemize}
\end{note}

As how we had the log-likelihood from the likelihood function, we can have the log relative likelihood.

\begin{defn}[Log Relative Likelihood]\index{Log Relative Likelihood}
\label{defn:log_relative_likelihood}
  The \hlnoteb{log relative likelihood} is the natural logarithm of the relative likelihood function:
  \begin{equation*}
    r(\theta) = r(\theta ; x) = \ln [ R(\theta) ] = log[ L(\theta) ] - \log[ L(\hat{\theta}) ] = \ell(\theta) - \ell(\hat{\theta})
  \end{equation*}
  for $\theta \in \Omega$.
\end{defn}

\begin{note}[Unimodality]
  If for $\theta \in \Omega$, $R(\theta)$ is unimodal as shown in the graph to the right,\marginnote{
		\begin{tikzpicture}
      \begin{axis}[
        no markers, domain=0:2, samples=100,
        axis lines*=left, xlabel=$\theta$, ylabel=$R(\theta)$,
        every axis y label/.style={at=(current axis.above origin),anchor=south},
        every axis x label/.style={at=(current axis.right of origin),anchor=west},
        height=5cm, width=5.5cm,
        xtick={4,6.5}, ytick=\empty,
        enlargelimits=false, clip=false, axis on top,
        grid = major
        ]
        \addplot [very thick,be-blue!50!dark] {gauss(1,0.5)};
      \end{axis}
      \draw[dashed] (4, 2) -- (0, 2) node[left] {$p$};
      \draw[dashed] (1, 2) -- (1, 0) node[below] {$a_p$};
      \draw[dashed] (3, 2) -- (3, 0) node[below] {$b_p$};
		\end{tikzpicture}
  } then $R(\theta) \geq p$ will give us the likelihood interval, in which this case we have that the interval is $[a_p, b_p]$.
\end{note}

\begin{eg}[Example 6.10]\label{eg:6_10}
  Let $X_1, ..., X_{100} \sim \Poi(\theta)$ be an IID sequence of rvs. Based on the observed values, you are given that $\sum_{i=1}^{100} x_i = 980$. Find a $10\%$ and $50\%$ likelihood intervals for $\theta$.
\end{eg}

\begin{solution}
  From \cref{eg:6_3}, we had
  \begin{equation*}
    L(\theta) = e^{-n \theta} \theta^{n \bar{x}} \left( \prod_{i=1}^{n} x_i! \right)^{-1} \quad \text{and } \hat{\theta} = \bar{x}.
  \end{equation*}
  The relative likelihood is therefore
  \begin{equation*}
    R(\theta) = \frac{L(\theta)}{L(\hat{\theta})} = \frac{e^{-n \theta} \theta^{n \bar{x}}}{e^{-n \hat{\theta} \hat{\theta}^{n \bar{x}}}} \left( \prod_{i=1}^{n} x_i! \right)^{-1} \left( \prod_{i=1}^{n} x_i! \right) = e^{-n ( \theta - \bar{x} )} \left( \frac{\theta}{\bar{x}} \right)^{n \bar{x}}.
  \end{equation*}
  We were given that
  \begin{equation*}
    \bar{x} = \frac{1}{100} \sum_{i=1}^{100} x_i = \frac{980}{100} = 9.8,
  \end{equation*}
  and so get
  \begin{equation*}
    R(\theta) = e^{- 100 ( \theta - 9.8 )} \left( \frac{\theta}{9.8} \right)^{980}.
  \end{equation*}
  Using WolframAlpha, we get that the $50\%$ likelihood interval for $\theta$ is $[9.436, 10.173]$. Unfortunately so, I cannot get the solution for the $10\%$ likelihood interval since the allowed free computation time is exceeded, and so I shall leave the likelihood interval as
  \begin{equation*}
    \left\{ \theta \in \Omega : e^{-100(\theta - 9.8)}\left( \frac{\theta}{9.8} \right)^{980} > 0.1 \right\}.
  \end{equation*}
\end{solution}

\begin{eg}[Example 6.11 (Course Notes 6.2.24)]\label{eg:6_11}
  Suppose $X_1, ..., X_n$ is a random sample from $\Exp(1, \theta)$. Plot the relative likelihood function for $\theta$ if $n = 20$ and $x_{(1)} = \min\{x_1, ..., x_n\} = 1$. Find the $10\%$ and $50\%$ likelihood intervals for $\theta$.
\end{eg}

\begin{solution}
  Recall the definition of the \hyperref[defn:double_parameter_exponential_distribution]{Double Parameter Exponential Distribution}. We have that the pdf of each $X_i$ is
  \begin{equation*}
    f_{X_i}(x_i) = e^{- \frac{x_i - \theta}{1}} = e^{- ( x_i - \theta )}.
  \end{equation*}
  Since the $X_i$'s form a random sample, we have that
  \begin{equation*}
    L(\theta) = \prod_{i=1}^{n} e^{- ( x_i - \theta )} \mathbb{1}_{\{x_i \geq \theta\}} = e^{- \sum_{i=1}^{n} x_i + n \theta} \; \mathbb{1}_{\{x_{(1)} \geq \theta\}}.
  \end{equation*}
  Firstly, note that $L(\theta)$ is an increasing function with respect to $\theta$. Now, note that if the indicator function is $1$, we would have that $\hat{\theta}_0 = \bar{x}$, where we denote the ``supposed'' ML estimate as $\hat{\theta}_0$. However, since the condition for the indicator function to be $1$ is that $\theta \leq x_{(1)}$, we must then have that $\hat{\theta} = x_{(1)}$, since $x_{(1)} \leq \bar{x}$ as $x_{(1)} = \min\{x_1, ..., x_n\}$.

  With that, we can calculate the relative likelihood function:
  \begin{equation*}
    R(\theta) = \frac{L(\theta)}{L(\hat{\theta})} = e^{n \left( \theta - x_{(1)} \right)} \mathbb{1}_{\{x_{(1)} \geq \theta\}}.
  \end{equation*}
  Given $n = 20$ and $x_{(1)} = 1$, we have that
  \begin{equation*}
    R(\theta) = e^{20 (\theta - 1)} \mathbb{1}_{\{\theta \leq 1\}}.
  \end{equation*}
  To get the $50\%$ likelihood interval, we have
  \begin{gather*}
    e^{20(\theta - 1)} \geq 0.5 \\
    \theta \geq 1 + \frac{1}{20} \ln 0.5 \; .
  \end{gather*}
  Note that $\ln 0.5 < 0$, so we have that the $50\%$ likelihood interval is $[1 + 0.05 \ln 0.5, \, 1]$.
  For the $10\%$ likelihood interval, we have
  \begin{gather*}
    e^{20(\theta - 1)} \geq 0.1 \\
    \theta \geq 1 + \frac{1}{20} \ln 0.1 \; .
  \end{gather*}
  Again, note that $\ln 0.1 < 0$, and so we have that the $10\%$ likelihood interval is $[1 + 0.05 \ln 0.1, \, 1]$.
\end{solution}

\newthought{We can extend} our methods analogously for the multiparameter case. Assume $\theta = ( \theta_1 \; ... \; \theta_k )^{T}$. Then the likelihood function, $L(\theta)$, is a function of $k$ parameters. Then the log-likelihood function is $\ell(\theta) = \ln L(\theta)$. From there, we can then use the same approach to get the ML estimate of $\theta$,
\begin{equation*}
  \hat{\theta} = \left( \hat{\theta}_1 \; \hdots \; \hat{\theta}_k \right)^{T} = \argmax_{\theta_1, \, ..., \, \theta_k} \ell(\theta).
\end{equation*}

We shall also note that we can extend \cref{thm:invariance_property_of_the_mle} to the multivariate case.

Furthermore, we will restate some of the definitions that were stated in univariate context for the multivariate case.

\begin{defn}[Score Vector]\index{Score Vector}
\label{defn:score_vector}
  Suppose $X_1, ..., X_n \sim f(x; \theta)$ is a sequence of IID rvs, where $\theta = (\theta_1 \; \hdots \; \theta_k)^T$. The \hlnoteb{score vector} is defined as
  \begin{equation*}
    S(\theta) = S(\theta; x) = \left[ \frac{\partial \ell}{\partial \theta_1} \; \hdots \; \frac{\partial \ell}{\partial \theta_k} \right]^T
  \end{equation*}
\end{defn}

\begin{defn}[Information Matrix]\index{Information Matrix}
\label{defn:information_matrix}
Suppose $X_1, ..., X_n \sim f(x; \theta)$ be a sequence of IID rvs, where $\theta = (\theta_1 \; \hdots \; \theta_k)^T$. The \hlnoteb{information matrix}, $I(\theta) = I(\theta; x)$, is a $k \times k$ symmetric matrix whose $(i, \, j)$ entry is given by
\begin{equation*}
  - \frac{\partial^2}{\partial \theta_i \partial \theta_j} \ell(\theta).
\end{equation*}
The constant matrix $I(\hat{\theta})$ is called the \hldefn{observed information matrix}.
\end{defn}

\begin{defn}[Fisher Information Matrix]\index{Fisher Information Matrix}
\label{defn:fisher_information_matrix}
  Suppose $X_1, ..., X_n \sim f(x; \theta))$ is a sequence of IID rvs, where $\theta = (\theta_1 \; \hdots \; \theta_k)^T$. The \hlnoteb{Fisher information matrix}, $J(\theta)$ is a $k \times k$ symmetric matrix whose $(i, \, j)$ entry is given by
  \begin{equation*}
    E\left[ - \frac{\partial^2}{\partial \theta_i \partial \theta_j} \ell(\theta; X) \right],
  \end{equation*}
  where $X = (X_1 \; \hdots X_k)^T$.
\end{defn}

\begin{defn}[Likelihood Region - Multivariate]\index{Likelihood Region}
\label{defn:likelihood_region_multivariate}
  Suppose $X_1, ..., X_n \sim f(x; \theta))$ is a sequence of IID rvs, where $\theta = (\theta_1 \; \hdots \; \theta_k)^T$. The set of $\theta = (\theta_1 \; \hdots \; \theta_k)^T$ for which $R(\theta) \geq p$, for some $0 \leq p \leq 1$, is called a $100p\%$ \hlnoteb{likelihood region} for $\theta$, which is a region in a $k$-dimensional space $\mathbb{R}^k$.
\end{defn}

\begin{note}
  Recall that in the univariate case, we can certainly check if our choice of the $\hat{\theta}$ is indeed a maximizing value by taking the second derivative of $\ell(\theta)$ or $L(\theta)$, and checking that $I(\hat{\theta}) > 0$. The analogous approach for the multivariate case is to ensure that the \hlnoteb{observed information matrix}, $I(\hat{\theta})$ is \href{https://en.wikipedia.org/wiki/Positive-definite_matrix}{positive definite}\sidenote{A matrix $M$ is called a \hlnoteb{positive definite} if the scalar $z^T M z > 0$ for every non-zero column vector $z$.}.

  We have two methods to verify that a matrix $M$ is positive definite:
  \begin{itemize}
    \item $\det M > 0$;
    \item eigenvalues of $M$ are all positive.
  \end{itemize}
\end{note}

\begin{eg}[Example 6.12 Part 1]\label{eg:6_12_pt1}
  Suppose $X_1, ..., X_n \sim \Nor\left( \mu, \sigma^2 \right)$ is a sequence of IID rvs.
  \begin{enumerate}
    \item Derive the MLE of the parameters $\mu$ and $\sigma^2$.
    \item Derive the MLE of the coefficient of variation $CV = \frac{\sigma}{\mu}$.
  \end{enumerate}
\end{eg}

\begin{solution}
  \begin{enumerate}
    \item Since the $X_i$'s are IID, we have
      \begin{equation*}
        L\left(\mu, \sigma^2\right) = ( 2 \pi )^{- \frac{n}{2}} \left( \sigma^2 \right)^{- \frac{n}{2}} \exp \left\{ - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} \left( x_i - \mu \right)^2 \right\}.
      \end{equation*}
      Then the log-likelihood is
      \begin{equation*}
        \ell\left( \mu, \sigma^2 \right) = - \frac{n}{2} \ln (2 \pi) - \frac{n}{2} \log \sigma^2 - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} ( x_i - \mu )^2.
      \end{equation*}
      Now, note that the first partial derivatives of $\ell\left(\mu, \sigma^2\right)$ are
      \begin{align*}
        \frac{\partial \ell}{\partial \mu} &= \frac{1}{\sigma^2} \sum_{i=1}^{n} ( x_i - \mu ) = \frac{1}{\sigma^2} \left( \sum_{i=1}^{n} x_i - n \mu \right) \\
        \frac{\partial \ell}{\partial \sigma^2} &= - \frac{n}{2 \sigma^2} + \frac{1}{2 \sigma^4} \sum_{i=1}^{n} (x_i - \mu)^2.
      \end{align*}
      So the score vector is
      \begin{equation*}
        S\left(\mu, \sigma^2\right) = \begin{bmatrix}
          \frac{\partial \ell}{\partial \mu} \\
          \frac{\partial \ell}{\partial \sigma^2}
        \end{bmatrix} = \begin{bmatrix}
          \frac{1}{\sigma^2} \left( \sum_{i=1}^{n} x_i - n \mu  \right) \\
          - \frac{n}{2 \sigma^2} + \frac{1}{2 \sigma^4} \sum_{i=1}^{n} ( x_i - \mu )^2
        \end{bmatrix}.
      \end{equation*}
      So the possible candidates for $\hat{\mu}$ and $\hat{\sigma}^2$ is
      \begin{equation*}
        \hat{\mu} = \bar{x} \quad \hat{\sigma}^2 = \frac{1}{n} \sum_{i=1}^{n} (x_i - \mu)^2.
      \end{equation*}
      It remains to verify that $\hat{mu}$ and $\hat{\sigma}^2$ are indeed the ``maximizers'', note that
      \begin{align*}
        \frac{\partial^2 \ell}{\partial \mu^2} &= - \frac{n}{\sigma^2} \\
        \frac{\partial^2 \ell}{\partial (\sigma^2)^2} &= \frac{n}{2 \sigma^4} - \frac{1}{\sigma^8} \sum_{i=1}^{n} (x_i - \mu)^2 \\
        \frac{\partial^2 \ell}{\partial \mu \partial \sigma^2} &= \frac{n \mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i,
      \end{align*}
      and so the information matrix is
      \begin{equation*}
        I\left(\mu, \sigma^2\right) = \begin{bmatrix}
          \frac{n}{\sigma^2} & \frac{n \mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i \\
          \frac{n \mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i & \frac{1}{\sigma^8} \sum_{i=1}^{n} (x_i - \mu)^2 - \frac{n}{2 \sigma^4}
        \end{bmatrix}.
      \end{equation*}
      Then the observed matrix is
      \begin{align*}
        I\left(\hat{\mu}, \hat{\sigma}^2\right) &= \begin{bmatrix}
          \frac{n}{\hat{\sigma}^2} & \frac{n \bar{x}}{\left(\hat{\sigma}^2\right)^2} - \frac{1}{\left(\hat{\sigma}^2\right)^2} \sum_{i=1}^{n} x_i \\
          \frac{n \bar{x}}{\left(\hat{\sigma}^2\right)^2} - \frac{1}{\left(\hat{\sigma}^2\right)^2} \sum_{i=1}^{n} x_i & \frac{1}{\left(\hat{\sigma}^2\right)^4} \sum_{i=1}^{n} (x_i - \bar{x})^2 - \frac{n}{2 \left(\hat{\sigma}^2\right)^2}
        \end{bmatrix} \\
        &= \begin{bmatrix}
        \frac{n}{\hat{\sigma}^2} & 0 \\
        0 & \frac{1}{\left(\hat{\sigma}^2\right)^2} \left( \hat{\sigma}^2 \right) - \frac{n}{2 \left(\hat{\sigma}^2\right)^2}
        \end{bmatrix} \\
        &= \begin{bmatrix}
        \frac{n}{\hat{\sigma}^2} & 0 \\
        0 & \frac{n}{\left(\hat{\sigma}^2\right)^2}
        \end{bmatrix}.
      \end{align*}
      We have that the determinant of the observed information matrix is
      \begin{equation*}
        \det I\left(\hat{\mu}, \hat{\sigma}^2\right) = \frac{n}{\hat{\sigma}^2} \cdot \frac{n}{\left(\hat{\sigma}^2\right)^2} > 0
      \end{equation*}
      since $\hat{\sigma}^2 > 0$.

    \item By \cref{thm:invariance_property_of_the_mle}, we have that the MLE of the coefficient of variation, $CV$, is\sidenote{Note that the function $f(x, y) = \frac{x}{y}$ is not injective if $x$ and $y$ are not coprime. Since both $\hat{\sigma}$ and $\hat{\mu}$ has a common multiple of $\sqrt{\frac{1}{n}}$, they are not coprime, so \hlwarn{I do not think we can actually invoke} \cref{thm:invariance_property_of_the_mle}. I shall, however, leave this answer here from the lecture.}
      \begin{equation*}
        \hat{CV} = \frac{\hat{\sigma}}{\hat{\mu}} = \frac{1}{\bar{x}} \sqrt{ \frac{\sum_{i=1}^{n} (x_i - \bar{x})^2}{n} } = \frac{\hat{\sigma}}{\sqrt{n} \bar{x}}
      \end{equation*}
  \end{enumerate}
\end{solution}

% subsection maximum_likelihood_estimation_continued_2 (end)

% section estimation_continued_2 (end)

% chapter lecture_19_july_5th_2018 (end)

\chapter{Lecture 20 Jul 10th 2018}%
\label{chp:lecture_20_jul_10th_2018}
% chapter lecture_20_jul_10th_2018

\section{Estimation (Continued 3)}%
\label{sec:estimation_continued_3}
% section estimation_continued_3

\subsection{Maximum Likelihood Estimation (Continued 3)}%
\label{sub:maximum_likelihood_estimation_continued_3}
% subsection maximum_likelihood_estimation_continued_3

\begin{eg}[Example 6.12 Part 2]\label{eg:6_12_pt2}
  Continuing with \cref{eg:6_12_pt1}, answer the following questions:
  \begin{itemize}
    \item Derive the Fisher information for the parameter vector $(\mu, \sigma^2)$.
  \end{itemize}
\end{eg}

\begin{solution}
  Recall that the information matrix was
  \begin{equation*}
    I\left(\mu, \sigma^2\right) = \begin{bmatrix}
      \frac{n}{\sigma^2} & \frac{n\mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i \\
      \frac{n\mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i & \frac{1}{\sigma^8} \sum_{i=1}^{n} (x_i - \mu)^2 - \frac{n}{2\sigma^4}
    \end{bmatrix}.
  \end{equation*}
  Thus the Fisher information matrix is
  \begin{align*}
    J\left(\mu, \sigma^2\right) &= \begin{bmatrix}
      E\left( \frac{n}{\sigma^2} \right) & E\left[ \frac{n\mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i \right] \\
      E\left[ \frac{n\mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} x_i \right] & E\left[ \frac{1}{\sigma^8} \sum_{i=1}^{n} (x_i - \mu)^2 - \frac{n}{2 \sigma^4} \right]
    \end{bmatrix} \\
    &= \begin{bmatrix}
    \frac{n}{\sigma^2} & \frac{n \mu}{\sigma^4} - \frac{1}{\sigma^4} \sum_{i=1}^{n} E[X_i] = 0 \\
    0 & \frac{n}{\sigma^6} - \frac{n}{2\sigma^4}
    \end{bmatrix}.
  \end{align*}
\end{solution}

\begin{note}
  Notice that we have, yet again, a similar scenario as when we first introduced the Fisher information in the univariate case: notice that
  \begin{equation*}
    \tilde{\mu} = \bar{X} \implies \Var(\tilde{\mu}) = \Var(\bar{X}) = \frac{\sigma^2}{n} = \frac{1}{J\left(\mu, \sigma^2\right)_{11}}
  \end{equation*}
  and
  \begin{align*}
    \tilde{\sigma}^2 = \frac{1}{n} \sum_{i=1}^{n} (X_i - \bar{X})^2 \implies& \Var\left(\tilde{ \sigma }^2\right) = \frac{1}{n^2} \Var\left((n - 1)S^2\right) \\
                                                                            &= \frac{\sigma^4}{n^2} \Var\left(\frac{n - 1}{\sigma^2} S^2\right) = \frac{\sigma^4 2(n - 1)}{n^2}
  \end{align*}
  by \hyperref[thm:properties_of_the_gaussian_distribution]{Cochran's Theorem}. Note that $\Var\left(\hat{\sigma}^2\right) \approx J(\mu, \sigma^2)_{22}$.
\end{note}

\begin{eg}[Example 6.13 (Course Notes 6.3.8)]
  Suppose $Y_i \sim \Nor\left( \alpha + \beta x_i, \sigma^2 \right)$, $i= 1, ..., n$, is a sequence of IID rvs. Show that the ML estimators of $\alpha, \beta$ and $\sigma^2$ are given by\marginnote{Note that $\tilde{\alpha}$ and $\tilde{\beta}$ are also the \hlnoteb{least squares estimators} of $\alpha$ and $\beta$.}
  \begin{align*}
    \tilde{\alpha} &= \bar{Y} - \tilde{\beta} \bar{x} \\
    \tilde{\beta} &= \frac{\sum\limits_{i=1}^{n} (x_i - \bar{x})(Y_i - \bar{Y})}{\sum\limits_{i=1}^{n} (x_i - \bar{x})^2} \\
    \tilde{\sigma}^2 &= \frac{1}{n} \sum_{i=1}^{n} \left(Y_i - \tilde{\alpha} - \tilde{\beta}x_i\right)^2
  \end{align*}
\end{eg}

\begin{solution}
  Since the $Y_i$'s constitute a random sample, the likelihood function is
  \begin{align*}
    L(\alpha, \beta, \sigma) &= \prod_{i=1}^{n} \frac{1}{\sqrt{2 \pi \sigma^2}} \exp\left\{ - \frac{(y_i - \alpha - \beta x_i)^2}{2 \sigma^2} \right\} \\
                             &= (2\pi)^{- \frac{n}{2}} \left( \sigma^2 \right)^{-\frac{n}{2}} \exp \left\{ - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} \left(y_i - \alpha - \beta x_i\right)^2 \right\}
  \end{align*}
  and consequently the log-likelihood is
  \begin{equation*}
    \ell(\alpha, \beta, \sigma) = - \frac{n}{2} \ln ( 2 \pi ) - \frac{n}{2} \ln \sigma^2 - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} \left( y_i - \alpha - \beta x_i \right)^2.
  \end{equation*}
  The first partial derivatives are
  \begin{align*}
    \frac{\partial \ell}{\partial \alpha} &= \frac{1}{\sigma^2} \sum_{i=1}^{n} (y_i - \alpha - \beta x_i) = \frac{n}{\sigma^2} \left[ \bar{y} - \alpha - \beta \bar{x} \right] \\
    \frac{\partial \ell}{\partial \beta} &= \frac{1}{\sigma^2} \sum_{i=1}^{n} \left[ x_i (y_i - \alpha - \beta x_i) \right] = \frac{1}{\sigma^2} \left[ \sum_{i=1}^{n} x_i y_i  - \alpha \sum_{i=1}^{n} x_i - \beta \sum_{i=1}^{n} x_i^2 \right] \\
    \frac{\partial \ell}{\partial \sigma} &= - \frac{n}{\sigma} + \frac{1}{\sigma^3} \sum_{i=1}^{n} \left(y_i - \alpha - \beta x_i\right)^2.
  \end{align*}
  The score vector is therefore
  \begin{equation*}
    S(\alpha, \beta, \sigma) = \begin{bmatrix}
      \frac{n}{\sigma^2} \left[ \bar{y} - \alpha - \beta \bar{x} \right] \\
      \frac{1}{\sigma^2} \left[ \sum_{i=1}^{n} x_i y_i  - \alpha \sum_{i=1}^{n} x_i - \beta \sum_{i=1}^{n} x_i^2 \right] \\
      \frac{1}{\sigma^3} \sum_{i=1}^{n} (y_i - \alpha - \beta x_i)^2 - \frac{n}{\sigma}
    \end{bmatrix}.
  \end{equation*}
  To get the candidates for respective MLE,
  \begin{align*}
    \hat{\alpha} &= \bar{y} - \hat{\beta} \bar{x} \\
    \hat{\beta} &= \frac{\sum_{i=1}^{n} x_i y_i - \hat{\alpha} \sum_{i=1}^{n} x_i}{\sum_{i=1}^{n} x_i^2} = \frac{\sum_{i=1}^{n} x_i y_i - ( \bar{y} - \hat{\beta} \bar{x} ) \sum_{i=1}^{n} x_i}{\sum_{i=1}^{n} x_i^2} \\
                &= \frac{\sum_{i=1}^{n} x_i y_i - n \bar{x}\bar{y} + n \hat{\beta} \bar{x}^2}{\sum_{i=1}^{n} x_i^2} = \frac{\sum_{i=1}^{n} x_i y_i - n \bar{x}\bar{y}}{\sum_{i=1}^{n} x_i^2 - n \bar{x}^2} \\
    \hat{\sigma}^2 &= \frac{1}{n} \sum_{i=1}^{n} \left( y_i - \alpha - \beta x_i \right)^2
  \end{align*}
  For $\hat{\beta}$, note that
  \begin{align*}
    \sum_{i=1}^{n} (x_i - \bar{x})(y_i - \bar{y}) &= \sum_{i=1}^{n} \left[ x_i y_i - \bar{x} y_i - \bar{y} x_i + \bar{x}\bar{y} \right] \\
                                                  &= \sum x_i y_i - \bar{x} \sum y_i - \bar{y} \sum x_i + n \bar{x} \bar{y} \\
                                                  &= \sum x_i y_i - \bar{x} (n \bar{y}) - \bar{y} ( n \bar{x} ) + n \bar{x} \bar{y} \\
                                                  &= \sum x_i y_i - n \bar{x} \bar{y}
  \end{align*}
  and similarly so we have
  \begin{equation*}
    \sum_{i=1}^{n} (x_i - \bar{x})^2 = \sum_{i=1}^{n} x_i^2 - n \bar{x}^2.
  \end{equation*}
  Therefore
  \begin{equation*}
    \hat{\beta} = \frac{\sum_{i=1}^{n} (x_i - \bar{x})(y_i - \bar{y})}{\sum_{i=1}^{n} (x_i - \bar{x})^2}
  \end{equation*}
  Thus the potential ML esimators are
  \begin{gather*}
    \tilde{\alpha} = \bar{Y} - \tilde{\beta}\bar{x} \\
    \tilde{\beta} = \frac{\sum_{i=1}^{n} (x_i - x)(Y_i - \bar{Y})}{\sum_{i=1}^{n} (x_i - \bar{x})^2} \\
    \tilde{\sigma}^2 = \frac{1}{n} \sum_{i=1}^{n} \left( Y_i - \tilde{\alpha} - \tilde{\beta}x_i \right)^2.
  \end{gather*}
  To verify that these are indeed maximizing estimates, we need the following second order partial derivatives:
  \begin{align*}
    \frac{\partial^2 \ell}{\partial \alpha^2} &= - \frac{n}{\sigma^2} \\
    \frac{\partial^2 \ell}{\partial \beta \partial \alpha} &= - \frac{n \bar{x}}{\sigma^2} \\
    \frac{\partial^2 \ell}{\partial \sigma \partial \alpha} &= - \frac{2n}{\sigma^3} \left[ \bar{y} - \alpha - \beta \bar{x} \right] \\
    \frac{\partial^2 \ell}{\partial \alpha \partial \beta} &= - \frac{n \bar{x}}{\sigma^2} \\
    \frac{\partial^2 \ell}{\partial \beta^2} &= - \frac{1}{\sigma^2} \sum_{i=1}^{n} x_i^2 \\
    \frac{\partial^2 \ell}{\partial \sigma \partial \beta} &= - \frac{2}{\sigma^3} \left( \sum x_i y_i - \alpha \sum x_i - \beta \sum x_i^2 \right) \\
    \frac{\partial^2 \ell}{\partial \alpha \partial \sigma} &= - \frac{2n}{\sigma^3} \left[ \bar{y} - \alpha - \beta \bar{x} \right] \\
    \frac{\partial^2 \ell}{\partial \beta \partial \sigma} &= - \frac{2}{\sigma^3} \left( \sum x_i y_i - \alpha \sum x_i - \beta \sum x_i^2 \right) \\
    \frac{\partial^2 \ell}{\partial \sigma^2} &= \frac{n}{\sigma^2} - \frac{3}{\sigma^4} \sum_{i=1}^{n} \left( y_i - \alpha - \beta x_i \right)^2.
  \end{align*}
  It remains to show that the observed information matrix is positive definite, which will not be shown since the working is too tedious.
\end{solution}

% subsection maximum_likelihood_estimation_continued_3 (end)

\subsection{Asymptotic Properties of ML Estimators}%
\label{sub:asymptotic_properties_of_ml_estimators}
% subsection asymptotic_properties_of_ml_estimators

\begin{thm}[Asymptotic Distribution of the ML Estimator]
\label{thm:asymptotic_distribution_of_the_ml_estimator}
Suppose $X = (X_1 \; \hdots \; X_n)$ be a random sample from $f(x; \theta)$. Let $\tilde{\theta}_n = \tilde{\theta}_n(X_1, ..., X_n)$ be the ML estimator of $\theta$ based on $X$. Then under certain regularity conditions\sidenote{See \cref{chp:regularity_conditions}.}:
\begin{enumerate}
  \item \hldefn{Consistency}:
    \begin{equation}\label{eq:consistency}
      \tilde{\theta}_n \convp \theta.
    \end{equation}
  \item \hldefn{Asymptotic Normality}:
    \begin{equation}\label{eq:asymptotic_normality}
      \sqrt{J(\theta)} ( \tilde{\theta}_n - \theta ) \convd Z \sim \Nor(0, 1).
    \end{equation}
  \item \hldefn{Asymptotic Distribution of Relative Likelihood}:
    \begin{equation*}
      - 2 \ln R(\theta; X) = 2 [ \ell(\tilde{\theta}_n ; X) - \ell(\theta; X) ] \convd W \sim \chi^2(1)
    \end{equation*}
    where $\theta$ is the unknown true parameter. If \hlnoteb{consistency} holds, then we call $\hat{\theta}_n$ a \hldefn{consistent estimator} of $\theta$.
\end{enumerate}
\end{thm}

For a proof of the above theorem see \textit{Casella \& Berger 2002}\cite{CasellaBerger2002}.

\begin{note}
  The above theorem implies that for sufficiently large $n$, $\hat{\theta}_n$ has, approximately, an $\Nor\left(\theta, [J(\theta)]^{-1}\right)$ distribution (using methods from \\ \noindent\cref{propo:other_limit_theorems}). $[J(\theta)]^{-1}$ is called the \hldefn{asymptotic variance} of $\hat{\theta}_n$. Consequently, for sufficiently large $n$, we have that
  \begin{equation*}
    \Var\left(\hat{\theta}_n\right) \approx [ J(\theta) ]^{-1}.
  \end{equation*}
  Note that \hlimpo{$J(\theta)$ is unknown since $\theta$ is unknown}.\sidenote{Comments from here on are my own after discussion with the lecturer. The explanation here will try to stay within the syllabus.} Since $\tilde{\theta}_n \convp \theta$, by \cref{propo:other_limit_theorems}, we have $J(\tilde{\theta}_n) \convp J(\theta)$ \sidenote{This is if $J$ is continuous on both $\theta$ and $\tilde{\theta}_n$.}. Then for large $n$, we have
  \begin{equation*}
    \Var(\tilde{\theta}_n) \approx [ J(\theta) ]^{-1} \approx [ J(\hat{\theta}_n) ]^{-1}
  \end{equation*}
  and so $\tilde{\theta}_n \sim \Nor( \theta, [ J(\theta) ]^{-1} ) \approx \Nor( \theta, [J(\tilde{\theta})]^{-1} )$, and therefore we get
  \begin{equation*}
    [J(\tilde{\theta}_n)]^{\frac{1}{2}} (\tilde{\theta}_n - \theta) \convd Z \sim \Nor(0, 1).
  \end{equation*}

  Now, note that the definition of the information function is
  \begin{equation*}
    I(\theta; X) = - \sum_{i=1}^{n} \frac{d^2}{d\theta^2} \ell(\theta; X_i).
  \end{equation*}
  According to Geyer \cite{Geyer2007}, by the \hlnotea{Law of Large Numbers},
  \begin{equation*}
    \frac{1}{n} I(\tilde{\theta}) \convp \frac{1}{n} J(\theta).
  \end{equation*}
  Therefore, by Limit Theorems, we have
  \begin{equation*}
    \frac{I(\tilde{\theta})}{J(\theta)} \convp 1
  \end{equation*}
  Then by the \hyperref[propo:other_limit_theorems]{Continuous Mapping Theorem}, we have
  \begin{equation*}
    [I(\tilde{\theta})]^{\frac{1}{2}} ( \tilde{\theta} - \theta ) \convd Z \sim \Nor(0, 1).
  \end{equation*}
  This result implies, by Limit Theorems, that for sufficiently large $n$, we have
  \begin{equation*}
    \Var(\tilde{\theta}_n) \approx [ I(\hat{\theta}) ]^{-1}.
  \end{equation*}
\end{note}

In the next section, we shall study how these results can be used to construct approximate \hlnoteb{confidence intervals} for $\theta$.

\begin{eg}[Example 6.14 (Course Notes 6.4.2)]
  Suppose $X_1, ..., X_n$ is a random sample from the $\Wei(\theta, 2)$ distribution. Verify that the \textit{consistency} and \textit{Normal distribution assumption} hold for this distribution. Also, show that
  \begin{equation*}
    [ I(\tilde{\theta}_n; X) ]^{\frac{1}{2}} ( \tilde{\theta}_n - \theta ) \convd Z \sim \Nor(0, 1).
  \end{equation*}
  Note that if $X \sim \Wei(\theta, 2)$, then $E(X^k) = \theta^k \Gamma\left( \frac{k}{2} + 1 \right)$ and its pf is
  \begin{equation*}
    f_{X_i}(x_i) = \frac{2}{\theta^2} x_i e^{- \left( \frac{x_i}{\theta} \right)^2}
  \end{equation*}
\end{eg}

\begin{solution}
  Since the $X_i$'s form a random sample, we have that the likelihood function is
  \begin{equation*}
    L(\theta) = \prod_{i=1}^{n} \frac{2}{\theta^2} x_i e^{- \left( \frac{x_i}{\theta} \right)^2} = \left( \frac{2}{\theta^2} \right)^n e^{- \frac{1}{\theta^2} \sum\limits_{i=1}^{n} x_i^2} \prod_{i=1}^{n} x_i.
  \end{equation*}
  Then the log-likelihood is
  \begin{equation*}
    \ell(\theta) = n \ln 2 - 2n \ln \theta - \frac{1}{\theta^2} \sum_{i=1}^{n} x_i^2 + \sum_{i=1}^{n} \ln x_i.
  \end{equation*}
  To get a candidate for the MLE, we have
  \begin{equation*}
    \frac{d \ell}{d \theta} = - \frac{2n}{\theta} + \frac{2}{\theta^3} \sum_{i=1}^{n} x_i^2,
  \end{equation*}
  and so
  \begin{equation*}
    \hat{\theta}^2 = \frac{1}{n} \sum_{i=1}^{n} x_i \text{ and } \tilde{\theta}^2 = \frac{1}{n} \sum_{i=1}^{n} X_i^2.
  \end{equation*}
  To verify that this candidate is indeed a maximum, note that we have
  \begin{equation}\label{eq:eg_6_14_eq1}
    \frac{d^2 \ell}{d \theta^2} = \frac{2n}{\theta^2} - \frac{6}{\theta^4} \sum_{i=1}^{n} x_i^2.
  \end{equation}
  Evaluating the above at $\hat{\theta}^2 = \frac{1}{n} \sum_{i=1}^{n} x_i^2$, we have
  \begin{equation*}
    \frac{2n}{\frac{1}{n} \sum\limits_{i=1}^{n} x_i^2} - \frac{6}{\left(\frac{1}{n}\right)^2 \left( \sum\limits_{i=1}^{n} x_i^2 \right)^2} \sum_{i=1}^{n} x_i^2 = - \frac{4n^2}{\sum\limits_{i=1}^{n} x_i^2} < 0.
  \end{equation*}
  Thus $\hat{\theta}$ is indeed the maximum likelihood estimate. Now to show consistency, note that we have
  \begin{equation*}
    E(X_i^2) = \theta^2 \Gamma(1 + 1) = \theta^2,
  \end{equation*}
  and so
  \begin{equation*}
    E\left( \tilde{\theta}^2 \right) = E\left( \frac{1}{n} \sum_{i=1}^{n} X_i^2 \right) = \frac{1}{n} \sum_{i=1}^{n} E\left(X_i^2\right) = \theta^2.
  \end{equation*}
  From here, to simplify notations, we shall write\sidenote{Note that $\bar{X^2} \neq \bar{X}^2$.}
  \begin{equation*}
    \bar{X^2} = \frac{1}{n} \sum_{i=1}^{n} X_i^2 \text{ and } \sum x_i^2 = \sum_{i=1}^{n} x_i^2.
  \end{equation*}
  By the \hyperref[eg:weak_law_of_large_numbers]{Weak Law of Large Numbers}, we have that $\bar{X^2} \convp \theta^2$. Since $\bar{X^2} \geq 0$, the function $g(x) = \sqrt{x}$ is continuous on its support, and so using \cref{propo:other_limit_theorems}, we have
  \begin{equation*}
    \tilde{\theta} = \sqrt{ \bar{X^2} } \convp \theta.
  \end{equation*}
  To show normality, note that the Fisher information is
  \begin{equation}\label{eq:eg_6_14_eq2}
    J(\theta) = E\left[ - \frac{d^2}{d \theta^2} \ell(\theta) \right] = \frac{6}{\theta^4} \sum_{i=1}^{n} E(X_i^2) - \frac{2n}{\theta^2} = \frac{4n}{\theta^2}
  \end{equation}
  where we shall note the use of \cref{eq:eg_6_14_eq1}. We want to invoke \\
  \noindent\hyperref[thm:central_limit_theorem]{CLT}, and so we also need $\Var(\bar{X^2})$.
  \begin{equation*}
    \Var(\bar{X^2}) = \frac{1}{n^2} \sum_{i=1}^{n} \Var(X_i^2) = \frac{1}{n^2} \sum_{i=1}^{n} \left[ E\left(X_i^4\right) - \left[ E\left(X_i^2\right) \right]^2 \right] = \frac{\theta^4}{n}.
  \end{equation*}
  Then by the CLT, we have
  \begin{equation*}
    \frac{\sqrt{n} ( \bar{X^2} - \theta^2 )}{\theta^2} \convd Z \sim \Nor(0, 1).
  \end{equation*}
  By \cref{propo:other_limit_theorems},
  \begin{equation*}
    \sqrt{n} ( \bar{X^2} - \theta^2 ) \convd \theta^2 Z \sim \Nor\left(0, \theta^4\right).
  \end{equation*}
  Using the same $g(x) = \sqrt{x}$ as above, by \cref{crly:_delta_method}, we have
  \begin{equation*}
    \sqrt{n}( \tilde{\theta} - \theta ) \convd \frac{\theta}{2} Z \sim \Nor\left(0, \frac{\theta^2}{4}\right)
  \end{equation*}
  Since \cref{eq:eg_6_14_eq2}, using \cref{propo:other_limit_theorems} again, we have
  \begin{equation*}
    \left[ J(\theta) \right]^{\frac{1}{2}} ( \tilde{\theta} - \theta ) = \frac{2\sqrt{n}}{\theta} ( \tilde{\theta} - \theta ) \convd \frac{2}{\theta} \cdot \frac{\theta}{2} Z = Z \sim \Nor(0, 1)
  \end{equation*}
  Now from \cref{eq:eg_6_14_eq1}, we have that the information function is
  \begin{equation*}
    I(\theta; X) = \frac{6}{\theta^4} \sum_{i=1}^{n} X_i^2 - \frac{2n}{\theta^2}.
  \end{equation*}
  Then
  \begin{equation*}
    I(\tilde{\theta}; X) = \frac{6n^2}{\left( \sum\limits_{i=1}^{n} X_i^2 \right)^2} \sum_{i=1}^{n} X_i^2 - \frac{2n^2}{ \sum\limits_{i=1}^{n} X_i^2 } = \frac{4n^2}{\sum_{i=1}^{n} X_i^2} = \frac{4n}{\tilde{\theta}^2}.
  \end{equation*}
  Since $\tilde{\theta} \convp \theta$, again by Limit Theorems, we have $\frac{\theta^2}{\tilde{\theta}^2} \convp 1$, and so
  \begin{equation*}
    \frac{\sqrt{I(\tilde{\theta}; X)}}{\sqrt{J(\theta)}} = \frac{\theta^2}{\tilde{\theta}^2} \convp 1.
  \end{equation*}
  Then
  \begin{equation*}
    [I(\tilde{\theta}; X)]^{\frac{1}{2}} ( \tilde{\theta} - \theta ) = \frac{\sqrt{I(\tilde{\theta}; X)}}{\sqrt{J(\theta)}} [ J(\theta) ]^{\frac{1}{2}} (\tilde{\theta} - \theta) \convd 1 \cdot Z = Z \sim \Nor(0, 1).
  \end{equation*}
\end{solution}

% subsection asymptotic_properties_of_ml_estimators (end)

% section estimation_continued_3 (end)

% chapter lecture_20_jul_10th_2018 (end)

\chapter{Lecture 21 Jul 12th 2018}%
\label{chp:lecture_21_jul_12th_2018}
% chapter lecture_21_jul_12th_2018

\section{Estimation (Continued 4)}%
\label{sec:estimation_continued_4}
% section estimation_continued_4

\subsection{Asymptotic Properties of ML Estimators (Continued)}%
\label{sub:asymptotic_properties_of_ml_estimators_continued}
% subsection asymptotic_properties_of_ml_estimators_continued

\begin{eg}[Example 6.15 (Course Notes 6.4.3)]
  Suppose $X_1, ..., X_n$ is a random sample from the $\Unif(0, \theta)$ distribution. We showed in \cref{eg:eg_where_derivatives_for_mle_fail} that $\tilde{\theta}_n = X_{(n)}$ is the ML estimator of $\theta$. Since the support of $X_i$ depends on $\theta$, \cref{thm:asymptotic_distribution_of_the_ml_estimator} does not hold. Show, however, that $X_{(n)}$ is still a consistent estimator of $\theta$.
\end{eg}

\begin{solution}
  Since $\theta$ is a fixed (but unknown) value, we have the option of use\\ \noindent \cref{propo:partial_converse_of_convp_implies_convd}. We have
  \begin{align*}
    F_{X_{(n)}}(t) &= P(X_{(n)} \leq t) = P(X_1 \leq t, X_2 \leq t, ..., X_n \leq t) \\
                   &= \prod_{i=1}^{n} P(X_i \leq t) \quad \because \text{ independence } \\
                   &= \prod_{i=1}^{n} \frac{t}{\theta} = \left( \frac{t}{\theta} \right)^n
  \end{align*}
  So the full expression for the CDF of $X_{(n)}$ is
  \begin{equation*}
    F_{X_{(n)}}(t) = \begin{cases}
      0 & t \leq 0 \\
      \left( \frac{t}{\theta} \right)^n & 0 < t < \theta \\
      1 & t \geq \theta
    \end{cases}.
  \end{equation*}
  Observe that
  \begin{equation*}
    \lim_{n \to \infty}F_{X_{(n)}}(t) = \begin{cases}
      0 & t < \theta \\
      1 & t \geq \theta
    \end{cases}
  \end{equation*}
  and so as $n \to \infty$, we have that $X_{(n)}$ has a denegerate distribution, and so the proof for consistency follows from \cref{propo:partial_converse_of_convp_implies_convd}.
\end{solution}

% subsection asymptotic_properties_of_ml_estimators_continued (end)

\subsection{Interval Estimators}%
\label{sub:interval_estimators}
% subsection interval_estimators

\begin{defn}[Interval Estimators]\index{Interval Estimators}\index{Interval Estimate}
\label{defn:interval_estimators}
  Suppose $X$ is an rv whose distribution depends on $\theta$. Suppose that $A(x)$ and $B(x)$ are functions such that $A(x) \leq B(x)$ for all $x \in \supp(X)$ and $\theta \in \Omega$. Let $x$ be the observed data. Then $(A(x), B(x))$ is an \hlnoteb{interval estimate} for $\theta$. The interval $(A(X), B(X))$ is an \hlnoteb{interval estimator} for $\theta$.
\end{defn}

\begin{note}
  Two interval estimators that we already know\sidenote{Well, one from STAT231, that is.}:
  \begin{enumerate}
    \item Confidence intervals
    \item Likelihood intervals
  \end{enumerate}
\end{note}

\newthought{We now consider} a general approach for constructing confidence intervals based on pivotal quantities.

\begin{defn}[Pivotal Quantity]\index{Pivotal Quantity}\index{Asymptotic Pivotal Quantity}
\label{defn:pivotal_quantity}
  Suppose $X$ is an rv whose distribution depends on $\theta$. The rv $Q(X; \theta)$ is called a \hlnoteb{pivotal quantity} if the distribution of $Q$ does not depend on $\theta$. $Q(X; \theta)$ is called an \hlnoteb{asymptotic pivotal quantity} if the limiting distribution of $Q$ as $n \to \infty$ does not depend on $\theta$.
\end{defn}

\begin{eg}[Example 6.16]\label{eg:6_16}
  Suppose $X_1, ..., X_n$ is a random sample from the $\Nor(\mu, \sigma^2)$ distribution. The following rvs are pivotal quantities.:\marginnote{
  \begin{ex}
    Show that the given rvs are pivotal quantities.
  \end{ex}}
  \begin{equation*}
    \frac{\bar{X} - \mu}{\sigma / \sqrt{n}}, \, \frac{\bar{X} - \mu}{S / \sqrt{n}}, \, \frac{\sum\limits_{i=1}^{n} (X_i - \mu)^2}{\sigma^2} \text{ and } \frac{(n - 1) S^2}{\sigma^2} = \frac{\sum\limits_{i=1}^{n} (X_i - \bar{X})^2}{\sigma^2}
  \end{equation*}
\end{eg}

\begin{eg}[Example 6.17 (Course Notes 6.5.4)]
  Suppose $X_1, ..., X_n$ is a random sample from the $\Poi(\theta)$ distribution. Show that
  \begin{equation*}
    \sqrt{n} \frac{(\bar{X}_n - \theta)}{\sqrt{\bar{X}_n}}
  \end{equation*}
  is an asymptotic pivotal quantity.
\end{eg}

\begin{solution}
  By \cref{thm:central_limit_theorem}, we have that
  \begin{equation*}
    \sqrt{n} \frac{\bar{X}_n - \theta}{\sqrt{\theta}} \convd Z \sim \Nor(0, 1).
  \end{equation*}
  By \cref{propo:other_limit_theorems}, we have\marginnote{This may be a little late but at this point, it should be clear how Limit Theorems work when it comes to playing with the Normal distribution as the asymptotic distribution. If this is not clear to you, read back from \cref{propo:other_limit_theorems}.}
  \begin{equation*}
    \sqrt{n} ( \bar{X}_n - \theta ) \convd \sqrt{\theta} Z \sim \Nor(0, \theta).
  \end{equation*}
  Now by the \hyperref[eg:weak_law_of_large_numbers]{Weak Law of Large Numbers}, we have that
  \begin{equation*}
    \bar{X}_n \convp \theta.
  \end{equation*}
  Since $\bar{X}_n$ takes only non-negative values, and $\theta > 0$, the function $g(x) = \sqrt{x}$ is well-defined on these ``points'', and so by \cref{propo:other_limit_theorems}, we have
  \begin{equation*}
    \sqrt{\bar{X}_n} \convp \sqrt{\theta}.
  \end{equation*}
  Then by \hyperref[propo:other_limit_theorems]{Slutsky's Theorem}, we have
  \begin{equation*}
    \sqrt{n} \frac{\bar{X}_n - \theta}{\sqrt{\bar{X}_n}} \convd Z \sim \Nor(0, 1).
  \end{equation*}
  We observe that the limiting distribution of $\sqrt{n}(\bar{X}_n - \theta) / \sqrt{\bar{X}_n}$ depends not on the unknown parameter $\theta$. Therefore it is a asymptotic pivotal quantity.
\end{solution}

\begin{defn}[Confidence Interval]\index{Confidence Interval}\index{Lower Confidence Limit}\index{Upper Confidence Limit}
\label{defn:confidence_interval}
  Suppose $A(X)$ and $B(X)$ are \hyperref[defn:statistic]{statistics}, and are called the \hlnoteb{lower} and \hlnoteb{upper confidence limits}, respectively. If $P[A(X) < \theta < B(X)] = p$, where $0 < p < 1$, then $(a(x), b(x))$ is called a $100p\%$ \hlnoteb{confidence interval} (CI) for $\theta$.
\end{defn}

Pivotal quantities can be used for constructing CIs in the following way: since the distribution of $Q(X; \theta)$ is known, we can write down a probability statement of the form
\begin{equation*}
  P(q_1 \leq Q(X; \theta) \leq q_2) = p.
\end{equation*}
If $Q$ is a monotone function of $\theta$, then this statement can be rewritten as
\begin{equation*}
  P[ A(X) \leq \theta \leq B(X) ] = p,
\end{equation*}
and the ``realized'' interval $[a(x), b(x)]$ is a $100p\%$ CI.

\begin{eg}[Example 6.18 (Course Notes 6.5.6)]
  Suppose $X_1, ..., X_n$ is a random sample from the $\Nor(\mu, \sigma^2)$ distribution. Use the pivotal quantities in \cref{eg:6_16} to find:
  \begin{enumerate}
    \item a $100p\%$ CI for $\mu$ if $\sigma^2$ is unknown;
    \item a $100p\%$ CI for $\sigma^2$ is $\mu$ is known.
  \end{enumerate}
\end{eg}

\begin{solution}
  \begin{enumerate}
    \item Since $\sigma^2$ is unknown, we cannot use
      \begin{equation*}
        \frac{\bar{X}_n - \mu}{\sigma / \sqrt{n}} \sim \Nor(0, 1)
      \end{equation*}
      as our pivotal quantity. We shall in fact, use
      \begin{equation*}
        \frac{\bar{X}_n - \mu}{S / \sqrt{n}} \sim t(n - 1)
      \end{equation*}
      since it contains the unknowm parameter whose confidence interval is that of what we want to construct, where we know that the rv follows a $t$-Distribution with degrees of freedom $n - 1$ from \cref{thm:gaussian_distribution}. Then since the $t$-Distribution is symmetric\sidenote{See \href{https://en.wikipedia.org/wiki/Student\%27s\_t\-distribution}{Wikipedia} for a graph of the $t$-Distribution.}, we have
      \begin{gather*}
        P\left[ -a \leq \frac{\bar{X}_n - \mu}{S / \sqrt{n}} \leq a \right] - p \\
        P\left[ \bar{X}_n - (S / \sqrt{n}) a \leq \mu \leq \bar{X}_n + (S / \sqrt{n}) a \right] = p.
      \end{gather*}
      And so the $100p\%$ CI for $\mu$ when $\sigma^2$ is unknown is
      \begin{equation*}
        \left[ \bar{X}_n - (S / \sqrt{n}) a , \bar{X}_n + (S / \sqrt{n}) a \right]
      \end{equation*}

    \item Since $\mu$ is known, we shall use the rv
      \begin{equation*}
        \frac{\sum\limits_{i=1}^{n} (X_i - \mu)^2}{\sigma^2} \sim \chi^2 (n - 1),
      \end{equation*}
      where the following of the chi-squared Distribution is from \hyperref[thm:gaussian_distribution]{Cochran's Theorem}. To construct the CI for $\sigma^2$,
      \begin{gather*}
        P\left(a \leq \frac{\sum\limits_{i=1}^{n} (X_i - \mu)^2}{\sigma^2} \leq b\right) = p \\
        P\left( \frac{1}{b} \sum_{i=1}^{n} (X_i - \mu)^2 \leq \sigma^2 \leq \frac{1}{a} \sum_{i=1}^{n} (X_i - \mu)^2 \right) = p.
      \end{gather*}
      Thus the CI for $\sigma^2$ is
      \begin{equation*}
        \left[ \frac{1}{b} \sum_{i=1}^{n} (X_i - \mu)^2, \, \frac{1}{a} \sum_{i=1}^{n} (X_i - \mu)^2 \right].
      \end{equation*}
  \end{enumerate}
\end{solution}

\begin{propo}[MLE of a Location/Scale Parameter as a Pivotal Quantity]
\label{propo:mle_of_a_location_scale_parameter_as_a_pivotal_quantity}
  Let $X = (X_1, ..., X_n)$ be a random sample from $f(x; \theta)$ and let $\tilde{\theta} = \tilde{\theta}(X)$ be the ML estimator of the scalar parameter $\theta$ based on $X$.
  \begin{enumerate}
    \item If $\theta$ is a location parameter, then $Q = \tilde{\theta} - \theta$ is a pivotal quantity.
    \item If $\theta$ is a scale parameter, then $Q = \tilde{\theta} / \theta$ is a pivotal quantity.
  \end{enumerate}
\end{propo}

\hlwarn{There is a proof out there and I wish to hunt for it but I do not have the time.}

\begin{eg}[Example 6.19]
  Suppse $X_1, ..., X_n \sim \Exp(\theta)$ is a random sample. Find a $100p\%$ equal tails CI for $\theta$. For the data $n = 15$ and $\sum_{i=1}^{15} x_i = 36$, find a $95\%$ equal tail CI for $\theta$.
\end{eg}

\begin{solution}
  Referring to \cref{eg:3_3_3}, we know that $\Exp(\theta)$ belongs to a scale parameter family of distributions. We shall proceed with finding the MLE of $\theta$ and then use \cref{propo:mle_of_a_location_scale_parameter_as_a_pivotal_quantity} to get a CI for $\theta$. Since the $X_i$'s are IID,
  \begin{align*}
    L(\theta) &= \prod_{i=1}^{n} \frac{1}{\theta} e^{- \frac{x_i}{\theta}} = \theta^{-n} \exp\left\{ - \frac{1}{\theta} \sum\limits_{i=1}^{n} x_i \right\} \\
    \ell(\theta) &= - n \ln \theta - \frac{1}{\theta} \sum_{i=1}^{n} x_i \\
    \frac{d \ell}{d \theta} &= - \frac{n}{\theta} + \frac{1}{\theta} \sum_{i=1}^{n} x_i \\
                            &\implies \hat{\theta} = \frac{1}{n} \sum_{i=1}^{n} x_i = \bar{x}.
  \end{align*}
  Thus the MLE of $\theta$ is $\tilde{\theta} = \bar{X}$. To get the $100p\%$ CI for $\theta$, we have by \cref{propo:mle_of_a_location_scale_parameter_as_a_pivotal_quantity}, $Y = \bar{X}/\theta$ is a pivotal quantity. Then
  \begin{gather*}
    P\left[ a \leq \frac{\bar{X}}{\theta} \leq b \right] = p \\
    P\left[ \frac{1}{b} \bar{X} \leq \theta \leq \frac{1}{a} \bar{X} \right] = p.
  \end{gather*}
  And so the $100p\%$ CI for $\theta$ is
  \begin{equation*}
    \left[ \frac{1}{b} \bar{X}, \, \frac{1}{a} \bar{X} \right].
  \end{equation*}
  To solve for $a$ and $b$, we would need to know what the distribution of $Y = \bar{X} / \theta$ is:
  \begin{align*}
    M_Y(t) &= E\left( \exp\left\{ \frac{t}{\theta n} \sum_{i=1}^{n} X_i\right\} \right) = E\left(e^{\frac{tX_1}{n \theta}}\right)^n \enspace \because \text{ IID } \\
           &= M_{X_1}\left( \frac{t}{n \theta} \right)^n = \left(1 - \frac{t}{n}\right)^{-n},
  \end{align*}
  which we observe that then $Y \sim \Gam\left(n, \frac{1}{n}\right)$.

  For $n = 15, \, \sum_{i=1}^{15} x_i = 36$ (and so $\bar{x} = 36/15 = 2.4$), and to find the $95\%$ equal tail CI for $\theta$, we have that
  \begin{gather*}
    P\left[ 0.559 \leq \frac{2.4}{\theta} \leq 1.566 \right] = 0.95 \\
    P\left[ 2.4/1.566 \leq \theta \leq 2.4/0.559 \right] = 0.95.
  \end{gather*}
  where the values of $a$ and $b$ are verified on $R$ using the following:
  \begin{lstlisting}[caption=R interactive code to get $a$ and $b$]
    qgamma(0.025, shape=15, rate=15)
    qgamma(0.975, shape=15, rate=15)
  \end{lstlisting}
  Thus
  \begin{gather*}
    \left[ \frac{2.4}{1.566}, \, \frac{2.4}{0.559} \right] \\
    \left[ 1.533, 4.293 \right]
  \end{gather*}
\end{solution}

\begin{eg}[Example 6.20 (Course Notes 6.5.11)]
  Suppose $X_1, ..., X_n \sim \Unif(0, \theta)$ is a random sample. Find an appropriate pivotal quantity. Determine $a$ such that
  \begin{equation*}
    \left[ \hat{\theta}, a \hat{\theta} \right]
  \end{equation*}
  is a $100p\%$ CI for $\theta$.
\end{eg}

\begin{solution}
  Since the $X_i$'s are IID, we have that
  \begin{align*}
    L(\theta) &= \prod_{i=1}^{n} \frac{1}{\theta} \mathbb{1}_{\{0 \leq x_i \leq \theta\}} = \theta^{-n} \mathbb{1}_{\{0 \leq x_{(1)} \leq x_{(n)} \leq \theta\}} \\
    \ell(\theta) &= - n \ln \theta + \ln \mathbb{1}_{\{0 \leq x_{(1)} \leq x_{(n)} \leq \theta\}}.
  \end{align*}
  Using a similar argument as in \cref{eg:6_11}, notice that $-n \ln \theta$ is a decreasing function, and so to get the maximum likelihood estimate, we want $\theta$ to be as close to $0$ as possible, but the closest $\theta$ can get to is $x_{(n)}$. Note that if the indicator function turns out to be $1$, then $\theta$ must also be $1$, but this does not maximize the likelihood function.

  So we have $\hat{\theta} = x_{(n)}$ and $\tilde{\theta} = X_{(n)}$. Note that $\Unif(0, \theta)$ is from a scale parameter family of distributions since
  \begin{equation*}
    f_{X_i}(x_i) = \frac{1}{\theta} \mathbb{1}_{\{0 \leq x_i \leq \theta\}} = \frac{1}{\theta} f_{X_i}\left(\frac{x_i}{\theta}\right).
  \end{equation*}
  Then by \cref{propo:mle_of_a_location_scale_parameter_as_a_pivotal_quantity}, $Y = \tilde{\theta} / \theta = X_{(n)} / \theta$ is a pivotal quantity. Note, then, that we want to solve for
  \begin{gather*}
    P\left[ \hat{\theta} \leq \theta \leq a \hat{\theta} \right] = p \\
    P\left[ \frac{1}{a} \leq \frac{\hat{\theta}}{\theta} \leq 1 \right] = p.
  \end{gather*}
  It suffices, from here, to find the CDF of $Y$:
  \begin{align*}
    F_Y(y) &= P(Y \leq y) = P\left( \frac{X_{(n)}}{\theta} \leq y \right) = P(X_{(n)} \leq y \theta) \\
           &= P(X_1 \leq y \theta)^n \quad \because \text{ IID } \\
           &= \left( \int_{0}^{y \theta} \frac{1}{\theta} \dif{x} \right)^n = y^n
  \end{align*}
  Thus
  \begin{gather*}
    F(1) - F\left(\frac{1}{a}\right) = p \\
    1 - \left( \frac{1}{a} \right)^n = p \\
    a = \frac{1}{\sqrt[n]{p - 1}}
  \end{gather*}
\end{solution}

% subsection interval_estimators (end)

% section estimation_continued_4 (end)

% chapter lecture_21_jul_12th_2018 (end)

\chapter{Lecture 22 Jul 17th 2018}%
\label{chp:lecture_22_jul_17th_2018}
% chapter lecture_22_jul_17th_2018

\section{Estimate (Continued 5)}%
\label{sec:estimate_continued_5}
% section estimate_continued_5

\subsection{Interval Estimators (Continued)}%
\label{sub:interval_estimators_continued}
% subsection interval_estimators_continued

\begin{eg}[Example 6.21]
  Suppose $X_1, ..., X_n \sim \Exp(1, \theta)$ is a random sample. Find an appropriate pivotal quantity. Determine $a$ such that
  \begin{equation*}
    [ \hat{\theta} - a, \, \hat{\theta} ]
  \end{equation*}
  is a $100p\%$ CI for $\theta$.
\end{eg}

\begin{solution}
  Recall from $\cref{eg:6_11}$ that the ML estimate is $\hat{\theta} = x_{(1)}$ and and the estimator is $\tilde{\theta} = X_{(1)}$. Note that $\Exp(1, \theta)$ is from a location parameter family of distributions, since
  \begin{equation*}
    f(x; 1, 0) = e^{-x} = f(x - \theta; 1, \theta).
  \end{equation*}
  We can then use \cref{propo:mle_of_a_location_scale_parameter_as_a_pivotal_quantity}, and use $Y = \tilde{\theta} - \theta$ as our pivotal quantity. Note that
  \begin{gather*}
    P\left( \hat{\theta} - a \leq \theta \leq \hat{\theta} \right) = p \\
    P( - \hat{\theta} \leq - \theta \leq a - \hat{\theta} ) = p \\
    P( 0 \leq \hat{\theta} - \theta \leq a ) = p \\
    P( 0 \leq Y \leq a ) = p
  \end{gather*}
  It remains, again, to find the CDF.
  \begin{align*}
    F_Y(y) &= P(X_{(1)} - \theta \leq y) = 1 - P(X_{(1)} > y + \theta) \\
           &= 1 - P(X_1 > y + \theta)^n \quad \because \text{ IID } \\
           &= 1 - \left[ \int_{y + \theta}^{\infty}e^{-(x + \theta)} \dif{x} \right]^n \\
           &= 1 - \left[ - e^{-(x + \theta)} \at{y + \theta}{\infty} \right]^n \\
           &= 1 - e^{-ny}
  \end{align*}
  Thus
  \begin{gather*}
    F(a) - F(0) = p \\
    1 - e^{-na} - 1 + 1 = p \\
    a = - \frac{1}{n} \ln (1 - p)
  \end{gather*}
\end{solution}

\newthought{In cases where} we cannot construct an exact pivotal quantity, we can use the limiting distribution of the ML estimator $\tilde{\theta} = \tilde{\theta}(X_1, ..., X_n)$ and construct approximate CIs.

\begin{propo}[Asymptotic Confidence Intervals]\index{Asymptotic Confidence Intervals}
\label{propo:asymptotic_confidence_intervals}
Since\sidenote{This is based on \cref{thm:asymptotic_distribution_of_the_ml_estimator}, which we could not prove, and its consequences are more than just fishy. I am also unsure whether this is presented as a proof or definition in the lecture, since there are no indications of that whatsoever both during the lectures and in the course notes.}
  \begin{equation*}
    [J(\tilde{\theta})]^{\frac{1}{2}} (\tilde{\theta} - \theta) \convd Z \sim \Nor(0, 1),
  \end{equation*}
  then $[J(\tilde{\theta})]^{\frac{1}{2}} (\tilde{\theta} - \theta)$ is an asymptotic pivotal quantity. An \hlnoteb{asymptotic $100p\%$ CI}\sidenote{Also called \hlnoteb{approximate $100p\%$ CI}.} based on this asymptotic pivotal quantity is given by
  \begin{equation*}
    [ \hat{\theta} - a[J(\hat{\theta})]^{-\frac{1}{2}}, \; \hat{\theta} + a[J(\hat{\theta})]^{-\frac{1}{2}} ]
  \end{equation*}
  where $P(-a < Z < a) = p$ and $Z \sim \Nor(0, 1)$.
\end{propo}

\begin{note}
  Similarly, since
  \begin{equation*}
    [I(\tilde{\theta})]^{\frac{1}{2}} (\tilde{\theta} - \theta) \convd Z \sim \Nor(0, 1),
  \end{equation*}
  then $[I(\tilde{\theta})]^{\frac{1}{2}} (\tilde{\theta} - \theta)$ is an asymptotic pivotal quantity. An asymptotic $100p\%$ CI based on this asymptotic pitoval quantity is given by
  \begin{equation*}
    [ \hat{\theta} - a[I(\hat{\theta})]^{-\frac{1}{2}}, \; \hat{\theta} + a[I(\hat{\theta})]^{-\frac{1}{2}} ]
  \end{equation*}
  where $I(\hat{\theta})$ is the observed information.
\end{note}

\begin{eg}[Example 6.22]
  Suppose $X \sim \Bin(n, \theta)$. Show how you would construct an approximate $100p\%$ CI for $\theta$.\end{eg}

\begin{solution}
  Since this is just a single rv, we have
  \begin{gather*}
    L(\theta) = \binom{n}{x} \theta^x ( 1 - \theta )^{n - x} \\
    \ell(\theta) = \ln \binom{n}{x} + x \ln \theta + (n - x) \ln (1 - \theta) \\
    \frac{d \ell}{d \theta} = \frac{x}{\theta} - \frac{n - x}{1 - \theta} \\
    \frac{d^2 \ell}{d \theta} = - \frac{x}{\theta^2} - \frac{n - x}{(1 - \theta)^2} < 0
  \end{gather*}
  where we note that the ML estimate is $\hat{\theta} = \frac{x}{n}$ and the estimator $\tilde{\theta} = \frac{X}{n}$, and we know that this will be the maximum since the second order derivative is always negative\sidenote{In fact, by rearranging $\frac{d \ell}{d \theta}$, we can show that it is a linear function.}. The information function is
  \begin{equation*}
    I(\theta; X) = \frac{X}{\theta} + \frac{n - X}{(1 - \theta)^2}
  \end{equation*}
  and so the Fisher information is
  \begin{equation*}
    J(\theta) = \frac{E(X)}{\theta^2} + \frac{n - E(X)}{(1 - \theta)^2} = \frac{n}{\theta} + \frac{n}{1 - \theta} = \frac{n}{\theta ( 1 - \theta )}.
  \end{equation*}
  Thus
  \begin{equation*}
    J(\hat{\theta}) = \frac{n^3}{x(n - x)}.
  \end{equation*}
  Then by \cref{propo:asymptotic_confidence_intervals}, we have that the asymptotic $100p\%$ CI for $\theta$ is
  \begin{equation*}
    \left[ \frac{x}{n} - zn \sqrt{\frac{x ( n - x )}{n}}, \; \frac{x}{n} + zn \sqrt{\frac{x (n - x)}{n}} \right],
  \end{equation*}
  where $P(-z \leq Z \leq z) = p$ where $Z \sim \Nor(0, 1)$.
\end{solution}

\begin{eg}[Example 6.23 (Course Notes 6.5.15)]\label{eg:6_23}
  Suppose $X_1, ... ,X_n$ is a random sample from the $\Poi(\theta)$ distribution. Show how you would construct an asymptotic $100p\%$ CI for $\theta$.
\end{eg}

\begin{solution}
  Since the $X_i$'s are IID, we have
  \begin{gather*}
    L(\theta) = \prod_{i=1}^{n} \frac{e^{-\theta} \theta^{x_i}}{x_i!} = e^{-n\theta} \theta^{\sum\limits_{i=1}^{n} x_i} \prod_{i=1}^{n} \frac{1}{x_i!} \\
    \ell(\theta) = -n \theta + \left( \sum_{i=1}^{n} x_i \right) \ln \theta - \sum_{i=1}^{n} \ln x_i! \\
    \frac{d \ell}{d \theta} = -n + \frac{1}{\theta} \sum_{i=1}^{n} x_i \\
    \frac{d^2 \ell}{d\theta^2} = - \frac{1}{\theta^2} \sum_{i=1}^{n} x_i.
  \end{gather*}
  The candidate for MLE from equating the score function to $0$ is $\hat{\theta} = \frac{1}{n} \sum_{i=1}^{n} x_i = \bar{x}$. We verify that it is indeed a maximum point since
  \begin{equation*}
    \frac{d^2 \ell}{d \theta^2} \at{\theta = \hat{\theta}}{} = - \frac{1}{\bar{x}} \sum_{i=1}^{n} = - \frac{1}{n} < 0.
  \end{equation*}
  The information function is
  \begin{equation*}
    I(\theta; X) = \frac{1}{\theta^2} \sum_{i=1}^{n} X_i
  \end{equation*}
  and so the Fisher information is
  \begin{equation*}
    J(\theta) = E[ I(\theta; X) ] = \frac{1}{\theta^2} n \theta = \frac{n}{\theta}.
  \end{equation*}
  Thus
  \begin{equation*}
    J(\hat{\theta}) = \frac{n}{\bar{x}}
  \end{equation*}
  Assuming that our random sample satisfies the regularity conditions, we have that the asymptotic $100p\%$ CI for $\theta$, by \cref{propo:asymptotic_confidence_intervals}, is
  \begin{equation*}
    \left[ \bar{x} - z \sqrt{\frac{\bar{x}}{n}}, \; \bar{x} + z \sqrt{\frac{\bar{x}}{n}} \right]
  \end{equation*}
\end{solution}

\begin{defn}[Likelihood Interval]\index{Likelihood Interval}
\label{defn:likelihood_interval}
  Given a random sample $X_1, ..., X_n$ from $f(x; \theta)$, the $100p\%$ \hlnoteb{likelihood interval} for $\theta$ is defined by
  \begin{equation*}
    \{ \theta : R(\theta) > p \}.
  \end{equation*}
\end{defn}

\begin{note}
  Recall the 3rd property from \cref{thm:asymptotic_distribution_of_the_ml_estimator} that\marginnote{The amount of things built on this shady theorem is insane...}
  \begin{equation*}
    -2 \log R(\theta; X) \convd W \sim \chi^2(1).
  \end{equation*}
  Since the $R(\theta)$ is a unimodal graph, we have
  \begin{align*}
    P[ R(\theta; X) \geq p ] &= P[ - 2 \ln R(\theta; X) \leq -2 \ln p ] \\
                             &\approx P(W \leq - 2 \ln p) \\
                             &= P(Z^2 \leq -2 \ln p) \\
                             &= P\left( - \sqrt{-2 \ln p} \leq Z \leq \sqrt{-2 \ln p} \right) \\
                             &= 2P\left(Z \leq \sqrt{-2 \ln p}\right) - 1.
  \end{align*}
  Consequently, if $p = 0.15$, then
  \begin{equation*}
    P[R(\theta; X) \geq 0.15] \approx 0.95
  \end{equation*}
  i.e. a $15\%$ likelihood interval is, approximately, a $95\%$ CI.
\end{note}

\begin{eg}[Example 6.24]
  In \cref{eg:6_10}, based on $X_1, ..., X_{100} \sim \Poi(\theta)$ is a random sample, where $\sum_{i=1}^{100} x_i = 980$, we derived the $10\%$ and $50\%$ likelihood intervals for $\theta$. Compare the approximate $95\%$ confidence interval based on the asymptotic distribution of the MLE to the $15\%$ likelihood interval for $\theta$.
\end{eg}

\begin{solution}
  Recall that
  \begin{equation*}
    R(\theta) = e^{-n ( \theta - \bar{x} )} \left( \frac{\theta}{\bar{x}} \right)^{n \bar{x}} = e^{-100 ( \theta - 9.8 )} \left( \frac{\theta}{980} \right)^{980}
  \end{equation*}
  and that the MLE is $\hat{\theta} = \bar{x}$. Using R \sidenote{This is actually a non-trivial process...}, one can find that the $15\%$ likelihood interval is
  \begin{equation*}
    9.2 < \theta < 10.4.
  \end{equation*}
  Again, assuming that the regularity conditions are satisfied, by \cref{propo:asymptotic_confidence_intervals}, and the previous example, the $95\%$ CI for $\theta$ is
  \begin{gather*}
    \left[ 9.8 - 1.96 \frac{\sqrt{980}}{100} , \, 9.8 + 1.96 \frac{\sqrt{980}}{100} \right] \\
    [ 9.186, 10.414 ]
  \end{gather*}
\end{solution}

% subsection interval_estimators_continued (end)

% section estimate_continued_5 (end)

% chapter lecture_22_jul_17th_2018 (end)

\chapter{Lecture 23 Jul 19th 2018}%
\label{chp:lecture_23_jul_19th_2018}
% chapter lecture_23_jul_19th_2018

\section{Hypothesis Testing}%
\label{sec:hypothesis_testing}
% section hypothesis_testing

\subsection{Introduction}%
\label{sub:introduction}
% subsection introduction

\begin{defn}[Hypothesis]\index{Hypothesis}
\label{defn:hypothesis}
  In inferential statistics, given a model $f(x; \theta)$, where $\theta = (\theta_1 \; \hdots \theta_n)^T \in \Omega$, and $\Omega$ is a parameter space, a \hlnoteb{hypothesis} is a general statement about the parameters for the model, denoted by
  \begin{equation*}
    H : \theta \in \Omega_0 \subseteq \Omega.
  \end{equation*}
\end{defn}

\begin{defn}[Null Hypothesis and Alternative Hypothesis]\index{Null Hypothesis}\index{Alternative Hypothesis}
\label{defn:null_hypothesis_and_alternative_hypothesis}
  A \hlnoteb{null hypothesis} is a hypothesis of which its default position is that there is no relationship between the two phenomena measured by the model $f(x; \theta)$. We usually denote the null hypothesis by
  \begin{equation*}
    H_0 : \theta \in \Omega_0.
  \end{equation*}
  The \hlnoteb{alternative hypothesis} is the other case of the null hypothesis; it is a hypothesis that states that there is a relationship between the two phenomena measured by the model $f(x; \theta)$. We usually denote the alternative hypothesis as
  \begin{equation*}
    H_a : \theta \notin \Omega_0.
  \end{equation*}
\end{defn}

\begin{defn}[Test of Hypothesis]\index{Test of Hypothesis}
\label{defn:test_of_hypothesis}
  A \hlnoteb{test of hypothesis} is a proedure for evaluating the strength of the evidence provided by the data against the null hypothesis.
\end{defn}

To measure the evidence against $H_0$ based on the observed data, we use a test statistic or a discrepancy measure.

\begin{defn}[Test Statistic]\index{Test Statistic}
\label{defn:test_statistic}
  A \hlnoteb{test statistic} is a \hyperref[defn:statistic]{statistic} that is used for measuring the evidence against $H_0$ using the observed data.
\end{defn}

\begin{note}
  \begin{enumerate}
    \item A small observed value of the test statistic shows close agreement between the observed data and the null hypothesis; while
    \item A large observed value of the test statistic shows poor agreement.
  \end{enumerate}
\end{note}

\begin{defn}[Significance Level and $p$-Values]\index{Significance Level}\index{$p$-Value}
\label{defn:significance_level_and_p_values}
  Assuming that $H_0$ is true, we compute the probability of observing a value of the test statistic being greater than or equal to the actual observed value. This probability is called the \hlnoteb{significance level} or \hlnoteb{$p$-value} of the data in relation with the null hypothesis.
\end{defn}

\begin{note}
  The $p$-value is the probability of observing a poor agreement between the null hypothesis and the data.
  \begin{itemize}
    \item A small $p$-value indicates that the probability that our observed test statistic occurs under the null hypothesis is low, i.e. an \hlnotec{evidence against the null hypothesis};
    \item A large $p$-value indicates that the probability that our observed test statistic occurs under the null hypothesis is high, i.e. we have \hlnotec{little evidence against the null hypothesis}.
  \end{itemize}
  This also means that the smaller the $p$-value, the stronger the evidence against the null hypothesis.
\end{note}

There are two types of errors that we may run into while performing hypothesis testing:

\begin{defn}[Type I and Type II Errors]\index{Type I Error}\index{Type II Error}
\label{defn:type_i_and_type_ii_errors}
  A \hlnoteb{Type I Error} is the case where we reject a true null hypothesis. A \hlnoteb{Type II Error} is the case where we fail to reject a false null hypothesis.

  \begin{tabular}{r | c | c}
                          & $H_0 : \theta \in \Omega_0$ & $H_a : \theta \notin \Omega_0$ \\
    \hline
    Reject $H_0$          & \hlnoteb{Type I Error}      & \faCheck \enspace (True Negatives) \\
    Fail to Reject $H_0$  & \faCheck \enspace (True Positives)   & \hlnoteb{Type II Error}
  \end{tabular}
\end{defn}

\begin{note}
  The common practice is to reduce the occurrence of a Type I Error. It is analogous to the \hlnotea{Presumption of Innocence}, where the ``null hypothesis'' is ``correct until proven wrong''. In other words, the goal of this practice is to reduce the occurrence of \hlnoteb{false positives}.
\end{note}

\begin{eg}[Example 7.11 (Course Notes 7.1.1)]\label{eg:7_1}
  Suppose $X_1, ..., X_n \sim \Nor\left(\mu, \sigma^2\right)$ is a random sample, where $\sigma^2$ is known. Suppose $n = 25, \, \sigma = 1, \, \bar{x} = 0.5$, and $\mu_0 = 0$. Perform a test of hypothesis on the null hypothesis $H_0 : \mu = \mu_0$ and state the conclusion.
\end{eg}

\begin{solution}
  By the CLT, we have
  \begin{equation*}
    Z = \frac{\bar{X} - \mu}{\sigma / \sqrt{n}} \sim \Nor(0, 1)
  \end{equation*}
  which contains all the parameters of our interest and is a statistic\sidenote{Note that it is also a pivotal quantity.}. We shall use this as a test statistic. Under $H_0$, we have $\mu = \mu_0$, i.e.
  \begin{equation*}
    \frac{\bar{X} - \mu_0}{\sigma / \sqrt{n}} \sim \Nor(0, 1).
  \end{equation*}
  Then, the observed test statistic is
  \begin{equation*}
    z_0 = \frac{\bar{x} - \mu_0}{\sigma / \sqrt{n}} = \frac{0.5 - 0}{1 / \sqrt{25}} = 2.5,
  \end{equation*}
  and so the $p$-value is
  \begin{equation*}
    p = 2P(Z \geq 2.5) \approx 0.01241923
  \end{equation*}
  by the $Z$-table. Therefore, we observe that the $p$-value is small, and so we have evidence against the null hypothesis $H_0 : \mu_0 = 0$.
\end{solution}

% subsection introduction (end)

\subsection{Likelihood Ratio Tests for Simple Hypothesis}%
\label{sub:likelihood_ratio_tests_for_simple_hypothesis}
% subsection likelihood_ratio_tests_for_simple_hypothesis

\begin{defn}[Simple Hypothesis]\index{Simple Hypothesis}
\label{defn:simple_hypothesis}
  Suppose $X_1, ..., X_n$ is a random sample from $f(x; \theta)$, where $\theta = (\theta_1 \; \hdots \; \theta_k)^T$. Suppose we wish to test $H : \theta = \theta_0$, where $\theta_0$ is a completely known $k \times 1$ vector. This hypothesis is called a \hlnoteb{simple hypothesis}, which specifies the values of all unknown parameters in the mode.
\end{defn}

\begin{defn}[Likelihood Ratio Statistic]\index{Likelihood Ratio Statistic}
\label{defn:likelihood_ratio_statistic}
The \hlnoteb{likelihood ratio} (LR) \hlnoteb{statistic} is defined as
\begin{equation*}
  \Lambda(\theta_0) = -2 \ln R(\theta_0; X) = -2 \ln \left[ \frac{L(\theta_0; X)}{L(\tilde{\theta}; X)} \right] = 2 \left[ \ell(\tilde{\theta}; X) - \ell(\theta_0; X) \right] \; ,
\end{equation*}
where $X = (X_1, ..., X_n)$ are the data and $\tilde{\theta} = \tilde{\theta}(X_1, ..., X_n)$ is the ML estimator of $\theta$. The observed likelihood ratio statistic is thus
\begin{equation*}
  \lambda(\theta_0) = - 2 \ln R(\theta_0; x)
\end{equation*}
where $x = (x_1, ..., x_n)$.
\end{defn}

Assuming $H_0$ is true, under certain regularity conditions\sidenote{This probably involves \cref{thm:asymptotic_distribution_of_the_ml_estimator}}, we have
\begin{equation*}
  \Lambda(\theta_0) = -2 \ln R(\theta_0, X) \convd W \sim \chi^2(k).
\end{equation*}
where $k$ is the number of parameters under the general hypothesis minus the number of parameters under $H_0$. Thus an approximate $p$-value is given by
\begin{equation*}
  p \approx P\left(W \geq -2 \ln R(\theta_0; x)]\right),
\end{equation*}
where $x = (x_1, ..., x_n)$ is the observed data.

\begin{eg}[Example 7.2 (Course Notes 7.2.1)]
  Continuing with \cref{eg:7_1}, find the LR statistic and compare it to the test statistic used in the example.
\end{eg}

\begin{solution}
  Let $H_0 : \mu = \mu_0$ and $H_a : \mu \neq \mu_0$. We need the MLE of $\mu$. Since the $X_i$'s forms a random sample, we have
  \begin{gather*}
    L(\mu) = \prod_{i=1}^{n} \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{(x_i - \mu)^2}{2 \sigma^2}} = (2 \pi)^{-\frac{n}{2}} \sigma^{-n} \exp\left\{ - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} (x_i - \mu)^2 \right\} \\
    \ell(\mu) = - \frac{n}{2} \ln (2 \pi) - n \ln \sigma - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} (x_i - \mu)^2 \\
    \frac{d \ell}{d \mu} = \frac{1}{\sigma^2} \sum_{i=1}^{n} (x_i - \mu) = \frac{1}{\sigma^2} ( \sum_{i=1}^{n} x_i - n \mu ) \\
    \frac{d^2 \ell}{d \mu^2} = - \frac{n}{\sigma^2} < 0
  \end{gather*}
  and so the ML estimate for $\mu$ is
  \begin{equation*}
    \hat{\mu} = \frac{1}{n} \sum_{i=1}^{n} x_i = \bar{x}.
  \end{equation*}
  Thus the likelihood ratio statistic is
  \begin{align*}
    \Lambda(\mu_0) &= -2 \ln R(\mu_0, X) = 2 [ \ell(\tilde{\mu}; X) - \ell(\mu_0; X) ] \\
                   &= \frac{1}{\sigma^2} \left( \sum_{i=1}^{n} (X_i - \mu_0)^2 - \sum_{i=1}^{n} (X_i - \tilde{\mu})^2 \right) \\
                   &= \frac{n}{\sigma^2} \left[ 2\bar{x}(\tilde{\mu} - \mu_0) + \mu_0^2 - \tilde{\mu}^2 \right]
  \end{align*}
  By our argument before this example, we have that
  \begin{align*}
    k &= \text{ number of unknown parameters } \\
      &\qquad- \text{ number of unknown parameters under } H_0 \\
      &= 1 - 0 = 1.
  \end{align*}
  Thus
  \begin{equation*}
    \Lambda(\theta_0) \convd W \sim \chi^2(1).
  \end{equation*}
  The observed likelihood ratio statistic is
  \begin{align*}
    \lambda(\mu_0) &= \frac{n}{\sigma^2} [ 2 \bar{x}( \bar{x} - \mu_0 ) + \mu_0^2 - \bar{x}^2 ] = \frac{n ( \bar{x}^2 - 2 \mu_0 \bar{x} + \mu_0^2 )}{\sigma^2} \\
                   &= \frac{n (\bar{x} - \mu_0)^2}{\sigma^2}.
  \end{align*}
  The $p$-value is therefore
  \begin{equation*}
    p \approx P(W \geq \lambda(\mu_0)) = P\left(W \geq \frac{n (\bar{x} - \mu_0)^2}{\sigma^2}\right).
  \end{equation*}
  Using the values given in \cref{eg:7_1}, we have
  \begin{equation*}
    p \approx P(W \geq 6.25) \approx 0.01241923,
  \end{equation*}
  where our value is obtained by the R code:
  \begin{lstlisting}
    1 - pchisq(6.25, 1)
  \end{lstlisting}
  We can now conclude that the two tests are similar.
\end{solution}

\begin{eg}[Example 7.3 (Course Notes 7.2.4)]
  Suppose $X_1, ..., X_n$ is a random sample from the $\Poi(\theta)$ distribution. Find the LR statistic for testing $H : \theta = \theta_0$. Another test statistic which could be used is
  \begin{equation*}
    T = \frac{\abs{\bar{X} - \theta_0}}{\sqrt{\theta_0 / n}}.
  \end{equation*}
  What is the approximate distribution of
  \begin{equation*}
    \frac{\bar{X} - \theta_0}{\sqrt{\theta_0 / n}}
  \end{equation*}
  for large $n$ if $H_0$ is true, and how could you use this to find an approximate $p$-value?
\end{eg}

\begin{solution}
  Let $H_0 : \theta \in \Omega_0$ and $H_a : \theta \notin \Omega_0$. By \cref{eg:6_23}, we have that $\hat{\theta} = \bar{x}$, and the log-likelihood is
  \begin{equation*}
    \ell(\theta) = -n\theta + \left( \sum_{i=1}^{n} x_i \right) \ln \theta - \sum_{i=1}^{n} \ln x_i! \; .
  \end{equation*}
  Thus the likelihood ratio statistic is
  \begin{equation*}
    \Lambda(\theta_0) = 2[ n\bar{X} \ln \tilde{\theta} - n \bar{X} \ln \theta_0 ] = 2n\bar{X} \ln \left( \frac{\tilde{\theta}}{\theta_0} \right)
  \end{equation*}
  and so the observed likelihood ratio statistic is
  \begin{equation*}
    \lambda(\theta_0) = 2n \bar{x} \ln \left( \frac{\bar{x}}{\theta_0} \right).
  \end{equation*}
  Thus the $p$-value under the assumption of $H_0$ is
  \begin{equation*}
    p \approx P(W \geq \lambda(\theta_0)).
  \end{equation*}
  On the other hand, by the CLT, we have that
  \begin{equation*}
    \frac{\bar{X} - \theta_0}{\sqrt{ \theta_0 / n }} \convd Z \sim \Nor(0, 1), \end{equation*}
  which then the $p$-value under this method is
  \begin{equation*}
    p \approx 2 P\left(Z \geq \abs{\frac{\bar{x} - \theta_0}{\sqrt{\theta_0 / n}}}\right).
  \end{equation*}
  However, it is not necessary that
  \begin{equation*}
    P(W \geq \lambda(\theta_0)) = 2 P\left(Z \geq \abs{\frac{\bar{x} - \theta_0}{\sqrt{\theta_0 / n}}}\right)
  \end{equation*}
\end{solution}

\begin{eg}[Example 7.4 (Course Notes 7.2.6)]
  The following table gives the observed frequencies of the six faces in $100$ rolls of a die:

  \begin{center}
  \begin{tabular}{r | c | c | c | c | c | c | c}
  Face: $i$   & 1  & 2  & 3  & 4  & 5  & 6  & Total \\
  \hline
  Freq: $f_i$ & 16 & 15 & 14 & 20 & 22 & 13 & 100
  \end{tabular}
  \end{center}

  Are these observations consistent with the hypothesis that the die is fair?
\end{eg}

\begin{solution}
  Let $\theta_i$ be the probability that we get face $i$, and let
  \begin{gather*}
    H_0 : \theta_1 = \theta_2 = \hdots = \theta_6 = \frac{1}{6} \\
    H_a : \text{ at least one of the } \theta_i \neq \theta_j \text{ for } i \neq j, \, 0 \leq i, j \leq 6
  \end{gather*}
  Note that the above experiment is of a multinomial model. Let $\theta = (\theta_1, ..., \theta_6)$. Thus we have
  \begin{equation*}
    L(\theta) = \frac{100!}{\prod_{i=1}^{6} x_i!} \theta_1^{x_1} \theta_2^{x_2} \hdots \theta_6^{x_6}.
  \end{equation*}
  With the constraint that $\sum_{i=1}^{6} \theta_i = 1$, we can use Lagrange's Multiplier\sidenote{A step of that can be found here: \href{https://math.stackexchange.com/questions/421105/maximum-likelihood-estimator-of-parameters-of-multinomial-distribution\#421838}{MathSE}.} to find that
  \begin{equation*}
    \hat{\theta}_i = \frac{x_i}{100}.
  \end{equation*}
  Then the likelihood ratio statistic is
  \begin{align*}
    \Lambda(\theta_0) &= -2 \ln \left[ \frac{L(\theta_0)}{L(\hat{\theta})} \right] = -2 \ln \left[ \frac{ \prod_{i=1}^{6} (\frac{1}{6})^{ X_i }}{\prod_{i=1}^{6} (\frac{X_i}{100})^{X_i}} \right] \\
                      &= 2 \sum_{i=1}^{6} X_i \ln \frac{6 X_i}{100} \convd W \sim \chi^2(5)
  \end{align*}
  where $\hat{\theta} = (\hat{\theta}_1, ..., \hat{\theta_6})$, and the degree of freedom is because $\theta_6 = 1 - \sum_{i=1}^{5} \theta_i$. The observed likelihood ratio statistic is
  \begin{align*}
    \lambda(\theta_0) &= 2 \sum_{i=1}^{6} x_i \ln \frac{6x_i}{100} \\
      &= -1.3063 - 3.1608 - 4.8819 + 7.2929 + 12.2158 - 6.4600 \\
      &= 3.6997 \; .
  \end{align*}
  And so the $p$-value, by
  \begin{lstlisting}
    pchisq(3.6997, 5, lower.tail=FALSE)
  \end{lstlisting}
  is
  \begin{align*}
    p &\approx P(W \geq 3.6997) \approx 0.5934.
  \end{align*}
  This shows that we do not have sufficient evidence to reject $H_0$, i.e. we have a fair die.
\end{solution}

\begin{eg}[Example 7.5]
  Suppose $X_1, ..., X_n \sim \Nor\left(\mu_0, \sigma^2\right)$ is a random sample. Test the hypothesis $H_0 : \sigma^2 = \sigma_0^2$, where the mean $\mu_0$ is known.
\end{eg}

\begin{solution}
  Let $H_a : \sigma^2 \neq \sigma_0^2$. Since the $X_i$'s are IID, we have
  \begin{gather*}
    L(\theta) = \prod_{i=1}^{n} \frac{1}{\sqrt{2 \pi \sigma^2}} \exp\left\{ - \frac{(x_i - \mu_0)^2}{2 \sigma^2} \right\} \\
    = (2 \pi)^{- \frac{n}{2}} (\sigma^2)^{-\frac{n}{2}} \exp\left\{ - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} (x_i - \mu_0)^2 \right\} \\
    \ell(\sigma^2) = - \frac{n}{2} \ln 2 \pi - \frac{n}{2} \sigma^2 - \frac{1}{2 \sigma^2} \sum_{i=1}^{n} (x_i - \mu_0)^2 \\
    \frac{d \ell}{d \sigma^2} = - \frac{n}{2 \sigma^2} + \frac{1}{2 \sigma^4} \sum_{i=1}^{n} (x_i - \mu_0)^2 \\
    \implies \hat{\sigma} = \frac{1}{n} \sum_{i=1}^{n} (x_i - \mu_0)^2 \\
    \frac{d^2 \ell}{d\left( \sigma^2 \right)^2} = \frac{n}{2 \sigma^4} - \frac{1}{\sigma^6} \sum_{i=1}^{n} (x_i - \mu_0)^2 \\
    \frac{d^2 \ell}{d\left( \sigma^2 \right)^2} \at{\sigma^2 = \hat{\sigma}^2}{} = \frac{n^3}{\left( \sum\limits_{i=1}^{n} (x_i - \mu_0)^2 \right)^2} ( \frac{1}{2} - n^3 ) < 0
  \end{gather*}
  Then, under the null hypothesis, note that
  \begin{align*}
    R(\sigma^2) &= \frac{L(\sigma_0^2)}{L(\tilde{\sigma}^2)} = \frac{\left( \sigma_0^2 \right)^{-\frac{n}{2}}}{\left( \tilde{\sigma}^2 \right)^{-\frac{n}{2}}} \exp\left\{ \left( \frac{1}{2 \tilde{\sigma}^2} - \frac{1}{2 \sigma_0^2} \right) \sum_{i=1}^{n} (X_i - \mu_0)^2 \right\} \\
                &= \left( \frac{\tilde{\sigma}^2}{\sigma_0^2} \right)^{\frac{n}{2}} \exp\left\{ \frac{n}{2} \left( 1 - \frac{\tilde{\sigma}^2}{\sigma_0^2} \right) \right\}
  \end{align*}
  and thus
  \begin{align*}
    \Lambda(\sigma_0^2) &= -2\ln R(\sigma_0^2) = - n \ln \left( \frac{\tilde{\sigma}^2}{\sigma_0^2} \right) - n \left(1 - \frac{\tilde{\sigma}^2}{\sigma_0^2}\right) \\
                        &= n \left[ \frac{\tilde{\sigma}^2}{\sigma_0^2} - 1 - \ln \left( \frac{\tilde{\sigma}^2}{\sigma_0^2} \right) \right] \convd W \sim \chi^2(1).
  \end{align*}
  The observed likelihood ratio statistic is therefore
  \begin{equation*}
    \lambda(\sigma_0^2) = n \left[ \frac{\hat{\sigma}^2}{\sigma_0^2} - 1 - \ln \left( \frac{\hat{\sigma}^2}{\sigma_0^2} \right) \right].
  \end{equation*}
  Then the $p$-value can be obtained by evaluating
  \begin{equation*}
    p \approx P\left(W \geq n \left[ \frac{\hat{\sigma}^2}{\sigma_0^2} - 1 - \ln \left( \frac{\hat{\sigma}^2}{\sigma_0^2} \right) \right]\right)
  \end{equation*}
\end{solution}

% subsection likelihood_ratio_tests_for_simple_hypothesis (end)

% section hypothesis_testing (end)

% chapter lecture_23_jul_19th_2018 (end)

\appendix

\chapter{Regularity Conditions}%
\label{chp:regularity_conditions}
% chapter regularity_conditions

Throughout this course, the phrase ``regularity conditions'' have been regularly coined but never formally introduced. To clarify that, I have sought out for the instructor and confirmed that these are the regularity conditions that are used. This is taken out from \textit{Casella \& Berger (2002)}\cite{CasellaBerger2002}.

\begin{displayquote}
  ``(R)egularity conditions'' are typically very technical, rather boring, and usually satisfied in most reasonable problems. But they are a necessary evil...
\end{displayquote}

\noindent Furthermore,

\begin{displayquote}
  These conditions mainly relate to differentiability of the density\sidenote{probability function} and the ability to interchange differentiation and integration.
\end{displayquote}

The following are the regularity conditions:

\begin{enumerate}
  \item $X_1, \, ..., \, X_n \sim f(x ; \theta)$ are IID;
  \item The unknown parameter is \hlnotea{identifiable}, i.e. if $\theta \neq \theta'$, then $f(x ; \theta) \neq f(x; \theta')$;
  \item The pf's $f(x; \theta)$ have common support, and $f(x; \theta)$ is differentiable in $\theta$;
  \item The parameter space $\Omega$ contains an open set $\omega \subset \Omega$ of which the true parameter value $\theta_0$ is an interior point\sidenote{$\omega$ is an open set, and so we can really only talk about interior points. The laymen definition of an interior point, is a point that is in a set but is not the limit point of the set.}.
  \item $\forall x \in \supp(X)$, the pf $f(x; \theta)$ is three times differentiable with respect to $\theta$, and the third derivative is continuous on $\theta$, and $\int f(x; \theta)$ can be differentiated three times under the integral sign.
  \item $\forall \theta_0 \in \Omega$, $\exists c \in \mathbb{R}_{> 0}$, $\exists$ a function $M(x)$ \sidenote{Note that both $c$ and $M(x)$ can depend on $\theta_0$, by the way that the statement is laid out.} such that $\forall x \in \supp(X)$,
    \begin{equation*}
      \abs{\theta - \theta_0} < c \implies \abs{\frac{\partial^3}{\partial \theta^3} \ln f(x; \theta)} \leq M(x)
    \end{equation*}
    with $E_{\theta_0}[M(X)] \leq \infty$ \sidenote{$E_{\theta_0}$ is the expected value operator that acts on the variable $\theta_0$.}
\end{enumerate}

% chapter regularity_conditions (end)

\chapter{Useful References}%
\label{chp:appendix}
% chapter appendix

\section{Commonly Used Distributions}%
\label{sec:commonly_used_distributions}
% section commonly_used_distributions

\begin{fullwidth}
\begin{longtable}{e m m m m}
\hline
\text{Distribution} & \text{pf} & \text{Mean} & \text{Var} & \text{mgf} \\
\hline
\multicolumn{5}{e}{\text{Binomial Distribution }: X \sim \Bin(n, p)} \\
\hline
x \in \mathbb{N} \cup \{0\}  & \multirow{3}{*}{$\binom{n}{x} p^x (1 - p)^{n - x}$} & \multirow{3}{*}{ $np$ } & \multirow{3}{*}{ $np(1 - p)$ } & \multirow{3}{*}{ $(pe^t + 1 - p)^n$ } \\
n \in \mathbb{N} & & & & \\
0 < p < 1 & & & & \\
\hline
\multicolumn{5}{e}{\text{Geometric Distribution }: X \sim \Geo(p)} \\
\hline
x \in \mathbb{N} \cup \{0\} &  \multirow{2}{*}{ $p (1 - p)^x$ } & \multirow{2}{*}{$\frac{1 - p}{p}$} & \multirow{2}{*}{ $\frac{1 - p }{p^2}$ } & \frac{p}{1 - (1 - p)e^t} \\
0 < p < 1 & & & & t < - \log ( 1- p ) \\
\hline
\multicolumn{5}{e}{\text{Poisson Distribution }: X \sim \Poi(\mu)} \\
\hline
x \in \mathbb{N} \cup \{0\} &  \multirow{2}{*}{ $\frac{e^{-\mu} \mu^x}{x!}$ } & \multirow{2}{*}{ $\mu$ } & \multirow{2}{*}{ $\mu$ } & \multirow{2}{*}{ $e^{\mu ( e^t - 1 )}$ } \\
\mu > 0 & & & & \\
\hline
\multicolumn{5}{e}{\text{Uniform Distribution } : X \sim \Unif(a, b)} \\
\hline
a \leq x \leq b & \multirow{2}{*}{ $\frac{1}{b - a}$ } & \multirow{2}{*}{ $\frac{a + b}{2}$ } & \multirow{2}{*}{ $\frac{(b - a)^2}{12}$ } & \frac{e^{bt} - e^{at}}{(b - a)^t} \\
a < b \in \mathbb{R} & & & & t \neq 0 \\
\hline
\multicolumn{5}{e}{\text{Normal Distribution } : X \sim \Nor(\mu, \sigma)} \\
\hline
x \in \mathbb{R} & \multirow{3}{*}{ $\frac{1}{\sqrt{2 \pi \sigma^2}}e^{- \frac{(x - \mu)^2}{2 \sigma^2}}$} & \multirow{3}{*}{ $\mu$ } & \multirow{3}{*}{ $\sigma^2$ } & \multirow{3}{*}{ $e^{\mu t + \frac{\sigma^2 t^2}{2}}$ } \\
\mu \in \mathbb{R} & & & & \\
\sigma > 0 & & & & \\
\hline
\multicolumn{5}{e}{\text{Gamma Distribution } : X \sim \Gamma(\alpha, \beta)} \\
\hline
x \in \mathbb{R}_{\geq 0} & \multirow{3}{*}{ $\frac{1}{\beta^\alpha \Gamma(\alpha)} x^{\alpha - 1} e^{- \frac{x}{\beta}}$ } & \multirow{3}{*}{ $\alpha \beta$ } & \multirow{3}{*}{ $\alpha \beta^2$ } & \multirow{3}{*}{ \tworow{m}{ (1 - \beta t)^{-\alpha} }{ t < \frac{1}{\beta} } } \\
\alpha > 0 & & & & \\
\beta > 0 & & & & \\
\hline
\multicolumn{5}{e}{\text{Exponential Distribution } : X \sim \Exp(\theta)} \\
\hline
x \in \mathbb{R}_{> 0} & \multirow{2}{*}{ $\frac{1}{\theta} e^{- \frac{x}{\theta}}$ } & \multirow{2}{*}{ $\theta$ } & \multirow{2}{*}{ $\theta^2$ } & (1 - \theta t)^{-1} \\
\theta > 0 & & & & t < \frac{1}{\theta} \\
\hline
\multicolumn{5}{e}{\text{Negative Binomial Distribution } : X \sim \NB(r, p)} \\
\hline
x \in \mathbb{N} \cup \{0\} & \multirow{3}{*}{ $\binom{x + r - 1}{x} \cdot ( 1 - p )^r p^{x}$ } & \multirow{3}{*}{$\frac{pr}{1 - p}$} & \multirow{3}{*}{$\frac{pr}{(1 - p)^2}$} & \left( \frac{1 - p}{1 - pe^t} \right)^r \\
r > 0 & & & & t < - \log p \\
p \in (0, 1) & & & & \\
\hline
\multicolumn{5}{e}{\text{Beta Distribution } : X \sim \BetaDist(\alpha, \beta)} \\
\hline
\tworow{m}{x \in [0, 1] or}{x \in (0, 1)} & \multirow{3}{*}{ $\frac{\Gamma(\alpha) \Gamma(\beta)}{\Gamma(a + b)} x^{\alpha - 1}(1 - x)^{\beta - 1}$ } & \multirow{3}{*}{ $\frac{\alpha}{\alpha + \beta}$ } & \multirow{3}{*}{ $\frac{\alpha \beta}{(\alpha + \beta)^2 ( \alpha + \beta + 1 )}$ } & \multirow{3}{*}{ $1 + \sum\limits_{k=1}^{\infty} \left( \prod_{r=0}^{k-1} \frac{\alpha + r}{\alpha + \beta + r} \right) \frac{t^k}{k!}$ } \\
\alpha > 0 & & & & \\
\beta > 0 & & & & \\
\hline
\multicolumn{5}{e}{\text{Chi-Squared Distribution } : X \sim \chi^2(k)} \\
\hline
k \in \mathbb{N} & \multirow{3}{*}{$\frac{1}{2^{\frac{k}{2}} \Gamma(\frac{k}{2})} x^{\frac{k}{2} - 1} e^{-\frac{x}{2}}$} & \multirow{3}{*}{ k } & \multirow{3}{*}{2k} & (1 - 2t)^{-\frac{k}{2}} \\
\tworow{e}{\text{if k = 1}}{ x \in (0, \infty)} & & & & t < \frac{1}{2} \\
\tworow{e}{\text{otherwise}}{ x \in [0, \infty]} & & & & \\
\end{longtable}
\end{fullwidth}

% section commonly_used_distributions (end)

% chapter appendix (end)

\backmatter
\pagestyle{plain}
\bibliography{references}

\printindex

\end{document}