\documentclass[notes,11pt, aspectratio=169]{beamer}
\usepackage[default]{lato}

\input{lec_style.tex}

%----------------------------------------------------------------------------------------
  %	TITLE PAGE
%----------------------------------------------------------------------------------------
\title[DAR]{Data Analytics with R}  % The short title appears at the bottom of
\author{Sumit Mishra} % Your name
\institute[IFMR] % Your institution as it will appear on the bottom of every slide, may be shorthand to save space
{
  Institute for Financial Management and Research, Sri City \\ % Your institution for the title page
  \medskip
  \medskip
  \textbf{Inference for numerical data} % Your email address
}
\date{19 December 2020} % Date, can be changed to a custom date

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  % Begin document
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Title page
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

 {
    \addtocounter{framenumber}{-1}
    {\removepagenumbers
      %\usebackgroundtemplate{\includegraphics[width=\paperwidth]{../OpenIntro_Grid_4_3-01.jpg}}
      \begin{frame}

      %\hfill \includegraphics[width=20mm]{../oiLogo_highres}

      \titlepage

      \end{frame}
    }
  }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{One-sample means with the $t$ distribution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Friday the 13$^{\text{th}}$}

\dq{Between 1990 - 1992 researchers in the UK collected data on traffic flow, accidents, and hospital admissions on Friday 13$^{\text{th}}$ and the previous Friday, Friday 6$^{\text{th}}$. Below is an excerpt from this data set on traffic flow. We can assume that traffic flow on given day at locations 1 and 2 are independent.}

{\scriptsize
\texttt{
\begin{center}
\begin{tabular}{rllrrrl}
  \hline
 & type & date & 6$^{\text{th}}$ & 13$^{\text{th}}$ & diff & location  \\
  \hline
1 & traffic & 1990,  July & 139246 & 138548 & 698 & loc 1 \\
  \rowcolor[gray]{.9}
  2 & traffic & 1990,  July & 134012 & 132908 & 1104 & loc 2 \\
  3 & traffic & 1991,  September & 137055 & 136018 & 1037 & loc 1 \\
  \rowcolor[gray]{.9}
  4 & traffic & 1991,  September & 133732 & 131843 & 1889 & loc 2 \\
  5 & traffic & 1991,  December & 123552 & 121641 & 1911 & loc 1 \\
  \rowcolor[gray]{.9}
  6 & traffic & 1991,  December & 121139 & 118723 & 2416 & loc 2 \\
  7 & traffic & 1992,  March & 128293 & 125532 & 2761 & loc 1 \\
  \rowcolor[gray]{.9}
  8 & traffic & 1992,  March & 124631 & 120249 & 4382 & loc 2 \\
  9 & traffic & 1992,  November & 124609 & 122770 & 1839 & loc 1 \\
  \rowcolor[gray]{.9}
  10 & traffic & 1992,  November & 117584 & 117263 & 321 & loc 2 \\
   \hline
\end{tabular}
\end{center}
}}

\vfill

\rule{2.5cm}{0.25pt} \\
{\tiny Scanlon, T.J., Luben, R.N., Scanlon, F.L., Singleton, N. (1993), ``Is Friday the 13th Bad For Your Health?," BMJ, 307, 1584-1586.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Friday the 13$^{\text{th}}$}

\begin{itemize}

\item We want to investigate if people's behavior is different on Friday 13$^{\text{th}}$ compared to Friday 6$^{\text{th}}$.

\pause

\item One approach is to compare the traffic flow on these two days.

\pause

\item $H_0:$ Average traffic flow on Friday 6$^{\text{th}}$ and 13$^{\text{th}}$ are equal. \\
$H_A:$ Average traffic flow on Friday 6$^{\text{th}}$ and 13$^{\text{th}}$ are different.

\pause

\end{itemize}

$\:$ \\

\dq{Each case in the data set represents traffic flow recorded at the same location in the same month of the same year: one count from Friday 6$^{\text{th}}$ and the other Friday 13$^{\text{th}}$. Are these two counts independent?}

\soln{\pause No}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Hypotheses}

\pq{What are the hypotheses for testing for a difference between the average traffic flow between Friday 6$^{\text{th}}$ and 13$^{\text{th}}$?}

\begin{enumerate}[(a)]
\item  \mathhl{H_0:} $\mu_{6th} = \mu_{13th}$ \\
\mathhl{H_A:} $\mu_{6th} \ne \mu_{13th}$
\item  \mathhl{H_0:} $p_{6th} = p_{13th}$ \\
\mathhl{H_A:} $p_{6th} \ne p_{13th}$
\solnMult{ \mathhl{H_0:} $\mu_{diff} = 0$ \\
\mathhl{H_A:} $\mu_{diff} \ne 0$ }
\item  \mathhl{H_0:} $\bar{x}_{diff} = 0$ \\
\mathhl{H_A:} $\bar{x}_{diff} = 0$
\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conditions}

\begin{itemize}

\item \hl{Independence:} We are told to assume that cases (rows) are independent.

\pause

\item \hl{Sample size / skew:} $\:$ \\

\pause

\twocol{0.75}{0.35}
{
{\tiny
\begin{itemize}

\item The sample distribution does not appear to be extremely skewed, but it's very difficult to assess with such a small sample size. We might want to think about whether we would expect the population distribution to be skewed or not -- probably not, it should be equally likely to have days with lower than average traffic and higher than average traffic.

\item We do not know $\sigma$ and $n$ is too small to assume $s$ is a reliable estimate for $\sigma$.
\end{itemize}
}
}
{
\includegraphics[width=0.9\textwidth]{graphs/trafficHist}
}

\end{itemize}

$\:$ \\

\pause

\dq{So what do we do when the sample size is small?}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Review: what purpose does a large sample serve?}

As long as observations are independent, and the population distribution is not extremely skewed, a large sample would ensure that...

\begin{itemize}

\item the sampling distribution of the mean is nearly normal

\item the estimate of the standard error, as $\frac{s}{\sqrt{n}}$, is reliable

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{The normality condition}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{The normality condition}

\begin{itemize}

\item The CLT, which states that sampling distributions will be nearly normal, holds true for \orange{any} sample size as long as the population distribution is nearly normal.

\pause

\item While this is a helpful special case, it's inherently difficult to verify normality in small data sets.

\pause

\item We should exercise caution when verifying the normality condition for small samples. It is important to not only examine the data but also think about where the data come from.
\begin{itemize}
\item For example, ask: would I expect this distribution to be symmetric, and am I confident that outliers are rare?
\end{itemize}

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Introducing the $t$ distribution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{The $t$ distribution}

\begin{itemize}

\item When the population standard deviation is unknown (almost always), the uncertainty of the standard error estimate is addressed by using a new distribution: the \hl{$t$ distribution}.

\pause

\item This distribution also has a bell shape, but its tails are \hl{thicker} than the normal model's.

\pause

\item Therefore observations are more likely to fall beyond two SDs from the mean than under the normal distribution.

\pause

\item Extra thick tails are helpful for resolving our problem with a less reliable estimate the standard error (since $n$ is small)

\end{itemize}

\begin{center}
\includegraphics[width=0.5\textwidth]{graphs/tDistCompareToNormalDist}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{The $t$ distribution (cont.)}

\begin{itemize}

\item Always centered at zero, like the standard normal ($z$) distribution.

\item Has a single parameter: \hl{degrees of freedom} (\mathhl{df}).

\end{itemize}

\begin{center}
\includegraphics[width=0.8\textwidth]{graphs/tDistConvergeToNormalDist}
\end{center}

\pause

\dq{What happens to shape of the $t$ distribution as $df$ increases?}

\soln{\pause Approaches normal.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Evaluating hypotheses using the $t$ distribution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Back to Friday the 13$^{\text{th}}$}

\vspace{-1cm}

{\footnotesize
\texttt{
\begin{center}
\begin{tabular}{rllrr || r || l}
  \hline
 & type & date & 6$^{\text{th}}$ & 13$^{\text{th}}$ & diff & location  \\
  \hline
1 & traffic & 1990,  July & 139246 & 138548 & 698 & loc 1 \\
  \rowcolor[gray]{.9}
  2 & traffic & 1990,  July & 134012 & 132908 & 1104 & loc 2 \\
  3 & traffic & 1991,  September & 137055 & 136018 & 1037 & loc 1 \\
  \rowcolor[gray]{.9}
  4 & traffic & 1991,  September & 133732 & 131843 & 1889 & loc 2 \\
  5 & traffic & 1991,  December & 123552 & 121641 & 1911 & loc 1 \\
  \rowcolor[gray]{.9}
  6 & traffic & 1991,  December & 121139 & 118723 & 2416 & loc 2 \\
  7 & traffic & 1992,  March & 128293 & 125532 & 2761 & loc 1 \\
  \rowcolor[gray]{.9}
  8 & traffic & 1992,  March & 124631 & 120249 & 4382 & loc 2 \\
  9 & traffic & 1992,  November & 124609 & 122770 & 1839 & loc 1 \\
  \rowcolor[gray]{.9}
  10 & traffic & 1992,  November & 117584 & 117263 & 321 & loc 2 \\
   \hline
\end{tabular}
\end{center}
}}

\begin{align*}
\hspace{6cm} &\orange{$\bar{x}_{diff} = 1836$} \\
& \orange{$s_{diff} = 1176$} \\
& \orange{$n = 10$}
\end{align*}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Finding the test statistic}

\formula{Test statistic for inference on a small sample mean}
{The test statistic for inference on a small sample ($n < 50$) mean is the $T$ statistic with $df = n - 1$.
\[ T_{df} = \frac{\text{point estimate} - \text{null value}}{SE} \]}

\pause

\vspace{-0.5cm}

\hl{in context...}
\begin{eqnarray*}
point~estimate &=& \bar{x}_{diff} = 1836 \\
\pause
SE &=& \frac{s_{diff}}{\sqrt{n}} = \frac{1176}{\sqrt{10}} = 372 \\
\pause
T &=& \frac{1836 - 0}{372} = 4.94 \\
\pause
df &=& 10 - 1 = 9
\end{eqnarray*}

\vspace{-0.25cm}

\Note{Null value is 0 because in the null hypothesis we set $\mu_{diff} = 0$.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{Finding the p-value}

\begin{itemize}

\item The p-value is, once again, calculated as the area tail area under the $t$ distribution.

\pause

\item Using R:
\begin{verbatim}
> 2 * pt(4.94, df = 9, lower.tail = FALSE)

[1] 0.0008022394
\end{verbatim}

\pause

\item Using a web app:

\href{https://gallery.shinyapps.io/dist_calc/}{\textcolor{oiB}{https://gallery.shinyapps.io/dist\_calc/}}

\pause

\item Or when these aren't available, we can use a $t$-table.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conclusion of the test}

\dq{What is the conclusion of this hypothesis test?}

\pause

Since the p-value is quite low, we conclude that the data provide strong evidence of a difference between traffic flow on Friday 6$^{\text{th}}$ and 13$^{\text{th}}$.

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Constructing confidence intervals using the $t$ distribution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{What is the difference?}

\begin{itemize}

\item We concluded that there is a difference in the traffic flow between Friday 6$^{\text{th}}$ and 13$^{\text{th}}$.

\pause

\item But it would be more interesting to find out what exactly this difference is.

\pause

\item We can use a confidence interval to estimate this difference.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Confidence interval for a small sample mean}

\begin{itemize}

\item Confidence intervals are always of the form
\[ \text{point estimate} \pm {ME} \]

\pause

\item ME is always calculated as the product of a critical value and SE.

\pause

\item Since small sample means follow a $t$ distribution (and not a $z$ distribution), the critical value is a $t^\star$ (as opposed to a $z^\star$).
\[ \text{point estimate} \pm t^{\star} \times SE \]

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{Finding the critical $t$ ($t^\star$)}

Using R:
\begin{verbatim}

> qt(p = 0.975, df = 9)

[1] 2.262157

\end{verbatim}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Constructing a CI for a small sample mean}

\pq{Which of the following is the correct calculation of a 95\% confidence interval for the difference between the traffic flow between Friday 6$^{\text{th}}$ and 13$^{\text{th}}$?}
\[ \bar{x}_{diff} = 1836 \qquad s_{diff} = 1176 \qquad n = 10 \qquad SE = 372 \]

\twocol{0.35}{0.65}
{
\begin{enumerate}[(a)]

\item $1836 \pm 1.96 \times 372$

\solnMult{ $1836 \pm 2.26 \times 372$}

\item $1836 \pm -2.26 \times 372$

\item $1836 \pm 2.26 \times 1176$

\end{enumerate}
}
{
\soln{\only<2>{\orange{$\rightarrow$ (995, 2677)}}}
\vspace{0.25cm}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Interpreting the CI}

\pq{Which of the following is the \orange{best} interpretation for the confidence interval we just calculated?
\[ \mu_{diff: 6th - 13th} = (995, 2677) \]
}

We are 95\% confident that ...

\begin{enumerate}[(a)]

\item the difference between the average number of cars on the road on Friday 6$^{\text{th}}$ and 13$^{\text{th}}$ is between 995 and 2,677.

\item on Friday 6$^{\text{th}}$ there are 995 to 2,677 fewer cars on the road than on the Friday 13$^{\text{th}}$, on average.

\item on Friday 6$^{\text{th}}$ there are 995 fewer to 2,677 more cars on the road than on the Friday 13$^{\text{th}}$, on average.

\solnMult{ on Friday 13$^{\text{th}}$ there are 995 to 2,677 fewer cars on the road than on the Friday 6$^{\text{th}}$, on average.}

\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Synthesis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Synthesis}

\dq{Does the conclusion from the hypothesis test agree with the findings of the confidence interval?}

$\:$ \\

\soln{\only<2->{Yes, the hypothesis test found a significant difference, and the CI does not contain the null value of 0.}}

$\:$ \\

\dq{Do you think the findings of this study suggests that people believe Friday 13$^{\text{th}}$ is   a day of bad luck?}

$\:$ \\

\soln{\only<3>{No, this is an observational study. We have just observed a significant difference between the number of cars on the road on these two days. We have not tested for people's beliefs.}}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Recap: Inference using the $t$-distribution}

\begin{itemize}

\item If $\sigma$ is unknown, use the $t$-distribution with $SE = \frac{s}{\sqrt{n}}$.

\pause

\item Conditions:
\begin{itemize}
\item independence of observations (often verified by random sample, and if sampling w/o replacement, $n < $ 10\% of population)
\item no extreme skew
\end{itemize}

\pause

\item Hypothesis testing:
\[ T_{df} = \frac{\text{point estimate} - \text{null value}}{SE}\text{, where }df = n - 1 \]

\pause

\item Confidence interval: $\text{point estimate} \pm t_{df}^\star \times SE$

\end{itemize}

\pause

\vspace{-0.25cm}

\Note{The example we used was for paired means (difference between dependent groups). We took the difference between the observations and used only these differences (one sample) in our analysis, therefore the mechanics are the same as when we are working with just one sample.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Paired data}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Paired observations}

\begin{frame}

\dq{200 observations were randomly sampled from the High School and Beyond survey. The same students took a reading and writing test and their scores are shown below. At a first glance, does there appear to be a difference between the average reading and writing test score?}

\begin{center}
\includegraphics[width=0.5\textwidth]{graphs/hsb2_read_write_box}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}

\pq{The same students took a reading and writing test and their scores are shown below. Are the reading and writing scores of each student independent of each other?}

{\small
\begin{center}
\begin{tabular}{rrrr}
  \hline
 & id & read & write \\
  \hline
1 & 70 & 57 & 52 \\
  2 & 86 & 44 & 33 \\
  3 & 141 & 63 & 44 \\
  4 & 172 & 47 & 52 \\
  $\vdots$ &   $\vdots$  &   $\vdots$ &   $\vdots$  \\
  200 & 137 & 63 & 65 \\
   \hline
\end{tabular}
\end{center}
}

\begin{multicols}{2}
\begin{enumerate}[(a)]
\item Yes
\solnMult{No}
\end{enumerate}
\end{multicols}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Analyzing paired data}

\begin{itemize}

\item When two sets of observations have this special correspondence (not independent), they are said to be \hl{paired}.

\pause

\item To analyze paired data, it is often useful to look at the difference in outcomes of each pair of observations.
\[ \text{diff} = \text{read} - \text{write} \]

\pause

\item It is important that we always subtract using a consistent order.

\end{itemize}

\twocol{0.5}{0.5}{
{\small
\begin{center}
\begin{tabular}{rrrr >{\columncolor[gray]{.9}}r}
  \hline
 & id & read & write & diff \\
  \hline
1 & 70 & 57 & 52 & 5 \\
  2 & 86 & 44 & 33 & 11 \\
  3 & 141 & 63 & 44 & 19 \\
  4 & 172 & 47 & 52 & -5 \\
  $\vdots$ &   $\vdots$  &   $\vdots$ &   $\vdots$ &   $\vdots$ \\
  200 & 137 & 63 & 65 & -2 \\
   \hline
\end{tabular}
\end{center}
}
}
{
\includegraphics[width=\textwidth]{graphs/hsb2_diff_hist}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Parameter and point estimate}

\begin{itemize}

\item \hl{Parameter of interest:} Average difference between the reading and writing scores of \orange{all} high school students.
\[ \mu_{diff} \]

$\:$ \\

\pause

\item \hl{Point estimate:} Average difference between the reading and writing scores of \orange{sampled} high school students.
\[ \bar{x}_{diff} \]

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Inference for paired data}

\begin{frame}
\frametitle{Setting the hypotheses}

\dq{If in fact there was no difference between the scores on the reading and writing exams, what would you expect the average difference to be?}

\pause

\soln{0}

$\:$ \\

\pause

\dq{What are the hypotheses for testing if there is a difference between the average reading and writing scores?}

\pause

\begin{itemize}
\item[$H_0$:] There is no difference between the average reading and writing score.
\[ \mu_{diff} = 0 \]
\item[$H_A$:] There is a difference between the average reading and writing score.
\[ \mu_{diff} \ne 0 \]
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Nothing new here}

\begin{itemize}

\item The analysis is no different than what we have done before.

\item We have data from \orange{one} sample: differences.

\item We are testing to see if the average difference is different than 0.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Checking assumptions \& conditions}

\pq{Which of the following is true?}

\begin{enumerate}[(a)]
\solnMult{Since students are sampled randomly and are less than 10\% of all high school students, we can assume that the difference between the reading and writing scores of one student in the sample is independent of another.}
\item The distribution of differences is bimodal, therefore we cannot continue with the hypothesis test.
\item In order for differences to be random we should have sampled with replacement.
\item Since students are sampled randomly and are less than 10\% all students, we can assume that the sampling distribution of the average difference will be nearly normal.
\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[shrink]
\frametitle{Calculating the test-statistic and the p-value}

\dq{The observed average difference between the two scores is -0.545 points and the standard deviation of the difference is 8.887 points. Do these data provide convincing evidence of a difference between the average scores on the two exams? Use $\alpha = 0.05$.}

\twocol{0.5}{0.5}
{
\begin{center}
\includegraphics[width=\textwidth]{graphs/hsb2_read_write_pval}
\end{center}
}
{
\pause
\begin{eqnarray*}
T &=& \frac{-0.545 - 0}{\frac{8.887}{\sqrt{200}}} \\
&=& \frac{-0.545}{0.628} = -0.87 \\
df &=& 200 - 1 = 199 \\
\pause
p-value &=& 0.1927 \times 2 = 0.3854
\end{eqnarray*}
}
\pause
$\:$ \\
Since p-value $>$ 0.05, fail to reject, the data do \underline{not} provide convincing evidence of a difference between the average reading and writing scores.

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Interpretation of p-value}

\pq{Which of the following is the correct interpretation of the p-value?}

\begin{enumerate}[(a)]
\item Probability that the average scores on the reading and writing exams are equal.
\item Probability that the average scores on the reading and writing exams are different.
\solnMult{Probability of obtaining a random sample of 200 students where the average difference between the reading and writing scores is at least 0.545 (in either direction), if in fact the true average difference between the scores is 0.}
\item Probability of incorrectly rejecting the null hypothesis if in fact the null hypothesis is true.
\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{HT $\leftrightarrow$ CI}

\pq{Suppose we were to construct a 95\% confidence interval for the average difference between the reading and writing scores. Would you expect this interval to include 0?}

\begin{enumerate}[(a)]
\solnMult{yes}
\item no
\item cannot tell from the information given
\end{enumerate}

\soln{\pause
\begin{eqnarray*}
-0.545 \pm 1.97 \frac{8.887}{\sqrt{200}} &=& -0.545 \pm 1.97 \times 0.628 \\
&=& -0.545 \pm 1.24 \\
&=& (-1.785, 0.695)
\end{eqnarray*}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Difference of two means}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Diamonds}

\begin{itemize}

\item Weights of diamonds are measured in carats.

\item 1 carat = 100 points, 0.99 carats = 99 points, etc.

\item The difference between the size of a 0.99 carat diamond and a 1 carat diamond is undetectable to the naked human eye, but does the price of a 1 carat diamond tend to be higher than the price of a 0.99 diamond?

\item We are going to test to see if there is a difference between the average prices of 0.99 and 1 carat diamonds.

\item In order to be able to compare equivalent units, we divide the prices of 0.99 carat diamonds by 99 and 1 carat diamonds by 100, and compare the average point prices.

\end{itemize}

\hfill \includegraphics[width=0.2\textwidth]{graphs/diamond}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{Data}

\begin{center}
\includegraphics[width=0.55\textwidth]{graphs/diamondBox.pdf}
\end{center}

{\small
\begin{center}
\begin{tabular}{l | c | c}
		& {\footnotesize \hl{0.99 carat}} &  {\footnotesize \hl{1 carat}}  \\
		& pt99	& pt100 \\
\hline
$\bar{x}$	& 44.50		& 53.43 \\
$s$		& 13.32		& 12.22 \\
$n$		& 23			& 30
\end{tabular}
\end{center}
}

\Note{These data are a random sample from the \texttt{diamonds} data set in \texttt{ggplot2} R package.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Parameter and point estimate}

\begin{itemize}

\item \hl{Parameter of interest:} Average difference between the point prices of \orange{all} 0.99 carat and 1 carat diamonds.
\[ \mu_{pt99} - \mu_{pt100} \]

$\:$ \\

\pause

\item \hl{Point estimate:} Average difference between the point prices of \orange{sampled}  0.99 carat and 1 carat diamonds.
\[ \bar{x}_{pt99} - \bar{x}_{pt100} \]

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Hypotheses}

\pq{Which of the following is the correct set of hypotheses for testing if the average point price of 1 carat diamonds ($_{pt100}$) is higher than the average point price of 0.99 carat diamonds ($_{pt99}$)?}

\begin{enumerate}[(a)]

\item  \mathhl{H_0:} $\mu_{pt99} = \mu_{pt100}$ \\
\mathhl{H_A:} $\mu_{pt99} \ne \mu_{pt100}$

\item  \mathhl{H_0:} $\mu_{pt99} = \mu_{pt100}$ \\
\mathhl{H_A:} $\mu_{pt99} > \mu_{pt100}$

\solnMult{  \mathhl{H_0:} $\mu_{pt99} = \mu_{pt100}$ \\
\mathhl{H_A:} $\mu_{pt99} < \mu_{pt100}$ }

\item  \mathhl{H_0:} $\bar{x}_{pt99} = \bar{x}_{pt100}$ \\
\mathhl{H_A:} $\bar{x}_{pt99} < \bar{x}_{pt100}$

\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conditions}

\pq{Which of the following does \underline{not} need to be satisfied in order to conduct this hypothesis test using theoretical methods?}

\begin{enumerate}[(a)]

\item Point price of one 0.99 carat diamond in the sample should be independent of another, and the point price of one 1 carat diamond should independent of another as well.

\item Point prices of 0.99 carat and 1 carat diamonds in the sample should be independent.

\item Distributions of point prices of 0.99 and 1 carat diamonds should not be extremely skewed.

\solnMult{Both sample sizes should be at least 30.}

\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Sampling distribution for the difference of two means}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Test statistic}

\formula{Test statistic for inference on the difference of two small sample means}
{The test statistic for inference on the difference of two means where $\sigma_1$ and $\sigma_2$ are unknown is the $T$ statistic.
\[ T_{df} = \frac{\text{point estimate} - \text{null value}}{SE} \]
where
\[ SE = \sqrt{ \frac{s_1^2}{n_1} + \frac{s_2^2}{n_2} } \qquad \text{ and } \qquad df = min(n_1 - 1, n_2 - 1) \]
}

\Note{The calculation of the $df$ is actually much more complicated. For simplicity we'll use the above formula to \underline{estimate} the true $df$ when conducting the analysis by hand.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Hypothesis testing for the difference of two means}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Test statistic (cont.)}

{\small
\begin{center}
\begin{tabular}{l | c | c}
		& {\footnotesize \hl{0.99 carat}} &  {\footnotesize \hl{1 carat}}  \\
		& pt99	& pt100 \\
\hline
$\bar{x}$	& 44.50		& 53.43 \\
$s$		& 13.32		& 12.22 \\
$n$		& 23			& 30
\end{tabular}
\end{center}
}

\hl{in context...}

\pause

{\small
\begin{eqnarray*}
T &=& \frac{\text{point estimate} - \text{null value} }{SE} \\
\pause
&=& \frac{(44.50 - 53.43) - 0}{ \sqrt{\frac{13.32^2}{23} + \frac{12.22^2}{30} }} \\
\pause
&=& \frac{-8.93}{3.56} \\
\pause
&=& -2.508
\end{eqnarray*}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Test statistic (cont.)}

\pq{Which of the following is the correct $df$ for this hypothesis test?}

\twocol{0.3}{0.7}
{
\begin{enumerate}[(a)]
\solnMult{ 22 }
\item 23
\item 30
\item 29
\item 52
\end{enumerate}
}
{
\soln{\only<2>{
\orange{$\rightarrow df = min(n_{pt99} - 1, n_{pt100} - 1)$ \\
$= min(23 - 1, 30 - 1)$ \\
$= min(22,29) = 22$} \\
\vspace{1cm}
}}}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{p-value}

\pq{Which of the following is the correct p-value for this hypothesis test?}

\[ T = -2.508 \qquad \only<1-2 | handout:0>{df = 22} \]

\begin{enumerate}[(a)]
\item between 0.005 and 0.01
\solnMult{between 0.01 and 0.025}
\item between 0.02 and 0.05
\item between 0.01 and 0.02
\end{enumerate}

\pause

\begin{verbatim}
> pt(q = -2.508, df = 22)
[1] 0.0100071
\end{verbatim}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Synthesis}

\dq{What is the conclusion of the hypothesis test? How (if at all) would this conclusion change your behavior if you went diamond shopping?}

\soln{\only<2>{
\begin{itemize}
\item p-value is small so reject $H_0$. The data provide convincing evidence to suggest that the point price of 0.99 carat diamonds is lower than the point price of 1 carat diamonds.
\item Maybe buy a 0.99 carat diamond? It looks like a 1 carat, but is significantly cheaper.
\end{itemize}
}}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Confidence intervals for the difference of two means}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Equivalent confidence level}

\pq{What is the equivalent confidence level for a one-sided hypothesis test at $\alpha = 0.05$?}

\twocol{0.3}{0.7}{
\begin{enumerate}[(a)]

\solnMult{90\%}

\item 92.5\%

\item 95\%

\item 97.5\%

\end{enumerate}
}
{
\soln{\only<2>{
\includegraphics[width=\textwidth]{graphs/middle90}
}}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{Critical value}

\pq{What is the appropriate $t^\star$ for a confidence interval for the average difference between the point prices of 0.99 and 1 carat diamonds?}

\begin{enumerate}[(a)]
\item 1.32
\solnMult{1.72}
\item 2.07
\item 2.82
\end{enumerate}

\pause

\begin{verbatim}
> qt(p = 0.95, df = 22)

[1] 1.717144
\end{verbatim}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Confidence interval}

\dq{Calculate the interval, and interpret it in context.}

\pause

\soln{
\[ \text{point estimate} \pm ME \]
\pause
\begin{eqnarray*}
(\bar{x}_{pt99} - \bar{x}_{pt1}) \pm t^\star_{df} \times SE &=& (44.50 - 53.43) \pm 1.72 \times 3.56 \\
\pause
&=& -8.93 \pm  6.12 \\
\pause
&=& (-15.05, -2.81)
\end{eqnarray*}
\pause
We are 90\% confident that the average point price of a 0.99 carat diamond is \$15.05 to \$2.81 lower than the average point price of a 1 carat diamond.
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Recap}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Recap: Inference using difference of two small sample means}

\begin{itemize}

\item If $\sigma_1$ or $\sigma_2$ is unknown, difference between the sample means follow a $t$-distribution with $SE = \sqrt{ \frac{s_1^2}{n_1} + \frac{s_2^2}{n_1} }$.

\pause

\item Conditions:
\begin{itemize}
\item independence within groups (often verified by a random sample, and if sampling without replacement, $n < $ 10\% of population) and between groups
\item no extreme skew in either group
\end{itemize}

\pause

\item Hypothesis testing:
\[ T_{df} = \frac{\text{point estimate} - \text{null value}}{SE}\text{, where }df = min(n_1 - 1, n_2 - 1) \]

\pause

\item Confidence interval:
\[ \text{point estimate} \pm t_{df}^\star \times SE \]

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Comparing means with ANOVA}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Aldrin in the Wolf River}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\begin{center}
\includegraphics[width=0.5\textwidth]{graphs/wolf}
\end{center}

{\small
\begin{itemize}

\item  The Wolf River in Tennessee flows past an abandoned site once used by the pesticide industry for dumping wastes, including chlordane (pesticide), aldrin, and dieldrin (both insecticides).

\pause

\item These highly toxic organic compounds can cause various cancers and birth defects.

\pause

\item The standard methods to test whether these substances are present in a river is to take samples at six-tenths depth.

\pause

\item But since these compounds are denser than water and their molecules tend to stick to particles of sediment, they are more likely to be found in higher concentrations near the bottom than near mid-depth.

\end{itemize}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Data}

Aldrin concentration (nanograms per liter) at three levels of depth. \\

\begin{center}
\begin{tabular}{r | c | c}
\hline
 	& aldrin 					& depth \\
\hline
1 	& \textcolor{darkGray}{3.80} 	& \textcolor{darkGray}{bottom}  \\
2 	& \textcolor{darkGray}{4.80} 	& \textcolor{darkGray}{bottom}  \\
...	&						& \\
10	& \textcolor{darkGray}{8.80} 	& \textcolor{darkGray}{bottom} \\
11	& \textcolor{blue}{3.20} 		& \textcolor{blue}{middepth}  \\
12	& \textcolor{blue}{3.80} 		& \textcolor{blue}{middepth} \\
...	&						& \\
20 	& \textcolor{blue}{6.60} 		& \textcolor{blue}{middepth} \\
21	& \textcolor{oiB}{3.10} 		& \textcolor{oiB}{surface} \\
22	& \textcolor{oiB}{3.60} 		& \textcolor{oiB}{surface} \\
...	&						& \\
30 	& \textcolor{oiB}{5.20} 		& \textcolor{oiB}{surface} \\
\hline
\end{tabular}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Exploratory analysis}

Aldrin concentration (nanograms per liter) at three levels of depth. \\

\begin{center}
\includegraphics[width=0.75\textwidth]{graphs/dotplot}
\end{center}

\begin{center}
\begin{tabular}{l | c c c}
		& n	& mean	& sd		\\
\hline
bottom	& 10	& 6.04	& 1.58 \\
middepth& 10	& 5.05	& 1.10 \\
surface	& 10	& 4.20	& 0.66 \\
\hline
overall	& 30	& 5.1	0	& 1.37
\end{tabular}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Research question}

\dq{Is there a difference between the mean aldrin concentrations among the three levels?}

\vspace{0.5cm}

\pause

\begin{itemize}

\item To compare means of 2 groups we use a Z or a T statistic.

\pause

\item To compare means of 3+ groups we use a new test called \hl{ANOVA} and a new statistic called \hl{F}.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{ANOVA}

ANOVA is used to assess whether the mean of the outcome variable is different for different levels of a categorical variable.

\pause

\begin{itemize}
\item[] \mathhl{H_0:} The mean outcome is the same across all categories,
\[\mu_1 = \mu_2 = \cdots = \mu_k, \]
where $\mu_i$ represents the mean of the outcome for observations in category $i$.
\item[]
\item[] \mathhl{H_A:} At least one mean is different than others.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conditions}

\begin{enumerate}

\item The observations should be independent within and between groups

\begin{itemize}
\item If the data are a simple random sample from less than 10\% of the population, this condition is satisfied.
\item Carefully consider whether the data may be independent (e.g. no pairing).
\item Always important, but sometimes difficult to check.
\end{itemize}

\pause

\item The observations within each group should be nearly normal.

\begin{itemize}
\item Especially important when the sample sizes are small.
\end{itemize}

\dq{How do we check for normality?}

\pause

\item The variability across the groups should be about equal.

\begin{itemize}
\item Especially important when the sample sizes differ between groups.
\end{itemize}

\dq{How can we check this condition?}

\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{$z$/$t$ test vs. ANOVA - Purpose}

\twocol{0.5}{0.5}
{
\[ \hl{$z$/$t$ test} \]
Compare means from \hl{two} groups to see whether they are so far apart that the observed difference cannot reasonably be attributed to sampling variability.
\[ H_0: \mu_1 = \mu_2 \]
}
{
\[ \hl{ANOVA} \]
Compare the means from \hl{two or more} groups to see whether they are so far apart that the observed differences cannot all reasonably be attributed to sampling variability.
\[ H_0: \mu_1 = \mu_2 = \cdots = \mu_k \]
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{$z$/$t$ test vs. ANOVA - Method}

\twocol{0.5}{0.5}
{
\[ \hl{$z$/$t$ test} \]
Compute a test statistic (a ratio).
\[ z / t = \frac{(\bar{x}_1 - \bar{x}_2) - (\mu_1 - \mu_2)}{SE(\bar{x}_1 - \bar{x}_2)} \]
}
{
\[ \hl{ANOVA} \]
Compute a test statistic (a ratio).
\[ F = \frac{\text{variability bet. groups}}{\text{variability w/in groups}} \]
}

\vspace{1cm}

\pause

\begin{itemize}

\item Large test statistics lead to small p-values.

\item If the p-value is small enough $H_0$ is rejected, we conclude that the population means are not equal.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{$z$/$t$ test vs. ANOVA}

\begin{itemize}

\item With only two groups t-test and ANOVA are equivalent, but only if we use a pooled standard variance in the denominator of the test statistic.

\pause

\item With more than two groups, ANOVA compares the sample means to an overall \hl{grand mean}.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Hypotheses}

\pq{What are the correct hypotheses for testing for a difference between the mean aldrin concentrations among the three levels?}

\begin{enumerate}[(a)]
\item $H_0: \mu_B = \mu_M = \mu_S$ \\
$H_A: \mu_B \ne \mu_M \ne \mu_S$ \\
\item $H_0: \mu_B \ne \mu_ M \ne \mu_S$ \\
$H_A: \mu_B = \mu_M = \mu_S$ \\
\solnMult{$H_0: \mu_B = \mu_M = \mu_S$ \\
$H_A:$ At least one mean is different.}
\item $H_0: \mu_B = \mu_M = \mu_S = 0$ \\
$H_A:$ At least one mean is different.
\item $H_0: \mu_B = \mu_M = \mu_S$ \\
$H_A: \mu_B > \mu_M > \mu_S$ \\
\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{ANOVA and the F test}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Test statistic}

\dq{Does there appear to be a lot of variability within groups? How about between groups?}

\[ F = \frac{\text{variability bet. groups}}{\text{variability w/in groups}}  \]

\begin{center}
\includegraphics[width=0.75\textwidth]{graphs/dotplot_var}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\begin{frame}
%\frametitle{Test statistic (cont.)}
%
%\[ F = \frac{\text{variability bet. groups}}{\text{variability w/in groups}} = \frac{\hl{MSG}}{\hl{MSE}}  \]
%
%\begin{itemize}
%
%\item \hl{MSG} is mean square between groups
%\[ df_G = k - 1 \]
%where $k$ is number of groups
%
%\item \hl{MSE} is mean square error - variability in residuals
%\[ df_E = n - k \]
%where $n$ is number of observations.
%
%\end{itemize}
%
%\end{frame}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{$F$ distribution and p-value}

\[ F =  \frac{\text{variability bet. groups}}{\text{variability w/in groups}} \]

\vspace{-1cm}

\begin{center}
\includegraphics[width=0.75\textwidth]{graphs/fdist}
\end{center}

\begin{itemize}

\item In order to be able to reject $H_0$, we need a small p-value, which requires a large F statistic.

\item In order to obtain a large F statistic, variability between sample means needs to be greater than variability within sample means.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{ANOVA output, deconstructed}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.5cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll >{\columncolor[gray]{.6}[.5\tabcolsep]}rrrrr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96 	& 8.48 		& 6.13 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{Degrees of freedom associated with ANOVA}
{
\begin{itemize}
\item groups: $df_G = k - 1$, where $k$ is the number of groups
\item total: $df_T = n - 1$, where $n$ is the total sample size
\item error: $df_E = df_T - df_G$
\end{itemize}
}

\pause

\begin{itemize}

\item $df_G = k - 1 = 3 - 1 = 2$ \\

\pause

\item $df_T = n - 1 = 30 - 1 = 29$

\pause

\item $df_E = 29 - 2 = 27$ \\

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll r>{\columncolor[gray]{.6}[.5\tabcolsep]}rrrr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& \orange{16.96} 	& 8.48 		& 6.13 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{Sum of squares between groups, SSG}
{
Measures the variability between groups
\vspace{-0.25cm}
\[ SSG = \sum_{i = 1}^{k} n_i (\bar{x}_i - \bar{x})^2 \]
where $n_i$ is each group size, $\bar{x}_i$ is the average for each group, $\bar{x}$ is the overall (grand) mean.
}

\pause

\vspace{-0.5cm}

\twocol{0.4}{0.5}
{
{\small
\begin{center}
\begin{tabular}{l | c c }
		& n	& mean		\\
\hline
bottom	& 10	& 6.04	 \\
middepth& 10	& 5.05	 \\
surface	& 10	& 4.2	 \\
\hline
overall	& 30	& 5.1
\end{tabular}
\end{center}
}
}
{
\pause
\begin{eqnarray*}
SSG &=& \pr{ 10 \times (6.04 - 5.1)^2 } \\
\pause
&+& \pr{ 10 \times (5.05 - 5.1)^2 } \\
\pause
&+& \pr{ 10 \times (4.2 - 5.1)^2 } \\
\pause
&=& 16.96 \\
\end{eqnarray*}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll r>{\columncolor[gray]{.6}[.5\tabcolsep]}rrrr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96	& 8.48 		& 6.13 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& \orange{54.29} \\
\end{tabular}
\end{center}
}

\formula{Sum of squares total, SST}
{
Measures the variability between groups
\vspace{-0.25cm}
\[ SST = \sum_{i = 1}^{n} (x_i - \bar{x})^2 \]
where $x_i$ represent each observation in the dataset.
}

\pause

\vspace{-0.75cm}

\begin{eqnarray*}
SST &=& (3.8 - 5.1)^2 + (4.8 - 5.1)^2 + (4.9 - 5.1)^2 + \cdots + (5.2 - 5.1)^2 \\
\pause
&=& (-1.3)^2 + (-0.3)^2 + (-0.2)^2 + \cdots + (0.1)^2 \\
\pause
&=& 1.69 + 0.09 + 0.04 + \cdots + 0.01 \\
\pause
&=& 54.29
\end{eqnarray*}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll r>{\columncolor[gray]{.6}[.5\tabcolsep]}rrrr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96	& 8.48 		& 6.13 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& \orange{37.33} 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{Sum of squares error, SSE}
{
Measures the variability within groups:
\[ SSE = SST - SSG \]
}

\pause

\[ SSE =  54.29 - 16.96 =  37.33 \]

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll rr>{\columncolor[gray]{.6}[.5\tabcolsep]}rrr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96	& \orange{8.48} 		& 6.13 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& \orange{1.38} 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{Mean square error}
{
Mean square error is calculated as sum of squares divided by the degrees of freedom.
}

\pause

\begin{eqnarray*}
MSG &=& 16.96 / 2 = 8.48 \\
\pause
MSE &=& 37.33 / 27 = 1.38
\end{eqnarray*}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll rrr>{\columncolor[gray]{.6}[.5\tabcolsep]}rr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96	& 8.48 		& \orange{6.14} 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{Test statistic, F value}
{
As we discussed before, the F statistic is the ratio of the between group and within group variability.
\[ F = \frac{MSG}{MSE} \]
}

\pause

\[ F = \frac{8.48}{1.38} = 6.14 \]

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\vspace{-0.25cm}

{\footnotesize
\begin{center}
\begin{tabular}{ll rrr>{\columncolor[gray]{.6}[.5\tabcolsep]}rr}
\hline
 			& 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
(\hl{G}roup) 	& depth 		& 2 	& 16.96	& 8.48 		& \orange{6.14} 	& 0.0063 \\
(\hl{E}rror) 	& Residuals 	& 27 	& 37.33 	& 1.38 		&  		&  \\
\hline
	 		& \hl{T}otal	& 29	& 54.29 \\
\end{tabular}
\end{center}
}

\formula{p-value}
{
p-value is the probability of at least as large a ratio between the ``between group" and ``within group" variability, if in fact the means of all groups are equal. It's calculated as the area under the F curve, with degrees of freedom $df_G$ and $df_E$, above the observed F statistic.
}

\pause

\includegraphics[width=0.65\textwidth]{graphs/f}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conclusion - in context}

\pq{What is the conclusion of the hypothesis test?}

$\:$ \\

The data provide convincing evidence that the average aldrin concentration
\begin{enumerate}[(a)]

\item  is different for all groups.

\item on the surface is lower than the other levels.

\solnMult{is different for at least one group.}

\item is the same for all groups.

\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Conclusion}

\begin{itemize}

\item  If p-value is small (less than $\alpha$), reject $H_0$. The data provide convincing evidence that at least one mean is different from (but we can't tell which one).

\pause

\item If p-value is large, fail to reject $H_0$. The data do not provide convincing evidence that at least one pair of means are different from each other, the observed differences in sample means are attributable to sampling variability (or chance).

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Checking conditions}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{(1) independence}

\dq{Does this condition appear to be satisfied?}

\soln{\only<2>{In this study the we have no reason to believe that the aldrin concentration won't be independent of each other.}}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{(2) approximately normal}

\dq{Does this condition appear to be satisfied?}

\begin{center}
\includegraphics[width=0.86\textwidth]{graphs/normal}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]
\frametitle{(3) constant variance}

\dq{Does this condition appear to be satisfied?}

\begin{center}
\includegraphics[width=0.7\textwidth]{graphs/homo}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Multiple comparisons \& Type 1 error rate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Which means differ?}

\begin{itemize}

\item Earlier we concluded that at least one pair of means differ. The natural question that follows is ``which ones?"

\pause

\item We can do two sample $t$ tests for differences in each possible pair of groups.

\pause

\end{itemize}

\dq{Can you see any pitfalls with this approach?}

\pause

\begin{itemize}

\item When we run too many tests, the Type 1 Error rate increases.

\item This issue is resolved by using a modified significance level.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Multiple comparisons}

\begin{itemize}

\item The scenario of testing many pairs of groups is called \hl{multiple comparisons}.

\pause

\item The \hl{Bonferroni correction} suggests that a more \orange{stringent} significance level is more appropriate for these tests:

\[ \alpha^\star = \alpha / K \]

where $K$ is the number of comparisons being considered.

\pause

\item If there are $k$ groups, then usually all possible pairs are compared and $K = \frac{k (k - 1)}{2}$.

\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Determining the modified $\alpha$}

\pq{In the aldrin data set depth has 3 levels: bottom, mid-depth, and surface. If $\alpha = 0.05$, what should be the modified significance level for two sample $t$ tests for determining which pairs of groups have significantly different means?}

\begin{enumerate}[(a)]
\item $\alpha^* = 0.05$
\item $\alpha^* = 0.05 / 2 = 0.025$
\solnMult{$\alpha^* = 0.05 / 3 = 0.0167$}
\item $\alpha^* = 0.05 / 6 = 0.0083$
\end{enumerate}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Which means differ?}

\pq{Based on the box plots below, which means would you expect to be significantly different?}

\twocol{0.6}{0.4}{
\includegraphics[width=\textwidth]{graphs/homo}
}
{
\begin{enumerate}[(a)]
\item bottom \& surface
\item bottom \& mid-depth
\item mid-depth \& surface
\item bottom \& mid-depth; mid-depth \& surface
\item bottom \& mid-depth; bottom \& surface; mid-depth \& surface
\end{enumerate}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{Which means differ? (cont.)}

If the ANOVA assumption of equal variability across groups is satisfied, we can use the data from all groups to estimate variability:

\begin{itemize}

\item Estimate any within-group standard deviation with $\sqrt{MSE}$, which is $s_{pooled}$

\item Use the error degrees of freedom, $n - k$, for $t$-distributions

\end{itemize}

\formula{Difference in two means: after ANOVA}{
\[ SE = \sqrt{  \frac{\sigma_1^2}{n_1} + \frac{\sigma_2^2}{n_2} } \approx \sqrt{ \frac{MSE}{n_1} + \frac{MSE}{n_2} } \]
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\dq{Is there a difference between the  average aldrin concentration at the bottom and at mid depth?}

\twocol{0.4}{0.7}{
{\scriptsize
\begin{center}
\begin{tabular}{l | c c c}
		& n	& mean	& sd		\\
\hline
bottom	& 10	& \orange{6.04}	& 1.58 \\
middepth& 10	& \orange{5.05}	& 1.10 \\
surface	& 10	& 4.2 	& 0.66 \\
\hline
overall	& 30	& 5.1		& 1.37
\end{tabular}
\end{center}
}
}
{
{\scriptsize
\begin{center}
\begin{tabular}{l rrrrr}
\hline
 			& Df 	& Sum Sq	& Mean Sq 	& F value 	& Pr($>$F) \\
\hline
depth 		& 2 	& 16.96 	& 8.48 		& 6.13 	& 0.0063 \\
Residuals 	& \orange{27} 	& 37.33 	& \orange{1.38} 		&  		&  \\
\hline
Total			& 29	& 54.29 \\
\end{tabular}
\end{center}
}
}

\begin{eqnarray*}
T_{df_E} &=& \frac{(\bar{x}_{bottom} - \bar{x}_{middepth})}{\sqrt{ \frac{MSE}{n_{bottom}} + \frac{MSE}{n_{middepth}} }} \\
\pause
T_{27} &=& \frac{( 6.04 - 5.05 )}{\sqrt{ \frac{1.38}{10} + \frac{1.38}{10} }} = \frac{0.99}{0.53}  =1.87 \\
\pause
0.05 &<& p-value < 0.10 \qquad \text{{\footnotesize (two-sided)}} \\
\pause
\alpha^\star &=& 0.05 / 3 = 0.0167
\end{eqnarray*}

\pause
{\small Fail to reject $H_0$, data do not provide convincing evidence of a difference between average aldrin concentrations at bottom and mid depth.}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}
\frametitle{}

\app{Pairwise comparisons}{Is there a difference between the  average aldrin concentration at the bottom and at surface?}

\pause

\soln{
\begin{eqnarray*}
T_{df_E} &=& \frac{(\bar{x}_{bottom} - \bar{x}_{surface})}{\sqrt{ \frac{MSE}{n_{bottom}} + \frac{MSE}{n_{surface}} }} \\
\pause
T_{27} &=& \frac{( 6.04 - 4.2 )}{\sqrt{ \frac{1.38}{10} + \frac{1.38}{10} }} = \frac{1.84}{0.53}  =3.47 \\
\pause
p-value = 0.0018 \qquad \text{{\footnotesize (two-sided)}} \\
\pause
\alpha^\star &=& 0.05 / 3 = 0.0167
\end{eqnarray*}
\pause
{\small Reject $H_0$, the data provide convincing evidence of a difference between the average aldrin concentrations at bottom and surface.}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}


\documentclass[11pt,]{article}
\usepackage[T1]{fontenc}
\usepackage{amssymb,amsmath}
\usepackage{txfonts}
\usepackage{microtype}
\usepackage{amssymb,amsmath}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{natbib}
\usepackage{paralist}
\usepackage{hyperref}
\usepackage{url}
\urlstyle{same}
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\newenvironment{Shaded}{}{}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
\newcommand{\RegionMarkerTok}[1]{{#1}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\NormalTok}[1]{{#1}}

\hypersetup{breaklinks=true,
            pdfauthor={},
            pdftitle={},
            colorlinks=true,
            citecolor=blue,
            urlcolor=blue,
            linkcolor=magenta,
            pdfborder={0 0 0}}

\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt plus 2pt minus 1pt}
\setlength{\emergencystretch}{3em}  % prevent overfull lines
\setcounter{secnumdepth}{0}

\usepackage[a4paper,body={170mm,250mm},top=25mm,left=25mm]{geometry}
\usepackage[sf,bf,small]{titlesec}
\usepackage{fancyhdr}

\pagestyle{fancy}
\lhead{\sffamily MLP Coursework 1}
\rhead{\sffamily Due: 30 October 2017}
\cfoot{\sffamily \thepage}

\author{}
\date{}

\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator{\sigmoid}{sigmoid}
\DeclareMathOperator{\sgn}{sgn}
\DeclareMathOperator{\relu}{relu}
\DeclareMathOperator{\lrelu}{lrelu}
\DeclareMathOperator{\elu}{elu}
\DeclareMathOperator{\selu}{selu}
\DeclareMathOperator{\maxout}{maxout}

\begin{document}

\section{Machine Learning Practical: Coursework
1}
\label{sec:machine-learning-practical-coursework-1}

\textbf{Release date: Monday 16th October 2017}\\
\textbf{Due date: 16:00 Monday 30th October 2017}

\subsection{Introduction}
\label{sec:introduction}
This coursework is concerned with training multi-layer networks to
address the MNIST digit classification problem. It builds on the
material covered in the first three lab notebooks and the first four
lectures. \textbf{You should complete the first three lab
notebooks before starting the coursework.} The aim of the coursework is
to investigate variants of the ReLU activation function for hidden units
in multi-layer networks, with respect to the validation set accuracies
achieved by the trained models.

\subsection{Code}
\label{sec:code}

You should run all of the experiments for the coursework inside the
Conda environment you set up in first labs.  The code for the coursework is available on the course
\href{https://github.com/CSTR-Edinburgh/mlpractical/}{Github repository}
on a branch \texttt{mlp2017-8/coursework1}. To create a local working
copy of this branch in your local repository you need to do the
following.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Make sure all modified files on the branch you are currently have been
  committed
  (\href{https://github.com/CSTR-Edinburgh/mlpractical/blob/mlp2017-8/master/notes/getting-started-in-a-lab.md}{see
  details here} if you are unsure how to do this).
\item
  Fetch changes to the upstream \texttt{origin} repository by running\\
  \texttt{git fetch origin}
\item
  Checkout a new local branch from the fetched branch using\\
  \texttt{git checkout -b coursework1 origin/mlp2017-8/coursework1}
\end{enumerate}

You will now have a new branch in your local repository with all the
code necessary for the coursework in it. In the \texttt{notebooks}
directory there is a notebook \texttt{Coursework\_1.ipynb} which is
intended as a starting point for structuring the code for your
experiments. You will probably want to add additional code cells to this
as you go along and run new experiments (e.g.~doing each new training
run in a new cell). You may also wish to use Markdown cells to keep
notes on the results of experiments.

There will also be a \verb+report+ directory which contains the LaTeX template and style files for the report.  You should copy all these files into the directory which will contain your report.

\subsection{Standard network
architecture}
\label{sec:standard-network-architecture}

To make the results of your experiments more easily comparable, you
should try to keep as many of the free choices in the specification of
the model and learning problem the same across different experiments. If
you vary only a small number of aspects of the problem at a time this
will make it easier to interpret the effect of those changes.

In these experiments you should use a multi-layer network with two hidden layers
(corresponding to three affine transformations) and a softmax output layer.  The initial baseline
should use a sigmoid activation function for the hidden layer;   other experiments will explore
different nonlinear activation functions.  The hidden layers should each contain 100 hidden units.
The baseline network can this be defined with the following code (which should be familiar to you from Lab 3):

\begin{Shaded}
\begin{Highlighting}[]
\CharTok{import} \NormalTok{numpy }\CharTok{as} \NormalTok{np}
\CharTok{from} \NormalTok{mlp.layers }\CharTok{import} \NormalTok{AffineLayer, SoftmaxLayer, SigmoidLayer}
\CharTok{from} \NormalTok{mlp.errors }\CharTok{import} \NormalTok{CrossEntropySoftmaxError}
\CharTok{from} \NormalTok{mlp.models }\CharTok{import} \NormalTok{MultipleLayerModel}
\CharTok{from} \NormalTok{mlp.initialisers }\CharTok{import} \NormalTok{ConstantInit, GlorotUniformInit}

\NormalTok{seed = }\DecValTok{10102016}
\NormalTok{rng = np.random.RandomState(seed)}

\NormalTok{input_dim, output_dim, hidden_dim = }\DecValTok{784}\NormalTok{, }\DecValTok{10}\NormalTok{, }\DecValTok{100}

\NormalTok{weights_init = GlorotUniformInit(rng=rng)}
\NormalTok{biases_init = ConstantInit(}\DecValTok{0}\NormalTok{.)}

\NormalTok{model = MultipleLayerModel([}
    \NormalTok{AffineLayer(input_dim, hidden_dim, weights_init, biases_init),}
    \NormalTok{SigmoidLayer(),}
    \NormalTok{AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init),}
    \NormalTok{SigmoidLayer(),}
    \NormalTok{AffineLayer(hidden_dim, output_dim, weights_init, biases_init)}
\NormalTok{])}

\NormalTok{error = CrossEntropySoftmaxError()}
\end{Highlighting}
\end{Shaded}

Here we are using the Glorot initialisation scheme, discussed in lecture 4.  In part 2B of this coursework you will explore the effect of different initialisation schemes.

The above code creates a network using sigmoid hidden layers; you should modify it to also create a network using ReLU activation functions (see Lab 3).  These two networks will form your baseline systems.

As well as standardising the network architecture, you should also fix
the hyperparameters of the training procedure not being investigated to
be the same across different runs. In particular for all experiments you
should use a \textbf{batch size of 50 and train for a total of 100
epochs} for all reported runs. You may of course use a smaller number of
epochs for initial pilot runs.

\subsection{Part 1: Implementing Activation Functions}
\label{sec:actfns}

In the first part of the assignment you will implement three further
activation functions, each of which is related to ReLU \citep{nair2010rectified}:  Leaky ReLU, ELU (Exponential Linear Unit), and SELU (Scaled Exponential Linear Unit).   Each of these units defines an activation function for which $f(x) = x$ when $x>0$, as for ReLU, but avoid having a zero gradient when $x<0$.

\textbf{Leaky ReLU} ($\lrelu(x)$) \citep{maas2013rectifier} has the following form:
\begin{equation}
  \lrelu(x) =
     \begin{cases}
      \alpha x     & \quad \text{if } x \leq  0 \\
      x       & \quad \text{if } x > 0 \\
    \end{cases}
\end{equation}
Where $\alpha$ is a constant;  typically $\alpha=0.01$, and you can use this value in this coursework.  Note that $\alpha$ can be taken to be a parameter which is learned by back-propagation along with the weights and biases -- this is called Parametric ReLU (PReLU).

\textbf{ELU} ($\elu(x)$) \citep{clevert2015fast} has the following form:
\begin{equation}
  \elu(x) =
     \begin{cases}
      \alpha (\exp(x) - 1)     & \quad \text{if } x \leq  0 \\
      x       & \quad \text{if } x > 0 \\
    \end{cases}
\end{equation}
Again $\alpha$ can be taken as a constant or a tunable parameter.  Typically $\alpha=1$, which results in a smooth function, and you can use this value in this coursework.

\textbf{SELU} ($\selu(x)$) \citep{klambauer2017self} has the following form:
\begin{equation}
  \selu(x) =
     \lambda \begin{cases}
      \alpha (\exp(x) - 1)     & \quad \text{if } x \leq  0 \\
      x       & \quad \text{if } x > 0 \\
    \end{cases}
\end{equation}
In the case of SELU, there is a theoretical argument for optimal values of the two parameters: $\alpha \approx 1.6733$ and $\lambda \approx 1.0507$, and you can use these values in this coursework.

\begin{enumerate}
  \item Implement each of these activations function as classes \verb+LeakyReluLayer+, \verb+EluLayer+ and \verb+SeluLayer+.  You need to implement \verb+fprop+ and \verb+bprop+ methods for each class.
  \item Verify the correctness of your implementation using the supplied unit tests in \verb+Activation\_Tests.ipynb+
  \item Automatically create a test file \verb+sXXXXXXX_test_file.txt+, by running the provided program \verb+generate_inputs.py+ which uses your code for  \verb+LeakyReluLayer+, \verb+EluLayer+ and \verb+SeluLayer+ to run your fprop and bprop methods for each layer on a unique test vector generated using your student ID number.
\end{enumerate}

For Part 1 of the coursework you need to submit the test file \verb+sXXXXXXX_test_file.txt+ (where sXXXXXXX is replaced with your student number) created in step 3 above.

\subsection{Part 2: MNIST Experiments}
\label{sec:expts}
In  Part 2 of the coursework you will experiment with \verb+LeakyReluLayer+, \verb+EluLayer+ and \verb+SeluLayer+ in multi-layer networks trained on MNIST.

\subsubsection{2A:  Comparing activation functions}
In this sub-part you should compare the behaviour of Leaky ReLU, ELU, and SELU activation functions on the MNIST task.  Carry out all experiments using 2 hidden layers, with 100 units per hidden layer.  You should compare the results with baseline systems of the same architecture using sigmoid units and using ReLU units.

\subsubsection{2B: Deep neural network experiments}
In this subpart you will explore the behaviour of deeper networks.  Based on the results of Part 2A, choose one activation function, and compare networks with 2--8 hidden layers, using 100 hidden units per hidden layer.

Also compare the effect of different initialisation strategies, as discussed in lecture 4.  First look at the effect of weight initialisation based on
\begin{compactitem}
\item Fan-in: $w_i \sim U(-\sqrt{3/n_{in}}, \sqrt{3/n_{in}}$
\item Fan-out: $w_i \sim U(-\sqrt{3/n_{out}}, \sqrt{3/n_{out}}$
\item Fan-in and Fan-out: $w_i \sim U \left(-\sqrt{6/(n_{in}+n_{out})}, \sqrt{6/(n_{in}+n_{out})}\right)$
\end{compactitem}
where $U$ is the uniform distribution.   The first of these corresponds to constraining the estimated variance of a unit to be independent of the number of incoming connections ($n_{in}$);  the second to  constraining the estimated variance of a unit's gradient to be independent of the number of outgoing connections ($n_{out}$);  the third corresponds to Glorot and Bengio's combined initialisation.

Additionally you could also explore the effect of drawing from a Gaussian distribution compared with a uniform distribution.  In particular you might like to explore initialising a SELU layer drawing from a Gaussian with mean 0 and variance $1/n_{out}$ as recommended by \cite{klambauer2017self}.

For Part 2 of the coursework you need to write and submit a report, using the template provided, in the directory \verb+report+.  Please read the template document \verb+mlp-cw1-template.pdf+ very carefully, as it provides advice and instructions on writing your report.  You can use the LaTeX source file \verb+mlp-cw1-template.tex+ as a template for your report (see below, in the section 'Report').

It is highly recommended that you use LaTeX for your report.  If you have not used LaTeX previously, now is a good time to learn how to use it!

\subsection{Backing up your work}
\label{sec:backing-up-your-work}

It is \textbf{strongly recommended} you use some method for backing up
your work. Those working in their AFS homespace on DICE will have their
work automatically backed up as part of the
\href{http://computing.help.inf.ed.ac.uk/backups-and-mirrors}{routine
backup} of all user homespaces. If you are working on a personal
computer you should have your own backup method in place (e.g.~saving
additional copies to an external drive, syncing to a cloud service or
pushing commits to your local Git repository to a private repository on
Github). \textbf{Loss of work through failure to back up
\href{http://tinyurl.com/edinflate}{does not consitute a good reason for
late submission}}.

You may \emph{additionally} wish to keep your coursework under version
control in your local Git repository on the \texttt{coursework1} branch.
% This does not need to be limited to the coursework notebook and
% \texttt{mlp} Python modules - you can also add your report document to
% the repository.

If you make regular commits of your work on the coursework this will
allow you to better keep track of the changes you have made and if
necessary revert to previous versions of files and/or restore
accidentally deleted work. This is not however required and you should
note that keeping your work under version control is a distinct issue
from backing up to guard against hard drive failure. If you are working
on a personal computer you should still keep an additional back up of
your work as described above.

\subsection{Report}
\label{sec:report}

Part two of your coursework submission, worth 70 marks will be a report. The directory
\verb+coursework1/report+ contains a template for your report (\verb+mlp-cw1-template.txt+);  the generated pdf file (\verb+mlp-cw1-template.pdf+) is also provided, and you should read this file carefully as it contains information about the required structure and experimentation. The template is written in LaTeX, and we strongly recommend that you write your own report using LaTeX, using the supplied document style \verb+mlp2017+ (as in the template).

You should copy the files in the \verb+report+ directory to the directory containing the LaTeX file of your report, as \verb+pdflatex+ will need to access these files when building the pdf document from the LaTeX source file.

Your report should be in a 2-column format, based on the document format used for the ICML conference. The report should be a \textbf{maximum of 6 pages long}, with a further page for references.  We will not read or assess any parts of the report beyond the allowed 6+1 pages.

Ideally, all figures should be included in your report file as
\href{https://en.wikipedia.org/wiki/Vector_graphics}{vector graphics}
rather than \href{https://en.wikipedia.org/wiki/Raster_graphics}{raster
files} as this will make sure all detail in the plot is visible.
Matplotlib supports saving high quality figures in a wide range of
common image formats using the
\href{http://matplotlib.org/api/pyplot_api.html\#matplotlib.pyplot.savefig}{\texttt{savefig}}
function. \textbf{You should use \texttt{savefig} rather than copying
the screen-resolution raster images outputted in the notebook.} An
example of using \texttt{savefig} to save a figure as a PDF file (which
can be included as graphics in
\href{https://en.wikibooks.org/wiki/LaTeX/Importing_Graphics}{LaTeX}
compiled with \texttt{pdflatex} and in Apple Pages and
\href{https://support.office.com/en-us/article/Add-a-PDF-to-your-Office-file-74819342-8f00-4ab4-bcbe-0f3df15ab0dc}{Microsoft
Word} documents) is given below.

\begin{Shaded}
\begin{Highlighting}[]
\CharTok{import} \NormalTok{matplotlib.pyplot }\CharTok{as} \NormalTok{plt}
\CharTok{import} \NormalTok{numpy }\CharTok{as} \NormalTok{np}
\CommentTok{# Generate some example data to plot}
\NormalTok{x = np.linspace(}\DecValTok{0}\NormalTok{., }\DecValTok{1}\NormalTok{., }\DecValTok{100}\NormalTok{)}
\NormalTok{y1 = np.sin(}\DecValTok{2}\NormalTok{. * np.pi * x)}
\NormalTok{y2 = np.cos(}\DecValTok{2}\NormalTok{. * np.pi * x)}
\NormalTok{fig_size = (}\DecValTok{6}\NormalTok{, }\DecValTok{3}\NormalTok{)  }\CommentTok{# Set figure size in inches (width, height)}
\NormalTok{fig = plt.figure(figsize=fig_size)  }\CommentTok{# Create a new figure object}
\NormalTok{ax = fig.add_subplot(}\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{)  }\CommentTok{# Add a single axes to the figure}
\CommentTok{# Plot lines giving each a label for the legend and setting line width to 2}
\NormalTok{ax.plot(x, y1, linewidth=}\DecValTok{2}\NormalTok{, label=}\StringTok{'$y = \textbackslash{}sin(2\textbackslash{}pi x)$'}\NormalTok{)}
\NormalTok{ax.plot(x, y2, linewidth=}\DecValTok{2}\NormalTok{, label=}\StringTok{'$y = \textbackslash{}cos(2\textbackslash{}pi x)$'}\NormalTok{)}
\CommentTok{# Set the axes labels. Can use LaTeX in labels within $...$ delimiters.}
\NormalTok{ax.set_xlabel(}\StringTok{'$x$'}\NormalTok{, fontsize=}\DecValTok{12}\NormalTok{)}
\NormalTok{ax.set_ylabel(}\StringTok{'$y$'}\NormalTok{, fontsize=}\DecValTok{12}\NormalTok{)}
\NormalTok{ax.grid(}\StringTok{'on'}\NormalTok{)  }\CommentTok{# Turn axes grid on}
\NormalTok{ax.legend(loc=}\StringTok{'best'}\NormalTok{, fontsize=}\DecValTok{11}\NormalTok{)  }\CommentTok{# Add a legend}
\NormalTok{fig.tight_layout()  }\CommentTok{# This minimises whitespace around the axes.}
\NormalTok{fig.savefig(}\StringTok{'file-name.pdf'}\NormalTok{) }\CommentTok{# Save figure to current directory in PDF format}
\end{Highlighting}
\end{Shaded}

(If you are using Libre/OpenOffice you should use Scalable Vector Format
plots instead using \\
\texttt{fig.savefig('file-name.svg')}. If the
document editor you are using for the report does not support including
either PDF or SVG graphics you can instead output high-resolution raster
images using \texttt{fig.savefig('file-name.png', dpi=200)} however note
these files will generally be larger than either SVG or PDF formatted
graphics.)

However to emphasise again: \textbf{It is highly recommended that you use LaTeX.}

If you make use of any any books, articles, web pages or other resources
you should appropriately cite these in your report. You do not need to
cite material from the course lecture slides or lab notebooks.

To create a pdf file \verb+mlp-cw1-template.pdf+ from a LaTeX source file (\verb+mlp-cw1-template.tex+), you can run the following in a terminal:
\begin{verbatim}
pdflatex mlp-cw1-template
bibtex mlp-cw1-template
pdflatex mlp-cw1-template
pdflatex mlp-cw1-template
\end{verbatim}
(Yes, you have to run pdflatex multiple times, in order  for latex to construct the internal document references.)

An alternative, simpler approach uses the \verb+latexmk+ program:
\begin{verbatim}
latexmk -pdf mlp-cw1-template
\end{verbatim}

It is worth learning how to use LaTeX effectively, as it is particularly powerful for mathematical and academic writing.  There are many tutorials on the web.

\subsection{Mechanics}
\label{sec:mechanics}

\textbf{Marks:}
This assignment will be assessed out of 100 marks and
forms 10\% of your final grade for the course.

\textbf{Academic conduct:}
Assessed work is subject to University
regulations on academic
conduct:\\\url{http://web.inf.ed.ac.uk/infweb/admin/policies/academic-misconduct}

\textbf{Submission:}
You can submit more than once up until the submission deadline. All
submissions are timestamped automatically. Identically named files
will overwrite earlier submitted versions, so we will mark the latest
submission that comes in before the deadline.

If you submit anything before the deadline, you may not resubmit
afterward. (This policy allows us to begin marking submissions
immediately after the deadline, without having to worry that some may
need to be re-marked).

If you do not submit anything before the deadline, you may submit {\em
exactly once} after the deadline, and a late penalty will be applied
to this submission unless you have received an approved extension.
Please be aware that late submissions may receive lower priority for
marking, and marks may not be returned within the same timeframe as
for on-time submissions.

{\em Warning:} Unfortunately the \verb+submit+ command will technically
allow you to submit late even if you submitted before the deadline
(i.e.\ it does not enforce the above policy). Don't do this! We will
mark the version that we retrieve just after the deadline, and (even
worse) you may still be penalized for submitting late because the
timestamp will update.

For additional information about late penalties and extension
requests, see the School web page below. Do {\bf not} email any course
staff directly about extension requests; you must follow the
instructions on the web page.

\url{http://web.inf.ed.ac.uk/infweb/student-services/ito/admin/coursework-projects/late-coursework-extension-requests}

\textbf{Late submission penalty:}
Following the University guidelines,
late coursework submitted without an authorised extension will be
recorded as late and the following penalties will apply: 5
percentage points will be deducted for every calendar day or part
thereof it is late, up to a maximum of 7 calendar days. After this
time a mark of zero will be recorded.

\subsection{Submission}
\label{sec:submission}

Your coursework submission should be done electronically using the
\href{http://computing.help.inf.ed.ac.uk/submit}{\texttt{submit}}
command available on DICE machines.

Your submission should include

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  the unit test file generated in part 1, \verb+sXXXXXXX_test_file.txt+, where your student number replaces \verb+sXXXXXXX+
\item
  your completed report as a PDF file, using the provided template
\item
  the notebook (\verb+.ipynb+) file you used to run the experiments in
\item
  and your local version of the \texttt{mlp} code including any changes
  you made to the modules (\texttt{.py} files).
\end{itemize}

You should copy all of the files to a single directory, \verb+coursework1+, e.g.

\begin{verbatim}
mkdir coursework1
cp notebooks/Coursework_1.ipynb mlp/*.py coursework1
cp reports/coursework1.pdf reports/sXXXXXXX_test_file.txt coursework1
\end{verbatim}

and then submit this directory using

\begin{verbatim}
submit mlp cw1 coursework1
\end{verbatim}

The \texttt{submit} command will prompt you with the details of the
submission including the name of the files / directories you are
submitting and the name of the course and exercise you are submitting
for and ask you to check if these details are correct. You should check
these carefully and reply \texttt{y} to submit if you are sure the files
are correct and \texttt{n} otherwise.

You can amend an existing submission by rerunning the \texttt{submit}
command any time up to the deadline. It is therefore a good idea
(particularly if this is your first time using the DICE submit
mechanism) to do an initial run of the \texttt{submit} command early on
and then rerun the command if you make any further updates to your
submisison rather than leaving submission to the last minute.

\subsection{Marking Scheme}
\label{sec:marking-scheme}

\begin{itemize}
\item
  Part 1, Activation function implementation (30 marks). Based on your submitted test file.
\item
  Part 2, Report (70 marks).  The following aspects will contribute to the mark for your report:
  \begin{itemize}
    \item Abstract - how clear is it? does it cover what is reported in the document
    \item Introduction - do you clear outline and motivate the paper, and describe the research questions investigated?
    \item Description of activation functions -- is it clear and correct?
    \item Experiments -- did you carry out the experiments correctly?  are the results clearly presented and described?
    \item Interpretation and discussion of results
    \item Conclusions
    \item Presentation and clarity of report
  \end{itemize}
\end{itemize}

\bibliographystyle{plainnat}
\bibliography{cw1-references}
\end{document}


\documentclass[letterpaper,10pt,twocolumn,twoside,printwatermark=false]{pinp}

%% Some pieces required from the pandoc template
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}

% Use the lineno option to display guide line numbers if required.
% Note that the use of elements such as single-column equations
% may affect the guide line number alignment.

\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}

% The geometry package layout settings need to be set here...
\geometry{layoutsize={0.95588\paperwidth,0.98864\paperheight},%
          layouthoffset=0.02206\paperwidth,%
		  layoutvoffset=0.00568\paperheight}

\definecolor{pinpblue}{HTML}{185FAF}  % imagecolorpicker on blue for new R logo
\definecolor{pnasbluetext}{RGB}{101,0,0} %

\title{DALITE Q5 - Bootstrap, Tests of Significance, Binomial Distribution,
Inference for Means and Proportions. Due October 8, 2019 by 10pm.}

\author[a]{EPIB607 - Inferential Statistics}

  \affil[a]{Fall 2019, McGill University}

\setcounter{secnumdepth}{5}

% Please give the surname of the lead author for the running footer
\leadauthor{Bhatnagar and Hanley}

% Keywords are not mandatory, but authors are strongly encouraged to provide them. If provided, please include two to five keywords, separated by the pipe symbol, e.g:
 \keywords{  Hypothesis testing |  Bootstrap |  t distribution |  One sample mean |  Normal calculations |  Confidence intervals |  Central Limit Theorem (CLT) |  Binomial distribution |  One sample proportion  }

\begin{abstract}
This DALITE quiz will cover the bootstrap, an introduction to
significance testing, and inference for a single mean using the t
distribution. This will also cover the binomial distribution and
inference for a sample proportion. You need to understand the binomial
distribution before moving on to the chapter on one sample proportions.
This is analgous to learning the normal distribution before inference
for a single mean.
\end{abstract}

\dates{This version was compiled on \today}

% initially we use doi so keep for backwards compatibility
\doifooter{\url{https://sahirbhatnagar.com/EPIB607/}}
% new name is doi_footer

\pinpfootercontents{DALITE Q5 due October 8, 2019 by 5pm}

\begin{document}

% Optional adjustment to line up main text (after abstract) of first page with line numbers, when using both lineno and twocolumn options.
% You should only change this length when you've finalised the article contents.
\verticaladjustment{-2pt}

\maketitle
\thispagestyle{firststyle}
\ifthenelse{\boolean{shortarticle}}{\ifthenelse{\boolean{singlecolumn}}{\abscontentformatted}{\abscontent}}{}

% If your first paragraph (i.e. with the \dropcap) contains a list environment (quote, quotation, theorem, definition, enumerate, itemize...), the line after the list may have some extra indentation. If this is the case, add \parshape=0 to the end of the list environment.

\hypertarget{marking}{%
\section*{Marking}\label{marking}}
\addcontentsline{toc}{section}{Marking}

Completion of this DALITE exercise will be availble to us automatically
through the DALITE website. Therefore \textbf{you do not need to hand
anything in}. Marks will be based on the number of correct answers. For
each question you will receive 0.5 marks for getting the correct answer
on the first attempt and an additional 0.5 marks if you stick with the
right answer or switch to the correct answer after seeing someone else's
rationale. Recall that access to your assignments is managed through
tokens sent to your e-mail address. You will be sent a new link
everytime a new assignment has been posted.

\hypertarget{bootstrap}{%
\section{Bootstrap}\label{bootstrap}}

\hypertarget{learning-objectives}{%
\subsection{Learning Objectives}\label{learning-objectives}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Understand that the bootstrap can be used to simulate a sampling
  distribution
\item
  Confidence intervals can subsequently be calculated directly from the
  bootstrap distribution
\item
  Bootstrap confidence intervals do not rely on the Central Limit
  Theorem
\end{enumerate}

\hypertarget{required-readings}{%
\subsection{Required Readings}\label{required-readings}}

\begin{enumerate}
\item \href{https://www.dropbox.com/s/cxiq70zxxtyxlb5/EfronDiaconisBootstrap.pdf?dl=0}{Computer-Intensive Methods in Statistics by Persi Diaconis and Bradley Efron, Scientific American 1983}
\end{enumerate}

\vspace*{0.25cm}

\hypertarget{tests-of-significance}{%
\section{Tests of Significance}\label{tests-of-significance}}

\hypertarget{learning-objectives-1}{%
\subsection{Learning Objectives}\label{learning-objectives-1}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Understand that a significance test answers the question ``Is this
  sample outcome good evidence that an effect is present in the
  population, or could it easily occur just by chance?''
\item
  Be able to formulate the null hypothesis and alternative hypothesis
  for tests about the mean of a population. Understand that the
  alternative hypothesis is the researcher's point of view.
\item
  Understand the concept of a p-value. Know that smaller p-values
  indicate stronger evidence against the null hypothesis.
\item
  Be able to calculate p-values as areas under a normal curve in the
  setting of tests about the mean of a normal population with known
  standard deviation.
\item
  Be able to test a population mean with a z-test.
\end{enumerate}

\hypertarget{videos}{%
\subsection{Videos}\label{videos}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  \href{https://www.learner.org/courses/againstallodds/unitpages/unit25.html}{Against
  All Odds Unit 25}
\end{enumerate}

\hypertarget{required-readings-1}{%
\subsection{Required Readings}\label{required-readings-1}}

\begin{enumerate}
\item \href{https://www.learner.org/courses/againstallodds/pdfs/AgainstAllOdds_StudentGuide_Unit25.pdf#page=1}{Against All Odds Unit 25, pages 1-12}
\item \href{http://www.medicine.mcgill.ca/epidemiology/hanley/BionanoWorkshop/P-Values.pdf}{JH notes on p-values}
\end{enumerate}

\hypertarget{small-sample-inference-for-one-mean}{%
\section{Small Sample Inference for One
Mean}\label{small-sample-inference-for-one-mean}}

\hypertarget{learning-objectives-2}{%
\subsection{Learning Objectives}\label{learning-objectives-2}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Understand when to use t-procedures for a single sample and how they
  differ from the z-procedures covered in Units 24 and 25.
\item
  Understand what a t-distribution is and how it differs from a normal
  distribution.
\item
  Know how to check whether the underlying assumptions for a t-test or
  t-confidence interval are reasonably satisfied.
\item
  Be able to calculate a t-confidence interval for a population mean.
\item
  Be able to test a population mean with a t-test. Be able to calculate
  the t-test statistic and to determine the p-value as an area under a
  t-density curve.
\item
  Be able to adapt one-sample t-procedures to analyze matched pairs
  data.
\end{enumerate}

\hypertarget{videos-1}{%
\subsection{Videos}\label{videos-1}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  \href{https://www.learner.org/courses/againstallodds/unitpages/unit26.html}{Against
  All Odds Unit 26}
\end{enumerate}

\hypertarget{required-readings-2}{%
\subsection{Required Readings}\label{required-readings-2}}

\begin{enumerate}
\item \href{https://www.learner.org/courses/againstallodds/pdfs/AgainstAllOdds_StudentGuide_Unit26.pdf#page=1}{Against All Odds Unit 26, pages 1-11}
\item DVB Chapter 23
\end{enumerate}

\hypertarget{binomial-distributions}{%
\section{Binomial Distributions}\label{binomial-distributions}}

\hypertarget{learning-objectives-3}{%
\subsection{Learning Objectives}\label{learning-objectives-3}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Be able to identify a binomial setting and define a binomial random
  variable.
\item
  Know how to find probabilities associated with a binomial random
  variable.
\item
  Know how to determine the mean and standard deviation of a binomial
  random variable.
\end{enumerate}

\hypertarget{videos-2}{%
\subsection{Videos}\label{videos-2}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  \href{https://www.learner.org/courses/againstallodds/unitpages/unit21.html}{Against
  All Odds Unit 21}
\end{enumerate}

\hypertarget{required-readings-3}{%
\subsection{Required Readings}\label{required-readings-3}}

\begin{enumerate}
\item \href{https://www.learner.org/courses/againstallodds/pdfs/AgainstAllOdds_StudentGuide_Unit21.pdf#page=1}{Against All Odds Unit 21, pages 1-10}
\item \href{https://www.dropbox.com/s/g1ui3xg8nup1p6b/proportion-model-inference-plan-2018.pdf?dl=0}{JH notes Section 1 on binomial distributions}
\item DVB Chapter 17 (Binomial Model)
\end{enumerate}

\vspace*{0.25cm}

\hypertarget{inference-for-proportions}{%
\section{Inference for Proportions}\label{inference-for-proportions}}

\hypertarget{learning-objectives-4}{%
\subsection{Learning Objectives}\label{learning-objectives-4}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Identify inference problems that concern a population proportion.
\item
  Know how to conduct a significance test of a population proportion.
\item
  Be able to calculate a confidence interval for a population
  proportion.
\item
  Understand that the \(z\)-inference procedures for proportions are
  based on approximations to the normal distribution and that accuracy
  depends on having moderately large sample sizes.
\end{enumerate}

\hypertarget{videos-3}{%
\subsection{Videos}\label{videos-3}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  \href{https://www.learner.org/courses/againstallodds/unitpages/unit28.html}{Against
  All Odds Unit 28}
\end{enumerate}

\hypertarget{required-readings-4}{%
\subsection{Required Readings}\label{required-readings-4}}

\begin{enumerate}
\item \href{https://www.learner.org/courses/againstallodds/pdfs/AgainstAllOdds_StudentGuide_Unit28.pdf#page=1}{Against All Odds Unit 28, pages 1-11}
\item \href{https://www.dropbox.com/s/g1ui3xg8nup1p6b/proportion-model-inference-plan-2018.pdf?dl=0}{JH notes Section 2 on Inference for a proportion}
\item DVB Chapters 19, 20 and 21
\end{enumerate}

%\showmatmethods

\bibliography{pinp}
\bibliographystyle{jss}

\end{document}


\begin{appendices}
Put stuff like Bootstrapping, kfold CV, OLS/Ridge/Lasso regression here.

\section{Refreshing linear regression}
% Move this to appendix!
\subsection{Ridge regression}
\subsection{Lasso regression}

\section{Bootstrapping}
\section{\texorpdfstring{$k$}{k}-fold Cross-Validation}
\end{appendices}