%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% UMB-CS114-2015F: Introduction to Programming in Java
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/UMB-CS114-2015F
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\def \topDirectory {../..}
\def \texDirectory {\topDirectory/src/main/tex}

\documentclass[10pt, compress]{beamer}

\usepackage{\texDirectory/template/style/directives}
\input{\texDirectory/template/config}
\usepackage{\texDirectory/template/style/beamerthemeUmassLecture}
\doc{number}{10}
%\setbeamertemplate{footline}[text line]{}

\begin{document}
\prepareCover

\section{Course Administration}

\begin{slide}
	\begin{itemize}
		\item[] Assignment 2 due. Solution released online.
		\item[] Assignment 3 released. Due on November 2, 2015 at 5:30 PM.
		\item[] Midterm exam to be held Ocober 21, 2015 at 5:30 PM.
	\end{itemize}
\end{slide}

\begin{slide}
	\begin{block}{Overview}
		\begin{itemize}
			\item[] Multi-Dimensional Arrays
		\end{itemize}
	\end{block}
\end{slide}

\section{Multi-Dimensional Arrays}

\begin{slide}
	\begin{block}{Dimension Definition}
		The dimension of an array is the number of indices needed to select an element.
	\end{block}
	\begin{block}{Initialization}
		\begin{minted}[fontsize=\small,tabsize=8]{java}
			int[][] matrix = new int[3][3];
			for (int i = 0; i < 3; i++)
			    for (int j = 0; j < 3; j++)
			        matrix[i][j] = 0;
		\end{minted}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{Objective}
		Write a program \texttt{MagicMatrix.java} that takes number $N$ and generates a magic matrix of size $N$ using numbers $1$ to $N^2$.
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{Simplification}
		\begin{itemize}
			\item[] Get matrix size N from user
			\item[] Build an array of length $N^2$ from 1 to N
			\item[] Shuffle the array
			\item[] Convert 1-D Array to 2-D Array
			\item[] Print elements of array
		\end{itemize}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{\texttt{MagicMatrix.java} (part 1)}
		\begin{minted}[fontsize=\small,tabsize=8, linenos, firstnumber=1]{java}
			public class Array {
			    public static void main(String[] args) {
			        // get matrix size
			        Scanner input = new Scanner(System.in);
			        int num = input.nextInt();
			        input.close();
			        // initialize array
			        int[] numbers = new int[num * num];
			        for (int i = 0; i < numbers.length; i++)
			            numbers[i] = i+1;
		\end{minted}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{\texttt{MagicMatrix.java} (part 2)}
		\begin{minted}[fontsize=\small,tabsize=8, linenos, firstnumber=12]{java}
			// shuffle array
			for (int i = 0; i < numbers.length; i++) {
			    int j = (int) (i + (numbers.length - i) * Math.random());
			    int temp = numbers[j];
			    numbers[j] = numbers[i];
			    numbers[i] = temp;
			}
		\end{minted}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{\texttt{MagicMatrix.java} (part 3)}
		\begin{minted}[fontsize=\small,tabsize=8, linenos, firstnumber=19]{java}
			        // convert shuffled array to matrix
			        int[][] matrix = new int[num][num];
			        for (int i = 0; i < numbers.length; i++) {
			            matrix[i/num][i%num] = numbers[i];
			        }
			        // show matrix
			        for (int i = 0; i < num; i++) {
			            for (int j = 0; j < num; j++)
			                System.out.print(matrix[i][j]);
			            System.out.println();
			        }
			    }
			}
		\end{minted}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{Higher Dimensional Arrays}
		\begin{minted}[fontsize=\small,tabsize=8, linenos, firstnumber=1]{java}
			int num = 1;
			int[][][] cubicMatrix = new int[3][3][3];
			for (int i = 1; i < 3; i++)
			    for (int j = 1; j < 3; j++)
			        for (int k = 1; k < 3; k++)
			            cubicMatrix[i][j][k] = num++;
		\end{minted}
	\end{block}
\end{slide}

\begin{slide}
	\begin{block}{Remember}
		\begin{itemize}
			\item[] Arrays are simple data structures for data organization
			\item[] Arrays provide an efficient way to access data
			\item[] Elements of array are of same type
			\item[] Arrays have fixed-size
		\end{itemize}
	\end{block}
\end{slide}

\plain{}{Keep Calm\\and\\Practice}

\end{document}


\documentclass[11pt,a4paper]{article}

% These are extra packages that you might need for writing the equations:
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{xcolor}
\lstset {language=C++,
		 basicstyle=\ttfamily,
         keywordstyle=\color{blue}\ttfamily,
         stringstyle=\color{red}\ttfamily,
         commentstyle=\color{purple}\ttfamily,
         morecomment=[l][\color{magenta}]{\#},
       	 basicstyle=\tiny}

% You need the following package in order to include figures in your report:
\usepackage{graphicx}

% With this package you can set the size of the margins manually:
\usepackage[left=2cm,right=2cm,top=2cm,bottom=2cm]{geometry}

\begin{document}

% Enter the exercise number, your name and date here:
\noindent\parbox{\linewidth}{
 \parbox{.25\linewidth}{ \large HPCSE (151-0116-00L), Project part 1}\hfill
 \parbox{.5\linewidth}{\begin{center} \large Beat Hubmann \end{center}}\hfill
 \parbox{.2\linewidth}{\begin{flushright} \large Mar 02, 2019 \end{flushright}}
}
\noindent\rule{\linewidth}{2pt}

\section{Task 1}
Done; nothing to report.

\section{Task 2}
Done; nothing to report.

\section{Task 3}
Done; nothing to report.

\section{Task 4}
\subsection{Subtasks a-d)}
Do complete these subtasks all together and after checking with a TA first if allowed to do so, I temporarily rewrote
\texttt{auxiliar.cpp} to accept command parameters for grid count, upcycle relaxations and downcycle
relaxations. I then wrote a few lines of Python code (shown below) to perform a
grid search on the parameter ranges some initial reading and manual expermenting made
seem promising. When it became obvious queueing times on Euler were to long to do this
on short notice, I ran the grid search locally on my personal machine with the smaller-sized problem 1.
To finalize my choice of parameters, I manually timed the top ten results from the grid search on the target problem 2 and
came up with \texttt{gridCount=5, downRelaxations=1, upRelaxations=7} as the
tuple consistently delivering the best runtimes below 5s.

\lstset{language=Python}
\begin{lstlisting}
	import subprocess
	p = 1
	avg = 3
	timings = {}
	for gc in range(2, 6):
		for ur in range(1, 6):
			for dr in range(1, 11):
				result = {}
				time = 0
				for _ in range(avg):
					output = subprocess.check_output(
						['./heat2d', '-p', str(p), '-v', '-gc', str(gc),
						 '-ur', str(ur), '-dr', str(dr)])
					for row in output.decode('utf-8').split('\n'):
						if ': ' in row:
							key, value = row.split(': ')
							result[key.strip()] = value.strip('s')
					time += float(result['Running Time'])
				time /= avg
				timings['gc= ' + str(gc) + ' ur= ' + str(ur) +
						' dr= ' + str(dr)] = time
				print(gc, ur, dr, time)
	[print(key , ' : ' , value, 's') for (key, value) in sorted(timings.items() ,  key=lambda x: x[1]  ) ]
\end{lstlisting}

\subsection{Subtask e)}
Given an initial discretisation of a problem, translating the problem to a coarser grid alllows
a 'good initial approximate guess' at the solution to be found cheaply in terms of computation (i.e. quickly in terms of time).
Refining the initial guess towards the final accepted solution on the finer original grid then can be done
faster compared to starting from scratch on the fine grid and approaching a solution iteratively.
This basic idea can be applied recursively leading to the stacking of multiple grids and traversing between them.

\subsection{Subtask f)}
Based on the parameters discovered above, using multiple Jacobi steps only improved convergence
speed when done in one direction. Nevertheless, more Jacobi steps at a given grid level help to smoothe out the
solution at that level so that the next level won't have to do it at a possibly smaller step size.

\subsection{Subtask g)}
Initial experiments with a W-cycle showed significantly worse convergence times.
Assuming that the problem size likely is too small to profit from such complications,
further investigation was abandoned.

\section{Task 5}

\subsection{Subtask a)}
Wherever possible throughout the code, loops where interchanged so that the fastest
running index (typically \texttt{j}) gets to be on the innermost loop.
Loop fusion was applied wherever possible except for the \texttt{applyProlongation} routine:
Measurements showed a difference of 0.1 - 0.2s while making the code almost unreadable. As the target
runtime could be reached easily with plenty of margin otherwise, I decided to refrain from fusing those loops
just to manually deal with all the edge cases in an ugly way.

\subsection{Subtask b)}
Cache access can be improved by allocating the pointers to the grid rows just after
allocating the double pointers so that the data gets contiguously placed in memory.

\subsection{Subtask c)}
Based on experience from loop tiling applied in the course HPC1, I know that results are
quite fickle and often not intuitively reproduced. Given the relatively modest problem size
and the ample margin with regards to the given convergence time target, I gave up further experiments after
a tile length of 8 (as a seemingly reasonable guess for dealing with \texttt{doubles}) showed no promising results.
This might have been due to the fact that initial memory allocation was not aligned and thus any potential tiling
benefits were eaten away by memory address jumps.

\section{Task 6}

\subsection{Subtask a)}

I replaced all divisions by multiplications. Furthermore, squared factors were
changed to be precalculated by simple multiplication outside the loop.

\section{Task 7}

\subsection{Subtask a-b)}
All hints given in the vectorization report were implemented.
The following loops were excluded from vectorization:
\begin{itemize}
	\item Saving the solution: Not performance relevant as only done once.
	\item Allocating memory: Inner \texttt{j} depending on outer loop; only done once.
	\item \texttt{while} loop until convergence: Number of iterations not know; unable to optimize.
\end{itemize}

\subsection{Subtask c)}
My local installation of \texttt{icc} somehow didn't want to play nice with Intel intrinsics' aligned memory allocation.
As no time was left for interactive development on Euler and local experiments with the functionally similar but portable \texttt{posix\_memalign}
didn't show any tangible benefits, implementing alignment wasn't pursued any further.
In theory, aligned SIMD operations would offer better performance, however reading through Intel's documentation on the subject, current-day CPUs and compilers seem to have done away
with the necessity to do this manually.

\end{document}


\documentclass[a4paper,10pt]{article}
\usepackage{graphicx,xspace}
\usepackage[utf8x]{inputenc}
\newcommand{\R}{\textsf{R}\xspace}

\usepackage{alltt}

\usepackage[left=2cm,top=2cm,bottom=2cm,right=2cm]{geometry}
\renewcommand{\abstractname}{Summary}
\newenvironment{rcode}{\begin{footnotesize}\begin{alltt}}{\end{alltt}\end{footnotesize}}
\newenvironment{rline}{\begin{small}\begin{ttfamily}}{\end{ttfamily}\end{small}}
%\setlength{\parskip}{10pt} \setlength{\parindent}{0mm}

\usepackage{/home/gavin/R/build/3.3-patched/share/texmf/tex/latex/Sweave}

\usepackage{mathpazo}
%\linespread{1.05}        % Palatino needs more leading (space between lines)
\normalfont
\usepackage[T1]{fontenc}

\begin{document}
\title{GLMs and GAMs}
\author{Gavin Simpson}
\date{February 2017}
\maketitle

\begin{abstract}
In this lab you will be introduced to the fitting generalised linear models and generalised additive models using \R.
\end{abstract}

\section{\textit{Darlingtonia}: Logistic Regression}
The first type of GLM we'll consider is a binomial GLM to fit a logistic regression model. We will use this model to test the hypothesis that the probability of wasp visitation at a leaf is related to the height of the left above the ground on specimens of the Cobra Lily (\textit{Darlingtonia californica}.

Begin by loading the data in the data frame \texttt{wasp} and convert the variable \texttt{visited} to a logical (\texttt{TRUE} or \texttt{FALSE})

\begin{Schunk}
\begin{Sinput}
> ## read data, skip 1 as first row contains a comment
> wasp <- read.csv("darlingtonia.csv", skip = 1)
> wasp <- transform(wasp, visited = as.logical(visited))
\end{Sinput}
\end{Schunk}

A simple summary of the data is given by a contingency table

\begin{Schunk}
\begin{Sinput}
> with(wasp, table(visited))
\end{Sinput}
\begin{Soutput}
visited
FALSE  TRUE
   32    10
\end{Soutput}
\end{Schunk}

\subsection*{Q and A}
\begin{enumerate}
\item How many observations in the data represent wasp visitations?
\item How many observations in total are there? (Hint: we want to get the number of rows using \texttt{nrow()})
\end{enumerate}

Produce a plot of the data

\begin{Schunk}
\begin{Sinput}
> plot(visited ~ leafHeight, data = wasp)
\end{Sinput}
\end{Schunk}

An alternative visualisation can be achieved by plotting kernel density estimates of the distribution of leaf heights for visited and un-visited leaves. For this we'll use the \textsf{ggplot2} package. Load the package

\begin{Schunk}
\begin{Sinput}
> library("ggplot2")
\end{Sinput}
\end{Schunk}

The plot can produced using

\begin{Schunk}
\begin{Sinput}
> plt <- ggplot(wasp, aes(x = leafHeight, colour = visited)) +
+     geom_density()
> plt
\end{Sinput}
\end{Schunk}

\subsection*{Q and A}
The kernel density estimates are a smooth representation of the density of the data at values of leaf height. These are a smoother version of a histogram and give an approximation of the distribution of values in a data set.

\begin{enumerate}
\item Describe the pattern shown in the figure you just drew.
\item What does this plot suggest in terms of our ability to discriminate or predict which leaves will be visited by wasps given their height above the ground?
\end{enumerate}

We'll now fit the logistic regression (binomial GLM with logit link function)

\begin{Schunk}
\begin{Sinput}
> ## fit a logistic regression
> mod <- glm(visited ~ leafHeight, data = wasp, family = binomial)
\end{Sinput}
\end{Schunk}

A likelihood ratio test can be performed using the \texttt{anova()} method

\begin{Schunk}
\begin{Sinput}
> anova(mod, test = "LRT")
\end{Sinput}
\begin{Soutput}
Analysis of Deviance Table

Model: binomial, link: logit

Response: visited

Terms added sequentially (first to last)

           Df Deviance Resid. Df Resid. Dev  Pr(>Chi)
NULL                          41     46.105
leafHeight  1   19.142        40     26.963 1.213e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
\end{Soutput}
\end{Schunk}

\subsection*{Q and A}
Using the likelihood ratio test output, answer the following question:

\begin{enumerate}
\item Does wasp visitation of leaves depend on the height of the leaf?
\end{enumerate}

The full model summary can be produced using the \texttt{summary()} method

\begin{Schunk}
\begin{Sinput}
> ## summary
> summary(mod)
\end{Sinput}
\begin{Soutput}
Call:
glm(formula = visited ~ leafHeight, family = binomial, data = wasp)

Deviance Residuals:
     Min        1Q    Median        3Q       Max
-2.18274  -0.46820  -0.23897  -0.08519   1.90573

Coefficients:
            Estimate Std. Error z value Pr(>|z|)
(Intercept) -7.29295    2.16081  -3.375 0.000738 ***
leafHeight   0.11540    0.03655   3.158 0.001591 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 46.105  on 41  degrees of freedom
Residual deviance: 26.963  on 40  degrees of freedom
AIC: 30.963

Number of Fisher Scoring iterations: 6
\end{Soutput}
\end{Schunk}

\subsection*{Q and A}
Using the likelihood ratio test output, answer the following question:

\begin{enumerate}
\item Does wasp visitation of leaves depend on the height of the leaf?
\end{enumerate}

Next we'll produce a plot of the fitted response or model. We do this by predicting from the model for 100 equally spaced values that cover the range of \texttt{leafHeight}. The \texttt{predict()} function is used, and because we want predictions on the probability (0--1) scale we use \texttt{type = "response"}. The final two lines show an alternative way to plot the observations, as rug plots on the upper and lower margins of the figure.

\begin{Schunk}
\begin{Sinput}
> ## predict for 100 equally spaced values
> newD <- with(wasp, data.frame(leafHeight = seq(min(leafHeight),
+                                 max(leafHeight), length = 100)))
> pred <- predict(mod, newdata = newD, type = "response")
> ## plot
> plot(pred ~ leafHeight, data = newD, type = "l", ylim = c(0,1),
+      ylab = "Probability of visitation",
+      xlab = "Leaf height (cm)")
> with(wasp, rug(leafHeight[visited == TRUE], side = 3, lwd = 1))
> with(wasp, rug(leafHeight[visited == FALSE], side = 1, lwd = 1))
\end{Sinput}
\end{Schunk}

Leave that plot for the moment. It would be useful to have confidence intervals on the fitted curve. We follow a similar recipe as before, predicting for the 100 new observations of \texttt{leafHeight}, but this time we want predictions on the scale of $\eta$, the linear predictor. The code below does the following things

\begin{enumerate}
\item We store the inverse of the link function in \texttt{ilogit}; we'll use this to map from the scale of $\eta$ back on the response scale later
\item We predict for the new observations, this time using \texttt{type = "link"}, so the inverse link function hasn't been applied yet. We also ask for the standard errors of the predicted values, again on the scale of $\eta$
\item \texttt{alpha} is our confidence limit ($1 - \alpha$, really)
\item \texttt{crit} is the critical value of the $t$ distribution which we'll use as the multiple to scale the standard error by. This will be close to 1.96 for reasonably sized data sets
\item Next, we convert the predicted values using \texttt{ilogit} on to the probability scale of the response, and create the confidence interval, mapping those on to the response scale as well
\item The final two lines of code just add the upper and lower intervals to the plot you made earlier.
\end{enumerate}

\begin{Schunk}
\begin{Sinput}
> ilogit <- family(mod)$linkinv
> predl <- predict(mod, newdata = newD, type = "link", se.fit = TRUE)
> alpha <- 0.05
> crit <- qt(1 - (alpha/2), df = mod$df.residual)
> pred2 <- with(predl,
+               data.frame(fitted = ilogit(fit),
+                          upper = ilogit(fit + (crit * se.fit)),
+                          lower = ilogit(fit - (crit * se.fit)),
+                          leafHeight = newD$leafHeight))
> ## plot
> lines(upper ~ leafHeight, data = pred2, lty = "dashed")
> lines(lower ~ leafHeight, data = pred2, lty = "dashed")
\end{Sinput}
\end{Schunk}

\section{Galap\'{a}gos species richness: Poisson GLM}

Now we'll look at the Galap\'{a}gos species richness data. Load the data and show the first few lines of data

\begin{Schunk}
\begin{Sinput}
> gala <- read.csv("galapagos.csv")
> head(gala)
\end{Sinput}
\begin{Soutput}
             Species Endemics  Area Elevation Nearest Scruz Adjacent
Baltra            58       23 25.09       346     0.6   0.6     1.84
Bartolome         31       21  1.24       109     0.6  26.3   572.33
Caldwell           3        3  0.21       114     2.8  58.7     0.78
Champion          25        9  0.10        46     1.9  47.4     0.18
Coamano            2        1  0.05        77     1.9   1.9   903.82
Daphne.Major      18       11  0.34       119     8.0   8.0     1.84
\end{Soutput}
\end{Schunk}

Produce a plot of the species richness versus predictor variables. Repeat this for other predictor variables/

\begin{Schunk}
\begin{Sinput}
> plot(Species ~ Elevation, data = gala)
\end{Sinput}
\end{Schunk}

In the next few code blocks you'll fit a linear model and a generalised linear model to the data and look at the residuals.

\begin{Schunk}
\begin{Sinput}
> gala.lm1 <- lm(Species ~ Area + Elevation + Nearest + Scruz + Adjacent,
+                data = gala)
> summary(gala.lm1)
\end{Sinput}
\begin{Soutput}
Call:
lm(formula = Species ~ Area + Elevation + Nearest + Scruz + Adjacent,
    data = gala)

Residuals:
     Min       1Q   Median       3Q      Max
-111.679  -34.898   -7.862   33.460  182.584

Coefficients:
             Estimate Std. Error t value Pr(>|t|)
(Intercept)  7.068221  19.154198   0.369 0.715351
Area        -0.023938   0.022422  -1.068 0.296318
Elevation    0.319465   0.053663   5.953 3.82e-06 ***
Nearest      0.009144   1.054136   0.009 0.993151
Scruz       -0.240524   0.215402  -1.117 0.275208
Adjacent    -0.074805   0.017700  -4.226 0.000297 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 60.98 on 24 degrees of freedom
Multiple R-squared:  0.7658,	Adjusted R-squared:  0.7171
F-statistic:  15.7 on 5 and 24 DF,  p-value: 6.838e-07
\end{Soutput}
\end{Schunk}

\begin{Schunk}
\begin{Sinput}
> plot(resid(gala.lm1) ~ predict(gala.lm1))
> abline(h = 0)
\end{Sinput}
\end{Schunk}

\begin{Schunk}
\begin{Sinput}
> gala.glm1 <- glm(Species ~ Area + Elevation + Nearest + Scruz + Adjacent,
+                  data = gala, family = poisson)
> summary(gala.glm1)
\end{Sinput}
\begin{Soutput}
Call:
glm(formula = Species ~ Area + Elevation + Nearest + Scruz +
    Adjacent, family = poisson, data = gala)

Deviance Residuals:
    Min       1Q   Median       3Q      Max
-8.2752  -4.4966  -0.9443   1.9168  10.1849

Coefficients:
              Estimate Std. Error z value Pr(>|z|)
(Intercept)  3.155e+00  5.175e-02  60.963  < 2e-16 ***
Area        -5.799e-04  2.627e-05 -22.074  < 2e-16 ***
Elevation    3.541e-03  8.741e-05  40.507  < 2e-16 ***
Nearest      8.826e-03  1.821e-03   4.846 1.26e-06 ***
Scruz       -5.709e-03  6.256e-04  -9.126  < 2e-16 ***
Adjacent    -6.630e-04  2.933e-05 -22.608  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for poisson family taken to be 1)

    Null deviance: 3510.73  on 29  degrees of freedom
Residual deviance:  716.85  on 24  degrees of freedom
AIC: 889.68

Number of Fisher Scoring iterations: 5
\end{Soutput}
\end{Schunk}

\begin{Schunk}
\begin{Sinput}
> layout(matrix(1:2, ncol = 2))
> plot(resid(gala.lm1) ~ predict(gala.lm1))
> abline(h = 0)
> plot(resid(gala.glm1) ~ predict(gala.glm1, type = "response"))
> abline(h = 0)
> layout(1)
\end{Sinput}
\end{Schunk}

\subsection*{Q and A}
\begin{enumerate}
\item Using the outputs generated in this section, comment on the differences in fit between the linear and the Poisson model.
\item Do the predictor variables appear to be releated to the species richness of Galap\'{a}gos islands? Which variables in particular?
\end{enumerate}

\section{Generalised additive models}

We now return to \textit{Darlingtonia} example and refit the logistic regression model used penalised regression splines (binomial GAM with logit link function)

\begin{Schunk}
\begin{Sinput}
> library("mgcv")                       # load mgcv
> ## fit a GAM logistic regression
> m1 <- gam(visited ~ s(leafHeight), data = wasp, family = binomial, method = "REML")
> summary(m1)
\end{Sinput}
\begin{Soutput}
Family: binomial
Link function: logit

Formula:
visited ~ s(leafHeight)

Parametric coefficients:
            Estimate Std. Error z value Pr(>|z|)
(Intercept)   -2.859      1.245  -2.296   0.0217 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                edf Ref.df Chi.sq p-value
s(leafHeight) 1.979  2.492  8.763  0.0355 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

R-sq.(adj) =  0.435   Deviance explained = 47.4%
-REML = 12.375  Scale est. = 1         n = 42
\end{Soutput}
\end{Schunk}

\subsection*{Q and A}
\begin{enumerate}
\item How many effective degrees of freedom are used by the smooth of \texttt{leafHeight}? What does this suggest about the shape of the fitted function?
\item Is their evidence to reject the null hypothesis that smooth of \texttt{leafHeight} is a flat function?
\item How much variation in the response is expain by the fitted model?
\end{enumerate}

Our checks include looking at the basis dimension and producing some diagnostics plots for the fitted model. These can be done using the \texttt{gam.check()} function:

\begin{Schunk}
\begin{Sinput}
> gam.check(m1)
\end{Sinput}
\begin{Soutput}
Method: REML   Optimizer: outer newton
full convergence after 2 iterations.
Gradient range [-3.102717e-07,-3.102717e-07]
(score 12.37474 & scale 1).
Hessian positive definite, eigenvalue range [0.1985074,0.1985074].
Model rank =  10 / 10

Basis dimension (k) checking results. Low p-value (k-index<1) may
indicate that k is too low, especially if edf is close to k'.

                k'  edf k-index p-value
s(leafHeight) 9.00 1.98    1.10    0.65
\end{Soutput}
\end{Schunk}
\includegraphics{03-glm-and-gam-prac-gam-check}

For binomial models the figures on the right, especially the top right are useless. But the ones on the left have some utility, especially the QQ-plot.

\subsection*{Q and A}
\begin{enumerate}
  \item Looking at the QQ-plot, do the residuals appear to follow the theortetical distribution?
  \item Look at the output printed to the console; is the default basis size sufficient?
\end{enumerate}

We can now plot the fitted smooth

\begin{Schunk}
\begin{Sinput}
> plot(m1, scheme = 1, seWithMean = TRUE, unconditional = TRUE)
\end{Sinput}
\end{Schunk}
\includegraphics{03-glm-and-gam-prac-gam-plot}

Remember, this is plot is drawn on the logit scale. We can use \texttt{predict()} as before to create a plot of the fitted model and confidence interval on the probability scale. In the last line, we add on the fitted GLM line for comparison

\begin{Schunk}
\begin{Sinput}
> pgam <- predict(m1, newdata = newD, type = "response")
> ## plot
> plot(pgam ~ leafHeight, data = newD, type = "l", ylim = c(0,1),
+      ylab = "Probability of visitation",
+      xlab = "Leaf height (cm)")
> with(wasp, rug(leafHeight[visited == TRUE], side = 3, lwd = 1))
> with(wasp, rug(leafHeight[visited == FALSE], side = 1, lwd = 1))
> ilogit <- family(m1)$linkinv
> pgaml <- predict(m1, newdata = newD, type = "link", se.fit = TRUE)
> crit <- 1.96
> pgam2 <- with(pgaml,
+               data.frame(fitted = ilogit(fit),
+                          upper = ilogit(fit + (crit * se.fit)),
+                          lower = ilogit(fit - (crit * se.fit)),
+                          leafHeight = newD$leafHeight))
> ## plot
> lines(upper ~ leafHeight, data = pgam2, lty = "dashed")
> lines(lower ~ leafHeight, data = pgam2, lty = "dashed")
> ## add the fitted GLM model
> lines(pred ~ leafHeight, data = newD, col = "forestgreen")
\end{Sinput}
\end{Schunk}
\includegraphics{03-glm-and-gam-prac-plot-fitted-gam-darlingtonia}

Do we need the smoother? To compare with the GLM, we can fit that model using \textbf{mgcv}. We do need to refit the GAM using maximum likelihood though as REML is not appropriate when you want to compare models with different fixed effects

\begin{Schunk}
\begin{Sinput}
> ## refit GAM using ML
> m1a <- gam(visited ~ s(leafHeight), data = wasp, family = binomial, method = "ML")
> ## fit a GLM logistic regression using mgcv
> m2 <- gam(visited ~ leafHeight, data = wasp, family = binomial, method = "ML")
> anova(m2, m1a, test = "LRT")
\end{Sinput}
\begin{Soutput}
Analysis of Deviance Table

Model 1: visited ~ leafHeight
Model 2: visited ~ s(leafHeight)
  Resid. Df Resid. Dev         Df   Deviance  Pr(>Chi)
1        40     26.963
2        40     26.963 5.7358e-06 7.9527e-06 3.401e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
\end{Soutput}
\end{Schunk}

In realty we probably didn't need to do that; the fact that the smoothness penalty resulted in a model with 2 degrees of freedom is strong evidence against the parametric model.

\end{document}


\documentclass{article}

\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{tikz}
\usepackage{physics}
\usepackage[plain]{algorithm}
\usepackage{hyperref}
\usepackage{algpseudocode}

\usetikzlibrary{automata,positioning}

%
% Basic Document Settings
%

\topmargin=-0.45in
\evensidemargin=0in
\oddsidemargin=0in
\textwidth=6.5in
\textheight=9.0in
\headsep=0.25in

\linespread{1.1}

\pagestyle{fancy}
\lhead{\hmwkAuthorName}
\chead{\hmwkClass\ : \hmwkTitle}
\rhead{\firstxmark}
\lfoot{\lastxmark}
\cfoot{\thepage}

\renewcommand\headrulewidth{0.4pt}
\renewcommand\footrulewidth{0.4pt}

\setlength\parindent{0pt}

%
% Create Problem Sections
%

\newcommand{\enterProblemHeader}[1]{
    \nobreak\extramarks{}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
}

\newcommand{\exitProblemHeader}[1]{
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \stepcounter{#1}
    \nobreak\extramarks{Problem \arabic{#1}}{}\nobreak{}
}

\setcounter{secnumdepth}{0}
\newcounter{partCounter}
\newcounter{homeworkProblemCounter}
\setcounter{homeworkProblemCounter}{1}
\nobreak\extramarks{Problem \arabic{homeworkProblemCounter}}{}\nobreak{}

%
% Homework Problem Environment
%
% This environment takes an optional argument. When given, it will adjust the
% problem counter. This is useful for when the problems given for your
% assignment aren't sequential. See the last 3 problems of this template for an
% example.
%
\newenvironment{homeworkProblem}[1][-1]{
    \ifnum#1>0
        \setcounter{homeworkProblemCounter}{#1}
    \fi
    \section{Problem \arabic{homeworkProblemCounter}}
    \setcounter{partCounter}{1}
    \enterProblemHeader{homeworkProblemCounter}
}{
    \exitProblemHeader{homeworkProblemCounter}
}

%
% Homework Details
%   - Title
%   - Due date
%   - Class
%   - Section/Time
%   - Instructor
%   - Author
%

\newcommand{\hmwkTitle}{Assignment\ \#1}
\newcommand{\hmwkDueDate}{Due on 7th September, 2021}
\newcommand{\hmwkClass}{Theory and Numerics of PDEs}
\newcommand{\hmwkClassTime}{}
\newcommand{\hmwkClassInstructor}{}
\newcommand{\hmwkAuthorName}{\textbf{Aditya Vijaykumar}}

%
% Title Page
%

\title{
    %\vspace{2in}
    \textmd{\textbf{\hmwkClass:\ \hmwkTitle}}\\
    \normalsize\vspace{0.1in}\small{\hmwkDueDate\ }\\
%    \vspace{3in}
}

\author{\hmwkAuthorName}
\date{}

\renewcommand{\part}[1]{\textbf{\large Part \Alph{partCounter}}\stepcounter{partCounter}\\}

%
% Various Helper Commands
%

% Useful for algorithms
\newcommand{\alg}[1]{\textsc{\bfseries \footnotesize #1}}

% For derivatives
\newcommand{\deriv}[1]{\frac{\mathrm{d}}{\mathrm{d}x} (#1)}

% For partial derivatives
\newcommand{\pderiv}[2]{\frac{\partial}{\partial #1} (#2)}

% Integral dx
\newcommand{\dx}{\mathrm{d}x}

% Alias for the Solution section header
\newcommand{\solution}{\textbf{\large Solution}}

% Probability commands: Expectation, Variance, Covariance, Bias
\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\Bias}{\mathrm{Bias}}

\begin{document}

\maketitle

%\pagebreak

\begin{homeworkProblem}
	\textbf{Question}: Show $ y(t)  = \sum_{n\ge0} \dfrac{a_0}{n!} t^n  $ converges.

	Let,
	\begin{equation}\label{key}
	x_n = \dfrac{a_0}{n!} t^n
	\end{equation}
	\begin{align}\label{key}
	\dfrac{x_{n+1}}{x_{n}} = \dfrac{t}{n+1}
	\end{align}
	which goes to 0 as $ n \rightarrow \infty $. So this passes the ratio test for convergence.

\end{homeworkProblem}

\begin{homeworkProblem}
	\textbf{Question}: Use $ y = \sum_{n\ge0} a_n t^n$ to solve $ y'' + y = 0 $.

	Substituting, we get,
	\begin{equation}\label{key}
	\sum_{n\ge0} a_n t^{n-2} + a_n t^n = 0 \implies \sum_{n\ge2} a_{n+2} t^{n-2} +  \sum_{n\ge0} a_n t^n = 0
	\end{equation}

	So we just need two constants, and the solution is fully specified. If $ a_0 = A $ and $ a_1 = B $,

	\begin{equation}\label{key}
	y = A \qty(1 - \dfrac{t^2}{2!} + \dfrac{t^4}{4!}+ \ldots) + B \qty(x - \dfrac{t^3}{3!} + \dfrac{t^5}{5!}) = A \cos(t) + B \sin(t)
	\end{equation}
\end{homeworkProblem}

\begin{homeworkProblem}
\textbf{Question}: Consider
\begin{equation}\label{q1}
t^2 y'' - 4 t y' + 6 y = 0 \qq{;} y(1) = 1  \qq{;} y'(1) = 2 \qq{.}
\end{equation}

\begin{enumerate}
	\item Show that, $y(t) = t^2$ is a solution to the differential equation.
	\item Use Picard iteration to solve the differential equation.\\
\end{enumerate}

\textbf{Solution:}\\
For $ y = t^2 $, $ y' = 2 t $ and $  y'' =2 $. Hence, the LHS of (\ref{q1}) becomes,

\begin{equation}\label{key}
y'' - \dfrac{4}{t} y' + \dfrac{6}{t^2} y = 0
\end{equation}
We write two iterates, for $ y' $ and $ y $.

\begin{align}\label{key}
y_{1}' &=  2 + \int_{1}^{t} \dd{t } \qty[ \dfrac{8}{t} - \dfrac{6}{t^2} ] = 2 +  8\log t +  \dfrac{6}{t} - 6 \\
y_{1} &= 1 + \int_{2}^t (1) \dd{t} = 1 + 2t - 2 = 2t - 1
\end{align}

\end{homeworkProblem}

\begin{homeworkProblem}
	\textbf{Question}: Consider
	\begin{equation}\label{q2}
	\dv{y}{t}  = y ( 1 -y) \qq{,} y(0) = 1/2
	\end{equation}

	\begin{enumerate}
		\item Show that,
		\begin{equation}\label{key}
		y(t) = \dfrac{e^t}{e^t + 1}
		\end{equation}
		solves the differential equation.
		\item Use Picard iterates to find the solution. Compare this solution to the true Taylor series.
		\item How many iterates do we need to obtain the true Taylor coefficient for $ t^k $.\\

	\textbf{Solution:}\\
	\begin{align}\label{key}
	\dv{y}{t} = \dfrac{e^t}{(e^t + 1)^2} &\qq{;} y(1-y) = \dfrac{e^t}{e^t + 1}  \dfrac{1}{e^t + 1} = \dfrac{e^t}{(e^t + 1)^2}\\
	\implies \dv{y}{t} &= y(1-y)
	\end{align}

	Also, $ y(0) = 1/2  $. So the solution satisfies both the differential equation as well as the initial condition.

	Let $ \phi_{k} $ be the $ k^\mathrm{th} $ Picard iterate. We start off with $ \phi_0 = 1/2 $. Then,
	\begin{align}\label{key}
	\phi_1 &= \dfrac{1}{2} + \int_{0}^{t} \dfrac{1}{2}  \dfrac{1}{2} \dd{s} =\dfrac{1}{2} + \dfrac{t}{4}\\
	\phi_2 &= \dfrac{1}{2} + \int_{0}^{t} \qty(\dfrac{1}{2} + \dfrac{s}{4} )\qty(\dfrac{1}{2} - \dfrac{s}{4}) \dd{s} = \dfrac{1}{2} + \dfrac{s}{4} - \dfrac{t^3}{48}\\
	\phi_3 &= \dfrac{1}{2} + \int_{0}^{t} \dd{t}\qty(\dfrac{1}{2} + \dfrac{s}{4} - \dfrac{s^3}{48})\qty(\dfrac{1}{2} - \dfrac{s}{4} + \dfrac{s^3}{48}) = \dfrac{t}{4} - \dfrac{t^3}{48} + \dfrac{t^5}{480} - \dfrac{t^7}{16128}
	\end{align}

	\end{enumerate}
\end{homeworkProblem}
\end{document}


%%
%% Automatically generated file from DocOnce source
%% (https://github.com/hplgit/doconce/)
%%
%%

%-------------------- begin preamble ----------------------

\documentclass[%
oneside,                 % oneside: electronic viewing, twoside: printing
final,                   % draft: marks overfull hboxes, figures with paths
10pt]{article}

\listfiles               %  print all files needed to compile this document

\usepackage{relsize,makeidx,color,setspace,amsmath,amsfonts,amssymb}
\usepackage[table]{xcolor}
\usepackage{bm,ltablex,microtype}

\usepackage[pdftex]{graphicx}

\usepackage[T1]{fontenc}
%\usepackage[latin1]{inputenc}
\usepackage{ucs}
\usepackage[utf8x]{inputenc}

\usepackage{lmodern}         % Latin Modern fonts derived from Computer Modern

% Hyperlinks in PDF:
\definecolor{linkcolor}{rgb}{0,0,0.4}
\usepackage{hyperref}
\hypersetup{
    breaklinks=true,
    colorlinks=true,
    linkcolor=linkcolor,
    urlcolor=linkcolor,
    citecolor=black,
    filecolor=black,
    %filecolor=blue,
    pdfmenubar=true,
    pdftoolbar=true,
    bookmarksdepth=3   % Uncomment (and tweak) for PDF bookmarks with more levels than the TOC
    }
%\hyperbaseurl{}   % hyperlinks are relative to this root

\setcounter{tocdepth}{2}  % levels in table of contents

% Tricks for having figures close to where they are defined:
% 1. define less restrictive rules for where to put figures
\setcounter{topnumber}{2}
\setcounter{bottomnumber}{2}
\setcounter{totalnumber}{4}
\renewcommand{\topfraction}{0.95}
\renewcommand{\bottomfraction}{0.95}
\renewcommand{\textfraction}{0}
\renewcommand{\floatpagefraction}{0.75}
% floatpagefraction must always be less than topfraction!
% 2. ensure all figures are flushed before next section
\usepackage[section]{placeins}
% 3. enable begin{figure}[H] (often leads to ugly pagebreaks)
%\usepackage{float}\restylefloat{figure}

% prevent orhpans and widows
\clubpenalty = 10000
\widowpenalty = 10000

% --- end of standard preamble for documents ---

% insert custom LaTeX commands...

\raggedbottom
\makeindex
\usepackage[totoc]{idxlayout}   % for index in the toc
\usepackage[nottoc]{tocbibind}  % for references/bibliography in the toc

%-------------------- end preamble ----------------------

\begin{document}

% matching end for #ifdef PREAMBLE

\newcommand{\exercisesection}[1]{\subsection*{#1}}

% ------------------- main content ----------------------

% ----------------- title -------------------------

\thispagestyle{empty}

\begin{center}
{\LARGE\bf
\begin{spacing}{1.25}
PHY321: Classical Mechanics 1
\end{spacing}
}
\end{center}

% ----------------- author(s) -------------------------

\begin{center}
{\bf Homework 10, due Friday  April 26${}^{}$} \\ [0mm]
\end{center}

\begin{center}
% List of all institutions:
\end{center}

% ----------------- end author(s) -------------------------

% --- begin date ---
\begin{center}
Apr 16, 2021
\end{center}
% --- end date ---

\vspace{1cm}

\paragraph{Practicalities about  homeworks and projects.}
\begin{enumerate}
\item You can work in groups (optimal groups are often 2-3 people) or by yourself. If you work as a group you can hand in one answer only if you wish. \textbf{Remember to write your name(s)}!

\item Homeworks are available  the week before the deadline.

\item How do I(we)  hand in?  Due to the corona virus and many of you not being on campus, we recommend that you scan your handwritten notes and upload them to D2L. If you are ok with typing mathematical formulae using say Latex, you can hand in everything as a single jupyter notebook at D2L. The numerical exercise(s) should always be handed in as a jupyter notebook by the deadline at D2L.
\end{enumerate}

\noindent
\paragraph{Introduction to homework 10.}
This homework is optional but gives an extra score of 10\% on top of
all you have done throughout the semester. The exercises are
essentially good old fashioned paper and pencil exercises and each
covers different aspects of what has been done after spring break.
The relevant reading background is marked in the different exercises (Taylor's text).

\paragraph{Exercise 1 (10pt), new reference frame (Taylor chapter 8).}
Show that if one transforms to a reference frame where the total
momentum is zero, $\bm{p}_1=-\bm{p}_2$, that the relative momentum
$\bm{q}$ corresponds to either $\bm{p}_1$ or $-\bm{p}_2$. This
means that in this frame the magnitude of $\bm{q}$ is one half the
magnitude of $\bm{p}_1-\bm{p}_2$.

\paragraph{Exercise 2 (10pt) Center of mass and relative coordinates (Taylor chapter 8).}
Given the center of mass and relative coordinates $\bm{R}$ and $\bm{r}$, respectively, for
particles of mass $m_1$ and $m_2$, find the coordinates $\bm{r}_1$
and $\bm{r}_2$ in terms of the masses, $\bm{R}$ and $\bm{r}$.

\paragraph{Exercise 3 (30pt),  Two-body problems (Taylor chapter 8).}
Consider a particle of mass $m$ moving in a potential
\[
V(r)=\alpha\ln(r/\alpha),
\]
where $\alpha$ is a constant.

\begin{itemize}
\item 3a (5pt)  If the particle is moving in a circular orbit of radius $R$, find the angular frequency $\dot{\theta}$. Solve this by setting $F=-m\dot{\theta}^2r$ (force and acceleration point inward).

\item 3b (5pt) Express the angular momentum $L$ in terms of the constant $\alpha$, the mass $m$ and the radius $R$. Also express $R$ in terms of $L$, $\alpha$ and $m$.

\item 3c (5pt) Sketch the effective radial potential, $V_{\rm eff}(r)$, for a particle with angular momentum $L$. (No longer necessarily moving in a circular orbit.)

\item 3d (5pt)  Find the position of the minimum of $V_{\rm eff}$ in terms of $L$, $\alpha$ and $m$, then compare to the result of (3b).

\item 3e (5pt)  What is the effective spring constant for a particle at the minimum of $V_{\rm eff}$? Express your answer in terms of $L$, $m$ and $\alpha$.

\item 3f (5pt)  What is the angular frequency, $\omega$, for small oscillations of $r$ about the $R_{\rm min}$?  Express your answer in terms of $\dot{\theta}$ from part (3a).
\end{itemize}

\noindent
\paragraph{Exercise 4 (10pt), Lagrangian formalism (Taylor chapters 6-7).}
Consider a mass $m$ connected to a spring with spring constant $k$. Rather than being fixed, the other end of the spring oscillates with frequency $\omega$ and amplitude $A$. For a generalized coordinate, use the displacement of the mass from its relaxed position and call it $y=x-\ell-A\cos\omega t$. In this system the potential energy of the spring is $ky^2/2$.

\begin{itemize}
\item 4a (5pt) Write the kinetic energy in terms of the generalized coordinate

\item 4b (5pt) Write down the Lagrangian and find the equations of motion for $y$
\end{itemize}

\noindent
\paragraph{Exercise 5 (40pt), Pendulum and Lagrangians (Taylor chapters 6-7).}
A mathematical pendulum consists of a point mass $m$ suspended by a massless thread/rod of length $l$ in a gravitational field, as shown in the figure here. The constraining force is labeled by $\bm{T}$
and the gravitational force is labeled $\bm{F}_g$.

\vspace{6mm}

% inline figure
\centerline{\includegraphics[width=0.6\linewidth]{figures/Simplependulum.png}}

\vspace{6mm}

We assume that the length $l$ is constant and we define the coordinates involved as

\[
\bm{r} = l\sin(\phi)\bm{\hat{x}}+l\cos(\phi)\bm{\hat{y}},
\]
where $\bm{\hat{x}}$ and $\bm{\hat{y}}$ are the unit vectors in the $x$ and $y$ directions, respectively.

\begin{itemize}
\item \textbf{5a (10pt):} Set up the forces acting on the system and show that the equation of motion is $m\ddot{\bm{r}}=\bm{F}_g+\bm{T}$.

\item \textbf{5b (10pt):} Show that you can rewrite the above equation of motion as two independent equations of motion, one for $\phi$ and one for the constraining force. Show that these equations are $\ddot{\phi}(t)=-\omega_0^2\sin{(\phi(t))}$ with $\omega_0^2=g/l$ and $T=ml\dot{\phi}^2+mg\cos{(\phi)}$.
\end{itemize}

\noindent
The equation for $\phi$ is a second-order differential equation
\[
\ddot{\phi}(t)=-\omega_0^2\sin{(\phi(t))}.
\]

This equation can be solved analytically if we assume that the angle $\phi$ is very small. Then we can approximate our equation as

\[
\ddot{\phi}(t)=-\omega_0^2\phi(t).
\]

\begin{itemize}
\item \textbf{5c (5pt):} Find the analytical solution for the last equation. Hint, look back at the solutions for the simple harmonic oscillator problem in one dimension in for example homework 7.

\item \textbf{5d (5pt):} Find the expressions for the kinetic and potential energies in terms of the variables $r$ and $\phi$.

\item \textbf{5e (10pt):} With the potential $V$  and kinetic $T$ energies, define the Lagrangian for the mathematical pendulum discussed here. Add the constraint $r=l$ via a Lagrange multiplier $\lambda$ and derive the equations of motion. Show that these result in  $\ddot{\phi}(t)=-\omega_0^2\sin{(\phi(t))}$ with $\omega_0^2=g/l$ and $\lambda=ml\dot{\phi}^2+mg\cos{(\phi)}$.  How would you interpret $\lambda$?
\end{itemize}

\noindent
\paragraph{Classical Mechanics Extra Credit Assignment: Scientific Writing and attending Talks.}
The following gives you an opportunity to earn \textbf{five extra credit
points} on each of the remaining homeworks and \textbf{ten extra credit points}
on the midterms and finals.  This assignment also covers an aspect of
the scientific process that is not taught in most undergraduate
programs: scientific writing.  Writing scientific reports is how
scientist communicate their results to the rest of the field.  Knowing
how to assemble a well written scientific report will greatly benefit
you in you upper level classes, in graduate school, and in the work
place.

The full information on extra credits is found at \href{{https://github.com/mhjensen/Physics321/blob/master/doc/Homeworks/ExtraCredits/}}{\nolinkurl{https://github.com/mhjensen/Physics321/blob/master/doc/Homeworks/ExtraCredits/}}. There you will also find examples on how to write a scientific article.
Below you can also find a description on how to gain extra credits by attending scientific talks.

This assignment allows you to gain extra credit points by practicing
your scientific writing.  For each of the remaining homeworks you can
submit the specified section of a scientific report (written about the
numerical aspect of the homework) for five extra credit points on the
assignment.  For the two midterms and the final, submitting a full
scientific report covering the numerical analysis problem will be
worth ten extra points.  For credit the grader must be able to tell
that you put effort into the assignment (i.e.~well written, well
formatted, etc.).  If you are unfamiliar with writing scientific
reports, \href{{https://github.com/mhjensen/Physics321/blob/master/doc/Homeworks/ExtraCredits/IntroductionScientificWriting.md}}{see the information here}

The following table explains what aspect of a scientific report is due
with which homework.  You can submit the assignment in any format you
like, in the same document as your homework, or in a different one.
Remember to cite any external references you use and include a
reference list.  There are no length requirements, but make sure what
you turn in is complete and through.  If you have any questions,
please contact Julie Butler at butler@frib.msu.edu.

\begin{quote}
\begin{tabular}{ccc}
\hline
\multicolumn{1}{c}{ HW/Project } & \multicolumn{1}{c}{ Due Date } & \multicolumn{1}{c}{ Extra Credit Assignment } \\
\hline
HW 3               & 2-8           & Abstract                   \\
HW 4               & 2-15          & Introduction               \\
HW 5               & 2-22          & Methods                    \\
HW 6               & 3-1           & Results and Discussion     \\
\textbf{Midterm 1} & \textbf{3-12} & \emph{Full Written Report} \\
HW 7               & 3-22          & Abstract                   \\
HW 8               & 3-29          & Introduction               \\
HW 9               & 4-5           & Results and Discussion     \\
\textbf{Midterm 2} & \textbf{4-16} & \emph{Full Written Report} \\
HW 10              & 4-26          & Abstract                   \\
\textbf{Final}     & \textbf{4-30} & \emph{Full Written Report} \\
\hline
\end{tabular}
\end{quote}

\noindent

You can also gain extra credits if you attend scientific talks.
This is described here.

\paragraph{Integrating Classwork With Research.}
This opportunity will allow you to earn up to 5 extra credit points on a Homework per week. These points can push you above 100\% or help make up for missed exercises.
In order to earn all points you must:

\begin{enumerate}
\item Attend an MSU research talk (recommended research oriented Clubs is  provided below)

\item Summarize the talk using at least 150 words

\item Turn in the summary along with your Homework.
\end{enumerate}

\noindent
Approved talks:
Talks given by researchers through the following clubs:
\begin{itemize}
\item Research and Idea Sharing Enterprise (RAISE): Meets Wednesday Nights Society for Physics Students (SPS): Meets Monday Nights

\item Astronomy Club: Meets Monday Nights

\item Facility For Rare Isotope Beam (FRIB) Seminars: Occur multiple times a week
\end{itemize}

\noindent
If you have any questions please consult Jeremy Rebenstock, rebensto@msu.edu.

All the material on extra credits is at \href{{https://github.com/mhjensen/Physics321/blob/master/doc/Homeworks/ExtraCredits/}}{\nolinkurl{https://github.com/mhjensen/Physics321/blob/master/doc/Homeworks/ExtraCredits/}}.

% ------------------- end of main content ---------------

\end{document}


\subsection{Small DB 2, the dynamic array version in Pascal} % (fold)
\label{sub:pas-small_db_2_the_dynamic_array_version}

\sref{sec:using_dynamic_memory_allocation}, \nameref{sec:using_dynamic_memory_allocation}, introduced a version of the Small DB program with a dynamic array structure, as opposed to the fixed array structure used to manage the rows in \cref{cha:more_data_types}. The Pascal code for the altered functions and procedures is shown in \lref{plst:dynamic-array-db}, the original version can be found in \lref{lst:pas-small-db}.

\straightcode{\pascode{plst:dynamic-array-db}{Pascal code for the dynamic array version of Small DB, see \lref{lst:pas-small-db} for the original version of this program}{code/pascal/dynamic-memory/ArrayDBforChap.pas}}

\mynote{
\begin{itemize}
  \item This version of the Small DB program includes the ability to add, delete, and print rows from the data store.
  \item The data store includes a dynamic array.
  \item See \sref{sec:using_dynamic_memory_allocation} for a discussion of how this works.
\end{itemize}
}


\chapter{Further results}
\label{cha:App-A}

\section{Baseline predictions}

\subsection{Structural time series}
\label{app:structts_preds}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_imp_preds-sts}
	\caption{Structural model baseline predictions}
	\label{fig:base_sts_preds}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_imp_residuals-sts}
	\caption{Structural model baseline residuals}
	\label{fig:base_sts_res}
\end{figure}

\subsection{Auto-ARIMA}
\label{app:arima_preds}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_imp_preds}
	\caption{Auto-ARIMA baseline predictions}
	\label{fig:base_arima_preds}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_imp_residuals}
	\caption{Auto-ARIMA baseline residuals}
	\label{fig:base_arima_res}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Univariate Prophet}
\label{app:uniprophet}

\subsection{Training fitting}

In training is obtained a very poor $R^2$ value, which from the plot in figure \ref{fig:prophet_uni_fitting} can be expected, since $\hat{y}$ looks like mostly an average of the seasonal components. Nonetheless, if it is considered that predicting an interval is acceptable, then the \ac{mobe} shows a very good result.

It can also be observed that the trend component did not overfit to the outliers.

\begin{table}[H]
	\centering
	\begin{tabular}{|c|c|}
		\hline
		\ac{mae} 	& 2.20 \\
		$R^2$ 		& 0.11 \\
		\ac{mobe} 	& 0.08 \\
		\hline
	\end{tabular}
	\caption{Univariate prophet training scores}
	\label{table:prophet_uni_train_scores}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{prophet_uni_fitting}
	\caption{Training fitting from univariate prophet}
	\label{fig:prophet_uni_fitting}
\end{figure}

\subsection{Test predictions}

When it comes to the predictions performance, the $R^2$ shows even worse results. The \ac{mobe} is also high since the trend component was not able to foresee the decreasing in the middle of the test set as shown in Figure \ref{fig:prophet_uni_preds}.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{prophet_uni_preds}
	\caption{Test predictions from the univariate Prophet model}
	\label{fig:prophet_uni_preds}
\end{figure}

\begin{table}[H]
	\centering
	\begin{tabular}{|c|c|}
		\hline
		\ac{mae} 	& 2.12 \\
		$R^2$ 		& -0.33 \\
		\ac{mobe} 	& 0.25 \\
		\hline
	\end{tabular}
	\caption{Univariate prophet prediction performance}
	\label{table:prophet_uni_test_scores}
\end{table}

\subsection{Residual analysis}

The residuals in the training set still show somehow resembles the original data pattern. This could be interpreted that there is still patterns to learn from data, i.e., this model is not complete.

\begin{figure}[hptb]
	\centering
	\begin{subfigure}{.8\textwidth}
		\includegraphics[width=\textwidth]{prophet_train_res_time}
		\caption{Residuals from train fit}
		\label{fig:prophet_train_res_time}
	\end{subfigure}%
	\hfill
	\begin{subfigure}{.8\textwidth}
		\includegraphics[width=\textwidth]{prophet_test_res_time}
		\caption{Residuals from test predictions}
		\label{fig:prophet_test_res_time}
	\end{subfigure}
	\caption{Univariate prophet residuals}
	\label{fig:prophet_res_time}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Multivariate Prophet}

\subsection{Test predictions}

The results of the predictions have notoriously improved in this case. The $R^2$ value is close to 0.9, which for such a complex signal shows how powerful is the model.

The \ac{mobe} has even decreased in the test set, this could be due to training outliers that still being difficult to approximate without overfitting the trend component.

\begin{table}[H]
	\centering
	\begin{tabular}{|c|c|}
		\hline
		\ac{mae} 	& 0.52 \\
		$R^2$ 		& 0.88 \\
		\ac{mobe} 	& 0.05 \\
		\hline
	\end{tabular}
	\caption{Multivariate prophet prediction performance}
	\label{table:prophet_multi_test_scores}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.6\linewidth]{prophet_multi_preds}
	\caption{Test predictions from multivariate prophet}
	\label{fig:prophet_multi_preds}
\end{figure}

\subsection{Residual analysis}

The residuals show that the patterns have been better learnt than in the univariate case.

\begin{figure}[hptb]
	\centering
	\begin{subfigure}{.8\textwidth}
		\includegraphics[width=\textwidth]{prophet_train_res_time_multi}
		\caption{Train}
		\label{fig:prophet_train_res_time_multi}
	\end{subfigure}%
	\hfill
	\begin{subfigure}{.8\textwidth}
		\includegraphics[width=\textwidth]{prophet_test_res_time_multi}
		\caption{Test}
		\label{fig:prophet_test_res_time_multi}
	\end{subfigure}
	\caption{Multivariate prophet residuals}
	\label{fig:prophet_res_time_multi}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Exhaustive experiments}

\subsection{Baselines predictions moving horizon}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_db-structts_R2}
	\caption{Baseline $R^2$}
	\label{fig:baseline_structts_r2}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{baseline_db-structts_MOBE}
	\caption{Baseline MOBE}
	\label{fig:baseline_structts_mobe}
\end{figure}

\subsection{Univariate Prophet prediction moving horizon}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{exp_prophet_uni_r2_time}
	\caption{Univariate moving prediction horizon: $R^2$}
	\label{fig:exp_prophet_uni_r2}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{exp_prophet_uni_mobe_time}
	\caption{Univariate moving prediction horizon: \ac{mobe}}
	\label{fig:exp_prophet_uni_mobe}
\end{figure}

\subsection{Multivariate Prophet prediction moving horizon}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\linewidth]{exp_prophet_mv_mobe_time}
	\caption{Multivariate moving prediction horizon: \ac{mobe}}
	\label{fig:exp_prophet_mv_mobe}
\end{figure}


\chapter{Conclusion}\label{chap:conclusion}

The main objective of our thesis, as described in chapter \ref{chap:introduction}, was to investigate the effects of sparsity in recurrent neural networks. In chapter \ref{chap:experiments}, we thoroughly explained the methodology followed, with results presented in chapter \ref{chap:results}. In this chapter, we briefly summarize our outcomes.

We followed two different methods to induce sparsity in recurrent networks. One method introduced sparsity by pruning a certain percent of input-to-hidden and hidden-to-hidden weights, and another method is to generate sparse structures based on random graphs.

Based on the results of pruning experiments, we conclude that it is possible to reduce the weight complexity of different RNN variants by more than 60\%. Our experiments also confirmed that it is possible to regain the accuracy of a pruned model with only one epoch in most cases. One big difference between RNN\_Tanh, RNN\_ReLU, and LSTM, GRU is that LSTM and GRU can regain the original performance even with 100\% hidden-to-hidden weight pruning.

Random structure experiments helped to identify the essential graph properties of a base random graph. In all four RNN variants we experimented with, two mutual graph properties are the number of nodes and node betweenness. The two most mutual and important graph properties are the number of edges and the number of source nodes.

The performance prediction experiment results identified that data from RNN\_Tanh has a weak fit with all three regressors used. Data from RNN\_ReLU and GRU has moderate and strong fit Random Forest regressor, respectively. Finally,  LSTM has a moderate fit with AdaBoost regressor.


\part{Fundamentals of Working in R}

This part of the course is an introduction to the different building
blocks in \R{}, objects.

\section{Objects, Modes, and Attributes \ldots \textit{Oh my!}}

The following description is typically not covered in introductions to
\R{}. It may seem weird, un-welcoming, and esoteric. In some sense, it
is. But, the reality is, if you can keep this in mind as you become
familiar with \R{} you will understand why and how things happen.

\subsection{Objects}

Within \R{}, the basic units or entities are called ``objects''. The
following are the basic objects that you will use as you learn
\R{}. There are many more.

\begin{description}

\item[vectors] These are exactly what you think they are. A sequence
  of elements of the same \textit{mode}. We'll talk about modes in a
  second. Vectors have order. You can have a vector of length 26
  (i.e.\ 26 elements) where each element is a letter in the
  alphabet. This vector's mode will be \texttt{character}.

\item[lists] Lists are similar to vectors. They are sequences of
  elements. They have order. However, each component of a list need
  not be of the same mode. You can have a list with two elements. The
  first element being a vector containing the 26 characters in the
  English alphabet and the second element being a vector contain the
  26 relative frequencies of these characters as they show up on the
  Political Science Department's website. These two vectors are not of
  the same mode and could not be combined into a single stacked
  vector. But, we can combined them in a list.

\item[matrices/arrays] Matrices are 2-dimensional rectangular
  objects. Arrays are higher dimensional ``rectangular'' objects. Each
  element must be of the same mode (e.g.\ everything is a character
  sting, logical value, numerical value, etc.)

\item[dataframes] Dataframes can be thought of as \textit{special}
  matrices. However, they technically are \textit{special} lists. In
  general, you will tend to store your empirical data in
  dataframes. These objects are two dimensional containers with the
  rows corresponding to ``observations'' and the columns corresponding
  to ``variables''.

\item[factors] These are vectors which are intended to help classify
  categorical data. They are handle differently than numerical,
  integer, or character vectors. Notice, though, factors are a
  different class of object, not a different mode.

\item[functions] Functions, although less obviously so, are also
  objects. The details of functions vary, but the general idea is
  simple. This object takes other objects (of any mode),
  performs even more basic functions on those objects and returns some
  final object (of some arbitrary mode).
\end{description}

\subsection{Modes}

\begin{description}
\item[logical] elements of either \texttt{TRUE} or \texttt{FALSE}

\item[integer] integer numbers like 1, 2, or -539

\item[numeric] real numbers; don't use 1.00, 2.00, and 3.00 when
  1, 2, and 3 will do

\item[complex] complex or imaginary numbers; you will most likely not
  use them much in applied political methodology but they can come up!

\item[character] elements made up of ``text''; each character string
  can be some arbitrary length; \texttt{``1''} is not \texttt{1} in \R{}

\item[raw] you can safely ignore this mode!

\end{description}

\subsection{Attributes}

The command \texttt{attributes()} can be used to set and extract the
attributes of an arbitrary \R{} object. Attributes can be accessed
through this general function or through particular functions which
are focused on a particular attribute. We won't cover how to do this
or why we would do this, here is an example of some attributes:
\begin{enumerate}
\item the length of a vector of any mode (\texttt{length()})
\item the size of a matrix or a higher dimensional array (\texttt{dim()})
\item the column names of a dataframe (\texttt{colnames()})
\end{enumerate}

\section{Assignment, References, Scope}
So far, we've superficially described objects in \R{}. Everything is
an object! Our data are an object. The results from a linear
regression are an object. The function that runs a linear regression
is an object.

Once we have created an object that we need, we can store it as its
own named object instead of re-computing it. So, if we want to save
the value of $2 \pi$ and recall it without having to continuously
multiple the two factors, we can create an object called
\texttt{vTwoPi} (or \texttt{BillRiker} for that matter).

After we have assigned this value to a name, we can recall it in
various ways.

\subsection{Assignment}

The standard assignment operator is \texttt{<-}. One \textit{could}
also use \texttt{=}, but you should not. We will keep \texttt{=} for
another use. Assignment can happen in the other direction,
\texttt{->}. The general form of the use of the assignment operator is
\verb= name <- object=. Although quotes aren't necessary,
\verb= "name" <- object= is identical.

An object is saved with the name \texttt{name} once this command is
run regardless of if an object of name \texttt{name} exists or
not. You can easily overwrite an object this way. You may want to or
you may not.

\subsubsection{Naming Rules}

However, when you name an object, you must follow certain rules.

Briefly, these are:

\begin{quote}
  Identifiers consist of a sequence of letters, digits, the period
  (`.') and the underscore. They must not start with a digit nor
  underscore, nor with a period followed by a
  digit.\footnote{\url{http://cran.r-project.org/doc/manuals/R-lang.html\#Identifiers}}
\end{quote}

In addition \texttt{TRUE}, \texttt{FALSE}, \texttt{Inf}, \texttt{NaN},
\texttt{NA}, \texttt{NULL}, \texttt{if}, \texttt{else}, \texttt{for},
\texttt{in}, \texttt{while}, \texttt{next}, and \texttt{break} are
reserved words.

I recommend the following convention for naming objects in \R{}. If I
want to associate the name ``foo'' or ``Foo'' with an object and it is
a vector (or factor vector), I name it \texttt{vFoo}. For objects of
the other classes, I use \texttt{lFoo}, \texttt{dfFoo}, and
\texttt{mFoo}.

The exception to this pattern is my naming of functions. Then, I use
action words and capitalization to name the function \texttt{DoFoo} or
\texttt{CalcFoo}.

\subsection{References}

If a vector object is named \texttt{vFoo} and you would like to
display or ``print'' the object, type \texttt{vFoo} and hit
\textit{enter}. Notice, \verb=print(vFoo)= will do the same
thing. \R{} automatically calls the \texttt{print()} function when you
simply enter the object name.

You can achieve the same thing with the more powerful function
\texttt{get()}.

If you are at the \R{} prompt, \R{} will help you choose the full name
of an object you are referring to. This is called tab completion. If
you type the beginning of a name of an object that \R{} knows about
and then hit \textit{tab} twice in succession, \R{} will print out a
list of object names (e.g. vectors, functions, etc.) that are
possible completions of the partial name you typed. For example, type
``\texttt{print}'' and invoke the tab completion. Look how many
objects \R{} knows about that begin with these five letters.

\subsection{Scope}

The scoping rules of the \R{} parser are slightly beyond this
tutorial. But see
\url{http://cran.r-project.org/doc/manuals/R-lang.html#Scope} to
understand them. The \R{} workspace is a hierarchy of layers. Objects
can exist in any of these layers (or environments/frames) and the
scoping rules determine which objects are being manipulated or
created. \texttt{.GlobalEnv} (an object) is the user's workspace and this is where
we will focus our attention for the time being.

\section{Asking for Help}

Asking for help in \R{} can be tricky for the following reasons:
\begin{itemize}
\item Depending on who you ask you may not like the response (e.g.\
  some people have been ridiculed on mailing out questions that are
  poorly formed). These communities have norms, if you want help,
  observe the norms before you start shooting off emails.

\item The language you use matters when you are searching. Knowing the
  right language is often half of the problem.

\item Not everyone is solving your exact problem. So you have to
  abstract away from your problem, find an applicable solution to the
  more general problem and then apply it to your specific case.
\end{itemize}

Follow these steps:
\begin{enumerate}
\item Begin looking for help/clarification in the official
  \R{} documentation.

\item If that fails, try a general web search with google. You will
  most likely want to append ``cran r'' to whatever topic you are
  looking for help with instead of just ``r''. The latter will not
  filter out unrelated sites too well.

\item Next, turn to the \R{} specific search engines.

\item Try re-wording your query. Read up on related topics to see if
  there are any hints on what might create a more successful search.

\item Ask a colleague in person. It is easier to explain these
  problems in person with code in front of you than communicating to
  someone else in the \R{} community.

\item Lastly, do your homework before you post in a public setting. It
  is not scary and people are helpful. But, they also tend not to be
  too patient with people who look for help without reading the
  mailing list or forum posting guidelines. No need to rock the boat.
\end{enumerate}

\subsection{R Documentation}
If you want to know about the object \texttt{foo}, you can type
\verb=help(``foo'')= or \verb=?foo=. If \texttt{foo} is not an object
in your current session with documentation, you will have no results.

If you want to search the \R{} help files for a concept or topic
\texttt{foo}, try \verb=??foo= or \verb=help.search(``foo'')=.

Next, try \texttt{help.start()}. This will bring up the HTML interface
to the help facilities. Here, you can find HTML versions of the \R{}
manuals which are worth searching. You can also browse
package-specific documentation through this interface.

\subsection{R Community}

\begin{description}
\item[Stack Overflow]
  <\url{http://stackoverflow.com/questions/tagged/r}> If you have
  looked for an answer to how to do something and have found nothing,
  try posting here first. Posting to the mailing list is a bad idea
  and can be embarrassing. Stack Overflow tends to be more friendly.

\item[R Meta Search] <\url{http://search.r-project.org/}> You can
  search just about every official web page and mailing list from this
  site.

\item[R Seek] <\url{http://www.rseek.org/}> Less impressive
  meta-search engine.

\item[R Graphics Gallery]
  <\url{http://addictedtor.free.fr/graphiques/}> Site with many
  examples of graphs and the code that generates them. Useful for
  getting started with fairly complex graphical output.

\item[inside-R] <\url{http://www.inside-r.org/}> Community site run by
  Revolution Analytics. Blog posts, a forum, snippets, etc.

\item[R-Forge] <\url{http://r-forge.r-project.org/}> Development
  platform for \R{} packages. You might find what you need here before
  it is found elsewhere.

\item[R-wiki] <\url{http://rwiki.sciviews.org/doku.php}> Pretty low
  traffic site site. However, many useful code snippets. Navigating
  this site can be an art.

\end{description}

\section{Exercises}
Enter all of the code in these exercises in a text file. Give it a
\texttt{.R} file extensions. Add some useful information at the top
about what this file by using comments. Run the code using RWinEdt's toolbar.

\begin{enumerate}
\item Create two numerical vectors:
\begin{verbatim}
vA <- c(1, 3, 5, 7)
vB <- (vA + 3) / 2
\end{verbatim}

\item Create a third vector by comparing the value of \texttt{vA} and \texttt{vB}:
\verb=vC <- vA > vB=

\item Use \texttt{print()} to inspect each vector. Now inspect each
  vector by referencing it on its own line.

\item What type of vector is \texttt{vC}? Try \texttt{is(vC)}.

\item What type of object is the output of \texttt{is(vC)}? Try \texttt{is(is(vC))}.

\item Coerce \texttt{vA} to a character vector with
  \texttt{as.character()}. What happens? If you get an error, use
  \texttt{?as.character}.

\item Run \verb=is.logical(is.logical(as.character(vA)))=. Does the
  output make sense? What kind of object is the output?

\item Create a matrix where \texttt{vA} and \texttt{vB} are the rows
  using \verb=rbind(vA, vB)=. Now do the same with the columns with
  \verb=cbind(vA, vB)=. Confirm that each of these is a matrix using
  either \texttt{is()} or \texttt{is.matrix()}. How do the output of
  these two differ?

\item Search for help on \texttt{data.frame()} using
  \texttt{?}. Knowing how to read \R{} help pages is a critical skill.

\item Create a dataframe from \texttt{vA}, \texttt{vB}, and \texttt{vC}.

\item Find two \R{} packages that would help you do \textit{ideal
    point estimation}.

\end{enumerate}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../full_course"
%%% End:


\documentclass[11pt, oneside]{article}
\usepackage[a4paper,bindingoffset=0.2in,%
            left=1in,right=1in,top=1in,bottom=1in,%
            footskip=.25in]{geometry}
\usepackage{graphicx}
\usepackage{enumerate}
\usepackage{url}
\usepackage{hyperref}
\usepackage{pbox}
\usepackage{CJKutf8}
\usepackage{mathtools}
\usepackage{microtype}
\usepackage[htt]{hyphenat}

\begin{document}
\title{Assignment 1: Naive Bayes (I)}
\author{Xiyou Zhou, 13307130189 \\ Computer Science and Technology}
\maketitle
\section{Problem 1}
\subsection{Question}
What's the relationship between Maximum-A-Posteriori (MAP) estimation and Maximum likelihood (ML) estimation in Naive Bayes? In what sense will the MAP estimation be equal to the ML estimation in Naive Bayes?
\subsection{Answer}
The Maximum-A-Posteriori estimation maximizes the posteriori distribution using likelihood while the Maximum likelihood estimation maximizes the likelihood function, i.e. fitting the data without considering the data bias. Thus the MAP estimation requires prior distribution to derive the actual posteriori distribution.

When \bf the prior function is a constant\rm, the MAP estimation would be equal to the ML estimation in Naive Bayes.

\section{Problem 2}
\subsection{Question}
Determine the mean, mode and variance for the beta distribution. Please write down your solution step-by-step.
\subsection{Answer}
The beta function is defined as follows:
$$B(\alpha, \beta) = \int_{0}^{1}{x^{\alpha-1}(1-x)^{\beta-1}}dx, \alpha > 0, \beta > 0$$
And the probability density function of beta distribution is:
$$p(x) = \frac{x^{\alpha-1}(1-x)^{\beta-1}}{B(\alpha, \beta)}$$
\subsubsection{Mean}
\begin{equation*}
\begin{multlined}
\mu=E(x)=\int_{0}^{1}{\frac{x^{\alpha-1}(1-x)^{\beta-1}}{B(\alpha, \beta)}xdx}=\frac{\int_0^1{x^\alpha(1-x)^{\beta-1}}dx}{B(\alpha, \beta)}=\frac{B(\alpha+1, \beta)}{B(\alpha, \beta)}\\
=\frac{\Gamma(\alpha+1)\Gamma(\beta)}{\Gamma(\alpha+\beta+1)}\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}=\frac{\alpha}{\alpha+\beta}
\end{multlined}
\end{equation*}

Here, $\Gamma(k)$ is Gamma function where $p(k) = \int_{0}^{\infty}{x^{k-1}e^{-x}dx}, k \in (0, \infty)$, thus:
$$\Gamma(k+1)=k\Gamma(k)$$since $$\Gamma(k + 1) = \int_0^\infty x^k e^{-x} \, dx = \left(-x^k e^{-x}\right)_0^\infty + \int_0^\infty k x^{k-1} e^{-x} \, dx = 0 + k \, \Gamma(k)$$
and$$B(\alpha, \beta) = \frac{\Gamma(\alpha)\Gamma(\beta)}{\Gamma(\alpha+\beta)}$$
\subsubsection{Mode}
Obviously, when $\alpha = \beta = 1$, the mode is 1. Otherwise, when $\alpha \leq 1$ or when $\beta \leq 1$ there is no mode.
When $\alpha > 1$ and $\beta > 1$, we differentiate the pdf and get:
$$p'(x) = \frac{(\alpha-1)x^{\alpha-2}(1-x)^{\beta-1}-x^{\alpha-1}(\beta-1)(1-x)^{\beta-2}}{B(\alpha, \beta)}$$
let $p'(x)=0$ and we get:
$$(\alpha-1)(1-x)-(\beta-1)x=0$$
$$\alpha-1+x-\alpha{}x-\beta{}x+x=0$$
$$(2-\alpha-\beta)x=1-\alpha$$
$$x=\frac{\alpha-1}{\alpha+\beta-2}$$
Therefore, the mode is at the peak of probability density function where $x=\frac{\alpha-1}{\alpha+\beta-2}$.
\subsubsection{Variance}
$E(x) = \frac{\alpha}{\alpha+\beta}$ and here we want $E(x^2)$.
\begin{equation*}
\begin{multlined}
E(x^2)=\int_{0}^{1}{\frac{x^{\alpha-1}(1-x)^{\beta-1}}{B(\alpha, \beta)}x^2dx}=\frac{\int_0^1{x^{\alpha+1}(1-x)^{\beta-1}}dx}{B(\alpha, \beta)}=\frac{B(\alpha+2, \beta)}{B(\alpha, \beta)}\\
=\frac{\Gamma(\alpha+2)\Gamma(\beta)}{\Gamma(\alpha+\beta+2)}\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}=\frac{(\alpha+1)\alpha}{(\alpha+\beta+1)(\alpha+\beta)}
\end{multlined}
\end{equation*}
Thus,
\begin{equation*}
\begin{multlined}
Var(x) = E(x^2)-E(x)^2 =\frac{(\alpha+1)\alpha}{(\alpha+\beta+1)(\alpha+\beta)}-(\frac{\alpha}{\alpha+\beta})^2\\
=\frac{(\alpha+1)\alpha}{(\alpha+\beta+1)(\alpha+\beta)}-\frac{\alpha^2}{(\alpha+\beta)^2}
=\frac{\alpha\beta}{(\alpha+\beta+1)(\alpha+\beta)^2}
\end{multlined}
\end{equation*}

\section{Problem 3}
\subsection{Question}
Show that the Maximum likelihood estimation for the Bernoulli / binomial model is
$$\hat{\theta}_{MLE} = \frac{N_1}{N}$$
by optimizing the log of the likelihood $p(D|\theta)= \theta^{N_1}(1-\theta)^{N_0}$. Please write down your solution step-by-step.
\subsection{Answer}
Here the likelihood function is given as $P(D|\theta) = \theta^{N_1}(1-\theta)^{N_0}$.\\
Thus, use log on this function and we get:
$$\log{P(D|\theta)} = \log{(\theta^{N_1}(1-\theta)^{N_0})}=N_1\log{\theta}+N_0\log{(1-theta)}$$
And then, we use the differentiation of the log likelihood:
$$\frac{d\log{P(D|\theta)}}{d\theta}=\frac{N_1}{\theta}-\frac{N_0}{1-\theta}$$
Let the differentiation equal to zero and we get:
$$\hat{\theta}_{MLE} = \frac{N_1}{N}$$

\section{Problem 4}
\subsection{Question (PRML exercise 1.22)}
Given a loss matrix with elements $L_{kj}$ , the expected risk is minimized if, for each $x$, we choose the class that minimizes $\sum_k{L_{kj}p(\mathcal{C}_k|x)}$. Verify that, when the loss matrix is given by $L_{kj} = 1-I_{kj}$ , where $I_{kj}$ are the elements of the identity matrix, this reduces to the criterion of choosing the class having the largest posterior probability. What is the interpretation of this form of loss matrix?
\subsection{Answer}
Since the sum of all posterior probability is 1 and we are using $L_{kj} = 1-I_{kj}$ as the corresponding loss function, the loss matrix would give 1 as a punishment for misclassification and 0 for correct classification to maximize $p(\mathcal{C}_j|x)$. Thus, misclassification rate is minimized using this loss matrix.

\section{Problem 5}
\subsection{Question (PRML exercise 2.3)}
Prove the probability function of binomial distribution
$$\rm Bin\it(m|N, \mu)=\binom{N}{m}\mu^m(1-\mu)^{N-m}$$
adds up to 1.
\subsection{Answer}
Here, since $(1+x)^{N+1} = \sum_{m=0}^{N+1}\binom{N+1}{m}x^m$ stands trivially for $N=0$, we can prove accordingly.
\begin{equation*}
\begin{multlined}
(1+x)^{N+1}=(1+x)(1+x)^N=(1+x)^N+x(1+x)^N\\
=\sum_{m=0}^{N}\binom{N}{m}x^m+x\sum_{m=0}^{N}\binom{N}{m}x^m\\
=\sum_{m=0}^{N}\binom{N}{m}x^m+\sum_{m=1}^{N+1}\binom{N}{m-1}x^m\\
=\binom{N}{0}x^0+\sum_{m=1}^{N}\left\{\binom{N}{m}+\binom{N}{m-1}\right\}x^m +\binom{N}{N}x^{m+1}\\
=\binom{N+1}{0}x^0+\sum_{m=1}^{N}\binom{N+1}{m}x^m +\binom{N+1}{N+1}x^{m+1}\\
=\sum_{m=0}^{N+1}\binom{N+1}{m}x^m
\end{multlined}
\end{equation*}
Therefore,
\begin{equation*}
\begin{multlined}
\sum_{m=0}^{N}\rm Bin\it(m|N, \mu)=\sum_{m=0}^{N}\binom{N}{m}\mu^m(1-\mu)^{N-m}\\
=(1-\mu)^N\sum_{m=0}^{N}\binom{N}{m}(\frac{\mu}{1-\mu})^m\\
=(1-\mu)^N(1+\frac{\mu}{1-\mu})^N
=1
\end{multlined}
\end{equation*}
Thus, the proof is complete.\\

\begin{thebibliography}{9}
\bibitem{WikipediaMap}
Wikipedia: Maximum a posteriori estimation,
\\\texttt{https://en.wikipedia.org/wiki/Maximum\char`_a\char`_posteriori\char`_estimation}

\bibitem{QuoraMapMleQuestion}
Quora: What is the difference between Maximum Likelihood (ML) and Maximum a Posteri (MAP) estimation?
\\\texttt{https://www.quora.com/What-is-the-difference-between-Maximum-Likelihood-ML-and-Maximum-a-Posteri-MAP-estimation}

\bibitem{StackExchangeBeta}
StackExchange: Mean And Variance Of Beta Distributions,
\\\texttt{http://math.stackexchange.com/questions/497577/mean-and-variance-of-beta-distributions}

\bibitem{UahGamma}
Uah: The Gamma Distribution,
\\\texttt{http://www.math.uah.edu/stat/special/Gamma.html}

\bibitem{WikipediaBinomial}
Wikipedia: Binomial distribution,
\\\texttt{https://en.wikipedia.org/wiki/Binomial\char`_distribution}

\end{thebibliography}

\end{document}


\documentclass{beamer}

\mode<presentation>{
\usetheme{Dresden}
\setbeamercovered{transparent}
\usecolortheme{lsc}
}

\mode<handout>{
  % tema simples para ser impresso
  \usepackage[bar]{beamerthemetree}
  % Colocando um fundo cinza quando for gerar transparências para serem impressas
  % mais de uma transparência por página
  \beamertemplatesolidbackgroundcolor{black!5}
}

\usepackage{amsmath,amssymb}
\usepackage[brazil]{varioref}
\usepackage[english,brazil]{babel}
\usepackage[utf8]{inputenc}
%\usepackage[latin1]{inputenc}
\usepackage{graphicx}
\usepackage{listings}
\usepackage{url}
\usepackage{colortbl}
\usepackage[ruled, linesnumbered]{algorithm2e}
\usepackage{amsmath}

\newcommand\Fontvi{\fontsize{6}{10}\selectfont}

\beamertemplatetransparentcovereddynamic

\title[Machine Learning for Probabilistic Robotics with Webots]{Machine Learning for Probabilistic Robotics with Webots}
\author[Joan Gerard]{%
  Joan Gerard\inst{1} \\
  Promotor: Prof. Gianluca Bontempi \inst{1}}
  \institute[ULB]{
  \inst{1}%
     Universit\'e Libre de Bruxelles}

% Se comentar a linha abaixo, irá aparecer a data quando foi compilada a apresentação
\date{June 7, 2019}

\AtBeginSection[]{
  \begin{frame}<beamer>
    \frametitle{Table of Contents}
    \tableofcontents[currentsection,currentsubsection]
  \end{frame}
}

\begin{document}

\begin{frame}
\titlepage
\end{frame}

\begin{frame}
\frametitle{Table of Contents}
\tableofcontents
\end{frame}

\section{Probabilistic Robotics}
\pgfdeclareimage[height=6cm]{RM}{figs/whereAmI.png}
\frame{
    \frametitle{Where am I?}
            \begin{itemize}
                \item Robot knows where it is
                \item Robot is kidnapped
                \item Robot does not know where it is
            \end{itemize}
            \pgfuseimage{RM}

}

\pgfdeclareimage[height=6cm]{BAYES-FILTER-1}{figs/bayes-filter-1.png}%
\pgfdeclareimage[height=6cm]{BAYES-FILTER-2}{figs/bayes-filter-2.png}%
\frame{
	\frametitle{Let's figure it out using Bayes Filter!}
	\begin{columns}
	    \column{5.5cm}
	        \begin{itemize}
	            \item Prediction
	        \end{itemize}
        	\pgfuseimage{BAYES-FILTER-2}

        \column{5.5cm}
            \begin{itemize}
	            \item Error Correction
	        \end{itemize}
        	\pgfuseimage{BAYES-FILTER-1}
	\end{columns}
}

\pgfdeclareimage[height=3cm]{PARTICLE-FILTER-1}{figs/particles-1.png}%
\pgfdeclareimage[height=3cm]{PARTICLE-FILTER-2}{figs/particles-2.png}%
\pgfdeclareimage[height=4.0cm]{PARTICLE-FILTER-3}{figs/particles-3.png}%
\frame{
	\frametitle{What about Particles Filter?}
	\centering

    	\pgfuseimage{PARTICLE-FILTER-1}
    	\pgfuseimage{PARTICLE-FILTER-2}\\
    	\pgfuseimage{PARTICLE-FILTER-3}
}

\begin{frame}
	\frametitle{Particles Filter Algorithm}

	\begin{itemize}
		\item Train/test a prediction model: robot state $\rightarrow$ distance sensor data
		\item Compare it with the true sensor data and determine a weight for each particle
		\item More close to the real sensor data more weight will have
	\end{itemize}
		\Fontvi
		\begin{columns}
			\column{9cm}
			\begin{algorithm}[H]

\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}
\Input{particles, controlAction, sensorData}
\Output{nextParticles}
$nextParticles \leftarrow \emptyset$\\
\BlankLine
  \ForEach{particle $\in$ particles}{
    particle.state $\leftarrow$ predictState(particle.state, controlAction) \\
    particle.weight $\leftarrow$ calculateWeight(particle.state, sensorData)\\
  }
  \For{$m=1 \text{ to } m=|partices|$}{
     $newParticle \leftarrow \text{draw } i \text{ from } particles \text{ with probability } \propto particles[i].weight$\\
     nextParticles.add(newParticle)\\
  }
  \Return nextParticles
\end{algorithm}

		\end{columns}

\end{frame}

\section{Webots}

\pgfdeclareimage[height=3cm]{WEBOTS}{figs/webots.png}%
\pgfdeclareimage[height=3cm]{ROBOT}{figs/robot.png}%
\pgfdeclareimage[height=3cm]{WORLD}{figs/world.png}%
\frame{
	\begin{columns}
    \frametitle{What tool can we use for simulating that problem?}
        \column{4cm}
            Webots!
            \begin{itemize}
            		\item Open Source
	                \item Python, C++, Java, etc.
	                \item 44 robot models
	                \item Robot model creation/customization
	                \item Robust documentation
            \end{itemize}

		\column{5cm}
			\pgfuseimage{WEBOTS}\\
			\pgfuseimage{ROBOT}
			\pgfuseimage{WORLD}

    \end{columns}
}

\section{Project Objectives}
\frame{
	\frametitle{Main Objective}
	\begin{itemize}
	    \item The objective will be to exploit the simulation benefits of Webots to introduce Machine Learning techniques together with non-parametric filters (such as Particles Filter) for robot positioning estimation, independently from the kind of robot used for in-door environments.
	\end{itemize}
}

\frame{
    \frametitle{How do we plan to do it?}
    \begin{itemize}
        \item Using Webots
        \begin{itemize}
            \item Create a custom robot
            \item Randomize control actions while moving avoiding obstacles
	        \item Use a realistic in-door environment
        \end{itemize}{}
	    \item Using Machine Learning and Particles Filters
	        \begin{itemize}
                \item Positioning estimation
    	        \item Error correction
        \end{itemize}
	    \item Test the developed technique with other robots
	\end{itemize}
}

\section{Preliminary Results}
\pgfdeclareimage[height=3.8cm]{RESULTS}{figs/position3.png}%
\pgfdeclareimage[height=3.5cm]{RESULTS1}{figs/arena.png}%
\frame{
	\frametitle{What have we done so far?}

	\begin{itemize}
	    \item Robot creation
	    \item Dummy environment creation
	    \item Collect distance sensor data + real position coordinates
	    \item Bayes Filter algorithm implementation
    	    \begin{itemize}
    	        \item \textit{Position prediction}: Odometry techniques
    	        \item \textit{Error correction}: Training a Random Forest model using collected data
    	    \end{itemize}
	\end{itemize}

	\pgfuseimage{RESULTS1}
	\pgfuseimage{RESULTS}
}

\frame{

    \frametitle{Are the results good?}
    \begin{itemize}
        \item Yes! but there are improvements to be done:
        \begin{itemize}
            \item Collect more data (+ Simulations)
            \item Randomize robot moves
            \item Train/test with other Machine Learning models
            \item Improve performance of the error correction algorithm
        \end{itemize}
    \end{itemize}
}

\pgfdeclareimage[height=6cm]{QUESTIONS}{figs/questions.png}%
\frame{
    \centering
    \pgfuseimage{QUESTIONS}
}

\end{document}