\documentclass[a4paper,12pt]{article}
\usepackage[left=2.5cm,right=2.5cm,top=2.5cm,bottom=2.5cm]{geometry}
\usepackage{color}
\usepackage[usenames,dvipsnames]{xcolor}
\usepackage{amsmath,amssymb,amsthm,algorithm,algorithmic,graphicx,yhmath,url,enumitem,lscape,mathtools}
\usepackage{wrapfig,subfigure}

\newcounter{problem}
\newenvironment{problem}{\refstepcounter{problem} \noindent {\bf Problem \arabic{problem}}}{\newpage}
\newenvironment{solution}{\vspace{0.3cm} \par \noindent {\bf Solution}}{}
\newenvironment{verification}{\vspace{0.3cm} \par \noindent {\bf Verification}}{}
\newenvironment{hint}{\vspace{0.3cm} \par {\bf Hint:}}{}

\newcounter{remark}
\newenvironment{remark}{\refstepcounter{remark} \vspace{0.3cm} \par \noindent {\bf Remark \arabic{remark}}}{\vspace{0.3cm}}
\newcounter{lesson}
\newenvironment{lesson}{\refstepcounter{lesson} \vspace{0.3cm} \par \noindent {\bf Lesson \arabic{lesson}}}{\vspace{0.3cm}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Rn}{\mathbb{R}^n}
\newcommand{\Rnn}{\mathbb{R}^{n \times n}}
\newcommand{\bes}{\begin{equation*}}
\newcommand{\ees}{\end{equation*}}
\newcommand{\be}{\begin{equation}}
\newcommand{\ee}{\end{equation}}
\newcommand{\eps}{\epsilon}
\newcommand{\fl}{\text{fl}}

\begin{document}

\title{5DV005, Fall 2018, Lab session 1}
\author{Carl Christian Kjelgaard Mikkelsen}

\maketitle
\tableofcontents

\section{The time and the place}
Our first lab session will take place on
\begin{center}
Wednesday, November 7th, 2018, (kl. 13.00-16.00), Room MA416-426.
\end{center}

\section{Setting the MATLAB path correctly}

\begin{enumerate}
\item Change directory to {\tt 5dv005ht18} and launch {\tt MATLAB} using the command {\tt matlab \&}.
\item Update {\tt MATLAB}'s search path to include the folders
\begin{verbatim}
  5dv005ht18/matlab
  5dv005ht18/exercises
\end{verbatim}
and all their subfolders. Save the path definition into the file {\tt 5dv005ht18/pathdef.m}
\end{enumerate}
Always launch {\tt MATLAB} from the directory {\tt 5dv005ht18}. This ensures that the search path is set correctly for each session.

\newpage
\section{The problems}

\begin{problem} Within {\tt MATLAB} do:
  \begin{enumerate}
  \item Issue the command {\tt help bal}. This prints a summary of the {\tt MATLAB} files contained in the folder {\tt bal}. These files are for solving problems in external ballistics, i.e. fire control of artillery.
  \item Issue the command {\tt help range\_rk1} and read through the documentation in detail.
  \item Execute the minimal working example {\tt range\_rk1\_MWE1}.
  \end{enumerate}

  \begin{remark} The function {\tt range\_rk1} demonstrates the standard that is  expect from you! In particular, all your codes must contain the call sequence, a complete description of all input and output variables, as well as a minimal working example. Moreover, all code must contain frequent and helpful comments. Software which does not adhere to this standard will not be accepted.
  \end{remark}
\end{problem}

\begin{problem}
  Copy the script {\tt lab1/scripts/l1p2.m} into {\tt lab1/work/my\_l1p2.m}. Your task is to modify {\tt my\_l1p2.m} so that it produces the output shown in Figure \ref{fig:output} and renders the graphics shown in Figure \ref{fig:trajectories}. These are the {\tt MATLAB} commands which you will need: {\tt fprintf}, {\tt plot}, {\tt axis equal}, {\tt xlabel}, {\tt ylabel}, {\tt grid}, {\tt axis}, {\tt print}.

\begin{figure}[ht]
  \begin{verbatim}
                  Flag    Range (meters)     TOI (seconds)
                    1        16844.662          66.33
                    1        14998.963          81.57
\end{verbatim} \caption{The output of {\tt l1p2} after completion.} \label{fig:output}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[width=16cm]{fig1.pdf} \caption{The trajectory of a two shells fired at using elevations of 45 degrees and 60 degrees} \label{fig:trajectories}
\end{figure}

\begin{remark}
  In general, I recommend that you export figures from MATLAB at {\tt .eps} files. You can convert the file {\tt fig.eps} to {\tt .pdf} format using the command {\tt ps2pdf -DEPSCrop fig.ps}. This crops the white space and produces a file called {\tt fig.pdf}.
\end{remark}

\end{problem}

\begin{problem} As demonstrated during yesterday's lecture the problem of computing a sum of positive numbers
  \bes
  s = \sum_{i=1}^m a_i
  \ees
  is substantially more complicated than it would appear! This exercise illustrates highlights the problem and shows how calculate an upper bound for the error.
  \begin{enumerate}
  \item Copy the script {\tt lab1/scripts/l1p3.m} to {\tt lab1/work/my\_l1p3.m}.
  \item Edit {\tt my\_l1p3} so that it uses {\tt simple\_sum} to compute
    \bes
    s = \sum_{i=1}^m \frac{1}{i}, \quad m = 2^{22}
    \ees
    using single/double precision and ascending/descending order, a total of 4 different calculations.
  \item Edit {\tt my\_l1p3} so that compute the error associated with each value of $s$. An exceedingly accurate value of the true sum $s$ can be obtained using double precision and the function {\tt kahan\_sum}.
  \item Copy function {\tt simple\_sum} to {\tt lab1/work/my\_simple\_sum}.
  \item Edit {\tt my\_simple\_sum} to include the computation of a running error bound $E_b$, such that
    \bes
    |s - \hat{s}| \leq E_b, \quad  E_b = \mu u
    \ees
    where $s$ is the true value of the sum, $\hat{s}$ is computed value of $s$ and $u$ is the unit roundoff, see \cite{mikkelsen2018}, Section 4.3, Algorithm 7. Be mindful of the fact that $u = 2^{-24}$ in single precision and $u = 2^{-53}$ in double precision.
  \item Edit {\tt my\_l1p3} so that it displays the running error bound right next to the actual error.
  \item Verify that the absolute value of the error is bounded by the running error bound.
  \item Which order of summation is the most accurate?
    \end{enumerate}
  \end{problem}

  \bibliographystyle{plain}
  \bibliography{../../../lecture-notes/refs}

\end{document}


\chapter{The purpose of this book}
This book is for people who like to understand how things work. It is
about getting to grips with computers and understanding how programming
languages really work by seeing not just what they do but how they do it.
The main language used here is Lisp. This is a simple enough language
that a very large part of its internal workings can be explained -- but
it is general enough that learning about how Lisp works can give insight
into the structure of almost all other programming systems. The aim
is to make the Lisp that is explained here Visible, and so perhaps
not surprisingly the version that is to be discussed is known as
Visible Standard Lisp, abbreviated as \vsl.

The term ``visible'' as applied here is intended
to indicate that all of its the inner workings are available for
inspection.  Lisp is one of the oldest
computer languages that is still in use (albeit in niche applications). It
it is also one that has been seriously influential in the development
of more modern languages, and so understanding it can give useful insight
into how we have got to where we are. The key application area that Lisp
dominated for many years was Artificial Intelligence, and perhaps in particular
the automation of logic, reasoning and symbolic mathematics. A second area
where it has been important has been as a notation embedded within some
other large program and used to describe customisation or scripting
capabilities there. The vsl{} system discussed here could serve quite
well in either of these broad areas.

Before explaining the organisation of the book here are a few taster
fragments that show what can be done with Lisp. You will find a fuller
explanation of each later on.

\setlength{\oddsidemargin}{1cm}
\setlength{\evensidemargin}{1cm}
(1) Arithmetic on big numbers works, so you can type in
{\small\begin{verbatim}
   (expt 7 510)
\end{verbatim}}
and obtain the output
{\small\begin{verbatim}
Value:
10000009377765355041159521080236291875500397545959064338020
91046611830071747256290251642675754113510381906538657851472
80048260730260174481888634961017465832153767083905055973115
17146314730511796799136227470799861707690686485640934223257
99898508099347892972049510888294887959312673650548919269730
26731194699108560193240351188856787007789047064042449190113
93827146105377053911155370062040100951494893263068969133321
6918712082217175249
\end{verbatim}}

Note how close to a power of ten this is, and so we have to a really
good approximation $7^{510} = 10^{431}$. From this we deduce that the
logarithm of $7$ is very close to $431/510$. Checking this with a calculator
reveals it has around 8 significant figures correct.

Of course big-number arithmetic has applications in encryption and
in keeping track of the bank balances of the very very rich as well as
for computing logarithms.

(2) One of the sample programs included here uses turtle
\begin{wrapfigure}{r}{2.5in}
{\centering
\includegraphics[width=2.4in]{snowflake.eps}}
\caption{A snowflake pattern}
\end{wrapfigure} graphics to draw snowflake patterns. The
key directives are {\tx turn} which changes the heading of the turtle
by a given number of degrees, and {\tx draw} which makes it march forward
a given distance leaving a visible trail. The particular graphics capabilities
shown here are not standardised across all versions of Lisp, but the
simple line-drawing capbility used here is available with \vsl{} and is
fully explained later on.

The full code has a few lines that reduce turtle graphics to
drawing instructions in ordinary cartesian coordinates, but the central part
of the Lisp to draw the image shown here is just
{\small\begin{verbatim}
   (setq a 5 b 0)
   (dotimes (i 2160)
      (draw 12.0)
      (setq a (plus a 7))
      (setq b (plus b a))
      (turn b))
\end{verbatim}}
\noindent and there is obviously scope for experimentation to see how changing the
numbers used alters the image created.

(3) The above two examples only use short fragments of Lisp code. The
final one here is based on use of many thousands of lines of Lisp to
implement an algebra system capable of integrating, differentiating and
performing all manner of other symbolic calculations. Here is a short
transaction that evaluates an integral:
{\small\begin{verbatim}}
1: int(x*(log x)^4, x);

  2          4           3           2
 x *(2*log(x)  - 4*log(x)  + 6*log(x)  - 6*log(x) + 3)
-------------------------------------------------------
                           4
\end{verbatim}}
\setlength{\oddsidemargin}{0cm}
\setlength{\evensidemargin}{0cm}

The first part of this book is a gentle introduction to Lisp and its
use. There will be those who view the term ``Lisp'' as referring to some
specific dialect. If that line were taken it would have been hard to avoid
the use of Common Lisp\cite{Steele:1990:CLL}. However Common Lisp is a large
and ambitious interpretation of the Lisp central ideas, and making its
implementation visible would be unfeasible. So the version of Lisp
used here preserves the key aspects of the language (for instance a notation
that uses unspeakable numbers of parentheses!), but is styled after
the Standard Lisp developed at the University of Utah\cite{StdLisp2},
which allows its scale to me much more reasonable. A section later in the book
provides some commentary about adapting everything here to match Common Lisp,
and the associated sample programs are provided in both Standard and Common
dialects.

The second section explains how the Lisp implementation used here works.
As well as making the source code accessible as a basis for learning about
Lisp and language implementation in greater generality it has comments
about the implementation issues of some features not included in the
code used here. As a result it amounts to a challenge to add some of
those, or to rework the existing code following alternate fundamental choices.
In this respect the aim is not to make this a book for passive readers, but
a starting point for active engagement with Lisp code. It should make it
possible both to write code in Lisp to do useful things, and also to
adjust and extent the implementation of Lisp to add extra capabilities or
tune it for particular needs. The source code of much of \vsl{} is coded
in the language C, and so as well as providing an introduction to Lisp this
book can be viewed as a resource showing C in use. But those who are not
fully fluent in C should still be able to follow the key aspects of how
\vsl's implementation works by reading the quoted C code as if it were
a sort of abstract pseudo-code and relying on explanations in the text to
sort out any places where C syntax could seem obscure.

Then there are a collection of miniaturised Lisp applications. These are
intended as starting points for more development rather that complete
polished code, but they illustrate the use of Lisp for a range of
tasks -- some frivolous and some serious.

Lisp has always tended to be the sort of language that a minority
of programmers get intensely attached to. Perhaps by learning it and
observing how simple it is, while at the same time how powerful it is,
will let anybody reading this join that minority! Discovering its history,
structure and use can give new insights into features of other
programming languages, and in many cases make them seem either
terribly limiting or dreadfully over-complicated.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS637: Database-Backed Websites
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 6}

\begin{enumerate}[label=(\alph*)]
\item In C, Java, and PHP, a non-negative integer constant consisting of a sequence of two or more digits is taken to be octal if the first digit is zero, Write a call to \texttt{preg\_match()} that checks if a string represents a non-negative octal number.

\item An octal number can be negative, with a minus sign before the digits. Write a call to \texttt{preg\_match()} that checks if a string represents an octal number.

\item A sequence of digits preceded by \texttt{0x} or \texttt{0X} is taken to be a hexadecimal integer, possibly preceded by a minus sign. The hexadecimal digits include \texttt{a} or \texttt{A} through \texttt{f} or \texttt{F} with values \texttt{10} through \texttt{15}. Write a call to \texttt{preg\_match()} that checks if a string represents a hexadecimal number, possibly negative.

\item Write a pattern that will match strings that are  either \texttt{messages} or \texttt{messages} followed by a dot character and a number. Write a call to \texttt{preg\_match()} that checks if a string represents this pattern.

\item A C-style comment is a sequence of characters that begin with \texttt{/*} and ends with \texttt{*/}. C-style comments do not nest. Write a call to \texttt{preg\_match()} that checks if the variable \$line represents a C-style comment.

\end{enumerate}

\subsection*{Solution}

\begin{enumerate}[label=(\alph*)]
\item Pattern \verb/(^[0]\d+$)/ will match all non-negative octal numbers as desired. The following code snippet shows how this pattern can be used inside the \texttt{preg\_match()} PHP function.

\lstset{language=php, tabsize=2}
\begin{lstlisting}
$pattern = '(^[0]\d+$)';
preg_match($pattern, $string);
\end{lstlisting}

\item Pattern \verb/(^[-]?[0]\d+$)/ will match all octal numbers as desired. The following code snippet shows how to use this pattern to check if \$string contains a string literal representation of an octal number.

\lstset{language=php, tabsize=2}
\begin{lstlisting}
$pattern = '(^[-]?[0]\d+$)';
preg_match($pattern, $string);
\end{lstlisting}

\item Pattern \verb/(^[-]?[0][x|X][0-9a-fA-F]+$)/ will match all string lateral representations of hexadecimal numbers. The following code snippet shows how to use this pattern to check if \$string contains a string literal representation of a hexadecimal number.

\lstset{language=php, tabsize=2}
\begin{lstlisting}
$pattern = '(^[-]?[0][x|X][0-9a-fA-F]+$)';
preg_match($pattern, $string);
\end{lstlisting}

\item Pattern \verb/(^\bmessages\b(?:[.]\d+)?$)/ will match string literals that begin with word \textit{messages} and may continue with a dot character followed by any number. The following code snippet shows how to use this pattern in PHP to see if an string literal matches the pattern.

\lstset{language=php, tabsize=2}
\begin{lstlisting}
$pattern = '(^\bmessages\b(?:[.]\d+)?$)';
preg_match($pattern, $string);
\end{lstlisting}

\item The simple pattern \verb#(\/\*.*\*\/)# would succeed in matching single-line block comments that are not nested. To resolve the nesting limitation, pattern \verb#(\/\*(\*(?!\/)|[^*])*\*\/)# is proposed. This pattern would match all single-line block comments. To further remove the limitation, pattern \verb#(\/\*([^*]|[\r\n]|(\*+([^*\/]|[\r\n])))*\*\/+)# is proposed that supports multi-line c-style block comments as well.

\end{enumerate}


\documentclass{article}
\usepackage{template}
\usepackage{url}

\title{Particle Filtering for Sequential Bayesian Inference}
\author{Zhangir Azerbayev}
\date{Spring 2020}
\numberwithin{equation}{section}

\begin{document}
\maketitle

\section{Problem Statement}
In many applied domains we wish to do Bayesian inference from noisy measurements sequentially and on-line. The
need to accomodate noisy measurements comes from measurement error or from our measurement being a stochastic
function of the quantity of interest. We wish to do inference sequentially and on-line so that we can make use of data
as it becomes available. For example, we may want to update a financial model according to live data or a robot to
update its
beliefs as it navigates an environment. Additionally, we may not want to store the entire history of data our model has
seen, for example in high-throughput high-dimensional settings such as biomedical data.

We provide a general formulation of our problem as follows. Let $(x_t)_{t\in\N}$ with $x_t\in\R^n$ be a discrete-time continuous-state
Markov process with transition kernel $p(x_t| x_{t-1})$ and prior $p(x_{0:t})$. The observations $(y_t)_{t\geq 1}$ are conditionally
independent given $(x_t)$ with known marginal distribution $p(y_t| x_t)$. Our goal is to solve the {\it filtering
problem}; that is, to compute $p(x_{0:t}| y_{1:t})$. We are particularly interested in the marginal distribution
$p(x_t| y_{1:t})$ and for some function $f_t(x_{0:t})$ the quantity
\[I(f_t) := \mathbb{E}_{p(x_{0:t}\mid y_{0:t})}[f_t(x_{0:t})]. \]

Whenever I play tennis, I must resist the temptation to run up to the ball and smack it into the net. For now, let us
make that mistake. Applying Bayes' rule, we see that the posterior distribution is given by
\begin{equation}
    p(x_{0:t}|y_{1:t}) = \frac{p(y_{1:t}|x_{0:t})p(x_{0:t})}{\int p(y_{1:t}|x_{0:t})p(x_{0:t})dx_{0:t}}.
\end{equation}
If we wish to estimate the posterior recursively, we get
\begin{IEEEeqnarray*}{rCl}
    p(x_{0:t+1}|y_{1:t+1}) &=& \frac{p(y_{1:t+1}|x_{0:t+1})p(x_{0:t+1})}{p(y_{1:t+1})}\\\\
                           &=& p(y_{1:t}|x_{1:t})\frac{p(y_{t+1}|x_{t+1})p(x_{0:t+1})}{p(y_{1:t+1})}\\\\
                           &=&
                           \frac{p(x_{0:t}|p(y_{1:t})}{p(x_{1:t})}\frac{p(y_{t+1}|x_{t+1})p(x_{0:t+1})}{p(y_{1:t+1})}\\\\
                           &=& p(x_{0:t}|y_{1:t})\frac{p(y_{t+1}|x_{t+1})p(x_{t+1})}{p(y_{t+1}|y_{1:t})}\\\\
                               &=& p(x_{0:t}|y_{1:t})\frac{p(y_{t+1}|x_{t+1})p(x_{t+1})}{\int
                               p(y_{t+1}|y_{1:t},x_{0:t+1})p(x_{0:t+1})dx_{0:t+1}}\\\\
                               &=& p(x_{0:t}|y_{1:t})\frac{p(y_{t+1}|x_{t+1})p(x_{t+1})}{\int p(y_{t+1}|x_{t+1})p(x_{0:t+1})dx_{0:t+1}}\\
\end{IEEEeqnarray*}
Here we run into a critical obstacle. Either method of computing $p(x_{0:t}|y_{0:t})$ from Bayes rule requires computing a normalizing
constant that is conditional on $y_{1:t}$, which in turns requires integrating over $x_{0:t}$. This is a complicated
and high-dimensional integral that is guaranteed to become intractable as $t$ grows large. At this point, we may proceed
in two ways.

First, we could try to make further assumptions about $(x_t)$ in order to obtain an analytic solution. This leads to the
{\it Kalman filter} \cite{anderson}. In
particular, we require Gaussian errors and linear state transitions. These assumptions are too strict for many
applications.

        The other approach is to find a way to approximate our high-dimensional integrals with an algorithm that does not
        scale with $t$. This leads to the powerful and general but compute-heavy method of {\it particle filtering} that is the focus of this article.

\section{Particle Filtering}
Particle filtering is a conceptually simple and powerful method for approaching the general filtering problem if one is willing to implement a
compute-heavy Monte-Carlo simulation. The basic idea of particle filtering is as follows. The intractable integrals in
section (1) become easy to approximate if we are able to take many samples from the posterior. In order to sample
from the posterior, we introduce the technique of {\it sequential importance sampling }(SIS). In practice, SIS tends to
converge to degenerate solutions, so we introduce a resampling trick that is enough to make our algorithm operationally
effective.
\subsection{Sequential Importance Sampling}
Let us introduce a proposal function $\pi(x_{0:t}|y_{1:t})$, conceptually similar to the proposal function used in the
Metropolis-Hastings algorithm. We define our proposal function based on our prior.
\[\pi(x_{0:t}|y_{1:t}) := p(x_{0:t}). \].
Define the \textit{importance weight}
$w(x_{0:t})$ by
\[w(x_{0:t}) = \frac{p(x_{0:t}|y_{1:t})}{\pi(x_{0:t}|y_{1:t})}. \]
Then we have that
\[I(f_t) = \frac{\int f_t(x_{0:t})w(x_{0:t})\pi(x_{0:t}|y_{1:t})d x_{0:t}}{\int w(x_{0:t})\pi(x_{0:t}|y_{1:t})
dx_{0:t}}. \]
Notice that
\begin{equation}\int f_t(x_{0:t})w(x_{0:t})\pi(x_{0:t}|y_{1:t})d x_{0:t} = \E_{x_{0:t}\sim
\pi(x_{0:t}|y_{1:t})}[f_t(x_{0:t})w(x_{0:t})]. \label{one}\end{equation}
And similarly that
\begin{equation}\int w(x_{0:t})\pi(x_{0:t}|y_{1:t})d x_{0:t} = \E_{x_{0:t}\sim
    \pi(x_{0:t}|y_{1:t})}[w(x_{0:t})].\label{two}
\end{equation}
Now suppose we draw $N$ iid samples $x_{0:t}^{(i)}$, called \textit{particles}, from $\pi(x_{0:t}|y_{1:t})$. Thus, by the
law of large numbers we can
approximate \ref{one} as
\[\frac{1}{N}\sum_{i=1}^N f_t\left(x_{0:t}^{(i)}\right) w\left(x_{0:t}^{(i)}\right). \]
and \ref{two} as
\[\frac{1}{N}\sum_{i=1}^N  w\left(x_{0:t}^{(i)}\right). \]
Thus we can obtain an estimate of $I(f_t)$ by
\[\hat{I}(f_t) = \frac{\frac{1}{N}\sum_{i=1}^N f_t(x_{0:t}^{(i)}) w(x_{0:t}^{(i)})}{\frac{1}{N}\sum_{i=1}^N
w(x_{0:t}^{(i)})} = \sum_{i=1}^Nf_t\left(x_{0:t}^{(i)}\right)\tilde{w}_t^{(i)},\]
where $\tilde{w}_t^{(i)}$ are normalized importance weights. It remains to show how we update our importance weights. We
do not know $p(x_{0:t}|y_{1:t})$, however from the definition of our importance weights  we can see that
\begin{equation}\tilde{w}_t^{(i)} \propto
    \tilde{w}_{t-1}^{(i)}\frac{p(y_t|x_t^{(i)})p(x_t^{(i)}|x_{t-1}^{(i)})}{\pi(x_t^{(i)}|x_{0:t-1}^{(i)}, y_{1:t})} =
\tilde{w}_{t-1}^{(i)} p\left(y_t|x_{t}^{(i)}\right).\label{recursive}\end{equation}

\subsection{Resampling: The Bootstrap Filter}
The SIS algorithm bypasses calculating high-dimensional integrals, however, it still scales poorly with respect
to $t$. To gain an intuition for why, think of approximating $p(x_{0:t}|y_{1:t})$ as consisting of two steps:
\begin{enumerate}
    \item Pick a good set of samples $\{x_{0:t}^{(i)}\}$.
    \item Calculate their likelihoods $p(x_{0:t}^{(i)}|y_{1:t})$.
\end{enumerate}
SIS allows to do (2), but does not help at all with (1) since we are still choosing our particles by sampling from the
prior. Thus, for any fixed $N$, there is a large enough $t$ such that $\{x_{0:t}^{(i)}\}_{i=1}^N$ is a sparse sample of
the state space and is not guaranteed to include any high-likelihood points. But if we fix $t$ and make $N$ large, we
still run into problems. Notice that if $p(x_{0:t}^{(i)}|y_{1:t})\ll p(x_{0:t}^{(j)}|y_{1:t})$, then we
should expect that $p(x_{0:k}^{(i)}|y_{1:k})\ll p(x_{0:k}^{(j)}|y_{1:k})$ for all $t<k$ due to the recursive nature of
our weight updates. In practice, when we run the SIS algorithms all but one of the importance weights rapidly converge to 0
(see accompanying code). In
order to avoid this problem we need some way of resampling our particles so that low importance particles are discarded
and replaced with high-likelihood particles.

There are many schemes for performing this resampling. We will consider the simplest: the \textit{bootstrap filter}. In
SIS, our approximation of $p(x_{0:t}|y_{1:t})$ is given by
\[p(x_{0:t}|y_{1:t}) = \sum_{i=1}^N \tilde{w}_i\delta_{x_{0:t}^{(i)}}(x_{0:t}),\]
where $\delta$ is the Dirac delta function. In the bootstrap filter, we will do away with importance weights as a part
of our model (even though they are still used in intermediate computations). Instead, we approximate
$p(x_{0:t}|y_{1:t})$ as
\[p(x_{0:t}|y_{1:t}) = \frac{1}{N}\sum_{i=1}^N \delta_{x_{0:t}^{(i)}}(x_{0:t}),\]
where the $\{x_{0:t}^{(i)}\}$ are now sampled from $p(x_{0:t}|y_{1:t})$ instead of the prior. To compute such
$\{x_{0:t}^{(i)}\}$, we modify the SIS algorithm with a \textit{selection step}, The selection step resamples particles so that those
inconsistent with $y_{1:t}$ are discarded and those consistent with $y_{1:t}$ are multiplied. In precise terms, The bootstrap filter's
Bayesian update at time $t$ is as follows:
\begin{enumerate}
    \item \textit{Importance sampling step}
        \begin{itemize}
            \item For each $i = 1, \dots, N$, sample $\tilde{x}_t^{(i)} \sim p(x_t|x_{t-1})$ and let
                $\tilde{x}_{0:t}^{(i)}=(x_{0:t}^{(i)}, \tilde{x}_t^{(i)})$.
            \item Compute normalized importance weights (cf. equation \ref{recursive})
                \[\tilde{w}_t^{(i)} \propto p(y_t|\tilde{x}_t^{(i)}). \]
        \end{itemize}
    \item \textit{Selection step}
        \begin{itemize}
            \item Resample with replacement $N$ particles $x_{0:t}^{(i)}$ from $\{\tilde{x}_{0:t}^{(i)}\}$ according to
                    their importance weights.
        \end{itemize}
\end{enumerate}
In the accompanying Python code, we see that this method is effective in drawing diverse samples from
$p(x_{0:t}|y_{1:t})$.

\section{Further Reading}
This expository article roughly follows chapter 1 of \cite{doucet}. The full text is a canonical reference for particle
filtering and sequential Monte Carlo. See section II for a number of convergence theorems concerning algorithms of the kind we
discuss. See section III for a discussion of more advanced particle filtering algorithms, particularly the resample-move
algorithm.

\textit{Probabilistic programming} is a programming paradigm designed to make expressing complex statistical inference
easy and natural. For an implementation of particle filtering in a probabilistic programming language, see \cite{gen}.
For an example of how particle filtering is used in modern large-scale statistical inference problems, see \cite{berke}.

The bearings-only tracking problem discussed in the accompanying code is described in \cite{gilks}.

\bibliographystyle{ieeetr}
\bibliography{project}

\end{document}


\documentclass[12pt]{article}
%% arXiv paper template by Flip Tanedo
%% last updated: Dec 2016

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%  THE USUAL PACKAGES  %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage{xcolor}
\usepackage{nopageno}
\usepackage{enumerate}
\usepackage{parskip}
\usepackage{tikzfeynman}

\renewcommand{\thesection}{}
\renewcommand{\thesubsection}{\arabic{subsection}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%  PAGE FORMATTING and (RE)NEW COMMANDS  %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\usepackage[margin=2cm]{geometry}   % reasonable margins

\graphicspath{{figures/}}	        % set directory for figures

% for capitalized things
\newcommand{\acro}[1]{\textsc{\MakeLowercase{#1}}}

\numberwithin{equation}{subsection}    % set equation numbering
\renewcommand{\tilde}{\widetilde}   % tilde over characters
\renewcommand{\vec}[1]{\mathbf{#1}} % vectors are boldface

\newcommand{\dbar}{d\mkern-6mu\mathchar'26}    % for d/2pi
\newcommand{\ket}[1]{\left|#1\right\rangle}    % <#1|
\newcommand{\bra}[1]{\left\langle#1\right|}    % |#1>
\newcommand{\Xmark}{\text{\sffamily X}}        % cross out

\let\olditemize\itemize
\renewcommand{\itemize}{
  \olditemize
  \setlength{\itemsep}{1pt}
  \setlength{\parskip}{0pt}
  \setlength{\parsep}{0pt}
}

% Commands for temporary comments
\newcommand{\comment}[2]{\textcolor{red}{[\textbf{#1} #2]}}
\newcommand{\flip}[1]{{\color{red} [\textbf{Flip}: {#1}]}}
\newcommand{\email}[1]{\texttt{\href{mailto:#1}{#1}}}

\newenvironment{institutions}[1][2em]{\begin{list}{}{\setlength\leftmargin{#1}\setlength\rightmargin{#1}}\item[]}{\end{list}}

\usepackage{fancyhdr}		% to put preprint number

%%%%%%%%%%%%%%%%%%%
%%%  HYPERREF  %%%%
%%%%%%%%%%%%%%%%%%%

%% This package has to be at the end; can lead to conflicts
\usepackage{microtype}
\usepackage[
	colorlinks=true,
	citecolor=black,
	linkcolor=black,
	urlcolor=green!50!black,
	hypertexnames=false]{hyperref}

%%%%%%%%%%%%%%%%%%%%%
%%%  TITLE DATA  %%%%
%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\begin{center}

    {\Large \textsc{Homework 1b:}
    \textbf{Linear Algebra}}

\end{center}

\vskip .4cm

\noindent
\begin{tabular*}{\textwidth}{rlcrll}
	\textsc{Course:}& Physics 231, \emph{Methods of Theoretical Physics} (2018)
	&
%	\hspace{1.2cm}
	&
	\\
	\textsc{Instructor:}& Professor Flip Tanedo (\email{flip.tanedo@ucr.edu})
	&
	%\hfill
	&
	&
	\\
	\textsc{Due by:}& Mon, October 15
	&
	%\hfill
	&
	%
\end{tabular*}

\subsection{Dimensional Analysis: explosive energy}

Suppose a large amount of energy is released in a small volume. This explosion produces a spherical shock wave where there is a sharp increase in air pressure. How does the characteristic size of the shock wave depend on time? Start by identifying the three relevant parameters for the problem and giving their dimensions in $L^\alpha M^\beta T^\gamma$ form. If you're stuck, see ``The Formation of a Blast Wave by a Very Intense Explosions''  by Geoffrey Taylor\footnote{\url{http://www.jstor.org/stable/98395}}.

\subsection{Discretized Second Derivatives}

\begin{enumerate}[(a)]
	\item Show that the discretized second derivative $D^2$ is written in matrix notation as
\begin{align}
	\left(D^2\right)^i_{\phantom{i}j}
	&=
	\frac{1}{\Delta x^2}
	\left(
	\delta_j^{(i-1)}
	- 2 \delta_j^{i}
	+
	\delta_j^{(i+1)}
	\right) \ .
	\label{eq:D2}
\end{align}
Specifically: if you have a continuous function $f(x)$ over some interval $[x_1, x_N]$. Break up the interval by defining $x_i = x_1 + (i-1)\Delta x$ with $\Delta x$ appropriately chosen. You can define a vector
\begin{align}
	\vec{f} = (f^1, f^2, f^3, \cdots, f^N)^T
\end{align}
where $f^i = f(x_i)$. Show that (\ref{eq:D2}) is the matrix representation of the second derivative acting on $\vec{f}$.

\textsc{Comment}: I will use $f^i$ to refer to particular component of $\vec{f}$ and  sometimes to also refer to the entire vectors, $\vec{f}$. Mathematicians hate me.

\item The above form is correct \emph{except} for the inclusion of boundary conditions. For \textbf{Dirichlet} boundary conditions, we could say that $f^0 = f^{N+1} = 0$ and instead focus on the components $\left(f^1, \cdots, f^N\right)^T$.

Now consider the case of \textbf{periodic} boundary conditions. This means that if you travel past one edge of the domain, you end up back at the opposite edge\footnote{Classic examples of periodic boundary conditions are \emph{Pacman} and \emph{Asteroids}. If you are not familiar with those classic video games, then be content that youth is wasted on the young.}.

Following the Dirichlet case, you may introduce fictitious `boundary' coordinates $f^0$ and $f^{N+1}$. What are these fixed to be? Write out what the $D^2$ matrix looks like for periodic conditions. Contrast this to the Dirichlet case.

\textsc{Comment}: observe that one of the key features of periodic boundary conditions is that there is no `special' location: there are effectively \emph{no} boundaries. This is why we like to use periodic boundary conditions when we do numerical simulations of large (potentially infinite) systems: we don't want the edges of our simulation to effect the bulk behavior, so we choose boundary conditions with no edges.

\textsc{Comment}: Recall from quantum mechanics that Hermiticity (self-adjoint-ness) is really important if you want real eigenvalues. Observe that the matrix representations of $D^2$ in this problem are both self-adjoint/Hermitian.

\end{enumerate}

\subsection{1D model of a molecule}

Consider the following \emph{spherical cow}\footnote{\url{https://en.wikipedia.org/wiki/Spherical_cow}} model of a water molecule (H$_2$O): imagine three point masses connected to each other by springs representing the atomic forces within the molecule. For this model, the point masses and springs are constrained to lie on a rigid hoop so that the problem is one dimensional.

\begin{center}
	\begin{tikzpicture}[line width=1.5 pt, scale=1]
		%
		\draw[gluon] (235:1) arc (235:125:1);
		\draw[gluon] (115:1) arc (115:5:1);
		\draw[gluon] (-5:1) arc (-5:-115:1);
		\draw[fill=black] (240:1) circle (.1);
		\draw[fill=black] (120:1) circle (.1);
		\draw[fill=black] (0:1) circle (.1);
		\node at (240:1.5) {$m_1$};
		\node at (120:1.5) {$m_2$};
		\node at (0:1.5) {$m_3$};
		\draw[<->, line width=1 pt] (225:1.5) arc (225:135:1.5);
		\draw[<->, line width=1 pt] (105:1.5) arc (105:15:1.5);
		\draw[<->, line width=1 pt] (-15:1.5) arc (-15:-105:1.5);
		\node at (60:2) {$x_{2}$};
		\node at (180:2) {$x_{1}$};
		\node at (300:2) {$x_{3}$};
	\end{tikzpicture}
\end{center}

For simplicity, assume that all of the masses and spring constants are the same,
\begin{align}
	m_i &= m & k_i &= k
	&
	\text{for } i=1,2,3
	\ .
\end{align}

\begin{enumerate}[(a)]
	\item Denote by $x$ the distance along the hoop. This is essentially an angular variable. Let $\vec{x} = (x^i, x^2, x^3)$ be a vector of positions along the hoop. The spring forces acting on mass $2$, for example, depends on $x_1$ and $x_2$. Using Hooke's law, write down the equation of motion in the following form:
\begin{align}
	\ddot{\vec{x}} &= V \vec{x} \ .
	\label{eq:eom}
\end{align}
Explicitly write the components of $V$ as a $3\times 3$ matrix in this basis.

	\item
\emph{This is the important part of the problem.}
Diagonalize $V$, identify the eigenvalues $\omega_i$ and eigenvectors $\vec{v}_i$. We refer to the set of eigenvalues as the \textbf{spectrum} of $V$. What is the physical interpretation of the spectrum? Draw a cartoon showing how each node `hoop molecule' moves with respect to the lowest eigenvector.

\textsc{Comment}: Diagonalizing matrices is tedious. Make sure you know how to do it. If you must, you can \emph{Mathematica} to do it for you---but you should feel a little bit embarrassed if you do.

\item
\emph{This is the tedious part of the problem so that you can write down ``equations 'n stuff.''}
Solve the eigenvalue problem in the $\left\{ \vec{v}_i\right \}$ basis. In this basis (\ref{eq:eom}) is simple to solve, right? Write down the general solution of the problem in the $\left\{ \vec{x}_i\right \}$ basis as a function of the initial positions and velocities in the system.  (If I were to choose to sacrifice a point for skipping a part of a problem, I would choose this sub-problem. But I'd make sure I understood how to do it.)

\end{enumerate}

\textsc{Comment}: I encourage you to think about how problem changes if the masses and spring constants are not degenerate.

\textsc{Comment}: You may enjoy reading a similar ``bead on a hoop with springs'' set up in \url{https://arxiv.org/abs/physics/0506195}. The paper discusses how the classical system exhibits what we call \textbf{critical behavior}\footnote{See, e.g.: \url{https://www.perimeterinstitute.ca/videos/conformal-bootstrap-magnets-boiling-water}}.

\subsection{\emph{What I want to be when I grow up...}}

In a few sentences or less, answer the following questions:
\begin{enumerate}[(a)]
\item Pick \emph{one} subfield that you are considering specializing in. Examples include cond-mat theory, AMO, biophysics, nuclear, etc.
\item Identify up to three potential PhD advisers at UCR for this discipline.
\item Write a question whose answer could be the basis for a PhD thesis. For example: \emph{Does dark matter interact with itself?}
\item Explain one \emph{big} idea in that field that could be the theme of a PhD. Use up to a \emph{maximum} of 4 sentences, and make it understandable to a high school student.
\end{enumerate}

\textsc{Comment}: if you have not done this exercise before, part (d) is more challenging than it seems.

\section{Extra Credit}

These problems are not graded and are for your edification. You are strongly encouraged to explore and discuss these topics, especially if they are in a field of interest to you.

\subsection{Error Estimates of High School Physics}

Use dimensional analysis to estimate the error on a simple high school physics calculation. To zeroth order (`high school order'). We would like to calculate the time it takes for an object to reach the ground after being dropped from rest at height $h$. Assume everything is human scale and on the surface of the Earth. The zeroth order solution is $t_0 = \sqrt{2h/g}$. Estimate the size of the correction from special relativity. If you need a hint, this problem came from ``Dimensional analysis, falling bodies, and the fine art of not solving differential equations'' by Craig Bohren\footnote{\emph{Am. J. Phys.} \textbf{72}  4 , April 2004 \url{http://dx.doi.org/10.1119/1.1574042}}.

\subsection{Natural Units}

In high energy physics one typically works in \textbf{natural units} where we work only in mass dimension\footnote{Actually, we work in dimensions of energy. Remind yourself of how energy and mass are related dimensionally.}, $[Q] = M^\beta$. High energy physicists are lazy, so they prefer to just write $[Q]=\beta$. Thus the mass of the electron has dimension $[m_e] = 1$.

This `gets rid' of the length and time dimensions. In order to do this \emph{without} throwing away information, one uses `universal constants' to convert between length/time and mass. Fortunately, nature provides us with such constants: the speed of light, $c$, and Planck's constant, $\hbar = h/2\pi$. When using natural units, experts often just say that we impose the following curious conditions:
\begin{align}
	c &= 1
	&
	\hbar = 1 \ .
\end{align}
This looks completely wrong from dimensional analysis. What gives?

\begin{enumerate}[(a)]
	\item First, get a sense of the numbers involved. Write out the numerical values of $c$ and $\hbar$ in cgs (centimeters, grams, seconds) units. Don't use more than 2 significant figures, that's just showing off.
	\item Suppose I have some quantity $Q$ with dimension $[Q] = L^\alpha M^\beta T^\gamma$ for some numbers $\alpha$, $\beta$, $\gamma$. What natural units is really saying is that we want to work with a different set of dimensions: energy $E$, $c$, and $\hbar$:
		\begin{align*}
			[Q] = E^x c^y \hbar^z .
		\end{align*}
		Find the values of $x$, $y$, and $z$ as a function of $\alpha$, $\beta$, $\gamma$.
	\item When a high energy physicist says that $[Q]$ has dimension $x$ in natural units, what they really mean is that $[Q] = E^x c^y \hbar^z$ for some $y$ and $z$. For a such a quantity, you already know what the `ordinary' dimensions are so that you can multiply by the appropriate powers of $c$ and $\hbar$ to get the number in any other units. For example, the mass of the proton $m_p$ is 1~GeV, where GeV is a unit of energy. We say that $[m_p] = 1$ in natural units. What are the values of $x$ and $y$? Convert 1~GeV into grams.
	\item Repeat problem 1 of Homework 1a in natural units.
	\item Convert the electron mass $m_e = 0.5$~MeV into a length. What does this length correspond to? (Think about what the possibilities are from quantum mechanics, check your answer.)
	\item Explain the following poetic statement: ``high-energy colliders are microscopes.''
\end{enumerate}

\subsection{Extra Credit: A 2D Model of a Molecule}

Consider the molecular model in Problem 3, except now the system is not constrained to live on a hoop. Instead, the three masses are free to move on a two-dimensional plane with springs connecting them to one another. This is a slightly more realistic model of the water molecule.

What is the dimensionality of the configuration space? In other words, if we write $\vec{x} = x^i \left|\vec{e}_i\right\rangle$, what is the range of $i$? Set up and solve the problem. Feel free to use \emph{Mathematica}'s \texttt{Eigensystem} function. Without doing any work, what is the interpretation of the eigenvectors with the lowest eigenvalue? What types of motion do these eigenvectors describe? You should be able to answer this without even doing any work on the problem.

\emph{Important comment}: This system exhibits $S_3$ permutation symmetry. The tools of representation theory (group theory) then allow one figure out the main features of the problem without doing any heavy lifting. A tool called \textbf{Schur's Lemma} (which sounds fancy, but is something you know intuitively in quantum mechanics) tells you how many different eigenvalues there are and the number of states with those eigenvalues.

\end{document}


\lab{Data Visualization}{Data Visualization}
\objective{Use data visualizations to explore data and communicate it to others.}
\label{lab:DataVis}

\noindent\emph{The purpose of visualization is insight, not pictures.} \small{---Card, Mackinlay, \& Shneiderman (after Hamming)}

\medskip

\noindent\emph{The ability to take data---to be able to understand it, to process it,
to extract value from it, to visualize it, to communicate it---that's
going to be a hugely important skill in the next decades,... because
now we really do have essentially free and ubiquitous data. So the
complimentary scarce factor is the ability to understand that data and
extract value from it.} \small{---Hal Varian (Google's Chief Economist)}
% in The McKinsey Quarterly, Jan 2009

\section*{What is data visualization?}
Data visualizations (or graphs) are used to understand and explore data as well as communicate results to others.  Often data is more easily interpreted or understood in graphical format than as a list of numbers or data points.

\section*{Types of visualizations}

Here are some of the most useful types of data visualizations.  We explain each of these in more detail below.

\begin{enumerate}
\item A \emph{bar chart} is generally the preferred method for displaying a small (discrete) collection of one-dimensional data points. Each discrete data point is represented as a bar whose length is determined by the value of the data.

\item A \emph{histogram} is a special type of  bar graph where the length of each bar corresponds to the number of data points in a certain range (sometimes called a \emph{bin}).
(see Figure \ref{fig:healthcare}).
Histograms are useful for revealing statistical distributions.

\item A \emph{line plot} plots $(x,y)$-pairs as points and connects them with a line.
You should use a line plot for plotting continuous functions or when there is a natural order to your two-dimensional data---for example when the data is a time series (a sequence of values over time).

\item A \emph{scatter plot} plots $(x,y)$ tuples as discrete points.  This should be used instead of line plot when the data is two-dimensional but has no natural order.

A scatter plot can reveal correlation (or lack thereof) between $x$ and $y$.

\item Three dimensional data can be displayed with on a two-dimensional page with a \emph{contour plot}.  This draws lines in the plane where the third value is constant---like a geological map.  Filling in the contours with successive colors gives a  \emph{heat map}.

\item A \emph{pseudocolor plot} or \emph{heat map} uses color to display a third dimension on a two-dimensional page (see Figure \ref{fig:heatmap}).

Three-dimensional data can also be plotted as a surface in 3-space, but it is often difficult to find a view where none of the important features of the data are obscured.
A contour plot or a heatmap can help avoid this problem and is often easier for the viewer to decode.

%
%
%\item A \emph{sparkline} is a word-sized graphic that is often embedded inline with the text.
%Created by Edward Tufte, an innovator and recognized leader in the field of data visualizations, sparklines can be used to quickly communicate large-scale trends in a data set. They are also ideal for binary data sets.
%Two sources for creating sparkplots in Python are \cite{sparkplots} and \cite{sparklineshtml}.
%%TODO: Add an example of a sparkline
%
%
%\item \emph{Small multiples} are several minature graphics of the same type in a single visualization.
%Like sparklines, they were pioneered by Edward Tufte. Any of the plots discussed so far may be assembled into a small multiple.
%Small multiples are useful for viewing trends over time, when the data associated to each time is more complex than a single number.
%More generally, they can be used to compare many sets of similar data (\cite{tufte1990} p. 67).
%Figure \ref{fig:log_plots} could be called a small multiple, though frequently they contain more subplots.
%Recall that this kind of plot can be created in matplotlib with the command \li{plt.subplot()}.
\end{enumerate}

\subsection*{Bar Charts}

Bar charts are best for relatively small sets of discrete, one-dimensional data.  They are usually best presented with the bars running horizontally because when the bars are vertical the labels tend to run together (and the other possible solution to that problem---making the text vertical---is harder to read).

You can create a histogram in matplotlib with the command \li{plt.hist()}.

TODO examples: Good example (horizontal). Bad examples: vertical problems, horizontal unsorted.

The data in a bar chart should generally be sorted in some natural way, based on the intended use.  If the chart will be used to compare the value of different data points, we usually sort by size; whereas if the chart will be used for looking up specific values, the data could be sorted alphabetically or in some other logical order that facilitates finding specific values.

As a general rule, bars should always start at zero, because people are conditioned to expect that.  Failing to do this can be confusing or misleading.  If you feel that you really must crop your bars so that they begin somewhere other than $0$, switch to a \emph{dot plot} instead.

TODO  example bar chart not starting at 0.  Give dot plot of same data for comparison \li{plt.plot(data,'o',markersize=4)}

\begin{problem}
TODO problem: get some data, plot a bar chart, horizontally,  order the data.TODO
\end{problem}

\subsection*{Histograms}

TODO more on what a histogram is and how it differs from a standard bar chart.

Histograms can give a clear view of the way data is distributed, but sometimes changing the size and position of the bins can change the picture substantially.

TODO example of a basic histogram.  Be sure to show the data so the reader can see how this is different from a standard bar.  Also give an example of the same data with bins a different size that gives a different picture.

When plotting histograms, it can sometimes be useful to include reference lines.  A nice way to do this without cluttering the figure is to make the reference lines white.  You can do this in matplotlib with the command \li{plt.grid(True, color='w',linestyle='-')}.

TODO example of previous histogram with white reference lines.

\begin{problem}
TODO problem: get some data, plot a histogram with white ref lines. Vary the bins and plot again and compare.\end{problem}

\subsection*{Line Plot}

Line plots should not be used for data that has no natural ordering or progression, because it gives the misimpression that there is an order or progression when there is none.

Recall that you can create a line plot in matplotlib with the command \li{plt.plot()}.

TODO give a bad example using a line graph with previous bar-chart data.

TODO give a good example of a line graph.

When plotting line graphs in matplotlib, the default line width is usually a little too thin.  It will help your reader (or you) if you set the line width to 2 or more with the argument \li{lw=2}.

\li{plt.plot(x,y,lw=2)}

TODO example of something too thin and the same with thicker line.

\begin{problem}
TODO problem: get some data, plot a line graph, with line thickness at least 2.
\end{problem}

\subsection*{Scatter Plots}

Scatter plots should be used instead of line plot when the data is two-dimensional but has no natural order.

 You can create a scatter plot in matplotlib with the command \li{plt.scatter()}.

TODO Example of scatter where there is a strong correlation. and one where there is none.

Varying the scale and window size can change your interpretation:

TODO example where different window gives a different sense of correlation.

\begin{problem}
TODO problem basic scatterplot. get data, plot, zoom in and out.
\end{problem}

\subsection*{Contour Plots and Heatmaps}

Three dimensional data can be displayed with on a two-dimensional page with a \emph{contour plot}.  This draws lines in the plane where the third value is constant---like a geological map.

You can create a contour plot in matplotlib with
\li{plt.contour()}.

TODO give more detail on how to do this.

TODO example.  Ideally with some meaningful data.

Filling in the contours with successive colors gives a  \emph{heat map}.

TODO give good example of filling in contours of previous example (using a good colormap).

Note that the default color map for matplotlib is the so-called \emph{rainbow color map}.  This is a very poor choice for a heat map or colored contour map because people do not naturally interpret one color to be greater than another.  To solve this problem in matplotlib, use the argument \li{cmap='NAME\_OF\_COLORMAP'}
where \li{NAME\_OF\_COLORMAP} is one of matplotlib's sequential or diverging colormaps.

TODO add in the list (and pix) of sequential and diverging colormaps from \url{http://matplotlib.org/examples/color/colormaps_reference.html}

\begin{problem}
TODO problem: get some 3-d data, make a contour plot and a filled contour plot with a good colormap.
\end{problem}

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\textwidth]{heatmap_color.png}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\textwidth]{heatmap_hot.png}
\end{subfigure}
%\includegraphics[width=\textwidth]{heatmap.png}
\caption{Both of these pseudocolor plots depict the function $z = \sin(x)\sin(y)$ on the domain $[-1,4] \times [-4,1]$.
The plot at left uses a rainbow gradient which is pretty to look at but whose choice of colors has no meaningful relationship to the data.
The plot at right uses the colormap called \li{afmhot}, which has the important benefit that the lighter colors are naturally associated with higher values, while the darker colors are naturally associated with lower values.}
\label{fig:heatmap}
\end{figure}

You can also create a pseudocolor plot in matplotlib with the \li{plt.pcolormesh()} command.

One situation where heatmaps are very useful is when data is naturally associated to a geographic location.  When a heatmap is plotted on a map it is often called a \emph{cloropleth}.

TODO example of a cloropleth---maybe percentage income donated to charity by county? (makes Utah look good).

As in the case of a histogram, if the data is relatively coarse then the way the data is divided up can have an effect on the overall picture of the heatmap.

TODO make an example where things are binned in two different ways  and give a different-looking result.

\begin{problem}
TODO problem where data looks different when binned differently.  Have them make different choices and compare the results.
\end{problem}

Finally, three-dimensional data can also be plotted as a surface in 3-space, but it is often difficult to find a view where none of the important features of the data are obscured.
A heatmap can help avoid this problem and is often easier for the viewer to decode.

TODO give example of how the surface can obscure your view and how the contour or heatmap gives a clearer picture.

\section*{Exploring data with visualizations}
When you visualize data you may notice trends that are not apparent from the numbers.
The following problem is a famous example of this phenomenon.

\begin{problem}\label{prob:anscombe}
The data sets I-IV in Table \ref{table:anscombe} are known as Anscombe's quartet.
Each dat set has identical statistical properties.
In each case,
\begin{itemize}
\item The mean of $x$ is 9 and the mean of $y$ is $7.5$.
\item The variance of $x$ is 11 and the variance of $y$ is 4.127.
\item The correlation between $x$ and $y$ is .816.
\item The linear regression line is $y=3+5x$.
\end{itemize}
Plot each data set. What do you notice?

\begin{table}[H]
\begin{tabular}{l l  |  l l  |  l l  |  l l }
I & & II & & III & & IV\\
x & y & x & y & x & y & x & y \\
\hline
10.0 & 8.04 & 10.0 & 9.14 & 10.0 & 7.46 & 8.0 & 6.58 \\
8.0 & 6.95 & 8.0 & 8.14 & 8.0 & 6.77 & 8.0 & 5.76 \\
13.0 & 7.58 & 13.0 & 8.74 & 13.0 & 12.74 & 8.0 & 7.71 \\
9.0 & 8.81 & 9.0 & 8.77 & 9.0 & 7.11 & 8.0 & 8.84 \\
11.0 & 8.33 & 11.0 & 9.26 & 11.0 & 7.81 & 8.0 & 8.47 \\
14.0 & 9.96 & 14.0 & 8.10 & 14.0 & 8.84 & 8.0 & 7.04 \\
6.0 & 7.24 & 6.0 & 6.13 & 6.0 & 6.08 & 8.0 & 5.25 \\
4.0 & 4.26 & 4.0 & 3.10 & 4.0 & 5.39 & 19.0 & 12.50 \\
12.0 & 10.84 & 12.0 & 9.13 & 12.0 & 8.15 & 8.0 & 5.56 \\
7.0 & 4.82 & 7.0 & 7.26 & 7.0 & 6.42 & 8.0 & 7.91 \\
5.0 & 5.68 & 5.0 & 4.74 & 5.0 & 5.73 & 8.0 & 6.89 \\
\end{tabular}
\caption{These four sets of data are known as Anscombe's quartet.}
\label{table:anscombe}
\end{table}
\end{problem}

As Problem \ref{prob:anscombe} demonstrates, a picture can quickly reveal properties of the data that are difficult to see in a list of numbers.

\subsection*{The Iterative Process of Exploratory Visualization}

Understanding a data set through visualization is often an iterative process. We start with an initial visualization---usually a broad overview.  Examining the result leads to observations that provoke questions.  We then filter or transform the data and adjust the visualization based on those questions and observations, and then repeat the process until we converge on a useful result.

Some of the questions that you should be asking in this iterative process include:
\begin{enumerate}
\item Are the data correct and reliable?
\item Are there any clear patterns, trends, or irregularities?
\item Would a different type of visualization give more information?
\item Is something unusual or interesting going on outside my view?
\item Would filtering the data to look at a particular subset give a more informative picture?
\item Would a transformation of the data give a more informative picture?
\end{enumerate}

\subsubsection*{Data integrity}

Visualization is a powerful way to identify problems with your data set.  Ask yourself, whether the results look like what you expect.  Are they the right magnitude? Are there missing values?

TODO examples of visualizations that reveal these types of problems.

\subsubsection*{Trends, patterns, and irregularities}

Some patterns you may look for:
\begin{enumerate}
\item Are there trends in time (up, down, flat, cyclical)?
\item Does the data look random, cyclical, or patterned?
\item Are there clear correlations between the $x$- and $y$- values?
\item Is the trend linear or curved?
\item Do lines intersect, or are they parallel?
\item Are the data symmetric or skewed?
\item What is the variance (wide vs narrow)?
\item Are there clusters and gaps?
\item Are the data dense or sparse?
\end{enumerate}

\subsubsection*{Using different types of visualization}

TODO examples

\begin{problem}
TODO problem: given one visualization, choose another type of vis and plot the data again with the new choice.  What are the relative merits of each?
\end{problem}

\subsubsection*{Changing the view}

By choosing to plot only a portion of your data, you may be able to focus your attention on the most relevant data.  On the other hand, you may also miss some interesting behavior in the data you didn't graph.

Another issue to consider is scale. If a number changes from 50,000 to 55,000 over some period, that  increase can appear small or large, depending on where we start the $y$-axis and how long we stretch out the $x$-axis (see Figure \ref{fig:dog_plots}).

You can modify the scale of an existing plot using \li{plt.yscale()} or \li{plt.xscale()}.

TODO more examples of zoom in or zoom out or different reference frame.

\begin{problem}
TODO problem changing the scale
\end{problem}

\subsubsection*{Filtering the data}

Sometimes important information about a subset of the data can be obscured by considering the whole data set.

TODO example where scatterplot of all people looks random, but separating by something like education level shows clear trends for one level.

\subsubsection*{Transforming the data}

Often a transformation of the data can reveal useful information that was previously hidden.

For example, data that grows very rapidly or that is tightly clustered may be better visualized by plotting the logarithm of the data, instead of the original data, A log-lin plot (also called a log plot) is constructed by by taking the logarithm of only the $y$-values of your data set.

This is easily done in matplotlib with the command \li{plt.semilogy(x,y)}.  The syntax is the same as \li{plt.plot}.

TODO example of basic line plot of $x^{14}$ versus $x^{13}$---looks identical inititally, but a log-lin plot shows they are very different.

Similarly, if the data grows very slowly, you may find that a lin-log plot (given by by taking the logarithm of only the $x$-values) gives a more useful view.
This is easily done in matplotlib with the command \li{plt.semilogy(x,y)}

TODO example of basic line plot of $x^{1/14}$ versus $x^{1/13}$---looks identical inititally, but a lin-log plot shows they are very different.

Transformations can also be useful for histograms.
As an example, let us look at the cost of health care claims.
Our data set is bogus but created to closely resemble real-life data.
An initial plotting of the data produces the histogram on the left in Figure \ref{fig:healthcare}.
From this picture, it is hard to see the nature of the data---the plot just looks like a single bar.
We may think that all health care claims are cheap.
However, when we take the logarithm of the claims prices, we get the histogram on the right in Figure \ref{fig:healthcare}.
Graphed on this scale, the data has a bell-shaped distribution similar to a normal distribution.
Moreover, we see that there are some very expensive claims being submitted, though they are few.
\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\textwidth]{healthcare_linscale.pdf}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\textwidth]{healthcare_logscale.pdf}
\end{subfigure}
\caption{The same data (meant to resemble the prices of health care claims) is plotted in both histograms above.
The plot at left is a lin-lin scale, whereas the plot at right is a log-lin scale.
In this case, changing scales revealed important information about the data.}
\label{fig:healthcare}
\end{figure}

As a general rule, log-lin plots are useful when the range of the $y$-values is orders of magnitude larger than the range of the $x$-values.
Our health care data was an example of this.
Similarly, lin-log plots are useful when the range of the $x$-values is much bigger than the range of the $y$-values.
In any case, you can try applying a log scale to one or both of your axes as a way to explore your data.

Let us analyze the mathematics behind log plots.
Suppose we have some data that roughly follows the line $y=cx^a$ where $c$ and $a$ are constants.
Taking the logarithm of both sides yields
\[
\log(y) = a\log(x) + \log(c).
\]
If we set new variables $X = \log(x)$ and $Y = \log(y)$, then this equation becomes $Y = aX + \log(c)$, which is the equation of a straight line.
So  on a log-log plot, polynomial-shaped data looks linear.
Similarly, taking the log of $y = a^x$ yields
\[
\log(y) = \log(a)x,
\]
so exponential data looks linear on a log-lin plot.
Finally, data of the form $y = a\log(x)+b$ will look linear on a lin-log plot.
See Figure \ref{fig:log_plots}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{log_plots.pdf}
\caption{Here are some functions plotted on various logarithmic and linear scales.
The dark blue line is $y=x$, the green line is $y=\log(x)$, the red line is $y=e^x$, and the light blue line is $y=x^3$.
We have used different ranges on the $y$-axes to highlight appropriate parts of the graphs.}
\label{fig:log_plots}
\end{figure}

Be warned that many different functions can look linear on log plots.
Thus, if you graph your data on a log-log plot and you see a line, you CANNOT conclude that your data must follow a polynomial.
In general, you need more information.

\begin{problem}
Graph your data from Problem 1
TODO: fix this reference so that it is automatic, otherwise it will break if we rearrange sections.---end TODO
 in Lab \ref{lab:complexity} (Matrices and Complexity) on a (a) log-log scale, (b) log-lin scale, and (c) lin-log scale.
Which plot makes the data easiest to understand?
\end{problem}

\begin{problem}

TODO: make a problem where they really explore some data, plot it in different ways, zoom in and out, and do a transformation.

\end{problem}

\section*{Communicating data with visualizations}

As the saying has it, ``a picture is worth a thousand words.''
Certainly, a data visualization is far more valuable than a thousand data points to a colleague who wants to understand the results of your analysis.
Data visualizations are critical for communication in both business and research.

\subsection*{Choosing the right visualization}
At this point, you have already used visualization to explore your data and draw conclusions, and you are ready to tell your results to someone else.
The first step in creating a graphic to do this is to choose the right type of visualization.
As we saw in the section ``Types of Visualizations,'' there are many possibilities, each with different strengths.
You should carefully consider which one best suits your data set and communication needs.  Your goal should always be communicating as clearly, effectively, and accurately as possible.

\subsubsection*{Visualizations to avoid}
There are also several popular types of visualizations that you should generally \textbf{not} use.  These may look pretty, but they tend to obscure information or are easily confused, because of natural difficulties humans have interpreting them.

For example, people have much more difficulty judging changes and variation in area than they do in length.
As a result, charts that depend on distinguishing variation in area will be less effective.  One of the most common charts that has this problem is the pie chart.

TODO include examples showing how hard it is to distinguish size of wedges.

TODO convert the pie data to a good bar chart and show see how much better it is.

As bad as the pie chart may be, there is a way to make things worse---make the chart three-dimensional.  It seems to be very popular for people to convert a visualization that is naturally 2 or 1 dimensional into one that is artificially three dimensional, but this inevitably makes things more confusing.  For example, we already have difficulty distinguishing the variation in area in a pie chart, and when we add a third dimension, it becomes even harder.

TODO give example of A 3-d pie chart with the same data rotated to different sides and show how the slice look very different from different angles.

Other visualization types that tend to be confusing and ineffective include the stacked bar and the radar graph.  The radar graph is just generally confusing.  While problem with stacking bars is that the second bar is not based at zero, so it becomes difficult to compare the different parts.

TODO give an example of a stacked bar chart.  and then swap the data so the other set is on top and see that the reader has a totally different takeaway depending on how it is laid out.

In these examples differences in design give completely different impressions of the data, so the reality of the data is obscured rather than illuminated by the design.  As the famous visualization guru, Edward Tufte, says, ``Show data variation, not design variation.''  Pie charts and stacked bar charts show design variation more than data variation.

\subsection*{Simplify}

\noindent\emph{A sentence should contain no unnecessary words, a paragraph no unnecessary sentences for the same reason that a drawing should contain no unnecessary lines and a machine no unnecessary parts.} \small{---Strunk and White}

Once you have chosen the visualization best suited to your data, you should design the details so that all elements contribute to the communication of data.

Edward Tufte offers two principles for simplifying graphics: (1) erase ink that does not communicate data, and (2) erase ink that communicates data redundantly.  (\cite{tufte2001} pp.96-100). According to these principles, decorative backgrounds, fancy lettering, and cute graphics in the corner of you plots should all be deleted.

Other opportunities for simplification are harder to notice and implement. As an example, let us examine the plot on the right of Figure \ref{fig:healthcare}.
This plot was created with the following code.

\begin{lstlisting}
import numpy as np
import scipy as sp
from matplotlib import pyplot as plt

m = 2.07
s = 0.63
num_samples = 10000
samples = []

for i in xrange(num_samples):
    samples.append(sp.random.lognormal(m, s))

sp_samples = sp.array(samples)

# Plot the histogram
plt.hist(sp_samples, 100)
\end{lstlisting}

For purposes of this example, the only important part of the above code is the line \li{plt.hist(sp_samples, 100)} that plots the histogram.
What ink in this plot can we erase because it communicates no data?

First, let's get rid of the vertical black lines that separate the bars of the histograph and the black outline. These are meaningless for our application and only distract.
We can do this by modifying our call to \li{plt.hist()} as follows.

\begin{lstlisting}
plt.hist(sp_samples, 100, edgecolor='none')
\end{lstlisting}

Next, let us turn off the top and right lines that box in the graph.
To do this, we need to access the ``axis'' object associated with the figure.
We can access the ``axis'' object with the command \li{plt.gca()} (get current axis).

\begin{lstlisting}
# Get current axis instance
axis = plt.gca()

# Hide top and right spines
axis.spines['right'].set_visible(False)
axis.spines['top'].set_visible(False)
\end{lstlisting}

These commands only turn off the sides of the box, not the tick marks.
To turn off the tick marks we run the following commands.

\begin{lstlisting}
# Only show bottom and left tick marks
axis.yaxis.set_ticks_position('left')
axis.xaxis.set_ticks_position('bottom')
\end{lstlisting}

Finally, we do not need so many tick marks on the $x$- and $y$-axes.
We adjust those along with specifying the range on each axis.

\begin{lstlisting}
# Fix x and y ranges
plt.xlim(0,70)
plt.ylim(0, 800)

# Use fewer axis ticks
plt.xticks(np.arange(0, 71, 35))
plt.yticks(np.arange(0, 801, 200))
\end{lstlisting}

The final graph is shown in Figure \ref{fig:simplify}.
Note how much cleaner this looks than the original.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{simplify.pdf}
\caption{A simplified version of the histogram in Figure \ref{fig:healthcare}.}
\label{fig:simplify}
\end{figure}

%TODO: Rewrite this problem to use a specific image with a specific context
\begin{problem}
Choose a graph you previously created that you think could use simplification.
Apply at least 3 of the following to simplify your plot:
\begin{enumerate}
\item Turn off some of the spines.
\item Fix the x or y range to better fit your data.
\item Change the number of x or y ticks.
\item Choose a better color scheme.
\end{enumerate}
Simplify your image in any other ways you can think of.
\end{problem}

\subsection*{Small Multiples}

Often we have too much information to display cleanly in one chart.  One indicator of this is if the reader must often look back and forth between a legend and the chart to decode the images.  A very useful way to deal with this problem is the method of \emph{small multiples}, an idea made famous by Edward Tufte.

The idea is to separate the data into many small graphics, placed near each other in a way that allows us to easily compare them.

For example, one might wish to compare the first six Chebyshev polynomials.  It would be natural to plot them together on the same graph, as in Figure TODO.  But this graph is too cluttered and one must constantly refer back to the legend to identify which colored line is which.

TODO insert cluttered chebyshev graph.

The small multiple approach is to plot each polynomial on its own set of axes, adjacent to each other and aligned in such a way that you can easily compare the different plots.

TODO insert small multiples of chebyshev

Note how even with 9 plots instead of 6, it is much easier to identify the different polynomials and to compare them.

TODO include the code for the small multiples.

The main tool you need to make small multiples is the command \li{ax = plt.subplot()}.  This allows you to set up multiple plots, and access them one at a time.

\begin{problem}

TODO make a problem where they plot small multiples of something and compare to the cluttered verison on top of each other.

\end{problem}

\subsection*{Tell the truth}

Just as graphics can tell your reader about your data, they can also give your viewer false impressions.
You should never use data visualizations to mislead your reader.
There are many ways pictures can be used to lie, and some of them can be unintentional.

\subsubsection*{2- and 3-d effects}
For example, if we see differences in flat shapes by area, not width or height.
Suppose you are creating a pictogram to demonstrate the recent increase in housing prices, which have doubled in the last $n$ years.
So, you draw two houses, one whose sides are 1cm and one whose sides are 2cm.
The area of the second house is now $2^2=4$ times as big as the area of the first. And your reader will likely interpret the picture as a four-fold increase rather than just double.
Hence, you should make two-dimensional pictures proportional in \emph{area} to the numbers they represent.

Even worse, volume is cubed rather than squared, so if a picture is intended to look like a three-dimensional object, doubling the side lengths will give the impression of an eight-fold increase.

TODO add figures with 2- and 3-d houses doubling in side length.

Three-dimensional special effects also tend to be misleading.  As mentioned above, a 3-D pie chart, for example, can be used to distort the sizes of its slices.
Similarly, a 3-D histogram can use perspective to distort the relative differences between bars.

\subsubsection*{Base, Scale, and Labels}

As mentioned above, bar charts should be based at 0 and all charts should be scaled in the way the most clearly conveys the most important and relevant information.  When there is a possibility of confusion, axes and key data values should be labeled for clarity.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{dog_plots.pdf}
\caption{The graph of some data can look very different depending on the scale.
Here are two graphs of the ``same'' line over the same range (1 to 4).
The only difference is the $y$-range. Notice easy it is to manipulate the y-scale to imply either a large or small increase in $y$.
These figures are especially misleading when we leave the range markers off of the $y$-axes.}
\label{fig:dog_plots}
\end{figure}

As in any form of communication, integrity is important in data visualization.
You will win more respect for yourself and those you represent if you avoid misleading diagrams.

\subsection*{Additional Resources}

There are many software packages that facilitate the visual exploration of data.
One Python library is Glue (see \cite{glue}).

Some packages for making nicer looking plots include \li{Seaborn} and \li{prettyplotlib}.  TODO get refs

For more about visualization of data, we highly recommend the following books:

TODO get full refs for the follwoing

\begin{itemize}
\item \emph{How to lie with statistics}
\item \emph{Envisioning Information} by Edward Tufte
\item \emph{The visual display of quantitative information} by Edward Tufte (2nd edition)
\item \emph{Now you see it} by Stephen Few
\end{itemize}

\printbibliography


\documentclass{article}

\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{tikz}
\usepackage{physics}
\usepackage{amssymb}
\usepackage[plain]{algorithm}
\usepackage{algpseudocode}

\usetikzlibrary{automata,positioning}

%
% Basic Document Settings
%

\topmargin=-0.45in
\evensidemargin=0in
\oddsidemargin=0in
\textwidth=6.5in
\textheight=9.0in
\headsep=0.25in

\linespread{1.1}

\pagestyle{fancy}
\lhead{\hmwkAuthorName}
\chead{\hmwkClass\ : \hmwkTitle}
\rhead{\firstxmark}
\lfoot{\lastxmark}
\cfoot{\thepage}

\renewcommand\headrulewidth{0.4pt}
\renewcommand\footrulewidth{0.4pt}

\setlength\parindent{0pt}

%
% Create Problem Sections
%

\newcommand{\enterProblemHeader}[1]{
    \nobreak\extramarks{}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
}

\newcommand{\exitProblemHeader}[1]{
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \stepcounter{#1}
    \nobreak\extramarks{Problem \arabic{#1}}{}\nobreak{}
}

\setcounter{secnumdepth}{0}
\newcounter{partCounter}
\newcounter{homeworkProblemCounter}
\setcounter{homeworkProblemCounter}{1}
\nobreak\extramarks{Problem \arabic{homeworkProblemCounter}}{}\nobreak{}

%
% Homework Problem Environment
%
% This environment takes an optional argument. When given, it will adjust the
% problem counter. This is useful for when the problems given for your
% assignment aren't sequential. See the last 3 problems of this template for an
% example.
%
\newenvironment{homeworkProblem}[1][-1]{
    \ifnum#1>0
        \setcounter{homeworkProblemCounter}{#1}
    \fi
    \section{Problem \arabic{homeworkProblemCounter}}
    \setcounter{partCounter}{1}
    \enterProblemHeader{homeworkProblemCounter}
}{
    \exitProblemHeader{homeworkProblemCounter}
}

%
% Homework Details
%   - Title
%   - Due date
%   - Class
%   - Section/Time
%   - Instructor
%   - Author
%

\newcommand{\hmwkTitle}{Assignment\ \#1}
\newcommand{\hmwkDueDate}{Due 31st August 2018}
\newcommand{\hmwkClass}{Classical Mechanics}
\newcommand{\hmwkClassTime}{}
\newcommand{\hmwkClassInstructor}{Prof.Manas Kulkarni}
\newcommand{\hmwkAuthorName}{\textbf{Aditya Vijaykumar}}

%
% Title Page
%

\title{
    %\vspace{2in}
    \textmd{\textbf{\hmwkClass:\ \hmwkTitle}}\\
    \normalsize\vspace{0.1in}\small{\hmwkDueDate\ }\\
%    \vspace{3in}
}

\author{\hmwkAuthorName}
\date{}

\renewcommand{\part}[1]{\textbf{\large Part \Alph{partCounter}}\stepcounter{partCounter}\\}

%
% Various Helper Commands
%

% Useful for algorithms
\newcommand{\alg}[1]{\textsc{\bfseries \footnotesize #1}}

% For derivatives
\newcommand{\deriv}[1]{\frac{\mathrm{d}}{\mathrm{d}x} (#1)}

% For partial derivatives
\newcommand{\pderiv}[2]{\frac{\partial}{\partial #1} (#2)}

% Integral dx
\newcommand{\dx}{\mathrm{d}x}

% Alias for the Solution section header
\newcommand{\solution}{\textbf{\large Solution}}

% Probability commands: Expectation, Variance, Covariance, Bias
\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\Bias}{\mathrm{Bias}}

\begin{document}

\maketitle

%\pagebreak

\begin{homeworkProblem}
\textbf{Part (a)}
\begin{flalign*}
V(x) = \alpha x^2/2 + \beta x^4/4\\
F(x) = -\pdv{V}{x} = -\alpha x - \beta x^3
\end{flalign*}
Including the damping term, we write the equation of motion as,
$$m\ddot{x} + \delta \dot{x} = -\alpha x - \beta x^3$$
$$m\ddot{x} + \delta \dot{x} + \alpha x + \beta x^3 = 0$$

The total energy of the system $E=T+V=m\dot{x}^2/2 + \alpha x^2/2 + \beta x^4/4$. Taking the time derivative, one gets
$$\dot{E} = m\ddot{x}\dot{x} + \alpha x \dot{x} + \beta x^3\dot{x}$$
Substituting from the equation of motion for $m\ddot{x}$,
$$\dot{E}=-(\delta \dot{x} + \alpha x + \beta x^3)\dot{x} + \alpha x \dot{x} + \beta x^3\dot{x}$$
$$\dot{E} = -\delta \dot{x}^2$$
Hence energy is dissipated from the system at a rate $\delta \dot{x}^2$.\\

\textbf{Part (b)}\\
The potential is plotted in in Figure 1. We see that for $\alpha = - 0.25$ and $\beta = -0.25$, there is only one critical point (maxima). So whatever energy is provided, the particle will not settle to a stable equilibrium point.

\begin{figure}[!h]
	\label{1a}
	\caption{Plots of $V(x)$ for different parameters}
	\centering
	\includegraphics[scale=0.7]{1.png}
\end{figure}

Now for the qualitative description of motion in various cases,
\begin{itemize}
	\item $\alpha > 0$ and $\beta > 0$ - There is just one equilibrium point at $x=0$, and it is a stable equilibrium point. A particle in this potential with a finite kinetic energy will exhibit periodic motion.

	\item  $\alpha > 0$ and $\beta > 0$ - There is just one equilibrium point at $x=0$, and it is an unstable equilibrium point. A particle at the origin will run off to infinity even if disturbed slightly. If the particle starts anywhere else, it will cross the potential barrier (ie. $x=0$) or not depending on whether it has enough energy.  Nevertheless, the particle will fly off to one of the infinities.This won't happen only if it is given exactly the amount of energy such that it stays at rest at $x=0$.

	\item  $\alpha > 0$ and $\beta < 0$ - There are two unstable and one stable equilibrium point ($x=0$). If the initial energy is greater than the value corresponding to the maxima, there will be no bound motion. If the particle starts off beyond the unstable equilibria with velocity enough to overcome the potential barrier, it will oscillate about the origin. Otherwise, the particle will fly off to the infinities. If the particle starts off between the unstable equilibria, it will execute oscillatory motion about the origin, unless it has enough energy to fly off to the infinities.

	\item  $\alpha < 0$ and $\beta > 0$ - The origin is an unstable equilibrium and there are two other stable equilibria. A particle starting off with a finite energy will always have bounded motion in this case, and will always execute oscillatory motion.
\end{itemize}

\textbf{Part (c)}\\
When $\delta = 0$, the energy can be written as $E = m\dot{x}^2/2 + \alpha x^2/2 + \beta x^4/4$. In the phase space, a constant value of energy could give a closed curve for all values of $\alpha$ and $\beta$, except for the case when $\alpha < 0$ and $\beta < 0$, where it will diverge whatever the values might be.

Figure 2 is the phase space plot analytically. The numerical integration is done in the \textit{attached codefile}.\\

\begin{figure}[H]
	\label{1c}
	\caption{Analytically obtained phase space plots}
	\centering
	\includegraphics[scale=0.7]{1c.png}
\end{figure}

\textbf{Part (d)}
$$E= mv^2/2 + \alpha x^2/2 + \beta x^4/4$$
$$\therefore v = \sqrt{\frac{2}{m}(E- \alpha x^2/2 - \beta x^4/4)}$$

At the turning points, $x$ will satisfy,
\begin{flalign*}
\alpha x^2/2 + \beta x^4/4 &= E\\
\alpha x^2/2 + \beta x^4/4 - E &= 0\\
\end{flalign*}
\text{The (possibly real) roots of which are }
$$x_1 = \sqrt{\frac{\sqrt{\alpha ^2+4 e \beta }}{\beta }-\frac{\alpha }{\beta }} \text{ and } x_2= - \sqrt{\frac{\sqrt{\alpha ^2+4 e \beta }}{\beta }-\frac{\alpha }{\beta }}$$

We know that,

$$v = \derivative{x}{t} \ , \  \therefore T = 2\int_{x2}^{x_1} \frac{1}{v}dx = 2\int_{x2}^{x_1} \frac{1}{\sqrt{\frac{2}{m}(E- \alpha x^2/2 - \beta x^4/4)}}dx$$

Using Mathematica to evaluate the indefinite integral, we get,

$$v = -2 \eval{-\frac{i \sqrt{\frac{\beta  x^2}{\alpha -\sqrt{\alpha ^2+4 \beta  \text{E}}}+1} \sqrt{\frac{x^2 \left(\sqrt{\alpha ^2+4 \beta  \text{E}}-\alpha \right)}{\text{E}}+4} F\left(i \sinh ^{-1}\left(x \sqrt{\frac{\beta }{\alpha -\sqrt{\alpha ^2+4 \text{E} \beta }}}\right)|\frac{\alpha -\sqrt{\alpha ^2+4 \text{E} \beta }}{\alpha +\sqrt{\alpha ^2+4 \text{E} \beta }}\right)}{\sqrt{\frac{\beta }{\alpha -\sqrt{\alpha ^2+4 \beta  \text{E}}}} \sqrt{\frac{8 \text{E}-2 \beta  x^4-4 \alpha  x^2}{m}}}}_{x_2}^{x_1}$$

where $F(x)$ is the \textit{elliptic integral of the first kind} and
$$x_1 = \sqrt{\frac{\sqrt{\alpha ^2+4 e \beta }}{\beta }-\frac{\alpha }{\beta }} \text{ and } x_2= - \sqrt{\frac{\sqrt{\alpha ^2+4 e \beta }}{\beta }-\frac{\alpha }{\beta }}$$

\textbf{Part (e)} - \textit{code files attached}\\

\textbf{Part (f)}\\
Substituting $x = A \cos(\omega t + \phi)$ in $m\ddot{x} + \delta \dot{x} + \alpha x  = \gamma \cos(\omega t) $
\begin{flalign*}
	(-m\omega^2 + \alpha )\cos(\omega t + \phi) - \delta \omega \sin(\omega t + \phi) = \frac{\gamma}{A}\cos(\omega t)
\end{flalign*}
Expanding the RHS and equating coefficients of $\cos(\omega t)$ and $\sin(\omega t)$, one gets
$$(-m\omega^2 + \alpha )\cos(\phi) - \delta \omega \sin(\phi) = \frac{\gamma}{A}$$
$$-(-m\omega^2 + \alpha )\sin(\phi) - \delta \omega \cos(\phi) = 0$$
Solving these, we get
$$A = \frac{\gamma}{\sqrt{\left(\alpha-m \omega ^2\right)^2+\delta^2 \omega ^2}} \text{ ; } \phi = \tan^{-1} \left(\frac{\delta \omega}{\alpha - m\omega^2}\right)$$

These are plotted below
\begin{figure}[h]
	\centering
	\includegraphics[scale=0.5]{A_om.png}
	\includegraphics[scale=0.5]{p_om.png}
\end{figure}

\textbf{Part (g)}\\
The equation to be solved now is $$m \ddot{x}(t)-\gamma  \cos (t \omega )+\delta  x'(t)+\alpha  x(t)+\beta  x(t)^3 = 0$$
Again, Substituting $x = A \cos(\omega t + \phi)$
$$3 A^3 \beta  \cos (t \omega +\phi )+A^3 \beta  \cos (3 t \omega +3 \phi )-4 A m \omega ^2 \cos (t \omega +\phi )+4 \alpha  A \cos (t \omega +\phi )-4 A \delta  \omega  \sin (t \omega +\phi )-4 \gamma  \cos (t \omega ) = 0$$

Expanding this in terms of trigonometric functions of $n \omega t$ and equating the coeffiecients of $\cos (\omega t)$ and $\sin(\omega t)$ to zero,

$$\frac{3}{4} A^3 \beta  \cos (\phi )+\alpha  A \cos (\phi )-A \omega ^2 \cos (\phi )-\delta  \omega  \sin (\phi ) = \gamma$$

$$\frac{3}{4} A^3 \beta  \sin (\phi )-\alpha  A \sin (\phi )+A \omega ^2 \sin (\phi )-\delta  \omega  \cos (\phi ) = 0$$

Solving the above two for the amplitude $A$, we get,
$$\frac{9 A^6 \beta ^2}{16}+\frac{3}{2} \alpha  A^4 \beta -\frac{3}{2} A^4 \beta  \omega ^2+\alpha ^2 A^2-2 \alpha  A^2 \omega ^2+A^2 \omega ^4+\delta ^2 \omega ^2 = \gamma^2$$
This is the governing equation for $A$.
\\

\textbf{Part (h)} - \textit{code files attached}\\
\end{homeworkProblem}

\begin{homeworkProblem}
	\textit{Solved completely in the attached code files}
\end{homeworkProblem}

\begin{homeworkProblem}
	\textbf{Part (a)}\\
	Let $(R_1,v_1)$ and $(R_2,v_2)$ be the separation and velocity and the perigee and apogee respectively. write the energy and momentum conservation equations at the apogee and perigee,
	$$m v_1 R_1 = m v_2 R_2 \implies v_1 = \frac{R_2}{R_1} v_2 = \frac{1+e}{1-e}v_2$$
	$$\frac{m v_1^2}{2} - \frac{GMm}{R_1} = \frac{m v_2^2}{2} - \frac{GMm}{R_2}$$
	Substituting for $v_1$ in the energy conservation equation, along with $R_1$ and $R_2$,
	$$ GMm\frac{R_1 - R_2}{R_2 R_1} = \frac{m v_2^2}{2} \left[1 - \left(\frac{R_2}{R_1}\right)^2\right]$$
	$$ GM\frac{2R_1}{R_2 (R_1 + R_2)} = v_2^2$$

	But the energy $E$ is just given by,
	\begin{flalign*}
	E &= \frac{m v_2^2}{2} - \frac{GMm}{R_2}\\
	&=  -\frac{GMm}{R_2}\left(\frac{R_2}{R_1 + R_2}\right)\\
	&=  -\frac{GMm}{R_1 + R_2}\\
	E&=  -\frac{GMm}{2R}\
	\end{flalign*}

	\textbf{Part (b)}\\
	The qualitative mechanism to change the orbit can be described as follows :-
	\begin{itemize}
		\item When the satellite is at apogee, it is given an impulse which changes the orbit into an ellipse of perigee separation equal to the current apogee separation ie. $R(1+e)$, and apogee separation $2R$.

		\item Then when the satellite is at the new apogee separation, it is given an impulse such that its orbit is changed into a circle of radius $2R$.
	\end{itemize}

	\textbf{Part (c)}\\
	Let us calculate the momentum imparted in each of the stages described above:-

	\begin{itemize}
		\item Using the formula derived in the first part, the energy is the new orbit is just
		$$E = -\frac{GMm}{R(3+e)}$$
		The change in energy is purely change in kinetic energy. Hence the new velocity is,

		\begin{flalign*}
			v^2 &= v_i^2 + \frac{2}{m}\Delta E\\
			&= \frac{GM(1-e)}{R(1+e)} + \frac{2}{m} \left( \frac{GMm}{2R} - \frac{GMm}{R(3+e)}\right)\\
			&= \frac{GM(1-e)}{R(1+e)} + \frac{2}{m}  \frac{GMm}{R}\left(\frac{1}{2} - \frac{1}{(3+e)}\right)\\
			&= \frac{GM(1-e)}{R(1+e)} + \frac{2}{m}  \frac{GMm}{R}\left(\frac{1+e}{2(3+e)}\right)\\
			&= \frac{GM(1-e)}{R(1+e)} +  \frac{GM}{R}\left(\frac{1+e}{3+e}\right)\\
			&= \frac{4GM}{R(1+e)(3+e)}\\
			v &= \sqrt{\frac{4GM}{R(1+e)(3+e)}}
		\end{flalign*}
		The impulse imparted in this step is,
		$$I_1= m \sqrt{\frac{GM}{R(1+e)}}\left[ \sqrt{\frac{4}{(3+e)}}-\sqrt{1-e}\right]$$
		\item The energy in the final orbit will be,
		$$E = -\frac{GMm}{4R}$$ and the final velocity will be,
		$$v_f = \sqrt{\frac{GM}{2R}}$$
		The impulse in this step can be calculated in a similar fashion and it is
		$$I_2= m{\sqrt{\frac{GM}{R}}}\left[\displaystyle{\sqrt{\frac{1}{2}} -\sqrt{\frac{(1+e)}{(3+e)}}}\right]$$
	\end{itemize}

	Total Impulse imparted to the satellite , \\
	$$I_{total} = I_1 + I_2$$
	 $$I_{total} = \displaystyle{ m \sqrt{\frac{GM}{R}}\left[ \frac{1}{\sqrt{2}} - \sqrt{\frac{(1+e)}{(3+e)}} +\frac{2}{\sqrt{(3+e)(1+e)}} - \sqrt{\frac{(1-e)}{(1+e)}}\right]}$$

	\textbf{Part (d)}\\

	Initial Angular Momentum,
	$$L_i = m v_2 R_2$$
	$$L_i = m \sqrt{GMR(1 - e^2)}$$

	Final Angular Momentum,
	$$L_f = 2m v_f R$$
	$$L_f = m\sqrt{2GMR}$$

	Hence, change in angular momentum,
	$$\Delta L = L_f - L_i = m\sqrt{GMR} (\sqrt{2}-\sqrt{1-e^2})$$
\end{homeworkProblem}

\begin{homeworkProblem}
	\textit{Solved completely in the attached code files}
\end{homeworkProblem}
\end{document}


\documentclass[letter]{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{ifthen}
\usepackage{fancyhdr}
\usepackage[usenames,dvipsnames,svgnames,table]{xcolor}
\usepackage{tikz}

%%%
% Set up the margins to use a fairly large area of the page
%%%
\oddsidemargin=.2in
\evensidemargin=.2in
\textwidth=6in
\topmargin=0in
\textheight=9.0in
\parskip=.07in
\parindent=0in
\pagestyle{fancy}

\expandafter\def\expandafter\quote\expandafter{\quote\sf\color{DarkGreen}}

%%%
% Set up the header
%%%
\newcommand{\setheader}[6]{
  \lhead{{\sc #1}\\{\sc #2} %({\small \it \today})
  }
  \rhead{
    {\bf #3}
    \ifthenelse{\equal{#4}{}}{}{(#4)}\\
    {\bf #5}
    \ifthenelse{\equal{#6}{}}{}{(#6)}%
  }
}

%%%
% Set up some shortcut commands
%%%
\newcommand{\R}{\mathbb{R}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\Proj}{\mathrm{proj}}
\newcommand{\Perp}{\mathrm{perp}}
\newcommand{\Span}{\mathrm{span}}
\newcommand{\Null}{\mathrm{null}}
\newcommand{\Rank}{\mathrm{rank}}
\newcommand{\mat}[1]{\begin{bmatrix}#1\end{bmatrix}}

%%%
% This is where the body of the document goes
%%%
\begin{document}
  \setheader{Math 211 (A01)}{Typed Homework 2}{Due Friday, February 20}{}{Bill Xiong, Arlene Zhang}{V00737042,V00802761}

\section{Question 1}
We know a system of linear equations can have $0,1$, or infinitely many solutions.
      \begin{enumerate}
        \item Explain why a system of linear equations cannot have exactly 2 solutions.
        \item For a homogeneous system of linear equations, what are the possibilities
          for the number of solutions?  Explain, and make sure to define
          \emph{homogeneous system}.
      \end{enumerate}

\subsection{Solution for Q1.1}
By definition, the individual equations that make up the system of equations in question have to be linear. This is, after all, \emph{linear} algebra.

Also by definition, lines can intercept other lines either at exactly one point, at infinite points (ie, two lines right on top of one another), or not at all (two lines parallel to each other). For a system to have exactly two solutions would require two points of interception, which would mean that one of the two equations plots to a parabola when graphed, making it nonlinear.

Thus, a system of linear equations cannot have exactly two solutions.

\subsection{Solution for Q1.2}
Recall that a system of linear equations is called \emph{homogeneous} if the right hand side is the zero vector. Since we can set all coefficients to 0 and get the zero vector as a result, a homogeneous system of equations must have at least one solution. (The one solution being the trivial solution.)

The system in question could also have infinite solutions, provided that the equations contained in the system are not all linearly independent.

Recall from 1.1 that a system of linear equations can have one solution, no solutions, or infinitely many solutions. Since the system in question is homogeneous, we are guaranteed to have at least one solution, thus a homogeneous system of linear equations can have exactly one or infinitely many solutions.

\section{Question 2}
Suppose $\mathcal M$ is a homogeneous system of 4 linear equations and 3 variables.
      Let $M$ be the non-augmented matrix of coefficients.
      \begin{enumerate}
        \item If $\Rank(M)=3$, how many solutions does the system $\mathcal M$ have?
          (You do not need to define \emph{reduced row echelon form}, but include
          all other relevant definitions.)
        \item If $\Rank(M)=2$, how many solutions does the system $\mathcal M$ have?
        \item Could $\Rank(M)=4$? Explain.
      \end{enumerate}
\subsection{Answer for Q2.1}
Remember that the \emph{rank} of a matrix is the number of leading ones that a matrix contains when it is put into reduced row echelon form. Since the matrix $\mathcal M$ is homogeneous, we are guaranteed at least one solution, the all-zero trivial solution.

\subsection{Answer for Q2.2}
If the rank of $\mathcal M$ is 2, then there are infinite solutions since there is a free variable.

\subsection{Answer for Q2.3}
$\mathcal M$ cannot have a rank of 4 because in Row-reduced echelon form, to have a rank of 4, $\mathcal M$ has to have 4 leading ones. Since $\mathcal M$ has only 3 variables, it cannot have 4 leading ones, and thus $\mathcal M$ can have at most a rank of 3.

\section{Question 3}
Consider the equation $\mathcal E$ given by
      \[
        x_1-x_2+x_3-x_4=0
      \]
      and let $V=\left\{\mat{x_1\\x_2\\x_3\\x_4}\in \R^4:\mat{x_1\\x_2\\x_3\\x_4}\text{ is a solution to }\mathcal E\right\}$.
      Find vectors $\vec v_1,\ldots,\vec v_n$ such that $V=\Span\{\vec v_1,\ldots,\vec v_n\}$.  Explain your
      process.
\subsection{Answer for Q3}
The Span of $\mat{1\\0\\0\\1}, \mat{0\\1\\1\\0}, \mat{1\\1\\0\\0}$ is equal to $V$. For the sake of simplicity, we'll call this set $U$.

Since the definition of a vector $\vec v \in V$ requires that the components of $\vec v$ satisfy $x_1-x_2+x_3-x_4=0$, it's easy to see that $x_1+x_3 = x_2+x_4$. If you view the equation as a balance, it's easy to see that any linear combination of the above vectors forces you to add $1$ to each side and thus never upsets the balance.

\subsubsection{Proof by dimensions}

Let's have a look at the dimensions of Span $U$ and $V$.
It's pretty obvious to see that dim (Span $U) = 3$ since the vectors of $U$ are lineally independent.

We can also see that dim $V \leq 4$ since $V$ is in $\R^4$.

But $V$ obviously doesn't span the entirety of $\R^4$ since there are vectors such as $\mat{3\\0\\0\\0}$ in $\R^4$ that doesn't satisfy the equation $x_1-x_2+x_3-x_4=0$, and thus are not in $V$. Therefore, dim $V \leq 3$.

Also, notice that $V$ contains $U$ since every vector in $U$ satisfies the equation $x_1-x_2+x_3-x_4=0$. Since Span $U$'s dimension is 3, by definition, the dimension of $V$ is at least $3$. We know from above that the dimension of $V$ is also at most $3$, thus, dim $V=3$.

Notice also that dim (Span $U)=3$. Since the dimensions of Span $U$ and $V$ are the same, Span $U$ = Span $V$ = $V$.

\subsubsection{Proof by Subsets}
\begin{enumerate}

\item $V \subset $ Span $U$

Let $\vec x \subset V$. We'll show that $\vec x$ can be made up of a linear combination of vectors in $U$. Let $\vec x = \mat{x_1\\x_2\\x_3\\x_4}$.

If this is true, then we should be able to write
$\vec x = \mat{x_1\\x_2\\x_3\\x_4} = a \mat{1\\0\\0\\1} + b\mat{0\\1\\1\\0} + c\mat{1\\1\\0\\0}$.

It's pretty obvious that for this equation to be true, $a = x_4$ and $b = x_3$. For the equation to hold, we must also set $c = x_1-x_4$. This results in the following:
\[
\vec x = \mat{x_1-x_4\\x_3+x_1-x_4\\x_3\\x_4}
\]

We go back to the definition: $x_1-x_2+x_3-x_4 = 0$, by rearranging this equation, we get $x_2 = x_1+x_3-x_4$, which is exactly what we got above.

Therefore $\vec x \subset$ span $U$.

\item Span $U \subset V$
Let $\vec u \subset $ span $U$. By definition, $\vec u$ is a linear combination of the vectors in $U$. Thus.

$\vec u = a \mat{1\\0\\0\\1} + b\mat{0\\1\\1\\0} + c\mat{1\\1\\0\\0} = \mat{a+c\\b+c\\c\\a}$. Does this satisfy the original requirement for $V$?

It does indeed! $(a+c)-(b+c)+c-a = a+c-b-c+c-a = 0$.

Therefore $\vec u \subset V$.

Because Span $U$ and $V$ are subsets of each other, Span $U=V$.
\end{enumerate}

\section{Addendum}
\includegraphics{ElvishPlea}\footnote{Originally I wanted to make all the variables Elvish too, but there were problems with the \LaTeX{} package and so I had to settle for including only the one sentence.}

\end{document}