\documentclass[11pt, fleqn]{amsart}
\usepackage[T1]{fontenc}
%\usepackage[latin1]{inputenc}
%\usepackage[english,francais]{babel}
\usepackage[retainorgcmds]{IEEEtrantools}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{pst-slpe}
\usepackage{auto-pst-pdf}
\usepackage{pst-node}
\usepackage{pst-plot}
\usepackage{pst-3dplot}
\usepackage{cancel}
\usepackage{empheq}
\usepackage{pst-grad,multido}
\usepackage{graphicx,url,etoolbox}
\usepackage{pifont}
\usepackage{subfig}
\usepackage{siunitx}
\usepackage{mathtools}
\usepackage{filecontents}
\usepackage{amsthm}
\usepackage{yhmath}
\usepackage{enumitem}
\usepackage{hyperref}
\hypersetup{
    colorlinks = true,
    linkbordercolor = {white},
    linkcolor=blue
}
%\usepgfplotslibrary{external}
\usepackage{standalone}
%\tikzexternalize
\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}
\usepackage{pstricks-add}
\let\clipbox\relax
%\usepackage{pgfplots}
\usepackage{stmaryrd}
\usepackage{lipsum}
%\usepackage{tikz}

\setlength{\textwidth}{\paperwidth}
\addtolength{\textwidth}{-2in}
\calclayout

\DeclareMathOperator{\vect}{vec}

\graphicspath{{./include/}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}

\setcounter{MaxMatrixCols}{20}

\theoremstyle{definition}
\newtheorem*{Lemma}{Lemma}
\theoremstyle{definition}
\newtheorem*{Theorem}{Theorem}
\theoremstyle{definition}
\newtheorem*{Definition}{Definition}
\theoremstyle{definition}
\newtheorem*{Proposition}{Proposition}
\theoremstyle{remark}
\newtheorem*{rmk}{Remark}

\patchcmd{\section}{\scshape}{\bfseries}{}{}
\makeatletter
\renewcommand{\@secnumfont}{\bfseries}
\makeatother
\newcommand{\overbar}[1]{\mkern 1.5mu\overline{\mkern-1.5mu#1\mkern-1.5mu}\mkern 1.5mu}

\newcommand{\Var}{\mathrm{Var}}
\newcommand{\E}{\mathrm{E}}
\newcommand{\Lip}{\mathrm{Lip}}
\newcommand{\Cov}{\mathrm{Cov}}
\renewcommand{\Re}{\operatorname{Re}}
\renewcommand{\Im}{\operatorname{Im}}
\newcommand{\atgt}{\mathrm{atan2}}
\newcommand{\KL}{\mathrm{KL}}
\newcommand{\id}{\mathrm{id}}
\newcommand{\I}{\mathrm{I}}
\newcommand{\p}{\mathrm{p}}

\usepackage{stmaryrd}
\usepackage{lipsum}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}

\addtolength{\voffset}{-3cm}
\addtolength{\textheight}{4cm}
\newcommand{\ud}{\,\mathrm{d}}

\makeatletter
\patchcmd{\@settitle}{center}{flushleft}{}{}
\patchcmd{\@settitle}{center}{flushleft}{}{}
%\patchcmd{\@setauthors}{\centering}{\raggedright}{}{}
\patchcmd{\abstract}{3pc}{0pt}{}{} % remove indentation
\makeatother
%\pgfplotsset{compat=1.16}

\title{%
		Probabilistic Robotics: \\
		Markov decision processes}

\begin{document}
\maketitle
\author{Pierre-Paul TACHER}

\setcounter{section}{1}
\section{} There are planning algorithms like Lifelong Planning $A^{*}$ and Dynamic $A^{*}$ Lite ~\cite{likhachev1} or $D^{*}$ lite~\cite{likhachev2} which adresse the context of time varying cost function and the problem of replanning reusing the results of previous searches.

\section{}
\subsection{} In this section the actions are deterministic and the state can be described using only the position, using for instance convention in figure~\ref{0}. Note there is no need to use a discount factor $(\gamma=1)$ for future payoff in this setting.  For the mathematically inclined reader, I give a proof of convergence of the value iteration algorithm in the stochastic shortest path setting (which encompass our simple example): cf.~\ref{ssp}. Algorithm is implemented in github repository. The quite obvious 2 equally optimal policies are represented in figure~\ref{1}.
\begin{figure}
\includegraphics[width=0.5\textwidth]{tikz1-figure0.pdf}
\caption{Problem setting}
\label{0}
\end{figure}
\begin{figure}
\includegraphics[width=0.5\textwidth]{tikz1-figure1.pdf}
\caption{Optimal policies}
\label{1}
\end{figure}
\subsection{} Optimal policies does not change by introducing a bit of stochasticity in action. See figure~\ref{2} for optimal expected payoff starting from each position.
\subsection{} We have to add a dimension to the state variable to handle the hidden state variable; there are 3 potential states regarding the knowledge of the position of the final reward. Thus we can modelize the state by
\begin{IEEEeqnarray*}{rCl}
x & = & (x_{1}, x_{2}) \in \mathbb{R}^{2}	\\
\end{IEEEeqnarray*}
where
\begin{IEEEeqnarray*}{rCl}
x_{1} & \in & \llbracket 0, 17 \rrbracket	\\
\end{IEEEeqnarray*}
\begin{equation*}
x_{2} = \left\{ \,
	\begin{IEEEeqnarraybox}[][c]{l?s}
		\IEEEstrut
		-1 & if it is known the reward is on the left, \\
		0 & in absence of any a priori information, \\
		+1 & if it is known the reward is on the right
		\IEEEstrut
	\end{IEEEeqnarraybox}
\right.
\end{equation*}
The knowledge $x_{2}$ can change only when reaching position $X$ or ending game in either position $H$; the transition are represented in figure~\ref{3}. The expected maximum payoff is represented in figure~\ref{4}. The optimal policy is represented in figure~\ref{5}.
\begin{figure}
\includegraphics[width=0.5\textwidth]{tikz1-figure6.pdf}
\includegraphics[width=0.5\textwidth]{tikz1-figure7.pdf}
\caption{State values}
\label{2}
\end{figure}
\begin{figure}
\includegraphics[width=0.7\textwidth]{tikz1-figure2.pdf}
\caption{Transition between $x_{2}=0$ and $x_{2}=\epsilon \in \{-1,1\}$}
\label{3}
\end{figure}
\begin{figure}
\includegraphics[width=0.7\textwidth]{tikz1-figure3.pdf}
\includegraphics[width=0.7\textwidth]{tikz1-figure5.pdf}
\caption{States values}
\label{4}
\end{figure}
\begin{figure}
\includegraphics[width=0.7\textwidth]{tikz1-figure4.pdf}
\caption{Optimal policy}
\label{5}
\end{figure}

\clearpage
\appendix
\section{}
\subsection{}
\label{ssp}
\begin{Proposition} To do.
\end{Proposition}
\begin{proof}
\end{proof}
\clearpage
{\small
\begin{thebibliography}{99}
\bibitem{likhachev1} Likhachev,~Maxim; Koenig,~Sven:
\emph{Lifelong Planning A* and Dynamic A* Lite: The proofs},
(2001)
\bibitem{likhachev2} Likhachev,~Maxim; Koenig,~Sven:
\emph{D* lite},
American Association for Artificial Intelligence (2002)
\end{thebibliography}
}
\end{document}


\lab{Matplotlib Syntax and Customization Guide}{Matplotlib Customization}

\objective{
The documentation for Matplotlib can be a little difficult to maneuver and basic information is sometimes difficult to find.
This appendix condenses and demonstrates some of the more applicable and useful information on plot customizations.
It is not intended to be read all at once, but rather to be used as a reference when needed.
For an interative introduction to Matplotlib, see the Introduction to Matplotlib lab in Python Essentials.
For more details on any specific function, refer to the Matplotlib documentation at \url{https://matplotlib.org/}.
}

\section*{Matplotlib Interface} %==================================
%This section is largely taken from MatplotlibIntro. I'm unsure about how important this part is.
Matplotlib plots are made in a \li{Figure} object that contains one or more \li{Axes}, which themselves contain the graphical plotting data.
Matplotlib provides two ways to create plots:
\begin{enumerate}
\item Call plotting functions directly from the module, such as \li{plt.plot()}.
This will create the plot on whichever \li{Axes} is currently active.
\item Call plotting functions from an \li{Axes} object, such as \li{ax.plot()}.
This is particularly useful for complicated plots and for animations.
\end{enumerate}

Table \ref{mpl:interface} contains a summary of functions that are used for managing \li{Figure} and \li{Axes} objects.

\begin{table}[H]
\centering
\begin{tabular}{r|l}
    Function & Description\\
    \hline
    \li{add_subplot()} & Add a single subplot to the current figure\\
    \li{axes()} & Add an axes to the current figure\\
    \li{clf()} & Clear the current figure\\
    \li{figure()} & Create a new figure or grab an existing figure\\
    \li{gca()} & Get the current axes\\
    \li{gcf()} & Get the current figure\\
    %\li{ion()} & Turn on interactive mode\\
    %\li{iof()} & Turn off interactive mode\\
    \li{subplot()} & Add a single subplot to the current figure\\
    \li{subplots()} & Create a figure and add several subplots to it\\
\end{tabular}
\caption{Basic functions for managing plots.}
\label{mpl:interface}
\end{table}

\li{Axes} objects are usually managed through the functions \li{plt.subplot()} and \li{plt.subplots()}.
The function \li{subplot()} is used as \li{plt.subplot(nrows, ncols, plot_number)}.
Note that if the inputs for \li{plt.subplot()} are all integers, the commas between the entries can be omitted.
For example, \li{plt.subplot(3,2,2)} can be shortened to \li{plt.subplot(322)}.
%This both returns that subplot and sets it as the active one.

The function \li{subplots()} is used as \li{plt.subplots(nrows, ncols)}, and returns a \li{Figure} object and an array of \li{Axes}.
This array has the shape \li{(nrows, ncols)}, and can be accessed as any other array.
Figure \ref{mpl:subplots-layout} demonstrates the layout and indexing of subplots.

\begin{figure}[H] % The layout created by subplots(23i).
%Taken from matplotlib intro
\captionsetup[subfigure]{justification=centering}
\centering
\begin{framed}
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_1.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_2.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_3.pdf}
\end{subfigure}
\\
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_4.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_5.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/layout_6.pdf}
\end{subfigure}
\end{framed}
\caption{The layout of subplots with \li{plt.subplot(2,3,i)} (2 rows, 3 columns), where \li{i} is the index pictured above. The outer border is the figure that the axes belong to.}
\label{mpl:subplots-layout}
\end{figure}

The following example demonstrates three equivalent ways of producing a figure with two subplots, arranged next to each other in one row:
\begin{lstlisting}
>>> x = np.linspace(-5, 5, 100)

# 1. Use plt.subplot() to switch the current axes.
>>> plt.subplot(121)
>>> plt.plot(x, 2*x)
>>> plt.subplot(122)
>>> plt.plot(x, x**2)

# 2. Use plt.subplot() to explicitly grab the two subplot axes.
>>> ax1 = plt.subplot(121)
>>> ax1.plot(x, 2*x)
>>> ax2 = plt.subplot(122)
>>> ax2.plot(x, x**2)

# 3. Use plt.subplots() to get the figure and all subplots simultaneously.
>>> fig, axes = plt.subplots(1, 2)
>>> axes[0].plot(x, 2*x)
>>> axes[1].plot(x, x**2)
\end{lstlisting}

\begin{warn}
Be careful not to mix up the following similarly-named functions:
\begin{enumerate}
    \item \li{plt.axes()} creates a new place to draw on the figure, while \li{plt.axis()} or \li{ax.axis()} sets properties of the $x$- and $y$-axis in the current axes, such as the $x$ and $y$ limits.
    \item \li{plt.subplot()} (singular) returns a single subplot belonging to the current figure, while \li{plt.subplots()} (plural) creates a new figure and adds a collection of subplots to it.
\end{enumerate}
\end{warn}

\section*{Plot Customization} %=====================================

\subsection*{Styles}%=====================================
Matplotlib has a number of built-in styles that can be used to set the default appearance of plots.
These can be used via the function \li{plt.style.use()}; for instance, \li{plt.style.use("seaborn")} will have Matplotlib use the "seaborn" style for all plots created afterwards.
A list of built-in styles can be found at \url{https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html}.

The style can also be changed only temporarily using \li{plt.style.context()} along with a \li{with} block:
\begin{lstlisting}
with plt.style.context('dark_background'):
    # Any plots created here use the new style
    plt.subplot(1,2,1)
    plt.plot(x, y)
    # ...
# Plots created here are unaffected
plt.subplot(1,2,2)
plt.plot(x, y)
\end{lstlisting}

\subsection*{Plot layout} %=====================================
\subsubsection*{Axis properties} %================================
Table \ref{mpl:axis} gives an overview of some of the functions that may be used to configure the axes of a plot.

\begin{table}%[H] %better for fitting stuff in pages
\centering
\begin{tabular}{r|l}
    Function & Description\\
    \hline
    \li{axis()} & set the $x$- and $y$-limits of the plot\\
    \li{grid()} & add gridlines\\
    \li{xlim()} & set the limits of the $x$-axis\\
    \li{ylim()} & set the limits of the $y$-axis\\
    \li{xticks()} & set the location of the tick marks on the $x$-axis\\
    \li{yticks()} & set the location of the tick marks on the $y$-axis\\
    \li{xscale()} & set the scale type to use on the $x$-axis\\
    \li{yscale()} & set the scale type to use on the $y$-axis\\
    \li{ax.spines[side].set_position()} & set the location of the given spine\\
    \li{ax.spines[side].set_color()} & set the color of the given spine\\
    \li{ax.spines[side].set_visible()} & set whether a spine is visible\\
    %\li{ax.xaxis}/\li{yaxis.set_ticks_position()} & set the position of the tick marks on the given axis\\
\end{tabular}
\caption{Some functions for changing axis properties. \li{ax} is an \li{Axes} object.}
\label{mpl:axis}
\end{table}

The functions \li{xlim()}, \li{ylim()}, and \li{axis()} are used to set one or both of the x and y ranges of the plot.
\li{xlim()} and \li{ylim()} each accept two arguments, the lower and upper bounds, or a single list of those two numbers.
\li{axis()} accepts a single list consisting, in order, of \li{xmin, xmax, ymin, ymax}.
Passing \li{None} instead of one of the numbers to any of these functions will make it not change the corresponding value from what it was.
Each of these functions can also be called without any arguments, in which case it will return the current bounds.
Note that \li{axis()} can also be called directly on an \li{Axes} object, while \li{xlim()} and \li{ylim()} cannot.

\li{axis()} also can be called with a string as its argument, which has several options.
The most common is \li{axis('equal')}, which makes the scale of the x- and y-scales equal (i.e. makes circles circular).

To use a logarithmic scale on an axis, the functions \li{xscale("log")} and \li{yscale("log")} can be used.
%There are several other scale types available, although used less frequenlty: \li{"linear"}, which creates a linear scale; \li{"symlog"}, which uses a logarithmic scale for both positive and negative values; and \li{"logit"}, which is based off of the logistic function.

The functions \li{xticks()} and \li{yticks()} accept a list of tick positions, which the ticks on the corresponding axis are set to.
Generally, this works the best when used with \li{np.linspace()}.
This function also optionally accepts a second argument of a list of labels for the ticks.
If called with no arguments, the function returns a list of the current tick positions and labels instead.

The spines of a Matplotlib plot are the black border lines around the plot, with the left and bottom ones also being used as the axis lines.
To access the spines of a plot, call \li{ax.spines[side]}, where \li{ax} is an \li{Axes} object and \li{side} is \li{'top'}, \li{'bottom'}, \li{'left'}, or \li{'right'}.
Then, functions can be called on the \li{Spine} object to configure it.

The function \li{spine.set_position()} has several ways to specify the position.
The two simplest are with the arguments \li{'center'} and \li{'zero'}, which place the spine in the center of the subplot or at an x- or y-coordinate of zero, respectively.
The others are a passed as a tuple \li{(position\_type, amount)}:
\begin{itemize}
\item \li{'data'}: place the spine at an x- or y-coordinate equal to \li{amount}.
\item \li{'axes'}: place the spine at the specified \li{Axes} coordinate, where 0 corresponds to the bottom or left of the subplot, and 1 corresponds to the top or right edge of the subplot.
\item \li{'outward'}: places the spine \li{amount} pixels outward from the edge of the plot area.
A negative value can be used to move it inwards instead.
\end{itemize}

\li{spine.set_color()} accepts any of the color formats Matplotlib supports.
Alternately, using \li{set_color('none')} will make the spine not be visible.
\li{spine.set_visible()} can also be used for this purpose.

The following example adjusts the ticks and spine positions to improve the readability of a plot of $\sin(x)$. The result is shown in Figure \ref{mpl:spines}.
\begin{lstlisting}
>>> x = np.linspace(0,2*np.pi,150)
>>> plt.plot(x, np.sin(x))
>>> plt.title(r"$y=\sin(x)$")

#Set the ticks to multiples of pi/2, make nice labels
>>> ticks = np.pi / 2 * np.array([0,1,2,3,4])
>>> tick_labels = ["$0$", r"$\frac{\pi}{2}$", r"$\pi$", r"$\frac{3\pi}{2}$",
...                 r"$2\pi$"]
>>> plt.xticks(ticks, tick_labels)

#Move the bottom spine to zero, remove the top and right ones
>>> ax = plt.gca()
>>> ax.spines['bottom'].set_position('zero')
>>> ax.spines['right'].set_color('none')
>>> ax.spines['top'].set_color('none')

>>> plt.show()
\end{lstlisting}
\begin{figure}[H]
\centering
\includegraphics[width=4in]{figures/axis_example.pdf}
\caption{Plot of $y=\sin(x)$ with axes modified for clarity}
\label{mpl:spines}
\end{figure}

\subsubsection*{Plot Layout} %===================================
The position and spacing of all subplots within a figure can be modified using the function \li{plt.subplots_adjust()}.
This function accepts up to six keyword arguments that change different aspects of the spacing.
\li{left}, \li{right}, \li{top}, and \li{bottom} are used to adjust the rectangle around all of the subplots.
In the coordinates used, 0 corresponds to the bottom or left edge of the figure, and 1 corresponds to the top or right edge of the figure.
\li{hspace} and \li{wspace} set the vertical and horizontal spacing, respectively, between subplots.
The units for these are in fractions of the average height and width of all subplots in the figure.
%
If more fine control is desired, the position of individual \li{Axes} objects can also be changed using \li{ax.get_position()} and \li{ax.set_position()}.
%Possibly too much info; can be learned from experimentation/documentation
%Note that \li{ax.get_position()} returns a Matplotlib \li{Bbox} (bounding box) object; the left, right, bottom, and top edges are accessed as \li{box.x0}, \li{box.x1}, \li{box.y0}, and \li{box.y1}, respectively.
%\li{ax.set_position()} accepts either a \li{Bbox} object or a list consisting of \li{[left, bottom, width, height]}.
%All of these positions are all specified in terms of the whole figure, with (0,0) being the bottom-left and (1,1) being the top-right.

The size of the figure can be configured using the \li{figsize} argument when creating a figure:
\begin{lstlisting}
>>> plt.figure(figsize=(12,8))
\end{lstlisting}
Note that many environments will scale the figure to fill the available space.
Even so, changing the figure size can still be used to change the aspect ratio as well as the relative size of plot elements.

The following example uses \li{subplots_adjust()} to create space for a legend outside of the plotting space. The result is shown in Figure \ref{mpl:positioning}.
\begin{lstlisting}
#Generate data
>>> x1 = np.random.normal(-1, 1.0, size=60)
>>> y1 = np.random.normal(-1, 1.5, size=60)
>>> x2 = np.random.normal(2.0, 1.0, size=60)
>>> y2 = np.random.normal(-1.5, 1.5, size=60)
>>> x3 = np.random.normal(0.5, 1.5, size=60)
>>> y3 = np.random.normal(2.5, 1.5, size=60)

#Make the figure wider
>>> fig = plt.figure(figsize=(5,3))

#Plot the data
>>> plt.plot(x1, y1, 'r.', label="Dataset 1")
>>> plt.plot(x2, y2, 'g.', label="Dataset 2")
>>> plt.plot(x3, y3, 'b.', label="Dataset 3")

#Create a legend to the left of the plot
>>> lspace = 0.35
>>> plt.subplots_adjust(left=lspace)
#Put the legend at the left edge of the figure
>>> plt.legend(loc=(-lspace/(1-lspace),0.6))
>>> plt.show()
\end{lstlisting}
\begin{figure}[H]
\centering
\includegraphics[width=4in]{figures/positioning_example.pdf}
\caption{Example of repositioning axes.}
\label{mpl:positioning}
\end{figure}

\subsection*{Colors} %========================================
The color that a plotting function uses is specified by either the \li{c} or \li{color} keyword arguments; for most functions, these can be used interchangeably.
There are many ways to specific colors.
The most simple is to use one of the basic colors, listed in Table \ref{mpl:colors}.
Colors can also be specified using an RGB tuple such as \li{(0.0, 0.4, 1.0)}, a hex string such as \li{"0000FF"}, or a CSS color name like \li{"DarkOliveGreen"} or \li{"FireBrick"}.
A full list of named colors that Matplotlib supports can be found at \url{https://matplotlib.org/stable/gallery/color/named_colors.html}.
If no color is specified for a plot, Matplotlib automatically assigns it one from the default color cycle.

\begin{table}[H] % Basic colors.
\begin{tabular}{r|l}
    Code & Color \\
    \hline
    \li{'b'} & blue\\
    \li{'g'} & green\\
    \li{'r'} & red\\
    \li{'c'} & cyan\\
    \li{'m'} & magenta\\
\end{tabular}
\qquad
\begin{tabular}{r|l}
    Code & Color \\
    \hline
    \li{'y'} & yellow\\
    \li{'k'} & black\\
    \li{'w'} & white\\
    \li{'C0'} - \li{'C9'} & Default colors
\end{tabular}
\caption{Basic colors available in Matplotlib}
\label{mpl:colors}
\end{table}

Plotting functions also accept an \li{alpha} keyword argument, which can be used to set the transparency. A value of \li{1.0} corresponds to fully opaque, and \li{0.0} corresponds to fully transparent.

The following example demonstrates different ways of specifying colors:
\begin{lstlisting}
#Using a basic color
>>> plt.plot(x, y, 'r')
#Using a hexadecimal string
>>> plt.plot(x, y, color='FF0080')
#Using an RGB tuple
>>> plt.plot(x, y, color=(1, 0.5, 0))
#Using a named color
>>> plt.plot(x, y, color='navy')
\end{lstlisting}

\subsubsection*{Colormaps}
Certain plotting functions, such as heatmaps and contour plots, accept a colormap rather than a single color.
A full list of colormaps available in Matplotlib can be found at \url{https://matplotlib.org/stable/gallery/color/colormap_reference.html}.
Some of the more commonly used ones are \li{"viridis"}, \li{"magma"}, and \li{"coolwarm"}.
A colorbar can be added by calling \li{plt.colorbar()} after creating the plot.

Sometimes, using a logarithmic scale for the coloring is more informative.
To do this, pass a \li{matplotlib.colors.LogNorm} object as the \li{norm} keyword argument:
\begin{lstlisting}
# Create a heatmap with logarithmic color scaling
>>> from matplotlib.colors import LogNorm
>>> plt.pcolormesh(X, Y, Z, cmap='viridis', norm=LogNorm())
\end{lstlisting}

\subsection*{Text and Annotations} %========================================
Matplotlib has several ways to add text and other annotations to a plot, some of which are listed in Table \ref{mpl:text}.
The color and size of the text in most of these functions can be adjusted with the \li{color} and \li{fontsize} keyword arguments.

\begin{table}%[H] %makes the layout better
\centering
\begin{tabular}{r|l|l}
    Function & Description & Usage\\
    \hline
    \li{annotate()} & adds a commentary at a given point on the plot & annotate('text',(x,y))\\
    \li{arrow()} & draws an arrow from a given point on the plot & arrow(x,y,dx,dy)\\
    %\li{axhline()} & draws a horizontal line at y from xmin to xmax & axhline(y=0, xmin=0, xmax=1)\\
    %\li{axvline()} & draws a vertical line at x from ymin to ymax & axvline(x=0, ymin=0, ymax=1)\\
    %\li{axhspan()} & draws a rectangle from xmin to xmax across the plot & axhspan(ymin, ymax, xmin=0, xmax=1)\\
    %\li{axvspan()} & draws a rectangle from ymin to ymax across the plot & axvspan(xmin, xmax, ymin=0, ymin=1)\\
    \li{colorbar()} & Create a colorbar & colorbar()\\
    \li{legend()} & Place a legend in the plot & legend(loc='best')\\
    \li{text()} & Add text at a given position on the plot & text(x,y,'text')\\
    \li{title()} & Add a title to the plot & title('text')\\
    \li{suptitle()} & Add a title to the figure & suptitle('text')\\
    % \li{xticks()} & set the location of the tick marks on the x axis, returns current locations if no arguments are given\\
    % \li{yticks()} & set the location of the tick marks on the y axis, returns current locations if no arguments are given\\
    \li{xlabel()} & Add a label to the $x$-axis & xlabel('text') \\
    \li{ylabel()} & Add a label to the $y$-axis & ylabel('text')
\end{tabular}
\caption{Text and annotation functions in Matplotlib}
\label{mpl:text}
\end{table}

%From DataVisualization
Matplotlib also supports formatting text with \LaTeX, a system for creating technical documents.%
\footnote{See \url{http://www.latex-project.org/} for more information.}
To do so, use an \li{r} before the string quotation mark and surround the text with dollar signs.
This is particularly useful when the text contains a mathematical expression.
For example, the following line of code will make the title of the plot be $\frac{1}{2}\sin(x^2)$:
\begin{lstlisting}
>>> plt.title(r"$\frac{1}{2}\sin(x^2)$")
\end{lstlisting}

The function \li{legend()} can be used to add a legend to a plot.
Its optional \li{loc} keyword argument specifies where to place the legend within the subplot.
It defaults to \li{'best'}, which will cause Matplotlib to place it in whichever location overlaps with the fewest drawn objects.
The other locations this function accepts are \li{'upper right'}, \li{'upper left'}, \li{'lower left'}, \li{'lower right'}, \li{'center left'}, \li{'center right'}, \li{'lower center'}, \li{'upper center'}, and \li{'center'}.
Alternately, a tuple of \li{(x,y)} can be passed as this argument, and the bottom-left corner of the legend will be placed at that location.
The point (0,0) corresponds to the bottom-left of the current subplot, and (1,1) corresponds to the top-right.
This can be used to place the legend outside of the subplot, although care should be taken that it does not go outside the figure, which may require manually repositioning the subplots.

The labels the legend uses for each curve or scatterplot are specified with the \li{label} keyword argument when plotting the object.
Note that \li{legend()} can also be called with non-keyword arguments to set the labels, although it is less confusing to set them when plotting.

The following example demonstrates creating a legend:
\begin{lstlisting}
>>> x = np.linspace(0,2*np.pi,250)

# Plot sin(x), cos(x), and -sin(x)
# The label argument will be used as its label in the legend.
>>> plt.plot(x, np.sin(x), 'r', label=r'$\sin(x)$')
>>> plt.plot(x, np.cos(x), 'g', label=r'$\cos(x)$')
>>> plt.plot(x, -np.sin(x), 'b', label=r'$-\sin(x)$')

# Create the legend
>>> plt.legend()
\end{lstlisting}

\begin{comment} %More information than needed
The functions \li{text()}, \li{arrow()}, and \li{annotate()} each place the text or arrow at the specified location in data coordinates.
With \li{arrow()}, the properties of the arrow can be configured with the \li{width}, \li{head\_width}, \li{head\_length}, and \li{overhang} parameters.
\li{annotate()} can also be used to create both an arrow and text, although it requires two more keyword arguments: \li{xytext}, which is a tuple specifying the text location; and \li{arrowprops}, which is a dictionary containing parameters for the arrow.
For example, the following code creates an arrow from (0,0) to (0.5,0.5) using \li{annotate()}: %
\footnote{This example is taken from \url{https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.arrow.html}.}
\begin{lstlisting}
>>> ax.annotate("", xy=(0.5, 0.5), xytext=(0, 0),
...             arrowprops={arrowstyle:"->"})
\end{lstlisting}
\end{comment}

\subsection*{Line and marker styles}
Matplotlib supports a large number of line and marker styles for line and scatter plots, which are listed in Table \ref{mpl:table-lmstyles}.
\begin{table}[H] % Line style
%\label{mpl:table-lmstyles}
\centering
\begin{tabular}{c|l}
    character & description \\
    \hline
    - & solid line style \\
    -{}- & dashed line style\\
    -. & dash-dot line style \\
    : & dotted line style \\
    . & point marker \\
    , & pixel marker \\
    o & circle marker \\
    v & triangle\_down marker \\
    \^{} & triangle\_up marker \\
    $<$ & triangle\_left marker \\
    $>$ & triangle\_right marker \\
    1 & tri\_down marker \\
    2 & tri\_up marker \\
\end{tabular}
	\quad
\begin{tabular}{c|l}
    character & description \\
    \hline
    3 & tri\_left marker \\
    4 & tri\_right marker \\
    s & square marker \\
    p & pentagon marker \\
    * & star marker \\
    h & hexagon1 marker \\
    H & hexagon2 marker \\
    + & plus marker \\
    x & x marker \\
    D & diamond marker \\
    d & thin\_diamond marker \\
    $|$ & vline marker \\
    \_{} & hline marker \\
\end{tabular}
\caption{Available line and marker styles in Maplotlib.}
\label{mpl:table-lmstyles}
\end{table}
The function \li{plot()} has several ways to specify this argument; the simplest is to pass it as the third positional argument.
The \li{marker} and \li{linestyle} keyword arguments can also be used.
The size of these can be modified using \li{markersize} and \li{linewidth}.
Note that by specifying a marker style but no line style, \li{plot()} can be used to make a scatter plot.
It is also possible to use both a marker style and a line style.
To set the marker using \li{scatter()}, use the \li{marker} keyword argument, with \li{s} being used to change the size.

The following code demonstrates specifying marker and line styles. The results are shown in Figure \ref{mpl:linestyles}.
\begin{lstlisting}
#Use dashed lines:
>>> plt.plot(x, y, '--')
#Use only dots:
>>> plt.plot(x, y, '.')
#Use dots with a normal line:
>>> plt.plot(x, y, '.-')
#scatter() uses the marker keyword:
>>> plt.scatter(x, y, marker='+')

#With plot(), the color to use can also be specified in the same string.
#Order usually doesn't matter.
#Use red dots:
>>> plt.plot(x, y, '.r')
#Equivalent:
>>> plt.plot(x, y, 'r.')

#To change the size:
>>> plt.plot(x, y, 'v-', linewidth=1, markersize=15)
>>> plt.scatter(x, y, marker='+', s=12)
\end{lstlisting}

\begin{figure}[H]
\centering
\includegraphics[width=\textwidth]{figures/linestyles_example.pdf}
\caption{Examples of setting line and marker styles.}
\label{mpl:linestyles}
\end{figure}

\section*{Plot Types} %=======================================
Matplotlib has functions for creaing many different types of plots, many of which are listed in Table \ref{mpl:plotfunctions}.
This section gives details on using certain groups of these functions.

\begin{table}[H] % Different plotting commands.
\centering
\begin{tabular}{l|p{8cm}|p{4cm}}
    Function & Description & Usage\\
    \hline
    \li{bar} & makes a bar graph & bar(x,height)\\
    \li{barh} & makes a horizontal bar graph & barh(y,width)\\
    \li{boxplots} & makes one or more boxplots & boxplots(data) \\
    \li{contour} & makes a contour plot & contour(X,Y,Z)\\
    \li{contourf} & makes a filled contour plot & contourf(X,Y,Z)\\
    \li{imshow} & shows an image & imshow(image) \\
    \li{fill} & plots lines with shading under the curve & fill(x,y)\\
    \li{fill\_between} & plots lines with shading between two given y
    values & fill\_between(x,y1, y2=0)\\
    \li{hexbin} & creates a hexbin plot & hexbin(x,y)\\
    \li{hist} & plots a histogram from data & hist(data)\\
    \li{pcolormesh} & makes a heatmap & pcolormesh(X,Y,Z) \\
    \li{pie} & makes a pie chart & pie(x)\\
    \li{plot} & plots lines and data on standard axes & plot(x,y)\\
    \li{plot_surface} & plot a surface in 3-D space & plot\_surface(X,Y,Z)\\
    \li{polar} & plots lines and data on polar axes & polar(theta,r)\\
    \li{loglog} & plots lines and data on logarithmic x and y axes &
    loglog(x,y)\\
    \li{scatter} & plots data in a scatterplot & scatter(x,y)\\
    \li{semilogx} & plots lines and data with a log scaled x axis &
    semilogx(x,y)\\
    \li{semilogy} & plots lines and data with a log scaled y axis &
    semilogy(x,y)\\
    \li{specgram} & makes a spectogram from data & specgram(x)\\
    \li{spy} & plots the sparsity pattern of a 2D array & spy(Z)\\
    \li{triplot} & plots triangulation between given points &
    triplot(x,y)\\
\end{tabular}
\caption{Some basic plotting functions in Matplotlib.}
\label{mpl:plotfunctions}
\end{table}

\subsection*{Line plots}
Line plots, the most basic type of plot, are created with the \li{plot()} function.
It accepts two lists of x- and y-values to plot, and optionally a third argument of a string of any combination of the color, line style, and marker style.
Note that this method only works with the single-character color codes; to use other colors, use the \li{color} argument.
By specifying only a marker style, this function can also be used to create scatterplots.

There are a number of functions that do essentially the same thing as \li{plot()} but also change the axis scaling, including \li{loglog()}, \li{semilogx()}, \li{semilogy()}, and \li{polar}.
Each of these functions is used in the same manner as \li{plot()}, and has identical syntax.

\subsection*{Scatter and hexbin plots}
Scatterplots can be created using either \li{plot()} or \li{scatter()}.
Generally, it is simpler to use \li{plot()}, although there are some cases where  \li{scatter()} is better.
In particular, \li{scatter()} allows changing the color and size of individual points within a single call to the function.
This is done by passing a list of colors or sizes to the \li{c} or \li{s} arguments, respectively.

Hexbin plots are an alternative to scatterplots that show the concentration of data in regions rather than the individual points.
They can be created with the function \li{hexbin()}.
Like \li{plot()} and \li{scatter()}, this function accepts two lists of x- and y-coordinates.

%The following example demonstrates using \li{scatter()} and \li{hexbin()}.
%The results is shown in Figure \ref{mpl:contour}
%\begin{lstlisting}
%\end{lstlisting}
%\begin{figure}[H]
%\centering
%\includegraphics[width=\textwidth]{figures/heat_contour_example.pdf}
%\caption{Example of heatmaps and contour plots.}
%\label{mpl:contour}
%\end{figure}

\subsection*{Heatmaps and contour plots}
Heatmaps and contour plots are used to visualize 3-D surfaces and complex-valued functions on a flat space.
Heatmaps are created using the \li{pcolormesh()} function.
Contour plots are created using \li{contour()} or \li{contourf()}, with the latter creating a filled contour plot.

Each of these functions accepts the x-, y-, and z-coordinates as a mesh grid, or 2-D array.
To create these, use the function \li{np.meshgrid()}:
\begin{lstlisting}
>>> x = np.linspace(0,1,100)
>>> y = np.linspace(0,1,80)
>>> X, Y = np.meshgrid(x, y)
\end{lstlisting}
The z-coordinate can then be computed using the x and y mesh grids.

Note that each of these functions can accept a colormap, using the \li{cmap} parameter.
These plots are sometimes more informative with a logarithmic color scale, which can be used by passing a \li{matplotlib.colors.LogNorm} object in the \li{norm} parameter of these functions.

With \li{pcolormesh()}, it is also necessary to pass \li{shading='auto'} or \li{shading='nearest'} to avoid a deprecation error.

The following example demonstrates creating heatmaps and contour plots, using a graph of $z=(x^2+y)\sin(y)$.
The results is shown in Figure \ref{mpl:contour}
\begin{lstlisting}
>>> from matplotlib.colors import LogNorm

>>> x = np.linspace(-3,3,100)
>>> y = np.linspace(-3,3,100)
>>> X, Y = np.meshgrid(x, y)
>>> Z = (X**2+Y)*np.sin(Y)

#Heatmap
>>> plt.subplot(1,3,1)
>>> plt.pcolormesh(X, Y, Z, cmap='viridis', shading='nearest')
>>> plt.title("Heatmap")

#Contour
>>> plt.subplot(1,3,2)
>>> plt.contour(X, Y, Z, cmap='magma')
>>> plt.title("Contour plot")

#Filled contour
>>> plt.subplot(1,3,3)
>>> plt.contourf(X, Y, Z, cmap='coolwarm')
>>> plt.title("Filled contour plot")
>>> plt.colorbar()

>>> plt.show()

\end{lstlisting}
\begin{figure}[H]
\centering
\includegraphics[width=\textwidth]{figures/heat_contour_example.png}
\caption{Example of heatmaps and contour plots.}
\label{mpl:contour}
\end{figure}

\subsection*{Showing images}
The function \li{imshow()} is used for showing an image in a plot, and can be used on either grayscale or color images.
This function accepts a 2-D $n\times m$ array for a grayscale image, or a 3-D $n\times m \times 3$ array for a color image.
If using a grayscale image, you also need to specify \li{cmap='gray'}, or it will be colored incorrectly.

It is best to also use \li{axis('equal')} alongside \li{imshow()}, or the image will most likely be stretched.
This function also works best if the images values are in the range $[0,1]$.
Some ways to load images will format their values as integers from 0 to 255, in which case the values in the image array should be scaled before using \li{imshow()}.

\subsection*{3-D Plotting}
Matplotlib can be used to plot curves and surfaces in 3-D space.
In order to use 3-D plotting, you need to run the following line:
\begin{lstlisting}
>>> from mpl_toolkits.plot3d import Axes3D
\end{lstlisting}
The argument \li{projection='3d'} also must be specified when creating the subplot for the 3-D object:
\begin{lstlisting}
>>> plt.subplot(1,1,1, projection='3d')
\end{lstlisting}

Curves can be plotted in 3-D space using \li{plot()}, by passing in three lists of x-, y-, and z-coordinates.
Surfaces can be plotted using \li{ax.plot_surface()}.
This function can be used similar to creating contour plots and heatmaps, by obtaining meshes of x- and y- coordinates from \li{np.meshgrid()} and using those to produce the z-axis.
More generally, any three 2-D arrays of meshes corresponding to x-, y-, and z-coordinates can be used.
Note that it is necessary to call this function from an Axes object.

The following example demonstrates creating 3-D plots. The results are shown in Figure \ref{mpl:3d}.

\begin{lstlisting}
#Create a plot of a parametric curve
ax = plt.subplot(1,3,1, projection='3d')
t = np.linspace(0, 4*np.pi, 160)
x = np.cos(t)
y = np.sin(t)
z = t / np.pi
plt.plot(x, y, z, color='b')
plt.title("Helix curve")

#Create a surface plot from np.meshgrid
ax = plt.subplot(1,3,2, projection='3d')
x = np.linspace(-1,1,80)
y = np.linspace(-1,1,80)
X, Y = np.meshgrid(x, y)
Z = X**2 - Y**2
ax.plot_surface(X, Y, Z, color='g')
plt.title(r"Hyperboloid")

#Create a surface plot less directly
ax = plt.subplot(1,3,3, projection='3d')
theta = np.linspace(-np.pi,np.pi,80)
rho = np.linspace(-np.pi/2,np.pi/2,40)
Theta, Rho = np.meshgrid(theta, rho)
X = np.cos(Theta) * np.cos(Rho)
Y = np.sin(Theta) * np.cos(Rho)
Z = np.sin(Rho)
ax.plot_surface(X, Y, Z, color='r')
plt.title(r"Sphere")

plt.show()
\end{lstlisting}
\begin{figure}[H]
\centering
\includegraphics[width=\textwidth]{figures/3d_example.pdf}
\caption{Examples of 3-D plotting.}
\label{mpl:3d}
\end{figure}

\section*{Additional Resources} % ==============================================
\subsection*{rcParams}
The default plotting parameters of Matplotlib can be set individually and with more fine control than styles by using \li{rcParams}.
\li{rcParams} is a dictionary that can be accessed as either \li{plt.rcParams} or \li{matplotlib.rcParams}.
%technically, it isn't actually a dictionary, but just looks like one

For instance, the resolution of plots can be changed via the \li{"figure.dpi"} parameter:
\begin{lstlisting}
>>> plt.rcParams["figure.dpi"] = 600
\end{lstlisting}

A list of parameters that can set via \li{rcParams} can be found at \url{https://matplotlib.org/stable/api/matplotlib_configuration_api.html#matplotlib.RcParams}.

\subsection*{Animations}
Matplotlib has capabilities for creating animated plots.
The Animations lab in Volume 4 has detailed instructions on how to do so.

\subsection*{Matplotlib gallery and tutorials}
The Matplotlib documentation has a number of tutorials, found at \url{https://matplotlib.org/stable/tutorials/index.html}.
It also has a large gallery of examples, found at \url{https://matplotlib.org/stable/gallery/index.html}.
Both of these are excellent sources of additional information about ways to use and customize Matplotlib.


\documentclass[a4paper,10pt]{article}
\usepackage[dvips]{color,graphicx}
\usepackage[dvips, bookmarks, colorlinks=false]{hyperref}
\usepackage[all]{xy}

%opening
\title{Math508 Final Exam}
\author{Yu Huang}
\date{2007-05-07}

\begin{document}

\maketitle

\section{Problem 1}
For Ornstein-Uhlenbeck stationary sequence, $E X_n=0$, $R(k) = a^{|k|}$. The best linear estimate of $X_m$ given $X_k, X_{k-1}, X_{k-2}, ...$ is
\begin{equation}
\hat{X_m} = k_0 + k_1*X_k + k_2*X_{k-1} + k_3*X_{k-2} + ... \\
k_0 = EX_m = 0
\end{equation}

The matrices in the normal equation (\ref{normal_eq}) would be infinite although it could be easily be seen that $k_1=a^{m-k}$ while other $k$'s are all zero. Hence, $\hat{X_m} = a^{m-k}X_k$.

\begin{equation} \label{normal_eq}
\left( \begin{array}{cccc}
a^{0} & a^{1} & a^{2} & \ldots \\
a^{1} & a{0} & a^{1} & \ldots \\
\vdots & \vdots & \ddots
\end{array} \right)
\left(\begin{array}{c}
k_1 \\
k_2 \\
\vdots
\end{array} \right) =
\left( \begin{array}{c}
a^{m-k} \\
a^{m-k+1} \\
\vdots
\end{array} \right)
\end{equation}

Instead guessing, i use the sequential project to prove this result. $M_n$ is a linear subspace generated by $\{1, X_k, X_{k-1}, \ldots, X{k-n+1}\}$.
\begin{equation}
\hat{X_{m,n}} = E(X_m|M_n)
\end{equation}
There's iteration that
\begin{eqnarray}
\hat{X_{m,n}} = \hat{X_{m,n-1}} + \frac{c_{n,n-1}}{u_{n,n-1}}(X_{k-n}-X_{k-n+1}) \nonumber \\
\hat{X_{k-n,n-1}} = E(X_{k-n}|M_{n-1}) \nonumber \\
c_{n,n-1} = E[(X_m - \hat{X_{m,n-1}})(X_{k-n}-\hat{X_{k-n,n-1}})] \nonumber
\end{eqnarray}

Now take $n=2$, $M_2$ is a linear subspace generated by $\{1, X_k, X_{k-1}\}$ and $M_1$ is a linear subspace generated by $\{1, X_k\}$. We can get following results easily.
\begin{eqnarray}
\hat{X_{m,n-1}} = \hat{X_{m,1}} = a^{m-k}*X_k \\
\hat{X_{k-n,n-1}} = \hat{X_{k-1,1}} = aX_{k}
\end{eqnarray}

We can prove $c_{n,n-1}=0$.
\begin{eqnarray}
 c_{n,n-1} = c_{2,1} = E[(X_m - a^{m-k}*X_k)(X_{k-1} - aX_{k})] \nonumber \\
= E(X_m X_{k-1} - aX_mX_k - a^{m-k}X_kX_{k-1} + a^{m-k+1}X_kX_k) \nonumber\\
= a^{m-k+1} - a*a^{m-k} - a^{m-k}*a + a^{m-k+1} = 0 \\
\hat{X_{m,2}} = \hat{X_{m,1}} = a^{m-k}*X_k
\end{eqnarray}

By induction,
\begin{equation}
\hat{X_m} = a^{m-k}X_k
\end{equation}

\section{Problem 2}
\subsection{part a}
\begin{eqnarray}
P(X_{n+1} \leq u, Y_{n+1} \leq v | X_n=r, Y_n=s) \nonumber \\
= \int_{-\infty}^u P(Y_{n+1} \leq v | X_{n+1}=x) f(X_{n+1} = x|X_n=r, Y_n=s) dx \nonumber \\
= \int_{-\infty}^u \int_{-\infty}^v l(X_{n+1}=x, Y_{n+1}=y) p(X_{n+1}=x|X_n=r) dy dx \nonumber
\end{eqnarray}

Differentiate both sides by $\frac{\partial^2}{\partial u \partial v}$ to get the transition kernel
\begin{equation}
 q^{r,u}(s,v) = p(r,u)l(u,v)
\end{equation}

\subsection{part b}
Given $X_n=r$, if $W_n \geq -r$, $Y_n$ is distributed as $N(r,1)$. If $W_n < -r$, $Y_n$ is distributed as $N(-r,1)$. Graphically, $Y_n$'s distribution is a combination of $N(r,1)$ and $N(-r,1)$ with the only positive domain. So,
\begin{equation}
 P(Y_n<u|X_n=r) = \int_0^u \rho(y-r)dy + \int_0^u \rho(y+r)dy \nonumber
\end{equation}

$\rho(x)$ is the standard normal density function. Differentiate both sides by $\frac{\partial}{\partial u}$ to get
\begin{equation}
l(r,u) = \rho(u-r) + \rho(u+r) \nonumber
\end{equation}

\section{Problem 3}
Professor, in this problem, i found a little problem with hint you gave us. Basically, the filtering procedure (i'll call it procedure 1) is more like a one step prediction. Following dependency structure will help illustrate the problem.

\begin{equation} \label{prob3_depend}
\xymatrix{  & X_0 \ar[d] \ar[r] & X_1 \ar[d] \ar[r] & \ldots & \ar[r] & X_n \ar[d] \ar[r] & X_{n+1} \ar[d] & \ldots \\
        Y_0 & Y_1               & Y_2               & \ldots &        & Y_{n+1}           & Y_{n+2}        & \ldots \\
            & W_1 \ar[u]        & W_2 \ar[u]        & \ldots &        & W_{n+1} \ar[u]    & W_{n+2} \ar[u] &\ldots \\ }
\end{equation}

The recursion for \textbf{procedure 1} is
\begin{eqnarray}
\gamma_0(H_0, X_0) = 0  \\
\gamma_0(X_0) = \frac{1}{0.1^2}\rho(x/0.1)  \\
\gamma_{n+1}(H_{n+1}, X_{n+1}) = \int_R p(X_n, X_{n+1}) \lambda^{-1}(Y_{n+1}, X_{n})[\gamma_n(H_n, X_n) + X_n \gamma_n(X_n)]dX_n  \\
\gamma_{n+1}(X_{n+1}) = \int_R p(X_n, X_{n+1}) \lambda^{-1}(Y_{n+1}, X_{n})\gamma_n(X_n)dX_n  \\
E[H_n|Y_{[0,n]}] = \frac{\int_R \gamma_n(H_n, X_n) dX_n}{\int_R \gamma_n(X_n)dX_n} \\
E[f(X_n) | Y_{[0,n]}] = \frac{\int_R f(X_n)\gamma_n(X_n)dX_n }{\int_R \gamma_n(X_n)dX_n}
\end{eqnarray}

Basically, to get the signal of $X_{n+1}$, we are $Y_{n+1}$, one step prediction from $X_n$ and all previous information. However, looking at diagram (\ref{prob3_depend}), we should incorporate $Y_{n+2}$ to do a full filtering. So after some calculation, i propose \textbf{procedure 2}
\begin{eqnarray}
\gamma_0(H_0, X_{-1}) = 0  \\
\gamma_0(X_0) = \lambda^{-1}(Y_{1}, X_{0})\frac{1}{0.1^2}\rho(x/0.1)  \\
\gamma_{n+1}(H_{n+1}, X_{n}) =  \lambda^{-1}(Y_{n+1}, X_{n}) \int_R p(X_{n-1}, X_n)[\gamma_n(H_n, X_{n-1})dX_{n-1} + X_n \gamma_n(X_n)  \\
\gamma_{n+1}(X_{n+1}) = \lambda^{-1}(Y_{n+2}, X_{n+1}) \int_R p(X_n, X_{n+1}) \gamma_n(X_n)dX_n  \\
E[H_n|Y_{[0,n]}] = \frac{E[Z_n^{-1}H_n|Y_{[0,n]}}{E[Z_n^{-1}|Y_{[0,n]}} = \frac{\int_R \gamma_n(H_n, X_{n-1}) dX_{n-1}}{\int_R \gamma_{n-1}(X_{n-1})dX_{n-1}} \\
E[f(X_n) | Y_{[0,n+1]}] = \frac{E[f(X_n)Z_{n+1}^{-1}|Y_{[0,n+1]}}{E[Z_{n+1}^{-1}|Y_{[0,n+1]}} = \frac{\int_R f(X_n)\gamma_n(X_n)dX_n }{\int_R \gamma_n(X_n)dX_n}
\end{eqnarray}

The idea is to get $\hat{H_n}$, only need information up to $Y_n$ because $H_n = \sum_{k=1}^{k=n} X_{k-1}$ and the last included $X$ is $X_{n-1}$. Looking at diagram (\ref{prob3_depend}), $X_{n-1}$ is depended by $Y_n$. To get $\hat{X_n}$, we need information up to $Y_{n+1}$. It created a little bit hardship in programming as two estimates are not synchronized in terms of sequential computing.

Figure~(\ref{f1}, \ref{f2}, \ref{f3}, \ref{f4}) are due to \emph{procedure 1}. Figure~(\ref{f5}, \ref{f6}, \ref{f7}, \ref{f8}) are due to \emph{procedure 2}. Figure~\ref{f5} from \emph{procedure 2} has smaller MSE than Figure~\ref{f1} from \emph{procedure 1}. By checking closely, you can actually see $\hat{X_n}$ in Figure~\ref{f1} is a little bit lagging behind $\hat{X_n}$ in Figure~\ref{f5}. This lagging phenomenon is due to the one-step-prediction in \emph{procedure 1}. It's evident in other figures as well.

\section{Problem 4}
By checking the notes, i realized there're some errors in both the notes and the hints although it won't hurt the result if $\frac{\alpha_{t_{i+1}}}{\alpha_{t_i}}$ is calculated approximately. Most errors are due to the swapping of $\lambda_1$ and $\lambda_2$ which depends on how to split the Ricotti equation. Ricotti equation is
\begin{eqnarray}
\lambda_1 = {(a^2B^4+b^2B^2)}^{1/2}+aB^2 \\
\lambda_2 = -{(a^2B^4+b^2B^2)}^{1/2}+aB^2 \\
\frac{dP}{dt} = 2aP_t + b^2 - \frac{P_t^2}{B^2}
\end{eqnarray}
So assume Ricotti equation is split in the way following
\begin{eqnarray}
\Rightarrow  \nonumber \\
\frac{dt}{dP} = \frac{C}{P - \lambda_1} - \frac{C}{P-\lambda_2} = \frac{-B^2}{(P^2-2aB^2P-b^2B^2)} \\
\Rightarrow C = \frac{B^2}{\lambda_2-\lambda_1} \label{eqn_C} \\
\Rightarrow Cln|\frac{P-\lambda_1}{P-\lambda_2}| = t + const \\
\Rightarrow |\frac{P-\lambda_1}{P-\lambda_2}| = Ke^{t/C} \\
\Rightarrow t=0, K=|\frac{P_0-\lambda_1}{P_0-\lambda_2}|
\end{eqnarray}

With $P_0 = 0 < \lambda_1 $, we'll get
\begin{eqnarray}
K = \frac{\lambda_1}{-\lambda_2} \\
\frac{P-\lambda_1}{P-\lambda_2} = -Ke^{t/C} \\
\Rightarrow P_t = \frac{\lambda_2Ke^{t/C}+\lambda_1}{Ke^{t/C}+1} = \lambda_2 + \frac{\lambda_1 - \lambda_2}{1+Ke^{t/C}} \\
\alpha_t = exp\{\int_0^t(a-\frac{P_s}{B^2})ds\} = exp\{(a-B^{-2}\lambda_1)t\}\frac{(K+1)e^{t/C}}{Ke^{t/C}+1}
\end{eqnarray}

One thing about $\alpha_t$, i'm not sure it should be $\lambda_1$ or $\lambda_2$ in the formula (i can't work out the complex integral.) So when calculating $\frac{\alpha_{t_{i+1}}}{\alpha_{t_i}}$, i tried both of them, calling them ${exactratio}_1$ and ${exactratio}_2$ and compared with the approximated version, which is
\begin{equation}
\frac{\alpha_{t_{i+1}}}{\alpha_{t_i}} \approx exp\{(a-\frac{P_{t_i}}{B^2})\delta t)\}
\end{equation}

So in figure~\ref{f9}, the ${exactratio}_1$ is the correct formula as it's closer to the approximated ratio. So Figure~(\ref{f10}, \ref{f11}, \ref{f12}, \ref{f13}) are for different settings. The $\hat{X_n}$ is calculated using both the approximated $\frac{\alpha_{t_{i+1}}}{\alpha_{t_i}}$ and the exact one. Previously, due to the opposite sign of $C$, $\hat{X_n}$ based on the approximated ratio (red curve) has smaller MSE than $\hat{X_n}$ based on the exact ratio (cyan curve). Now with the correct formula for $C$ (\ref{eqn_C}), $\hat{X_n}$ based on the exact ratio (cyan curve) has smaller MSE, which is within expectation.

\section{Figures}
\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_Xn_Un_Xn_hat_gap_0.01.eps}
\caption{}\label{f1}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_Hn_Hn_hat_gap_0.01.eps}
\caption{}\label{f2}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_Xn_Xn_hat_Xn_hat_max_gap_0.01.eps}
\caption{}\label{f3}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_3d_pi_FDF.eps}
\caption{}\label{f4}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_corrected_Xn_Un_Xn_hat_gap_0.01.eps}
\caption{}\label{f5}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_corrected_Hn_Hn_hat_gap_0.01.eps}
\caption{}\label{f6}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_corrected_Xn_Xn_hat_Xn_hat_max_gap_0.01.eps}
\caption{}\label{f7}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_3_corrected_3d_pi_FDF.eps}
\caption{}\label{f8}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_4_alpha_ratio_comp_a_0.1_b_1_B_0.3_gap_0.01.eps}
\caption{}\label{f9}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_4_Xn_Yn_Xn_hat_a_0.1_b_1_B_0.3_gap_0.01.eps}
\caption{}\label{f10}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_4_Xn_Yn_Xn_hat_a_0.1_b_1_B_10_gap_0.01.eps}
\caption{}\label{f11}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_4_Xn_Yn_Xn_hat_a_-1_b_1_B_0.3_gap_0.01.eps}
\caption{}\label{f12}
\end{figure}

\begin{figure}
\includegraphics[width=1\textwidth]{Final_4_Xn_Yn_Xn_hat_a_-1_b_2_B_0.3_gap_0.01.eps}
\caption{}\label{f13}
\end{figure}

\end{document}


\section{The framework}
\label{sec:framework}

The work of this thesis is designated as ``a front-end for LISA''. The term ``front-end'' is generally used for layered architectures to outline a layer that is the closest to the user. Often this corresponds to some kind of interface, though what is understood as a user and an interface can vary depending on the context. In our case, we understand front-end as a programmatic interface for the end-user of LISA. More precisely, we were interested in designing a framework that depended on the LISA kernel and strove to provide a higher-level interface to develop proofs. The use of an indefinite article in our designation is to emphasize the fact that such a layer is by essence guided by opinionated design choices.

One could wonder about the necessity of relying on an existing system. The fundamental reason is to keep the core of the system sufficiently small and simple enough that it could be understood, reasoned about and perhaps formally proved by another system. We refer to this part as the \textbf{kernel} of LISA. The constraint on the complexity of the kernel gives us better reasons to trust its correctness. On the other side, the front-end does not make such claims and can thus be made arbitrarily more sophisticated. The only requirement is for the front-end to be able to reconstruct all of its proofs in the kernel, such that one could check them with the same level of confidence as if they were initially written there. We define soundness as the property for a proof in the front to have the same meaning as one in the kernel, and to be valid there. By the requirement, soundness thus relies on the only assumption that statements in the front are logically equivalent to statements in the kernel. As a consequence such a system would satisfy the De Bruijn criterion, in the sense that the underlying proofs can be checked by a reasonably simple procedure: the kernel.

Another aspect of the front is that despite its dependency on the kernel, it does not expose it to the user. Instead, it redefines most of its components and provides mappings for uni or bi-directional translations. The motivation is two-fold. First of all, it allows us to extend the existing functionality. Practically, for example, it was decided that the kernel would not provide schematic connectors as they were not strictly needed. However the front requires such a functionality to represent its pattern variables and it is not possible to emulate them at low cost. Consequently we implement this functionality in the front and provide conversions. We can also strengthen some of the kernel features, for instance by representing sequents as indexed sequences rather than sets. We may additionally redefine the method \code{toString} on all of our classes. Secondly, it allows us to enrich the language and bring type safety guarantees. Some types in the kernel are (purposely) weak and require runtime checks to ensure safety. This aspect contributes to building a strong framework and is compatible with the mid-term objective of having LISA exist as a Scala library.

\begin{figure}[H]
  \centering
  \begin{tikzpicture}[box/.style = {draw, rectangle, minimum width=3.75cm, minimum height=0.75cm}, group/.style = {draw, rectangle, dashed}, label/.style = {fill=white, inner sep=1,outer sep=1}]
  \node[box, minimum width=3.75cm] at (-1, 2) (string) {Character stream};
  \node[box] at (-4.5, 0) (front-fol) {Front FOL};
  \node[box] at (4.5, 0) (kernel-fol) {Kernel FOL};
  \node[box] at (-4.5, -1.5) (front-proof-mode) {Proof mode};
  \node[box] at (4.5, -1.5) (kernel-proof) {Kernel proof};
  \node[box] at (-4.5, -3) (front-justification) {Front justification};
  \node[box] at (4.5, -3) (kernel-justification) {Kernel justification};
  \draw[group] (-6.75, 1.25) rectangle (1, -4.25);
  \draw[group] (2.25, 1.25) rectangle (6.75, -4.25);
  \draw[<-] (front-fol.east) -- ++(3.6, 0);
  \draw[<-, densely dotted] (kernel-fol.west) -- ++(-1.6, 0);
  \draw[<->] (-2.62, 0.25) -| (-1.25, 1.625);
  \draw[<->] (-0.75, 1.625) |- (2.62, 0.25);
  \path[label] (-1.2, 0.2) rectangle (-0.8, -0.2);
  \draw[<->] (-1, 1.625) |- (2.62, -1.25);
  \draw[->] (front-proof-mode.east) -- (kernel-proof.west);
  \draw[->] (kernel-justification.west) -- (front-justification.east);
  \draw[<-] (2.62, -2.75) -| (-1, -1.75) -- (2.62, -1.75);
  \draw[->] (front-justification) -- (front-proof-mode);
  \node[label] at (-1, 0.8) (label1) {\scriptsize{parsing/printing}};
  \node[label] at (0.1, 0.03) (label2) {\scriptsize{conversion}};
  \node[label] at (-1, -1.47) (label3) {\scriptsize{reconstruction}};
  \node[label] at (-1, -2.25) (label4) {\scriptsize{contextualization}};
  \node[label] at (-1, -3) (label5) {\scriptsize{lifting}};
  \node[label] at (-4.5, -2.3) (label6) {\scriptsize{usage}};
  \node[fill=white] at (-2.9, -4.25) (label7) {Our framework};
  \node[fill=white] at (4.5, -4.25) (label8) {LISA};
  \end{tikzpicture}
  \caption[Data flow between the front and the kernel]{Illustration of the data flow between the front and the kernel.}
  \label{fig:front-kernel}
\end{figure}

The interaction between the front and the kernel is relatively limited as the front handles its own state independently (\autoref{fig:front-kernel}). The common ground between the two are justified statements (axioms, theorems, definitions), which means that such objects can be converted back and forth. Whenever a theorem is proven in the front, it is translated to the kernel and checked for consistency. The front obtains the mirror of that justified statement and can be quite certain about its correctness. This enables full compatibility with LISA, making it possible to lift theorems that were not proven in the front.

\subsection{Language and tools}

The implementation was done in Scala version 3\footnote{Also known as Dotty: \href{https://dotty.epfl.ch}{dotty.epfl.ch}.}. The choice of version was a requirement, following the decision to upgrade LISA from version 2 to 3. Since this project does not work directly on the LISA codebase but instead depends on it as a library, both versions must be the same. Regarding the version, it turns out that some of the features implemented in this project would not have had an equivalent encoding in the older version, which lets us to confidently argue that Scala version 3 is an appropriate language for the design of DSL\footnote{DSL: domain specific language} libraries.

During the implementation, eight bugs were discovered in different areas of the Dotty compiler. All the issues that could be minimized and reproduced were reported\footnote{
\href{https://github.com/lampepfl/dotty}{github.com/lampepfl/dotty}:
\href{https://github.com/lampepfl/dotty/issues/14667}{\#14667},\
\href{https://github.com/lampepfl/dotty/issues/14707}{\#14707},\
\href{https://github.com/lampepfl/dotty/issues/14765}{\#14765},\
\href{https://github.com/lampepfl/dotty/issues/14818}{\#14818},\
\href{https://github.com/lampepfl/dotty/issues/14834}{\#14834},\
\href{https://github.com/lampepfl/dotty/issues/14858}{\#14858},\
\href{https://github.com/lampepfl/dotty/issues/14907}{\#14907},\
\href{https://github.com/lampepfl/dotty/issues/15145}{\#15145}.
}\textsuperscript{,}\footnote{
\href{https://github.com/scalameta/scalameta}{github.com/scalameta/scalameta}:
\href{https://github.com/scalameta/scalameta/issues/2741}{\#2741}.
}; most of them have since been fixed or are still in the process of getting fixed.

\subsection{Structure}

\begin{figure}[H]
  \centering
  % https://tex.stackexchange.com/q/515582
  \begin{tikzpicture}[grow via three points={one child at (1.0,-0.6) and two children at (1.0,-0.6) and (1.0,-1.2)}, edge from parent path={(\tikzparentnode.south)|-(\tikzchildnode.west)}]
  \node{(root package)}
  child{node{\code{example}}}
  child{node{\code{front}}
      child{node{\code{fol}}}
      child{node{\code{parser}}}
      child{node{\code{printer}}}
      child{node{\code{proof}}}
      child{node{\code{theory}}}}
  child [missing] {}
  child [missing] {}
  child [missing] {}
  child [missing] {}
  child [missing] {}
  child{node{\code{util}}}
  child{node{\code{legacy}}};
  \node at (6, -0.6) {\parbox{7cm}{\hspace*{0pt}\hfill(examples)}};
  \node at (6, -1.2) {\parbox{7cm}{\hspace*{0pt}\hfill(the front-end framework)}};
  \node at (6, -1.8) {\parbox{7cm}{\hspace*{0pt}\hfill(first-order logic)}};
  \node at (6, -2.4) {\parbox{7cm}{\hspace*{0pt}\hfill(FOL parsers)}};
  \node at (6, -3.0) {\parbox{7cm}{\hspace*{0pt}\hfill(FOL and proof printers)}};
  \node at (6, -3.6) {\parbox{7cm}{\hspace*{0pt}\hfill(logic for proofs, depends on FOL)}};
  \node at (6, -4.2) {\parbox{7cm}{\hspace*{0pt}\hfill(available theories)}};
  \node at (6, -4.8) {\parbox{7cm}{\hspace*{0pt}\hfill(utilities that only rely on the kernel)}};
  \node at (6, -5.4) {\parbox{7cm}{\hspace*{0pt}\hfill(older experiments)}};
  \end{tikzpicture}
  \caption[Source code packages structure]{Structure of the packages in the source code. The reader is invited to review the package \code{example}, which contains a collection of examples giving an overview of the framework.}
  \label{fig:packages}
\end{figure}

The code is organized hierarchically to reduce coupling. This is achieved through Scala dependent members: the components are implemented as children of traits, these traits can then extend and mix with other traits to benefit from their features (included the inherited ones). Finally, all the traits are aggregated in a singleton object that acts as a module. All the components can then be put into scope using a single import. Note that if a same member is exported from two distinct modules it will be considered as a different entity: this is because the types are path-dependent.

\subsection{DSL}

Our framework introduces its own DSL\footnote{DSL: Domain-Specific Language, a custom language designed for a particular purpose.} implemented in Scala. In this section we briefly describe it and explain in more details two interesting problems.

\input{figures/fol-erd}

The major function of the DSL to assist the developer in using the first-order logic package (\autoref{fig:fol-erd}), namely to provide methods to construct objects in an intuitive way. The second aspect is to provide compile-time safety guarantees when doing so. For instance the sequent $\vdash f(a, b), c \land d$ can be written as \code{|- (f(a, b), c /\textbackslash~d)} in our Scala DSL. Scala allows custom infix operators to be defined, therefore the operators of propositional logic are straightforward to implement. The first interesting problem is about converting the tuple \code{(t1:~T1, ..., tn:~TN)} into a sequence of values of type \code{T1 {|} ...~{|} TN}. While tuples have a method to iterate over their members, this solution is unsound because it loses all type information. Instead, we rely on a mechanism offered by the language: \textit{given} instances and conversions (\autoref{lst:generic-tuples-1}).

\begin{lstlisting}[language=Scala,caption={[Generic tuples (1)]{Generic tuples: converting a tuple of \code{T} into a \code{Seq[T]}.}},label={lst:generic-tuples-1}]
trait Converter[R, S] extends (R => Seq[S])

// Tuple: base case
given[S]: Converter[EmptyTuple, S] with {
  def apply(t: EmptyTuple): Seq[S] = Seq()
}
// Tuple: inductive case
given[S, H <: S, T <: Tuple](using c: Converter[T, S]):
  Converter[H *: T, S] with {
  def apply(t: H *: T): Seq[S] = t.head +: c(t.tail)
}
// Sugar for `()'
given[S]: Converter[Unit, S] with {
  def apply(u: Unit): Seq[S] = Seq()
}

given [R, S](using c: Converter[R, S]):
  Conversion[R, Seq[S]] = c.apply

// Tests
(): Seq[Int]        // ok
(1, 2, 3): Seq[Int] // ok
(1, "2"): Seq[Int]  // error
\end{lstlisting}

The idea is to inductively deconstruct the tuple at compilation-time, using the polymorphic helper trait \code{Converter} and its \textit{given} implementations. If such an instance can be constructed, then the value can be converted into a sequence thanks to the \code{Conversion} instance. With this method the types are preserved and we obtain a representation of the tuple as a sequence of values which we can work with at runtime.

The second problem we encountered was about checking the arguments when applying labels, namely ensuring that the correct number of arguments is passed with respect to the label's arity. Labels represent unapplied functions, predicates or connectors, and they store information about the expected arity. A naive solution would be to manually encode common arities, but it is not satisfactory. The solution we adopted was once again a generic one (\autoref{lst:generic-tuples-2}) that works for any arity.

\begin{lstlisting}[language=Scala,caption={[Generic tuples (2)]{Generic tuples: enforcing correct function application arity. The type \code{HTuble} represents an homogeneous tuple, \code{Label} is a function label and \code{Applied} is an applied function.}},label={lst:generic-tuples-2}]
type Arity = Int & Singleton

type HTuple[T, N <: Arity] <: Tuple = N match {
  case 0 => EmptyTuple
  case _ => T *: HTuple[T, N - 1]
}

given [T, N <: Arity]: Conversion[HTuple[T, N], Seq[T]] =
  _.productIterator.toSeq.asInstanceOf[Seq[T]]

case class Label[N <: Arity](arity: N) {
  def apply(args: HTuple[Applied, N]): Applied =
    Applied(this, args)
}
case class Applied(label: Label[?], args: Seq[Applied])

// Tests
val a: Applied = ...
val l2 = Label(2)
l2(a, a)    // ok
l2(a, a, a) // error
\end{lstlisting}

The type \code{Arity} is defined as the intersection between the primitive \code{Int} type and the special \code{Singleton} synthetic type, which enforces a type to be the same as the value it models. For example the singleton type of the literal value \code{1} would be the literal type \code{1}. We constrain labels to express their arity in terms of this precise type, in order for us to use this information later. We then introduce the recursively defined type alias \code{HTuple} representing an homogeneous tuple of arity \code{N} and for which all the members are of type \code{T}. Then, an \code{apply} method is added to the label taking an homogeneous tuple as its own argument, representing the arguments passed to this label. The arity of the tuple is naturally the one specified by the label. Optionally we can protect the regular constructor of \code{Apply} to protect it from unsound usage.

The DSL also supports string interpolation which we describe later, in \autoref{sec:parsing-printing-string-interpolation}.