% --------------------------------------------------------------
% This is all preamble stuff that you don't have to worry about.
% Head down to where it says "Start here"
% --------------------------------------------------------------

\documentclass[12pt,answers]{exam}

\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amsthm,amssymb}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{float}
\usepackage{color}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage{verbatim}
\usepackage{listings}
\usepackage[comma, authoryear]{natbib}
\bibliographystyle{plainnat}
\usepackage{mathtools}
\usepackage[shortlabels]{enumitem}
\usepackage{pythontex}
\usepackage[makeroom]{cancel}

\usepackage{listings}
\usepackage{xcolor}

\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\definecolor{darkblue}{rgb}{0.1,0.1,1}

\lstdefinestyle{mystyle}{
    backgroundcolor=\color{backcolour},
    commentstyle=\color{codegreen},
    keywordstyle=\color{darkblue},
    numberstyle=\tiny\color{codegray},
    stringstyle=\color{codepurple},
    basicstyle=\ttfamily\footnotesize,
    breakatwhitespace=false,
    breaklines=true,
    captionpos=b,
    keepspaces=true,
    numbers=left,
    numbersep=1pt,
    showspaces=false,
    showstringspaces=false,
    showtabs=false,
    tabsize=2
}

\lstset{style=mystyle}

\graphicspath{{../figures/}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand\tab[1][1cm]{\hspace*{#1}}
\newcommand{\K}{\mathbb{K}}
\newcommand{\zero}{\mathbb{O}}

\newenvironment{theorem}[2][Theorem]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{lemma}[2][Lemma]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{exercise}[2][Exercise]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{reflection}[2][Reflection]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{proposition}[2][Proposition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{corollary}[2][Corollary]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}

\begin{document}

% --------------------------------------------------------------
%                         Start here
% --------------------------------------------------------------

%\renewcommand{\qedsymbol}{\filledbox}

\title{\textbf{Advanced Numerical Methods for PDEs}\\ \Large{Homework 3}}%replace X with the appropriate number
\author{Francisco Castillo}

\maketitle
Consider the 2D problem
\begin{align*}
\partial_tu(x,y,t) + \partial_xf^x &+ \partial_yf^y = 0,~~ x,y\in[0,1]\\
f^x = u - \partial_xu,&~~ f^y = - \partial_yu
\end{align*}
with boundary conditions
\begin{align*}
f^x(x=0,y,t) = 0,~~f^x(x=1,y,t) = 0,\\
f^y(x,y=0,t) = 0,~~f^y(x,y=1,t) = 0,
\end{align*}
and initial conditions
\begin{align*}
u(x,y,t=0) = u^I(x,y) = \delta(x-0.5)\delta(y-0.5)
\end{align*}
\section*{Problem 1}
\input{problem1}
\newpage
\section*{Problem 2}
\input{problem2}
\newpage

Consider the Schroedinger equation
\begin{align*}
i\hbar \frac{\partial\psi(x, t)}{\partial t}  = -\frac{\hbar^2}{2m}\frac{\partial^2}{\partial x^2}\psi(x,t) + V (x)\psi(x,t).
\end{align*}
Here $\hbar$ is Planck’s constant, $m$ is the mass of the particle,
and $V (x)$ is the external potential energy. $\psi$ is the wave
function. The particle density is given by $u(x, t) = |\psi(x, t)|^2$.
Assume units for space, time and mass such that $\hbar = m = 1$ holds.

\section*{Problem 3}
\input{problem3}
\newpage
\section*{Problem 4}
\input{problem4}

\end{document}


% 9.5.07
% This is a sample documentation for Compass in the tex format.
% We restrict the use of tex to the following subset of commands:
%
% \section, \subsection, \subsubsection, \paragraph
% \begin{enumerate} (no-nesting), \begin{quote}, \item
% {\tt ... }, {\bf ...}, {\it ... }
% \htmladdnormallink{}{}
% \begin{verbatim}...\end{verbatim} is reserved for code segments
% ...''
%

\section{Control Variable Test Against Function}
\label{ControlVariableTestAgainstFunction::overview}

This checker detects if there exists a for loop that tests its control (induction) variable against a function. One can get better performance by pulling out the function call before the loop and use a constant value for the test. That is,

\begin{verbatim}
  for(int i = 0; i < constSize(); ++i)
  { // do something }
\end{verbatim}

The code above can be improved as the following:

\begin{verbatim}
  const int size = constSize();
  for(int i = 0; i < size; ++i)
  { // do something }
\end{verbatim}

\subsection{Parameter Requirements}

%Write the Parameter specification here.
None

\subsection{Implementation}

%Details of the implementation go here.
This checker uses a simple traversal. For every \texttt{for} statement, the checker examines whether or not there is a function call inside the test expression.

\subsection{Non-Compliant Code Example}

% write your non-compliant code subsection

\begin{verbatim}
int bar();

void foo()
{
  int j=2;

  for(int i = 0; i < bar(), j < 10; ++i)
  {
    j += 2;
  }
}
\end{verbatim}

\subsection{Compliant Solution}

% write your compliant code subsection

\begin{verbatim}
int bar();

void foo()
{
  int j=2;

  for(int i = 0; i < 10; ++i)
  {
    j += 3;
  }
}
\end{verbatim}

\subsection{Mitigation Strategies}
\subsubsection{Static Analysis}

Compliance with this rule can be checked using structural static analysis checkers using the following algorithm:

\begin{enumerate}
\item Check if a node is a for statement
\item Check if the test expression for the for statement contains a function call
\end{enumerate}

\subsection{References}

% Write some references
% ex. \htmladdnormallink{ISO/IEC 9899-1999:TC2}{https://www.securecoding.cert.org/confluence/display/seccode/AA.+C+References} Forward, Section 6.9.1, Function definitions''
The Programming Research Group, High-Integrity C++ Coding Standard Manual, Item 5.7: ``The control variable in a for loop should be tested against a constant value, not a function''


\chapter{\label{ch:formal-languages}Formal languages and automata}

Natural languages can hardly be considered `formal' languages.%
\sidenote{?? Quote from Montague: ``no important theoretical difference between natural languages and the artificial languages of logicians.''}
From a practical engineering perspective,
our efforts to come up with (hand-crafted) formal specifications of
languages often often resulted in sub-optimal results.
However, we learned a lot about trying to do so,
and many of the concepts/methods developed are at the hart of computer science,
and many (practical) methods we use in computational linguistics.

In this lecture,
we will go through a gentle introduction to the theory of computation.
Our focus will be on \emph{automata} and corresponding formal languages.
However, we will necessarily touch on the other two subfields of
theory of computation, \emph{computability} and \emph{complexity}.

\section{Some definitions: Languages, strings, grammars}

\newthought{An alphabet} is a set of symbols.
We often use letters for our examples,
so $\{a, b, c\}$ is an alphabet consisting of three letters.
For many problems we may be interested in a binary alphabet like $\{1, 0\}$.
And for some of natural language problems our alphabet may consist of words,
such as $\{\text{cat}, \text{dog}, \text{sat}, \text{on}, \text{mat}\}$.
By convention, the Greek capital letter sigma $\Sigma$ for representing
an alphabet.

\newthought{A string} is a finite sequence of symbols from an alphabet.
$a$, $ac$ or $abcaab$
are example string over alphabet $\{a, b, c\}$.
A special string that does not contain any symbols,
the \emph{empty string} is denoted by Greek letter epsilon, $\epsilon$.
We denote the set of all string over $\Sigma$,
including the empty string, with $\Sigma^{*}$.
$\Sigma^{+}$ is often used for the set of non-empty string in $\Sigma^{*}$
(a shorthand for $\Sigma^{*} - {\epsilon}$).
Note that both $\Sigma^{*}$ and $\Sigma^{+}$ have infinite number of members.
We indicate the length of a string with vertical bars on both sides,
$\lvert{}ab\rvert = 2$, or if $u = 0100$, $\lvert{}u\rvert{} = 4$.
For two string $u$ and $v$, we indicate their concatenation with $uv$.
If $u = 0100$ and $v = 11$, $uv = 010011$.

\newthought{A language} is a set of strings over an alphabet.
Empty set, $\emptyset$, is a language,
as well as the set containing only the empty string $\{\epsilon\}$.
Similarly $\Sigma^{*}$ is another language over alphabet $\Sigma$.
We are generally interested in more interesting languages,
such as%
\sidenote{Although we define these languages informally here,
  all of them can be defined more formally.
  For example, the language by the first item
  can be defined as $\{u\mid \lvert{}u\rvert = 2\}$.}
\begin{itemize}
  \item The set of strings of length \num{2}
    over $\{0, 1\}$: $\{00, 01, 10, 11\}$
  \item The set of strings with even number of \num{1}'s over $\{0, 1\}$:
      $\{\epsilon, 101, 0, 11, 111110, \ldots\}$
  \item The set of string that retain alphabetical ordering
    over $\{a, b, c\}$: $\{a, ab, abc, ac, abcc, \ldots\}$
  \item The set of strings of words
    that form a grammatically correct English sentences
\end{itemize}
The strings that are in a language are called \emph{sentences} of the language.

\newthought{A grammar} is a finite description of a language.
Although the set-based definitions above is flexible,
it is more convenient to express a language with a finite set of rules.
The most common method for formalizing grammars is based on
a set of \emph{rewrite rules}.
A rewrite rule two sides delimited by an arrow ($\ra$).
The grammar generates strings in the language,
by rewriting sequence of symbols matching right-hand-side (RHS) of the rule
as the left-hand-side (LHS) of the rule,
starting from a special `start' symbol denoted by `S'.
The sequences of symbols in a rewrite rule is may consist of
\emph{terminal} symbols that are member of the alphabet
and \emph{non-terminal} symbols.
The non-terminal symbols are denoted using capital letter by convention.
For example, the grammar in Figure~\ref{fig:exmpl-grammar}
produces a language over the alphabet $\{a, b, c\}$
whose last symbol (word) is always an $a$.
This is not the only way to express the same language.
There are multiple (better, more concise) ways of writing
the same grammar.
The rule number 2\ in the grammar shown in Figure~\ref{fig:exmpl-grammar}
is recursive, it allows infinite number applications of this rule,
and hence, the grammar defines an infinite language with a finite set of rules.

\begin{marginfigure}
  \begin{tcolorbox}
    \begin{enumerate}
      \item $ S \ra A a$
      \item $ A \ra B A$
      \item $ A \ra B$
      \item $ B \ra a$
      \item $ B \ra b$
      \item $ B \ra c$
    \end{enumerate}
  \end{tcolorbox}
  \caption{\label{fig:exmpl-grammar}%
    An example grammar producing string over $\{a, b, c\}$
    that end with symbol $a$.
    A common short-hand for writing last three rules is a single rule
    of the form `$B \ra a | b | c$'.
  }
\end{marginfigure}

Formally,
a phrase structure grammar is a tuple
$G = (\Sigma, N, S, R)$, where
  \begin{itemize}
    \item[$\Sigma$] is an alphabet of terminal symbols
    \item[$N$] are a set of non-terminal symbols
    \item[$S$] is a special `start' symbol $\in N$
    \item[$R$] is a set of rules of the form $\alpha \ra \beta$,
      where  $\alpha$ and $\beta$ are strings from $(\Sigma \cup N)^{*}$.
  \end{itemize}

For our example in Figure~\ref{fig:exmpl-grammar},
$\Sigma = \{a, b, c\}$, $N = \{S, A, B\}$, $S = S$,
and $R = \{ S \ra A a, A \ra B A, A \ra B,
      B \ra a, B \ra b, B \ra c\}$.

\begin{marginfigure}
  \begin{tcolorbox}[halign=center]
    \begin{tabular}{l@{\Ra}ll}
      $S      $ & $A a$     &(rule 1)\\
      $A a    $ & $B A a$   &(rule 2)\\
      $B A a  $ & $B B A a$ &(rule 3)\\
      $B B A a$ & $B B B a$ &(rule 3)\\
      $B B B a$ & $B B c a$ &(rule 6)\\
      $B B c a$ & $B b c a$ &(rule 5)\\
      $B b c a$ & $a b c a$ &(rule 4)\\
    \end{tabular}
  \end{tcolorbox}
  \caption{\label{fig:example-derivation}%
    A derivation of string $abca$ using the grammar
    in Figure~\ref{fig:exmpl-grammar}.
  }
\end{marginfigure}
Given the grammar in Figure~\ref{fig:exmpl-grammar},
the string \texttt{abca} can be derived with the derivation sequence
presented in Figure~\ref{fig:example-derivation}.
We could write the same derivation sequence as
'$S \Ra A a \Ra B A a \Ra B B A a \Ra B B B a \Ra B B c a \Ra B b c a \Ra a b c a$'.
In general, if a string can be derived from the start symbol S
using a sequence of rewrite rules as above
(sometimes notes as `S $\overset{*}{\Ra}$ a b c a',
we say that the grammar \emph{generates} the string,
or the string is \emph{recognized} by the grammar.
Hence, the string is a valid sentence in the language.

The above definitions are important for understanding formal languages,
and also used in study of natural languages in formal linguistics.

\section{Chomsky hierarchy of languages}

\begin{marginfigure}
  \begin{center}
    \tikzset{external/export next=false}%
    \begin{tikzpicture}[x=4mm,y=8mm]
      \draw (0,0) ellipse (3.0 and 1.0);
      \node[font=\footnotesize] at (0,0) {Regular};
      \draw (0,0.5) ellipse (4.0 and 1.5);
      \node[font=\footnotesize] at (0,1.3) {Context Free};
      \draw (0,1.0) ellipse (5.0 and 2.0);
      \node[font=\footnotesize] at (0,2.3) {Context Sensitive};
      \draw (0,1.5) ellipse (6.0 and 2.5);
      \node[font=\footnotesize] at (0,3.2) {Recursively Enumerable};
      \draw (-6.1,-1.1) rectangle (6.1,4.1);
      %\draw (4,3.8) node {All};
    \end{tikzpicture}
  \end{center}
  \caption{\label{fig:chomsky-h}%
    Chomsky hierarchy of language classes.
    The classes are also known as \emph{type-0}
    (recursively enumerable) to \emph{type-3} (regular).
  }
\end{marginfigure}

The Chomsky hierarchy of languages
is one of Chomsky's most important contributions
to both linguistics and computer science \parencite{chomsky1959b}.
For his theory of natural language syntax,
Chomsky defined a set of formal language classes
(depicted in Figure~\ref{fig:chomsky-h}).
Each class in this hierarchy is the proper subset of the larger class.
The larger classes are more descriptive,
but their computational processing is more demanding.
This particular classification of the language classes
has some attractive formal properties,
and each class corresponds to
a certain type of abstract computational device
that is capable of recognizing
and generating the languages in the corresponding class.
As a result, as well as its importance for (computational) linguistics,
this hierarchy is also an important part theory of computation.
We will define the individual language classes and the
properties of the rewrite grammars that expresses them.
In general, we will describe each language class
based on the restriction they put in the rewrite grammars that describe them.

We will next discuss at these language classes with in detail.
However, we note that these language classes are not
the only formally defined language classes.
There are other well-studied language classes
some of which can be placed
in the set-inclusion hierarchy in Figure~\ref{fig:chomsky-h},
while other may cross-cut parts of this hierarchy.

\subsection{A brief divergence: complexity theory}

As noted earlier,
study of formal languages strongly linked to theory of computation,
particularly complexity theory.
Here, we will briefly review a part of the complexity theory briefly,
since some of the concepts will be instrumental in our discussion
of the language classes and the computational problems
related to these language classes.

In analysis of algorithms,
and algorithms worst time order of complexity
is indicated a notation called \emph{big-O notation}.
Big-O notation aims to measure an algorithms complexity
as its input grows in size.
Since the aim of the big-O notation is indicating
the worst case with large number of input,
we discard all constant and lower order complexities from
the analysis of our algorithm.
For example, if our analysis indicate an algorithm to require
$T(n) = 100\times{}2^{n} + n^{10} + \log{n} + 10^{100}$ steps
(e.g., a simple operation on a CPU) to complete,
the complexity we are interested in is indicated by $O(2^{n})$.
In general, we drop all terms except $2^{n}$ above,
since all other figure will dwarfed by this term
as the size of the input ($n$) is increased.
Although even a constant factor is often important in practice,
we may be able get reasonable running times by parallelization,
or hope that increase in computing power will allow us
to use these algorithms.
The order of complexity measured with big-O notation has
more important theoretical consequences.
A few common examples of complexities measured using big-O notation
are given below.

\begin{itemize}[nosep,labelindent=1em,labelwidth=1.3cm,labelsep*=1em,leftmargin =!]
  \item[\textcolor{blue!50!green}{$O(1)$}] constant,
    an algorithm that does not depend on the size of its input,
    e.g., accessing a single element of an array
  \item[\textcolor{blue!50!green}{$O(\log n)$}] logarithmic,
    e.g., binary search
  \item[\textcolor{blue!50!green}{$O(n)$}] linear,
    e.g., finding minimum or maximum value of an array
  \item[\textcolor{blue}{$O(n \log n)$}] log linear,
    e.g., some sorting algorithms
  \item[\textcolor{blue}{$O(n^{2})$}] quadratic,
    e.g., selection sort (and most arithmetic operations)
  \item[\textcolor{blue}{$O(n^{3})$}] cubic
  \item[\textcolor{red}{$O(2^{n})$}] exponential,
    e.g., graph search algorithms involving backtracking
    (note that the base is not important)
  \item[\textcolor{red}{$O(n!)$}] factorial,
    e.g., enumerating all binary trees spanning $n$ items (words)
\end{itemize}

\begin{marginfigure}
  \begin{tikzpicture}
    \begin{axis}[
        ymax=2000,
        axis x line=bottom,
        axis y line=left,
        xlabel=$n$,
        xlabel style={font=\tiny},
        ylabel style={font=\tiny},
        title style={font=\tiny},
        ticklabel style={font=\tiny},
        x=0.4mm,
        y=0.025mm,
%        cycle list={colormap/Spectral-7},
        thick,
        grid,
        legend style={font=\tiny,fill=none,draw=none},
        legend cell align=left,
      ]
      \addplot[blue,dashed, domain=1:100, samples=100] {ln(x)};
      \addplot[green,domain=1:100, samples=100] {x};
      \addplot[black,dashed, domain=1:100, samples=100] {x*ln(x)};
      \addplot[orange,domain=1:100, samples=100] {x^2};
      \addplot[brown,dashed,domain=1:15, samples=100, smooth,unbounded coords=jump] {x^3};
      \addplot[purple,domain=1:12, samples=100, smooth,unbounded coords=jump] {2^x};
      \addplot[red,dashed,domain=1:8, samples=7, unbounded coords=jump] {x!};
      \legend{
          $O(\log n)$,
          $O(n)$,
          $O(n \log n)$,
          $O(n^{2})$,
          $O(n^{3})$,
          $O(2^{n})$,
          $O(n!)$,
      };
    \end{axis}
  \end{tikzpicture}
  \begin{tikzpicture}
    \begin{semilogyaxis}[
        ymax=1000000000000000000000000000,
        ytickten={0,5,...,25},
        axis x line=bottom,
        axis y line=left,
        xlabel=$n$,
        xlabel style={font=\tiny},
        ylabel style={font=\tiny},
        title style={font=\tiny},
        ticklabel style={font=\tiny},
        x=0.4mm,
        y=0.75mm,
%        cycle list={colormap/Spectral-7},
        thick,
        grid,
        legend style={font=\tiny,fill=none,draw=none, yshift=-1cm},
        legend cell align=left,
      ]
      \addplot[blue,dashed, domain=1:100, samples=100] {ln(x)};
      \addplot[green,domain=1:100, samples=100] {x};
      \addplot[black,dashed, domain=1:100, samples=100] {x*ln(x)};
      \addplot[orange,domain=1:100, samples=100] {x^2};
      \addplot[brown,dashed,domain=1:100, samples=100, smooth,unbounded coords=jump] {x^3};
      \addplot[purple,domain=1:100, samples=100, smooth,unbounded coords=jump] {2^x};
      \addplot[red,dashed,domain=1:30, samples=30, unbounded coords=jump] {x!};
      \legend{
          $O(\log n)$,
          $O(n)$,
          $O(n \log n)$,
          $O(n^{2})$,
          $O(n^{3})$,
          $O(2^{n})$,
          $O(n!)$,
      };
    \end{semilogyaxis}
  \end{tikzpicture}
  %TODO: more print-friendly graphs
  \caption{\label{fig:computational-complexity}%
    Order of operations required by different algorithmic complexity
    measured by big-O notation.
    Top graph represents this linearly,
    while the y-axis of bottom graph is logarithmic
    (changes in the graph indicate an exponential changes
    in the number of operations),
    allowing us to visualize later compare complexity over
    a larger interval of input size $n$.
  }
\end{marginfigure}
Naturally, we prefer the lower-complexity algorithms
(earlier items in the above list).
However, the behavior of the last two items in the list
and the earlier ones are drastically different when the input size is increased.
The ones up to $O(n^{3})$ above,
in general any algorithm whose complexity can be explained
a polynomial of any degree,
are called \emph{polynomial} time algorithms.
In general, polynomial time algorithms are said to be \emph{tractable}.
And algorithms with worse complexity than polynomial time
are called \emph{intractable}.
Intractable algorithms scale badly with increased input size,
Even with reasonable increase in computing power,
we do not have any hopes of solving intractable algorithms
for larger input sizes.
Figure~\ref{fig:computational-complexity}
presents order of operations needed for algorithms
with different complexities.
The top graph in Figure~\ref{fig:computational-complexity},
showing how drastic the complexity may change.
Note that an algorithm with $O(n^{3})$ requires more operations
for small input sizes,
but in time non-polynomial algorithms more costly for larger input sizes.
The bottom graph shows this more drastically,
as the input size increased,
the polynomial and non-polynomial algorithms becomes much clear.

A fundamental activity in computer science is to find
algorithms with lower big-O complexity.
For example,
a brute-force approach to recognizing context-free languages
is exponential in the length of the sentence (or computer program).
However, we have clever solutions that work in polynomial time
using a modest amount of additional storage.
The class of problems that has a known polynomial-time algorithm is
indicated by P.
For some problems we do not know a tractable algorithm.
However, they are not provably intractable,
we may hope to find some polynomial time algorithm for solving
these problems and they may become a member of P.
An interesting subclass of these problems
for which we do not have a polynomial-time algorithm to solve it,
we can identify the solution (in polynomial time) when we see the solution.
This class of problems are called
\emph{non-deterministic polynomial time} (NP).
P is a subset of NP.
However, one of the big questions in theory of computation is
whether P is equal to NP or not.
If it is,
then there are tractable algorithms waiting to be discovered
for many of the problems in NP.

An interesting sub-class of NP is called NP-complete.
If we find a polynomial time solution to any of the NP-complete problems,
we can solve all NP problems in polynomial time,
since solution of any NP problem can be `converted' obtained using
the solution of an NP-complete problem
with a polynomial overhead for the conversion.
Finding a polynomial algorithm for an NP-complete problem
would prove P = NP,
as well as solving many difficult problems in a wide range of fields.
However,
the fact we do not know tractable algorithms for NP-complete problems
despite decades of efforts by many clever people
suggests that P is probably not equal to NP.
Hence, when we face a problem that is NP-complete,
or in a more difficult complexity class,
chances of finding a scalable solution is rather unlikely.

The concepts introduced above should be sufficient to understand
some of the complexity issues we will discuss below.
However, there are many other complexity classes,
and many unknowns, interesting problems,
about the relation between these classes.
A reasonable overview of computational complexity is
out of the scope of this lecture.
The additional resources at end of the chapter points to
a few textbooks covering the complexity theory is covered.

\subsection{Regular languages}

Regular languages is the simplest
(least expressive, easy to process computationally)
class in Chomsky hierarchy.
The regular languages are defined by \emph{regular grammars}
which are (mostly) equivalent to \emph{regular expressions}
that are used in many programming environments
for searching patterns in text.
We will study regular languages in detail in Chapter~\ref{ch:fst}.

\begin{marginfigure}
  \begin{tcolorbox}[rounded corners]
    \begin{multicols}{2}
      \begin{enumerate}
        \item $A \ra a$
        \item $A \ra Ba$
        \item $A \ra \epsilon$
      \end{enumerate}
      \begin{enumerate}
        \item $A \ra a$
        \item $A \ra aB$
        \item $A \ra \epsilon$
      \end{enumerate}
    \end{multicols}
  \end{tcolorbox}
  \caption{\label{fig:regular-grammars}%
    The possible rules for left-regular (left)
    and right-regular (right) grammars.
    For both, $A$ and $B$ are non-terminal,
    $a$ is a terminal sybol ($a \in \Sigma$),
    and $\epsilon$ is the empty string.
  }
\end{marginfigure}
A \emph{regular grammar} is a grammar that generates a regular language.
There are two flavors of regular grammars.
The \emph{left-regular grammars} can have only the rules listed in left side
of Figure~\ref{fig:regular-grammars}.
The rules allowed in a \emph{right-regular grammar} is listed
in the right side of Figure~\ref{fig:regular-grammars}.
In simple words, regular grammar can have three types of rules
\begin{itemize}
  \item A single non-terminal symbol rewritten as a single terminal symbol
  \item A single non-terminal symbol as one terminal
    and one non-terminal symbol
    (the order determines whether the grammar left or right regular)
  \item A single non-terminal symbol as empty string ($\epsilon$)
\end{itemize}
Note that both definitions allow defining all (and only) regular languages.
Some definitions do not allow transition to empty string
(3rd rule in both definitions).

A regular grammar only defines regular languages.
However, note that regular languages can also be defined
with non-regular grammars (e.g., context free grammars we will define next).

%TODO: examples, exercises

\subsection{Context-free languages}

Context-free languages are the next (more expressive) language
after regular languages in the Chomsky hierarchy.
They are used in describing (and parsing) programming languages.
The syntax of all programming languages are based on context-free grammars.%
\sidenote{They are generally defined by a restricted
  (deterministic, non-ambiguous) subset of context-free grammars
  for efficiency of parsing,
  and in a limited/restrict set of non-context freeness may be used.
}
Similarly,
many syntactic theories of natural languages are based on
a context-free backbone
(we will discuss this further in Section~\ref{sec:nl-and-fl}).

The restrictions on the rewrite rules
for context-free grammars are simple:
left-hand-side (LHS) of the rule has to contain a single non-terminal.
Unlike regular grammars,
we do not have any restrictions on right-hand-side (RHS)
of the rewrite rules.
They can contain any combination of non-terminal and terminal symbols
(or $\epsilon$).
The context-free languages are called so,
since the rewrite rules express
expanding a non-terminal anywhere independent of their context.

%TODO: examples, exercises

\subsection{Context sensitive languages}

Context sensitive languages are a superset of context-free languages.
The rules of context sensitive language are required to be in the following form.
\[
  \alpha A \beta \ra \alpha \gamma \beta
\]
where $A$ is a single non-terminal symbol,
$\alpha$ and $\beta$ are
possibly empty strings of terminals and non-terminals,
and $\gamma$ is a non-empty string of terminal and non-terminal symbols.
Note that the above definition does not allow
a LHS non-terminal to be deleted (rewritten as $\epsilon$).
However, a rule of the form $S \ra \epsilon$ is allowed
to ensure that the context-sensitive languages are
a superset of context-free languages.
Another way to define a context sensitive grammars is
requiring the RHS of all rules to be longer than or equal to the LHS.%
\sidenote{The grammars defined this way is known as
  \emph{non-contracting grammars} and are
  known to be \emph{weakly equivalent} to context-sensitive grammars.
  That is, we can convert any non-contracting grammar to
  a context-sensitive grammar.
}

Many problems with context-sensitive grammars do not have
efficient solutions (see Section~\ref{sec:formallang-automata}).

\subsection{Recursively enumerable languages}

The recursively enumerable languages are the most expressive language
in the Chomsky hierarchy.
Recursively enumerable languages are languages over an alphabet that are,
well, recursively enumerable.
That is, there exist an algorithm that can enumerate (list)
all strings in the language.

The grammars that define recursively enumerable languages
do not pose any restrictions on their rewrite rules,
except that their LHS being non-empty.
The class of grammars that generate recursively enumerable languages
are called \emph{unrestricted grammars}.

\section{\label{sec:formallang-automata}Formal languages and automata}

The simplest question we ask about a grammar,
or the corresponding language,
is recognizing strings that belong to the language.
That is, given a string,
can we determine whether it belongs to the language or not?

This question links the language classes described above to
abstract computation devices, \emph{automata}.
In other words,
for each language class discussed above,
there is an type of automaton that recognizes the strings of
a language in the class.
We will briefly discuss these correspondences,
starting from the most-expressive language class.

Strings in a recursively enumerable language,
the most expressive class in the Chomsky hierarchy,
can be recognized by a \emph{Turing machine}.
The Turing machine is a simple abstract computing device
that can compute any computable function.
In other words, any algorithm can be simulated by a Turing machine.
The relationship between recursively enumerable languages
and Turing machines mean
that a Turing machine can generate all (and only) string
in the corresponding recursively enumerable language.
However, the question is whether a string is in
a recursively enumerable language or not is undecidable,
since it is equal to a \emph{halting problem},
a well known undecidable problem.

The machines that recognize the context-sensitive languages
are a restricted version of Turing machines
called \emph{linear-bounded automata}.
Although whether a string is generated by a context-sensitive grammar
or not is decidable,
it is known to be computationally complex.%
\footnote{More formally,
  it is PSPACE-complete, and PSPACE is a superset of NP.
}

Context-free grammars can be recognized by \emph{push-down automata}.
A push-down automaton has a finite number of sates that controls
the computation
(similar to the finite-state machines we will summarize next),
and a stack (with access to only the top element) as memory.
Although the complexity of first two
the grammars and machines discussed above are computationally intractable,
there are many practical algorithms for solving problems
involving context-free grammars.
We will revisit the context-free grammars
when we discuss parsing algorithms later.

\begin{marginfigure}
  \begin{center}
    \begin{tikzpicture}[
      ]
      \node[draw,thick,state,initial,initial text={}] (s0) {0};
      \node[draw,thick,right=of s0,state] (s1) {1};
      \node[draw,thick,right=of s1,state,accepting] (s2) {2};
      \draw[->] (s0) edge[above] node{a} (s1)
                (s1) edge[loop above] node{b} (q1)
                (s1) edge[above] node{c} (s2);
    \end{tikzpicture}
  \end{center}
  \caption{\label{fig:fsa-example}%
    An example automata recognizing the strings over alphabet
    $\{a, b, c\}$ which begins with $a$, followed by zero or more
    $b$s, and ends with $c$.
  }
\end{marginfigure}
Finally, the regular languages are recognized by finite-state automata.
Finite state automata (FSA) can be described by a directed graph
with a finite nodes (states) and edges are
labeled with symbols of the alphabet.
A finite-state automaton
can accept (or rejects) its input in time complexity
linear in the length of input.
Furthermore, the FSA are `memoryless',
except the automaton itself, they do not need any additional memory.
Figure~\ref{fig:fsa-example} shows an example finite state automata.
We will discuss finite-state automata in Chapter~\ref{ch:fsa}.

\section{Where do natural languages fit}

One of the motivations behind building the classes of languages
in the Chomsky hierarchy was to characterize
the type of formal grammars that are adequate for natural languages,
primarily natural language syntax.
By `adequate', we mean classes of grammars that can represent
the phenomena we observe in natural languages,
yet computationally not so demanding.
The latter condition, computational efficiency,
is not only desirable
from the perspective of computational processing of natural languages.
It is also important for cognitive science
where one of the main tenets is that cognition is a form of computation.
Since human cognitive system is a computing device
with limited/finite resources,
there are bounds on its computational capabilities.
As a result, the computational efficiency of language processing is
also important for study of human languages from a cognitive perspective.

Initially,
context-free languages were considered to be adequate
for representing natural languages.
However,
a small number of linguistic constructions cannot be represented
using context-free languages.
A typical example is the cross-serial dependencies found
in Dutch and Swiss German.
Figure~\ref{fig:cross-serial} presents
an example cross-serial dependency from Swiss German.
The dependencies with the verbs and their arguments cross.
As a result, context free grammars cannot represent this structure.
The problem is clearer if you consider
the operations of a push down automaton.
If we push the case marking on each noun on the stack,
by the time we process the verb \xmpl{hälfed},
we do not have access to case marking of its argument
as it is not on the top of the stack anymore.

\begin{figure*}
  \tikzset{external/export next=false}
  \begin{center}
    \begin{tikzpicture}
      \matrix[matrix of nodes] (m) {
        Jan & säit  & das & mer &\color{blue} em Hans & \color{red}es huss & \color{blue} hälfed & \color{red}aastriiche \\
        Jan & said & that & we & Hans (\textsc{dat}) & the house (\textsc{acc}) & helped & paint \\
      };
      \draw[blue] (m-1-5) to[in=north, out=north, looseness=0.5] (m-1-7);
      \draw[red] (m-1-6) to[in=north, out=north, looseness=0.5] (m-1-8);
    \end{tikzpicture}
  \end{center}
  \caption{
    An example cross-serial dependency from Swiss German
    \parencite{shieber1985}.
    The case marking of \xmplt{em Hans}{Hans} agrees with the verb
    \xmplt{hälfed}{helped},
    while the case marking of \xmplt{es huss}{house} agrees with the verb
    \xmplt{aastriiche}{paint}.
    Note that this resembles
    $a^{\color{blue}n}b^{\color{red}m}c^{\color{blue}n}d^{\color{red}m}$.
  }\label{fig:cross-serial}%
\end{figure*}

Although the syntactic phenomena such as the one presented
in Figure~\ref{fig:cross-serial} are clearly non-context free,
their existence do not require the full representation power
(and computational complexity) of context-sensitive grammars.
A common theoretical consensus seems that
a grammar class, called \emph{mildly context sensitive} grammars
are adequate for representing natural languages
(see Figure~\ref{fig:chomsky-h-mcs}).
In practice,
many computationally oriented theories of grammars use
a context-free backbone with some extensions that increase their
expressive power.
This extension may range from mildly context-sensitive grammars%
\sidenote{E.g., Combinatory categorial grammar (CCG)
and tree adjoining grammar (TAG).}
to Turing machine equivalence.%
\sidenote{E.g., Head-driven phrase structure grammars (HPSG),
  and (according to some analyses lexical-functional grammar (LFG).
}
Although these extensions increase
their worst-case computational complexity drastically,
these formalisms often work acceptably in practice due to their
reasonable expected/average time complexity.

Before closing this brief discussion of natural languages
and formal languages,
their representational power and computational complexity,
we should note that the non-context-free extensions
to the theories of natural language grammars are not
motivated only by the representation power.
Besides being able to represent the constructions
like the cross-serial dependencies,
extensions requiring higher
than context-free power are often proposed
when they match better with linguistic intuitions.
In other words, even if a context-free grammar may be adequate
to represent a language,
an extended version may match the linguists' intuition
of how the linguistic structures are derived better.

\begin{marginfigure}
  \begin{center}
    \tikzset{external/export next=false}%
    \begin{tikzpicture}[x=4mm,y=8mm]
      \draw[dashed] (0,.6) ellipse (4.2 and 1.6);
      \draw (0,0) ellipse (3.0 and 1.0);
      \node[font=\footnotesize] at (0,0) {Regular};
      \draw (0,0.5) ellipse (4.0 and 1.5);
      \node[font=\footnotesize] at (0,1.3) {Context Free};
      \draw (0,1.0) ellipse (5.0 and 2.0);
      \node[font=\footnotesize] at (0,2.4) {Context Sensitive};
      \draw (0,1.5) ellipse (6.0 and 2.5);
      \node[font=\footnotesize] at (0,3.2) {Recursively Enumerable};
      \draw (-6.1,-1.1) rectangle (6.1,4.1);
      %\draw (4,3.8) node {All};
    \end{tikzpicture}
  \end{center}
  \caption{\label{fig:chomsky-h-mcs}%
    A depiction of Chomsky hierarchy of language classes
    (Figure~\ref{fig:chomsky-h},
    with the addition of \emph{mildly context-sensitive languages}
    represented with the dashed ellipse.
  }
\end{marginfigure}

\section{\label{sec:formal-lang-learnability}%
        Formal languages and learnability}

A central question in modern linguistics, surrounded with a big debate,
has been learnability of natural languages.
Some researchers, primarily based on Noam Chomsky's ideas,
championed the idea that the natural languages are not learnable.
Hence, only way for a human child to learn the languages they are exposed to
is to have an innate linguistic component.
The debate is linked to an even bigger, and much older one:
nature vs.\ nurture debate.
The debate at large  has little practical use in a course
on natural language processing.
It also seems the heated discussion on the topic has slowed down after 2000s.
However, part of the debate is linked with some of the results regarding
learnability of classes of formal languages,
and together with the fact that this discussion has
a historical significance in the field,
it deserves a brief mention here.

There has been a number of formal proofs
indicating that none of the languages in the Chomsky hierarchy is
learnable only from positive examples.%
\footnote{Mainly due to \textcite{gold1967}.}
These results frequently used as an argument for the innateness hypothesis.
However, the formal arguments are often too general,
and do not necessarily match the settings in which children learn languages.
For example, the results cited above are obtained for the language classes
in the Chomsky hierarchy,
while the human languages do not necessarily match with
one of these language classes.
It is more likely that the class of natural languages,
if such a class exists,
cross cut the hierarchy (as in Figure~\ref{fig:chomsky-h-cross}).
In other words,
even regular languages allow some constructions
that are not useful for representing structures in human languages.
%TODO: example
Furthermore,
the condition that the languages should be learned from
the positive examples alone is likely to be too strict
for a realistic setting for child language acquisition.

\begin{marginfigure}
  \begin{center}
    \tikzset{external/export next=false}%
    \begin{tikzpicture}[x=4mm,y=8mm]
      \definecolor{vlgray}{rgb}{0.9,0.9,0.9}
      \filldraw[vlgray,draw opacity=0.1] (0,.6) ellipse (1.5 and 1.6);
      \draw[dashed] (0,.6) ellipse (4.2 and 1.6);
      \draw (0,0) ellipse (3.0 and 1.0);
      \node[font=\footnotesize] at (0,0) {Regular};
      \draw (0,0.5) ellipse (4.0 and 1.5);
      \node[font=\footnotesize] at (0,1.3) {Context Free};
      \draw (0,1.0) ellipse (5.0 and 2.0);
      \node[font=\footnotesize] at (0,2.4) {Context Sensitive};
      \draw (0,1.5) ellipse (6.0 and 2.5);
      \node[font=\footnotesize] at (0,3.2) {Recursively Enumerable};
      \draw (-6.1,-1.1) rectangle (6.1,4.1);
      %\draw (4,3.8) node {All};
    \end{tikzpicture}
  \end{center}
  \caption{\label{fig:chomsky-h-cross}%
    A possible set (indicated by the shaded region in the figure)
    for class of natural languages in the Chomsky hierarchy,
    to demonstrate that learnability results on a particular language
    class in the hierarchy is not necessarily
    a positive or negative indication for learnability of natural languages.
    As in Figure~\ref{fig:chomsky-h-mcs},
    the dashed ellipse represents
    the class of mildly context-sensitive languages.
  }
\end{marginfigure}

Perhaps more interesting for computational linguistics is
the question of learnability of languages is an empirical question
that can be answered by a computational model.
Theoretical claims aside,
if we can build models that can learn natural languages
from the input available to children,
that is a good indication that they are learnable.
Even if the computational model may not mimic
the way children learn languages,
they can be used to show that learning is in principle possible.
Furthermore,
if we constrain such models the way human learners are constrained,
we may be able to answer even more interesting questions.

In summary,
although it is not part of the mainstream
computational linguistics or natural language processing fields,
the methods used typically in these fields can also be instrumental
in study of languages from cognitive science perspective.

\section{Where to go from here}

This lecture is a brief overview of formal languages
and automata theory.
We also noted some topics in theory of computation
(e.g., computational complexity and complexity classes),
and even some related issues in theoretical linguistics
and cognitive science.
All of these topics are, necessarily, covered briefly, on the surface.
Some of these topics will be revisited in later lectures.
Here, we point to some of the relevant literature.

\textcite{hopcroft1979} (and its successive editions) has been
the most influential textbook on formal languages and automata theory.
\textcite{sipser2006} is another good textbook
on the topic
covering formal languages and automata as well as
complexity theory and other interesting issues in the theory of computation.

The discussion of learnability
briefly touched in Section~\ref{sec:formal-lang-learnability}
started in 1950s and 1960s based on Chomsky's work.
The literature in the field is vast,
ranges from psycholinguistics to computational linguistics.
Probably most popular work
on the innateness side of the debate is \textcite{pinker1994},
focusing mainly claims based on theoretical and experimental work.
\textcite{a.clark2011book} provides a more recent review of the field
from the opposite side of the debate.
This book also is more closely related
to the formal languages discussed in this chapter.

\section{What to add}

\begin{itemize}
  \item About complexity: infinity is overrated
  \item A bit of clarification/emphasis on `generative' side of grammars
\end{itemize}


\documentclass[a4paper]{article}

\def\npart{IB}

\def\ntitle{Markov Chains}
\def\nlecturer{G.\ R.\ Grimmett}

\def\nterm{Michaelmas}
\def\nyear{2017}

\input{header}

\newcommand*{\tot}{\leftrightarrow}

\begin{document}

\input{titlepage}

\tableofcontents

\section{Definition}

Let $X:\Omega \to S$ be a random variable. Then $(X_0,X_1, \ldots)$, a sequence of random variables, is called a \emph{stochastic/random process}. The problem we are interested in is whether there is any dependence between the random variables. Another example of a stochastic process is $(X_t, t \in \mathbb{R})$, representing, for example, the evolution of a system with respect to time.

\begin{definition}
  Let $X=(X_n:n=0,1,2,\ldots)$ be a sequence taking values in some \emph{state space} $S$, which is either finite or countably infinite. $X$ is a \emph{Markov chain} if it satisfies the \emph{Markov condition}:
  \begin{multline*}
    \prob(X_{n+1}=i_{n+1}|X_0=i_0,X_1=i_1,\ldots,X_n=i_n) = \prob(X_{n+1}=i_{n+1}|X_n=i_n) \\
    \forall n\geq0, i_0,\ldots,i_{n+1}\in S
  \end{multline*}

  $X$ is called \emph{homogeneous} if $\prob(X_{n+1}=j|X_n=i)$ does not depend on the value of $n$.
\end{definition}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Random walk is a Markov chain: let $Z_1, Z_2, \ldots$ be independent, $\prob(Z_i=1) = p,\prob(Z_i=-1) =1-p$, $X_n=Z_1+\cdots+Z_n$.
  \item Branching process: let $X_n$ be the size of the $n$th generation, then $(X_n)$ is a Markov chain.
  \end{enumerate}
\end{eg}

\begin{convention}
  Henceforth, unless contradicted, all chains are assumed to be homogeneous.
\end{convention}

Two quantities associated with a chain are:
\begin{enumerate}
\item initial distribution $\lambda = (\lambda_i: i\in S)$ where $\lambda_i = \prob(X_0=i)$, the probability mass function at $0$,
\item transition matrix $P = (p_{i,j}: i,j\in S)$ given by $p_{i,j} = \prob(X_1=j|X_0=i)$.
\end{enumerate}

\begin{proposition}\leavevmode
  \begin{enumerate}
  \item $\lambda$ is a distribution in that $\lambda_i\geq 0$ and $\sum_i \lambda_i = 1$.
  \item $P$ is a \emph{stochastic matrix} in that $p_{i,j} \geq 0, \sum_j p_{i,j} = 1$.
  \end{enumerate}
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item $\lambda_i = \prob(X_0=i) \geq 0$, $\sum_i \lambda_i = \sum_i\prob(X_0=i)=1$, i.e. $\{X_0=i\}_{i\in X}$ partitions $\Omega$.
  \item $p_{i,j} = \prob(X_1=j|X_0=i) \geq 0$, $\sum_j p_{i,j} = \sum_j \prob(X_1=j|X_0=i) = 1$.
  \end{enumerate}
\end{proof}

\begin{theorem}
  Let $\lambda$ be a distribution on $S$ and $P$ be a stochastic matrix. The sequence $X=(X_n:n \geq 0)$ is a Markov chain with initial distribution $\lambda$ and transition matrix $P$ if and only if
  \begin{multline}\label{eqn:joint probability}
    \prob(X_0=i_0, X_1=i_1, \ldots, X_n=i_n) = \lambda_{i_0} p_{i_0,i_1} p_{i_1,i_2} \cdots p_{i_{n-1}, i_n} \\
    \forall n\geq 0, i_0,\cdots, i_n \in S \tag{$\ast$}
  \end{multline}

\end{theorem}

\begin{proof}
  Let $A_k = \{X_k=i_k\}$. Equation~\eqref{eqn:joint probability} is
\begin{equation}
  \label{eqn:joint probability2}
  \prob(A_0\cap A_1 \cap \cdots \cap A_n) = \lambda_{i_0} p_{i_0,i_1} p_{i_1,i_2} \cdots p_{i_{n-1}, i_n}
  \tag{$\star$}
\end{equation}

Suppose $X$ is a $(\lambda,P)$ Markov chain. Proof of equation~\eqref{eqn:joint probability2} by induction on $n$. When $n=0$, $\prob(X_0=i_0)=\lambda_{i_0}$. Suppose equation~\eqref{eqn:joint probability2} holds for $n<N$.
\begin{align*}
  \prob(A_0\cap\cdots\cap A_N) &= \prob(A_0\cap\cdots \cap A_N|A_0\cap\cdots\cap A_{N-1}) \prob(A_0\cap\cdots \cap A_{N-1}) \\
                            &= \prob(A_N|A_0\cap\cdots\cap A_{N-1})\prob(A_0\cap\cdots\cap A_{N-1}) \\
                            &\stackrel{\text{MP}}{=} \prob(A_N|A_{N-1}) \lambda_{i_0} p_{i_0,i_1} \cdots p_{i_{N-2},i_{N-1}}
  \end{align*}

Conversly, suppose equation~\eqref{eqn:joint probability2} holds. By the equation, when $n=0$,
\[
  \prob(X_0=i_0) = \lambda_{i_0},
\]
so $X_0$ has p.m.f. $\lambda$. Then
\[
  \prob(A_{n+1}|A_0\cap\cdots\cap A_n) = \frac{\prob(A_0\cap\cdots\cap A_{n+1})}{\prob(A_0\cap\dots\cap A_n)} = p_{i_n,i_{n+1}}.
    \]
    Therefore $X$ is a Markov chain with transition matrix $P$.
\end{proof}

\begin{theorem}[Extended Markov Property]
  Let $X$ be a Markov chain and $n\geq1$. Let $H$ be a historic event, i.e. $H$ is given in terms of $\{X_0,X_1,\ldots,X_{n-1}\}$, and let $F$ be a future event, i.e. $F$ is given in terms of $\{X_{n+1},X_{n+2},\ldots\}$. Then
  \[
    \prob(F|H, X_n=i) = \prob(F|X_n=i).
  \]
\end{theorem}

\begin{proof}
  For $F$ that depends only on finitely many of the future variables,
  \begin{align*}
    \prob(F|H,X_n=i) &= \frac{\sum_{>n}\sum_{<n}\lambda_{i_0}p_{i_0,i_1}\cdots p_{i_n,i}p_{i,i_{n+1}}\cdots}{\sum_{<n} \lambda_{i_0}p_{i_0,i_1}\cdots p_{i_{n-1},i}} \\
                  &= \sum_{>n} p_{i,i_{n+1}} \cdots \\
                  &= \prob(F|X_n=i)
  \end{align*}
  The case for infinite variables can be deduced using continuity of probability measure.
\end{proof}

\begin{notation}
  $\sum_{<n}$ denotes the summation over all $i_0,\ldots, i_{n-1}$ contibutions to $H/F$.
\end{notation}

\section{Transition Probabilities}

The one-step transition probability is $p_{i,j} = \prob(X_1=j|X_0=i)$. The $n$-step transition probability is $p_{i,j}(n) = \prob(X_n=j|X_0=i)$.

\begin{question}
  How to compute the $n$-step probabilities from the one-step probabilities?
\end{question}

The answer is: matrix. The one-step transition matrix is $P=(p_{i,j})_{i,j\in S}$. Similarly $P(n) = (p_{i,j}(n))_{i,j\in S}$.

\begin{theorem}
  \[
    P(n) = P^n.
  \]
\end{theorem}

\begin{proposition}[Chapman-Kolmogorov equations]
  \[
    p_{i,j}(m+n) = \sum_{k\in S} p_{i,k}(m) p_{k,j}(n).
  \]

\end{proposition}

\begin{proof}
  \begin{align*}
    \prob(X_{m+n}=j|X_0=i) &= \sum_{k\in S} \prob(X_{m+n}=j,X_m=k|X_0=i) \\
    \intertext{Use the equality $\prob(A\cap B|C) = \prob(A|B\cap C)\prob(B|C)$,}
                        &= \sum_{k\in S} \prob(X_{m+n}=j| X_m=k,X_0=i)\prob(X_m=k|X_0=i) \\
    \intertext{By Markov property, the first term can be simplified}
                        &= \sum_{k\in S} p_{i,j}(m) p_{k,j}(n)
  \end{align*}
\end{proof}

\begin{proof}[Proof of Theorem]
  By Chapman-Kolmogorov equation, $P(m+n) = P(m)P(n)$. Thus $P(n) = P(1)P(n-1)=\cdots =P(1)^n=P^n$.
\end{proof}

\begin{eg}
  Let $S = \{1,2\}$, $P=\begin{psmallmatrix} 1-\alpha & \alpha \\ \beta & 1-\beta\end{psmallmatrix}$ where $0<\alpha, \beta < 1$.
  \begin{enumerate}
  \item Diagonalisation method: $\det(P-\kappa I)=0$, has roots $\kappa_1=1,\kappa_2=1-\alpha-\beta$. Then
  \[
    P = U^{-1}
    \begin{pmatrix}
      1 & 0\\
      0 & 1-\alpha-\beta
    \end{pmatrix}U,
  \]
  for some invertible $U$. Then
  \[
    P^n = U^{-1}
    \begin{pmatrix}
      1^n & 0 \\
      0 & (1-\alpha-\beta)^n
    \end{pmatrix} U
    \]
    Thus we may write
    \[
      P_{1,1}(n) = A+B(1-\alpha-\beta)^n,
    \]
    with $P_{1,1}(0)=1,P_{1,1}(1)=1-\alpha$. Solve to get
    \begin{align*}
      A &= \frac{\beta}{\alpha+\beta} \\
      B &= \frac{\alpha}{\alpha+\beta}
    \end{align*}
    For the other entries, note $P_{1,2}(n)=1-P_{1,1}(n)$ and $P_{2,1}(n)$ and $P_{2,2}(n)$ can be obtained by exchanging $\alpha$ and $\beta$ due to symmetry.

  \item Difference equation method:
    \begin{align*}
      p_{1,1}(n+1) &= \sum_{k=1,2} p_{1,k}(n)p_{k,1}(1) \\
                   &= p_{1,1}(n)(1-\alpha) + p_{1,2}(n)\beta \\
                   &= p_{1,1}(n)(1-\alpha) + (1-p_{1,1}(n))\beta
    \end{align*}

    Thus $\pi_{n}=p_{1,1}(n)$ satisfies
    \[
      \pi_{n+1} = (1-\alpha-\beta)\pi_n+\beta
    \]
    which can be solved.
  \end{enumerate}
\end{eg}

It is convenient to think of \(\lambda\) as a row vector and \(P\) as a matrix. Then \(\prob(X_1=j) = \sum \lambda_ip_{i,j}\) becomes \((\prob(X_i=j):j\in S) = \lambda P\), i.e. pre-multiplying \(P\) by \(\lambda\).

\section{Class Structure}

\begin{definition}
  We say a state \(i\) \emph{leads} to a state \(j\) if \(p_{i,j}(n) > 0\) for some \(n\geq0\), write \(i \to j\).

  If \(i\to j\) and \(j\to i\), we say \(i\) and \(j\) \emph{communicate} and write \(i \tot j\).
\end{definition}

\begin{proposition}
  \(\tot\) is an equivalence relation.
\end{proposition}

\begin{proof}\leavevmode
  \begin{itemize}
  \item reflexive: \(p_{i,i}=1>0\),
  \item symmetric: if \(i\tot j\) then \(j\tot i\) by definition,
  \item transitive: if \(i\tot j, j\tot k\) then exist \(m, n\) such that \(p_{i,j}(m)>0, p_{j,k}(n)>0\). Then
    \[
      p_{i,k}(m+n) \stackrel{\text{CK}}{=} \sum_{r}^{ }p_{i,r}(m)p_{r,k}(n) \geq p_{i,j}(m)p_{j,k}(n) >0
      \]
      so \(i\to k\). Similarly \(k\to i\).
  \end{itemize}
\end{proof}

\begin{definition}
The equivalence classes of \(\tot\), i.e. subsets of \(S\) of the form \(C_i=\{j\in S: i\tot j\}\), are called \emph{communicating classes}.
\end{definition}

\begin{definition}
  If there is a unique equivalence class \(S\), call \(S\) (or the chain) \emph{irreducible}.
\end{definition}

\begin{definition}
  A subset \(C \subseteq S\) is called \emph{closed} if
  \[
i\in C, i\to j \Rightarrow j\in C.
\]

  If \(i \in S\) is such that \(\{i\}\) is closed, \(i\) is called \emph{absorbing}.
\end{definition}

A closed set is different from communicating class as it is ``one-way''.

\begin{proposition}
  \(C \subseteq S\) is closed if and only if
  \begin{equation}
    \label{eqn:closed subset}
    p_{i,j}=0 \text{ for } i\in C, j\notin C.
    \tag{\(\ast\)}
    \end{equation}
\end{proposition}

\begin{proof}
  Let \(C \subseteq S\). If \eqref{eqn:closed subset} fails, then there exists \(i\in C, j\notin C\) such that \(p_{i,j}>0\) so \(i\to j\) and so \(C\) is not closed.

  Suppose \eqref{eqn:closed subset} holds and exists \(m>0\) such that \(p_{i,j}(m) >0\). Then
  \[
0 < p_{i,j}(m) = \sum_{x_1,\ldots,x_{m-1}\in S} p_{i,x_1}p_{x_1,x_2}\cdots p_{x_{m-1},j}.
  \]
  Thus exists \(x_1,\ldots,x_{m-1}\in S\) such that a summand on RHS is larger than \(0\). By \eqref{eqn:closed subset} \(x_1,\ldots,x_{m-1},j \in C\) so \(C\) is closed.
\end{proof}

\begin{eg}
  \begin{minipage}[t]{0.45\textwidth}
  \[
    P=
    \begin{pmatrix}
      \frac{1}{2} & \frac{1}{2} & 0& 0&0 &0 \\
      0& 0&1&0&0&0 \\
      \frac{1}{3} &0&0&\frac{1}{3} & \frac{1}{3} &0 \\
      0&0&0&\frac{1}{2}&\frac{1}{2}& 0 \\
      0&0&0&0&0& 1 \\
      0&0&0&0&1&0
    \end{pmatrix}
  \]
\end{minipage}
\hfill
\begin{minipage}[t]{0.45\linewidth}
  \begin{center}
   \begin{tikzpicture}[->,auto,node distance=1.5cm]
    \node[state] (1) {\(1\)};
    \node[state,above right of=1] (2) {\(2\)};
    \node[state,right of=1] (3) {\(3\)};
    \node[state,below right of=3] (4) {\(4\)};
    \node[state,right of=3] (5) {\(5\)};
    \node[state,right of=5] (6) {\(6\)};

    \draw
    (1)
    edge[loop left] node{} (1)
    edge[] node{} (2)
    (2)
    edge[] node{} (3)
    (3)
    edge[] node{} (1)
    edge[] node{} (4)
    edge[] node{} (5)
    (4)
    edge[loop left] node{} (4)
    edge[] node{} (5)
    (5)
    edge[bend right] node{} (6)
    (6)
    edge[bend right] node{} (5);
  \end{tikzpicture}
  \end{center}
\end{minipage}
  has communicating classes \(\{1,2,3\}, \{4\},\{5,6\}\), only the last of which is closed.
\end{eg}

\section{Recurrence \& Transience}

\subsection{Definition}

Introduce notation \(\prob_i(\cdot) = \prob(\cdot| X_0 = i), \E_i(\cdot) = \E(\cdot|X_0=i)\).

\begin{definition}
The \emph{first-passage time} of \(j\in S\) is
\[
T_j = \min \{n\geq 1: X_n=j\}.
\]
The \emph{first-passage probability} are
\[
f_{i,j}(n) = \prob_i(T_j=n).
\]
\end{definition}

\begin{definition}
  \(i\in S\) is \emph{recurrent (or persistent)} if
  \[
\prob_i(T_i < \infty) = 1,
  \]
  and \emph{transient} otherwise.
\end{definition}

\subsection{Recurrent Condition}

\begin{theorem}
  \label{thm:recurrent}
  \(i\) is recurrent if and only if
  \[
    \sum_{n\geq0}^{}p_{i,i}(n) = \infty.
  \]
\end{theorem}

Before we prove the theorem, introduce two generating functions:
\begin{align*}
  P_{i,j}(s) &= \sum_{n\geq0}^{ }p_{i,j}(n)s^n \\
  F_{i,j}(s) &= \sum_{n\geq0}^{ }f_{i,j}(n)s^n
\end{align*}
with the convention that
\begin{align*}
  p_{i,j}(0) &= \delta_{i,j} =
                 \begin{cases}
                   1 & i = j \\
                   0 & i \neq j
                 \end{cases}\\
  f_{i,j}(0) &= 0 \, \forall i,j
\end{align*}

Note that
\begin{align*}
  p_{i,j}(n) &= \sum_{m=1}^{n}\prob_i(X_n=j|T_j=m)\prob_i(T_j=m) \\
             &\stackrel{\text{MP}}{=} \sum_{m=1}^{n}p_{j,j}(n-m)f_{i,j}(m)
\end{align*}
which is the convolution of \(p\) and \(f\).

It follows that for \(n\geq 1\),
\begin{align*}
  \sum_{n\geq1}^{ } p_{i,j}(n)s^n &= \sum_{n\geq1}^{ } \sum_{m=1}^{n} \Big( f_{i,j}(m)s^m \Big) \Big(p_{j,j}(n-m)s^{n-m} \Big) \\
  P_{i,j}(s) -\delta_{i,j} &= \sum_{m=1}^{\infty} \sum_{n=m}^{\infty} \Big( f_{i,j}(m)s^m \Big) \Big(p_{j,j}(n-m)s^{n-m} \Big) \\
                                  &= \sum_{m\geq1}^{ } f_{i,j}(m)s^m \sum_{r\geq0}^{} p_{j,j}(r)s^r \\
                                  &= F_{i,j}(s) P_{j,j}(s)
\end{align*}
be careful when dealing with double summations.

\begin{theorem}
  \[
    P_{i,j}(s) = \delta_{i,j} + F_{i,j}(s)P_{j,j}(s) \text{ for } |s| < 1.
  \]
\end{theorem}

The \(|s| < 1\) condition: since \(|F_{i,j}(s)| < \infty\) if \(|s| < 2\),
\[
  |P_{i,j}(s)| \leq \sum_{n}^{ }|s|^n < \infty
\]

A lemma from analysis before we finally prove Theorem~\ref{thm:recurrent}:
 \begin{lemma}[Abel's Lemma]
  Let \((u_i)_{i\geq0}\) be a non-negative sequence such that
  \[
    \mathcal U(s) = \sum_{i=0}^{\infty}u_is^i
  \]
  converges when \(|s| <1\), then
  \[
    \sum_{i=0}^{\infty}u_i = \lim_{s\to 1^-} \mathcal U(s),
  \]
  whether or not this limit is finite.
\end{lemma}

\begin{proof}
  Exercise.
\end{proof}

\begin{proof}[Proof of Theorem~\ref{thm:recurrent}]
  For \(|s| < 1\), then
  \begin{align*}
    p_{i,i}(s) &= 1+ F_{i,i}P_{i,i} \\
    P_{i,i} &= \frac{1}{1-F_{i,j}}
  \end{align*}
Let \(s \to 1^-\), then by Abel's Lemma
\begin{equation}
  \label{eqn:recurrent P}
  P_{i,i}(1) = \infty \Leftrightarrow F_{i,i}(1) =1
\end{equation}
i.e. \(i\) is recurrent.
\end{proof}

\subsection{Properties of Recurrence}

\begin{theorem}[Recurrence as a class property]
  Let \(C\) be a communicating class, then
  \begin{enumerate}
  \item either every state in \(C\) is recurrent or every state is transient,
  \item if \(C\) contains some recurrent state then \(C\) is closed.
  \end{enumerate}
\end{theorem}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Let \(i,j \in C\), \(i \neq j\). There exist \(m,n\geq 1\) such that
    \[
      \alpha = p_{i,j}(m)p_{j,i}(n) >0.
    \]
    This is simply a restatement of the communicating property. Then
    \begin{align*}
      p_{i,j}(m+k+n) \geq p_{i,j}(m)p_{j,j}(k)p_{j,i}(n) = \alpha p_{j,j}(k)
    \end{align*}
    Intuitively, the middle term is the probability of going from \(i\) to \(i\) by passing \(j\) at step \(m\) and \(m+k\).
    Thus
    \[
      \sum_{k}^{}p_{i,i}(m+k+n) \geq \alpha \sum_{k}^{ }p_{j,j}(k).
    \]
    Thus \(j\) recurrent implies that \(i\) is recurrent. Vice versa.
  \item Suppose \(i\in C\) is recurrent but \(C\) is not closed. Then there exists \(j\in C, k\notin C\) with \(p_{j,k} > 0\). By the previous part \(j\) is recurrent so
    \[
      1 = \prob_j(T_j < \infty) = 1 - \prob_j(\text{no return to } j) \leq 1- p_{j,k} < 1
    \]
    Absurd.
  \end{enumerate}
\end{proof}

\begin{theorem}
  \label{thm:recurrent irreducible chain}
  Assume \(|S| < \infty\), then
  \begin{enumerate}
  \item \(S\) contains some recurrent state,
  \item if the chain is irreducible, all states are recurrent.
  \end{enumerate}
\end{theorem}

Similar as before, we need a proposition before the proof:
  \begin{proposition}
    If \(j\) is a transient state then
    \[
      \forall i,\, p_{i,j}(n) \to 0 \text{ as } n\to\infty
    \]
  \end{proposition}

  \begin{proof}
    Asuume \(j\) is transient. By \eqref{eqn:recurrent P},
    \[
      P_{j,j}(1) < \infty.
    \]
    By Theorem~\ref{thm:recurrent},
    \[
      \sum_{n}^{ }p_{i,j}(n) = P_{i,j}(1) < \infty
    \]
    so \(n\)th term \(p_{i,j}(n)\) tends to \(0\) as \(n\to \infty\).
  \end{proof}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item \(\sum_{j\in S}^{ }p_{i,j}(n) = 1 \) since \(P\) is a stochastic matrix. If \(j\) is transient then each summand tends to \(0\) as \(n\to \infty\), which is absurd since \(|S| < \infty\).
  \item Obvious.
  \end{enumerate}
\end{proof}

\subsection{Random Walks and Pólya's Theorem}

In this section, we discuss simple symmetric random walk on \(d\)-dimensional lattices, i.e.\ \(\Z^d\), in particular answering the question when the chain is recurrent.\footnote{Note that this chain is irreducible so by Theorem~\ref{thm:recurrent irreducible chain} either we can talk about recurrence as a chain property.}  It turns out that there is a surprisingly beautiful result.

We call two points \(x\) and \(y\) \emph{neighbours} if
\[
  \sum_{i}^{ }|x_i-y_i| = 1,
\]
i.e.\ they differ by \(1\) in only one coordinate. Let \(X\) be a symmetric random walk on \(\Z^d\) where \(d \geq 1\), i.e.\ \(X = (X_1, X_2, \dots)\) is a Markov chains with state space \(S = \Z^d\) and transition probability
\[
  \prob(X_{n+1} = y | X_n = x) =
  \begin{cases}
    0 & \text{ if } x \text{ and } y \text{ are not neighbours} \\
    \frac{1}{2d} & \text{ if } x \text{ and } y \text{ are neighbours}
  \end{cases}
\]

\begin{theorem}[Pólya's]
  \(X\) is recurrent if \(d \leq 2\) and transient if \(d \geq 3\).
\end{theorem}

\begin{proof}
  First set \(d = 1\). Recall that \(0\) is recurrent if and only if \(\sum_n p_{0,0}(n) = 0\). However, it is not possible to return to the same place after an odd number of steps so the expression simplifies to
  \begin{equation}
    \label{eqn:polya}
    \sum_n p_{0, 0}(2n)
    \tag{\(\ast\)}
  \end{equation}
  In the even case, the random walk returns to \(0\) if and only if there are equal number of movement to either direction so by applying the binomial distribution,
  \[
    p_{0,0}(2n) = \left( \frac{1}{2} \right)^{2n} \binom{2n}{n}
  \]
  To simplify this, recall Sterling's formula
  \[
    n! \sim \left( \frac{n}{e} \right)^n \sqrt{2\pi n} \text{ as } n \to \infty.
  \]
  So
  \[
    p_{0,0}(2n) \sim \frac{1}{\sqrt{\pi n}}
  \]
  and the sum~\eqref{eqn:polya} tend to infinity.

  Now let \(d = 2\). By the same reasoning and generalising binomial to multinomial coefficients, we get
  \begin{align*}
    p_{0,0}(2n) &= \left( \frac{1}{4} \right)^{2n} \sum_{m=0}^{n} \binom{2n}{m, m, n-m, n-m} \\
                &= \left( \frac{1}{4} \right)^{2n} \sum_{m=0}^{n} \frac{(2n)!}{(m!)^2 ((n-m)!)^2} \\
                &= \left( \frac{1}{4} \right)^{2n} \frac{(2n)!}{(n!)^2} \sum_{m=0}^{n} \binom{n}{m} \binom{n}{n-m} \\
    \intertext{Now pause and think: the summation represents the number of ways to take \(m\) balls from a bag of \(n\) balls and take \(n-m\) balls from another bag of \(n\) balls, for \(0 \leq m \leq n\), but this is precisely the number of ways to take \(n\) balls from \(2n\) balls!}
                &= \left( \frac{1}{4} \right)^{2n} \binom{2n}{n} \binom{2n}{n} \\
                &= \left( p_{0,0}^{d=1}(2n) \right)^2
  \end{align*}
  thus the sum~\eqref{eqn:polya} also tends to infinity and \(0\) is recurrent.

  Let \(d = 3\) (similar for \(d \geq 4\)) and we have
  \begin{align*}
    p_{0,0}(2n) &= \left( \frac{1}{6} \right)^{2n} \sum_{i+j+k = n} \binom{2n}{i,i,j,j,k,k} \\
                &= \left( \frac{1}{6} \right)^{2n} \sum_{i+j+k = n}^{}\frac{(2n)!}{(i! j! k!)^2} \\
                &= \left( \frac{1}{6} \right)^{2n} \binom{2n}{n} \sum_{i+j+k = n}^{} \left( \frac{n!}{i! j! k!} \right)^2 \\
                &=  \left( \frac{1}{2} \right)^{2n} \binom{2n}{n} \sum_{i+j+k = n}^{} \left( \frac{n!}{3^n i! j! k!} \right)^2 \\
                &\leq \left( \frac{1}{2} \right)^{2n} M_n \sum_{i+j+k = n}^{}\frac{1}{3^n i! j! k!} \\
    \intertext{ where \(M_n = \max \{ \frac{n!}{3^n i!j!k!}, i+j+k = n \} \).}
    \intertext{The reason we introduce \(3^n\) becomes apparent in this step: \( \frac{n!}{3^n i! j! k!}\) is the probability of, upon throwing \(n\) balls into \(3\) urns, finding \(i, j, k\) in each respectively. Thus they sum up to \(1\).}
                &\leq \left( \frac{1}{2} \right)^{2n} \binom{2n}{n} \frac{n!}{3^n \left( \floor{n/3}! \right)^3}
  \end{align*}
  The upper bound of \(M_n\) is left as an exercise. By Sterling's formula,
  \[
    p_{0,0}(2n) \leq \frac{C}{n^{3/2}}
  \]
  so the sum is finite and \(0\) is recurrent.
\end{proof}

We have seen that the probability \(p_{0, 0}(2n)\) when \(d = 2\) is the square of the probability when \(d = 1\), but when \(d = 3\) it doesn't become cubed. It should inspire us to suspect that the \(d = 2\) case is simple enough such that the random walks in two directions are ``independent'', but when \(n \geq 3\) there is some hidden structure that destroys such independence. What is so special about dimension two?

There is an alternative way to tackle this problem that might be more lucid and shed some light on the magical property of \(d = 2\). Instead of cartesian coordinates \(X_n = (A_n, B_n)\), rotate the axes by \(45^\circ\) clockwise. The new coordinates, scaled by a constant factor for convenience, are
\[
  Y_n =
  \begin{pmatrix}
    U_n \\
    V_n
  \end{pmatrix}
  = \sqrt 2
  \begin{pmatrix}
    \cos 45^\circ & -\sin 45^\circ \\
    \sin 45^\circ & \cos 45^\circ
  \end{pmatrix} X_n
  =
  \begin{pmatrix}
    A_n - B_n \\
    A_n + B_n
  \end{pmatrix}
\]
\iffalse
\begin{align*}
  U_n &= A_n - B_n \\
  V_n &= A_n + B_n \\
  Y_n &= (U_n, V_n)
\end{align*}
\fi

Claim \(U = (U_n)\) and \(V = (V_n)\) are \emph{independent} random walks on \(\Z\):

\begin{proof}
  \begin{align*}
    \prob(Y_{n+1} &= Y_n+ (1,1)) = \prob(X_{n+1} = X_n + (1,0)) = \frac{1}{4} \\
                  &= \prob(U_{n+1} - U_n = 1, V_{n+1} - V_n = 1)
  \end{align*}
  since during one step \(X_n\) can only change by \(1\) in one coordinate. Similar for the other three cases.

  So
  \[
    \prob(U_{n+1} - U_n = \alpha, V_{n+1} - V_n = \beta) = \left( \frac{1}{2} \right)^2 = \prob(U_{n+1} - U_n = \alpha) \prob(V_{n+1} - V_n = \beta)
  \]
  for \(\alpha, \beta = \pm 1\). \(U_n\) and \(V_n\) are independent and each generates a random walk on \(\Z\).
\end{proof}

By independence
\[
  \prob_0(X_n = (0,0)) = \prob_0(Y_n = (0,0)) = \prob_0(U_n = 0) \prob_0(V_n = 0)
\]
so
\[
  p_{0,0}^{d=2}(2n) = \left( p_{0,0}^{d=1} \right)^2.
\]

The moral of this calculation is, two dimensional random walk is indeed the product of two one dimensional cases, but in a not-entirely-straightforward way.

\section{Hitting Time and Probability}

\begin{definition}[Hitting Time]
  Given a subset \(A \subseteq S\), the \emph{hitting time} of \(A\) is
  \[
    H^A = \inf\{n\geq 0: X_n \in A \}
  \]
\end{definition}
Note that \(\inf \emptyset = \infty\) so \(H^A: \Omega \to \{0, 1, \dots\} \cup \{\infty\}\).

\begin{definition}[Hitting probability]
  The \emph{hitting probability} is \(h_i^A = \prob_i(H^A < \infty)\).
\end{definition}

By Markov property, hitting probability satisfies the equation
\begin{equation}
  \label{eqn:hitting prob}
  h_i^A =
  \begin{cases}
    1 & i \in A \\
    \sum_{j \in S} p_{ij}h_j^A & i \notin A
  \end{cases}
  \tag{\(\ast\)}
\end{equation}

\begin{theorem}
  The vector \(h^A = (h_i^A: i \in S)\) is the minimal non-negative solution to~\eqref{eqn:hitting prob} in that for any \(x = (x_i: i \in S)\) satisfy
  \[
    x_i =
    \begin{cases}
      1 & i \in A \\
      \sum_j p_{ij}x_j & i \notin A
    \end{cases}
  \]
  and \(x_i \geq 0\) for \(i \in S\), \(h_i^A \leq x_i\) for \(i \in S\).
\end{theorem}

\begin{proof}
  By MP, \(h^A\) satisfies~\eqref{eqn:hitting prob}. Suppose \(x = (x_i: i \in S)\) satisfies the hypothesis in the theorem. If \(i \in A\), \(x_i = 1 = h_i^A\) so \(h_i^A \leq x_i\). Let \(i \notin A\), then
  \begin{align*}
    x_i &= \sum_j p_{ij}x_j \\
        &= \sum_{j \in A}p_{ij}\cdot 1 + \sum_{j \notin A} p_{ij}x_j \\
        &\geq \sum_{j \in A} p_{ij} \\
        &= \prob_i(H^A = 1)
  \end{align*}
  Similarly
  \begin{align*}
    x_i &= \prob_i(H^A = 1)  + \sum_{j \notin A}^{ } p_{ij} \left( \sum_{k \in A}^{ } p_{jk}x_k + \sum_{k \notin A}^{ } p_{jk}x_k \right) \\
        &\geq \prob_i(H^A = 1) + \prob_i(H^A = 2)
  \end{align*}
  so by induction
  \[
    x_i \geq \sum_{m = 1}^{n} \prob_i(H^A = m) = \prob_i(H^A \leq n) \to \prob_i(H^A < \infty) = h_i^A
  \]
  as \(n \to \infty\).
\end{proof}

\begin{definition}[Mean hitting time]
  The \emph{mean hitting time} is
  \[
    k_i^A = \E_i(H^A).
  \]
\end{definition}

\begin{note}
  \(k_i^A = \infty\) if \(h_i^A < 1\).
\end{note}

\begin{theorem}
  The vector \(k^A = (k_i^A: i \in S)\) is the minimal non-negative solution to the equation
  \begin{equation}
    \label{eqn:hitting time}
    y_i =
    \begin{cases}
      0 & i \in A \\
      1 + \sum_{j}^{ } p_{ij}y_j & i \notin A
    \end{cases}
    \tag{\(\dag\)}
  \end{equation}
\end{theorem}

\begin{proof}
  By MP, \(k^A\) satisfies~\eqref{eqn:hitting time}. Let \(y = (y_i: i \in S)\) be a non-negative solution to~\eqref{eqn:hitting time}. Let \(i \in A\) then \(y_i = 0 = k_1\). Let \(i \notin A\).
  \begin{align*}
    y_i &= 1 + \sum_{j \in S}^{ } p_{ij}y_j \\
        &= 1 + \sum_{j \notin A}^{ } p_{ij}y_j \\
        &= 1 + \sum_{j \notin A}^{ } p_{ij} \left( 1+ \sum_{k \notin A}^{ } p_{jk}y_k \right) \\
        &\geq \prob_i(H^A \geq 1) + \prob_i(H^A \geq 2)
  \end{align*}
  By induction
  \[
    y_i \geq \sum_{m = 1}^{n} \prob_i(H^A \geq m) \to \sum_{m = 1}^{\infty} \prob_i(H^A \geq m) = \E_i(H^A)
  \]
  as \(n \to \infty\).
\end{proof}

\begin{eg}[Gambler's ruin]
  Let \(S = \{0, 1, 2, \dots \}\) and \(0 < p < 1\). Take a random walk on \(S\) which moves one step right with probability \(p\) and left with probability \(q = 1 - p\). \(0\) is an \emph{absorbing barrier} as as soon as the random walk hits \(0\) it ends.

  \begin{question}
    What is the probability of absorption at \(0\) starting at \(i\)?
  \end{question}

  \begin{answer}
    Let \(h_i = h_i^{\{0\}}\), then
    \begin{align*}
      h_0 &= 1 \\
      h_i &= p h_{i+1} + q h_{i-1}
    \end{align*}
    which can be regarded as a second order difference equation with initial conditions.

    First suppose \(p \neq q\). The general solution is
    \[
      h_i = A + B \left( \frac{q}{p} \right)^i \text{ for } i \geq 0.
    \]
    Suppose \(p < q\). Since \(h^i \leq 1\), \(B = 0\) or otherwise for large \(i\), \(h^i\) will blow up. Thus \(h_i = h_0 = 1\) for \(i \geq 0\). Now suppose \(p > q\). Eliminate \(B\) to get
    \[
      h_i = \left( \frac{q}{p} \right)^i + A \left( 1 - \left( \frac{q}{p} \right)^i \right)
    \]
    The minimality of \(h^i\) requires that \(A = 0\) so
    \[
      h_i = \left( \frac{q}{p} \right)^i
    \]

    Finally, consider the case \(p = q = \frac{1}{2}\). As above \(B = 0, A = 1\) so \(h_i = 1\) for \(i \geq 0\).
  \end{answer}
\end{eg}

\begin{eg}[Birth-death chain]
  This is similar to gambler's ruin but with inhomogeneuous transition probabilites: \(p_i + q_i = 1, p_i \in (0, 1)\). The governing equations are
  \begin{align*}
    h_0 &= 1 \\
    h_i &= p_i h_{i+1} + q_i h_{i-1}
  \end{align*}
  At first glance, it seems nothing like a second order differential equation. However, rearrange to get
  \[
    p_i(h_i - h_{i + 1}) = q_i(h_{i - 1} -h_i)
  \]
  Define \(u_i = h_{i - 1} - h_i\) then
  \[
    p_i u_{i + 1} = q_i u_i
  \]
  so
  \begin{align*}
    u_i &= \frac{q_i q_{i - 1} \dots q_1}{p_i p_{i - 1} \dots p_1} u_1 \\
        &= \gamma_i u_1
  \end{align*}
  where
  \[
    \gamma_i = \frac{\prod_{j = 1}^{i} q_j}{\prod_{j = 1}^{i} p_j}.
  \]
  As
  \begin{align*}
    h_i &= 1 - (u_1 + u_2 + \dots + u_i) \\
        &= 1 - u_1(\gamma_0 + \dots + \gamma_{i - 1}) \text{ for } i \geq 1
  \end{align*}
  where \(\gamma_0 = 1\). Let \(S = \sum_{i = 0}^{\infty} \gamma_i \). If \(S = \infty\) then \(u_1 = 0\) and \(h_i = 1\). If \(S < \infty\), \(1 - u_1 S = 0\) so
  \[
    h_i = 1 - \frac{\sum_{j = 0}^{i - 1}\gamma_j}{\sum_{j = 0}^{\infty}\gamma_j}
  \]
\end{eg}

\section{Stopping Times \& Strong Markov Property}

\begin{definition}[Stopping time]
  Let \(X\) be a Markov chain. A \emph{stopping time (or Markov time)} is a random variable \(T: \Omega \to \{0, 1, \dots\}\cup \{\infty\}\) such that for \(n \geq 0\), the event \(\{T = n\}\) is given in terms of \(X_0, \dots, X_n\).
\end{definition}

\begin{note}
  The definition can be equivalently formulated using languages of meausure theory. Recall that in a probability space \(\Omega, \mathcal F, \prob\), a random variable \(X: \Omega \to \R\) is measurable if
  \[
    X^{-1}((-\infty, n]) \in \mathcal F \, \forall n \in \R.
  \]
  Define the \(\sigma\)-field generated by \(X\) to be
  \[
    \sigma(X) = \sigma(\{X^{-1}((-\infty, n]): n \in \R \}) \subseteq \mathcal F
  \]
  and the definition basically says \(\{T = n \} \in \sigma(\{X_0, \dots X_n\})\).
\end{note}

\begin{theorem}[Strong Markov property]
  Let \(X\) be a Markov chain with transition matrix \(P\) and let \(T\) be a stopping time. Given \(T < \infty\) and \(X_T = i\),
  \[
    Y := (X_T, X_{T + 1}, X_{T + 2}, \dots),
  \]
  the future process of \(X\), is a Markov chain with transition matrix \(P\) and \(Y\) is independent of \(X_0, \dots, X_{T - 1}\).
\end{theorem}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Hitting time \(H^A\) is a stopping time:
    \[
      \{H^A = n\} = \{X_n \in A\} \cap \left( \bigcap_{m = 0}^{n - 1} \{X_m \notin A \} \right)
    \]
  \item \(H^A + 1\) is a stopping time.
  \item \(H^A - 1\) is \emph{not} a stopping time: \(\{H^A - 1 = n\}\) obviously depends on \(X_{n + 1}\).
  \end{enumerate}
\end{eg}

\begin{eg}[Gambler's ruin]
  Let \(H = H^{\{0\}}\). We have proved that
  \[
    \prob_i(H < \infty) =
    \begin{cases}
      = 1 & q \geq p \\
      < 1 & q < p
    \end{cases}
  \]
  if \(i \geq 1\). However, we want to find the probability mass function of \(H\) given \(X_0 = 1\). Use probability generating function:
  \begin{align*}
    G(s) &= \E_1(s^H) \\
    \intertext{where \(|s| < 1\) so that \(s^\infty\) can be interpreted as \(0\)}
         &= \sum_{n = 1}^{\infty} s^n \prob_1(H = n) \\
  \end{align*}
  where the limit \(s \to 1^-\) is studied via Abel's Lemma.

  \begin{align*}
    G(s) &= \E_1(s^H | X_1 = 0)q + \E_1(s^H | X_1 = 2)p \\
         &= qs + p \E_1(s^{1 + H' + H''}) \\
           \intertext{where \(H'\) is the hitting time of \(1\) starting at \(2\) and \(H''\) is the subsequent time needed to reach \(0\). By strong Markov property, \(H'\) and \(H''\) are independent and distributed as \(H\) so}
         &= qs + ps G(s)^2
  \end{align*}
  Hence
  \[
    G(s) = \frac{1 \pm \sqrt{1 - 4pqs^2}}{2ps}
  \]
  Note that \(G\) is continuous on \((-1, 1)\) since it is the a power series. Since \(\sqrt{1 - 4pqs^2} \neq 0\) for \(|s| < 1\), we must choose a sign and stick with it on \((-1, 1)\). Since \(G\) convergens \((-1, 1)\), \(+\) is impossible as otherwise \(G\) does not converge at \(s = 0\). Thus
  \[
    G(s) = \frac{1 - \sqrt{1 - 4pqs^2}}{2ps} = \sum_{n = 1}^{\infty} s^n \prob_1(H = n)
  \]
  hence \(\prob_1(H = n) \) can be found by expanding \(G\).

  In addition
  \begin{align*}
    \lim_{s \to 1^-} G(s) &= \sum_{n = 1 }^{\infty} \prob_1(H = n) \\
                          &= \prob_1(H < \infty) \\
                          &= \frac{1 - \sqrt{1- 4pq}}{2p} \\
                          &= \frac{1 - |p - 1|}{2p} \\
                          &=
                            \begin{cases}
                              1 & q \geq p \\
                              \frac{q}{p} & q < p
                            \end{cases}
  \end{align*}

  To find \(\E_1(H)\) when \(q \geq p\), differentiate to get
  \[
    G' = q + p G^2 + 2ps G G'
  \]
  so
  \[
    G'(s) = \frac{q + pG^2}{1 - 2psG}
  \]
  By \Cref{tbf} Abel's theorem
  \[
    E_1(H) = \lim_{s \to 1^-} G'(s) = \frac{q + p}{1 - 2p} = \frac{1}{q - p}.
  \]
\end{eg}

\section{Classification of States}

Depending on ht mean time to return to the recurrent state: finite or infinite

\begin{theorem}
  Let \(X_0 = i\) and \(V_i = |\{n \geq 1: X_n = i\}|\). Then \(V_i\) has a geometric distribution
  \[
    \prob_i(V_i = r) = f^r (1 - f), r \geq 1
  \]
  where \(f = f_{i, i} \in [0, 1]\).
\end{theorem}

\begin{proof}
  \begin{align*}
    \prob_i(V_i \geq r) &= \prob_i(T^r < \infty) \\
                        &= \prob_i(T^4 < \infty | T^{r - 1} < \infty) \prob_i(T^{r - 1} < \infty) \\
                        &= f \cdot \prob_i(V_i \geq r - 1) SMP \\
                        &= f^r
  \end{align*}
  where
  \[
    T^r =
    \begin{cases}
      time of rth return & \\
      \infty & \text{ if } V_i < r
    \end{cases}
  \]
  Then
  \[
    \prob_i(V_i = r) = \prob_i(V_i \geq r) - \prob_i(V_i \geq r + 1) = f^r(1 - r)
  \]
\end{proof}

\begin{note}
  If \(f < 1\), \(\prob_i(V_i < \infty) = 1\) and if \(f = 1\), \(\prob_i(V_i = \infty) = 1\).
\end{note}

\begin{definition}[Mean recurrence time]
  The \emph{mean recurrence time} of \(i \in S\) is
  \[
    \mu_i = \E_i(T_i) =
    \begin{cases}
      \infty & \text{if \(i\) is transient} \\
      \sum_{n = 1}^{\infty} n f_{i,i}(n) &
    \end{cases}
  \]

  Let \(i\) be recurrnet. Then \(i\) is \emph{null} if \(\mu_i = \infty\) and \emph{positive} if \(\mu_i < \infty\).

  The \emph{period} of \(i \in S\) is
  \[
    d_i = \gcd\{n: p_{i,i}(n) > 0 \}.
  \]

  \(i\) is \emph{aperiodic} if \(d_i = 1\).

  \(i\) is \emph{ergodic} if it is recurrent, positive and aperiodic.
\end{definition}

\begin{theorem}
  Let \(i \tot j\). Then
  \begin{enumerate}
  \item \(d_i = d_j\), i.e.\ period is a class property.
  \item \(i\) is recurrent if and only if \(j\) is recurrent.
  \item \(i\) is positive recurrent if and only if \(j\) is positive recurrent.
  \item \(i\) is ergodic if and only if \(j\) is ergodic.
  \end{enumerate}
\end{theorem}

\begin{proof}
  2 has already been proved and 4 follows from 1, 2 and 3. 3 will be proved later. To prove 1:

  Let \(i \tot j\) and \(i \neq j\).
  \begin{align*}
    D_k = \{n \geq 1: p_{k,k}(n) > 0 \} \\
    d-k = \gcd\{D_k\}
  \end{align*}
  Since \(i \to j\), there exists \(m, n \geq 1\) such that \(\alpha = p_{i,j}(m)p_{i,j}(n) > 0\). By CK,
  \[
    p_{i,i}(m + r + n) \geq \alpha p_{j,j}(r).
  \]
  Thus if \(r \in D_j \cup \{0\}\), \(p_{i,i} (m + r + n) > 0\) and hence \(d_i \divides m + r + n\) and hence \(d_i \divides r\) since if \(r = 0\), \(d_i \divides m + n\).

  Thus \(d_i \divides d_j\). Similarly \(d_j \divides d_i\) and hence \(d_i = d_j\).
\end{proof}

\begin{proposition}
  If a chain is irreducible and let \(j \in S\) be recurrent. Then
  \[
    \prob(T_j < \infty) = \prob(X_n = j \text{ for some } n \geq 1) = 1.
  \]
\end{proposition}
Compare to the definition of recurrence, \(\prob_j\)

\begin{proof}
  \(f_{i,j} = \prob_i(T_j < \infty)\). Let \(i \neq j\). Claim that \(p_{j, i}(m)(1 - f_{i,j}) \leq 1 - f_{j,j}\) where \(m= \inf\{r: p_{j,i}(r) > 0\}\). Then \(p_{j, i}(m) = \prob_j(X_m = i, X_r \neq j \text{ for } 1 \leq r < m)\). We have \(f_{j,j} = 1\) and hence \(f_{i,j} = 1\).

  Thus \(\prob_i(T_j < \infty) = 1\). Let \(\lambda_i = \prob(X_0 = i)\),
  \[
    \prob(T_j < \infty) = \sum_{i \in S} \prob_i(T_j < \infty) \lambda_i = 1.
  \]
\end{proof}

\section{Invariant Distributions}

What happens to \(X_n\) as \(n \to \infty\)? Random variables are functions so we are talking about convergence of a sequence of functions. There are lots of modes of convergence on function space. When studying Markov chains, it turns out there is a unique convergence that we need: does \(\prob(X_n = i)\) converge as \(n \to \infty\).

\[
  \prob(X_{n + 1} = j) = \sum_i \prob(X_{n + 1} = j | X_n = 0) \prob(X_n = i)
\]

\[
  \pi_j = \sum_i p_{i,j} \pi_i
\]
\(\pi = \pi P\), eigenvalue problem.

\begin{definition}[Invariant distribution]
  \(X\) is aMarkov chain with transition matrix \(P\). The vector \(\pi = (\pi_i: i \in S)\) is an \emph{invariant distribution} if
  \begin{enumerate}
  \item \(\pi_i \geq 0, \sum_i \pi_i = 1\),
  \item \(\pi = \pi P\)
  \end{enumerate}
\end{definition}

If \(X_0\) has distribution \(\pi\), \(X_n\) has distribution
\[
  \pi P^n = (\pi P) P^{n - 1} = \pi P^{n - 1} = \cdots \pi
\]

\begin{theorem}
  Let \(X\) be an irreducible Markov chain. Then
  \begin{enumerate}
  \item There exists an invariant distribution if and only if some state of the chain is positive recurrent.
  \item If there exists an invariant distribution \(\pi\) then every state is positive recurrent and
    \[
      \pi_i = \frac{1}{\mu_i}
    \]
    for \(i \in S\) where \(\mu_i\) is the mean recurrence time. In particular \(\pi\) is unique.
  \end{enumerate}
\end{theorem}

Fix \(k \in S\), start at \(k\). Let \(W_i\) be the number of visits to \(i\) up to the first return time to \(k\), i.e.
\[
  W_i = \sum_{m = 1}^{\infty} \V 1(X_m = i, T_k \geq m) = \sum_{m = 1}^{T_k} \V 1(X_m = i)
\]
where \(\V 1(\cdot)\) is the indicator function.

Let \(\rho(i) = \E_k(W_i)\).

\begin{proposition}
  Suppose the chain is irreducible and recurrent, \(k \in S\). \(\rho = (\rho_i: i \in S)\) satisfies
  \begin{enumerate}
  \item \(\rho_k = 1\).
  \item \(\sum_i \rho(i) = \mu_k\) whether or not \(\mu_k < \infty\).
  \item \(\rho = \rho P\).
  \item \(0 < \rho_i < \infty\) for \(i \in S\).
  \end{enumerate}
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Immediate from the definition.
  \item
    \[
      \E_k \sum_{i \in S}^{ } W_i = E_k T_k
    \]
    Assuming we can interchange summations (since \(\E\) is the limit of a series), since all summands are non-negative,
    \[
      \sum_{i \in S}^{ } E_k(W_i) = \mu_k
    \]
    but the summand on LHS is precisely \(\rho_i\).
  \item
    \begin{align*}
      \rho_j &= \E_k(W_j) \\
             &= \sum_{m \geq 1}^{ } \prob_k(X_m = j, T_k \geq m) \text{ again we interchange the summations} \\
             &= \sum_{m \geq 1}^{ } \sum_{i \in S}^{ } \prob_k(X_m = j, X_{m - 1} = i, T_k \geq m) \\
             &= \sum_{m \geq 1}^{ } \sum_{i \in S}^{ } \prob_k(X_m = j| X_{m - 1} = i, T_k \geq m)\prob_k(X_{m - 1} = i, T_k \geq m) \\
             &= \sum_{m \geq 1}^{ } \sum_{i \in S}^{ } \prob_k(X_m = j| X_{m - 1} = i, T_k \geq m)\prob_k(X_{m - 1} = i) \text{ Markov property} \\
             &= \sum_{m \geq 1}^{ } \sum_{i \in S}^{ } p_{i,j} \prob_k(X_{m - 1} = i, T_k \geq m) \\
             &= \sum_{i \in S}^{ } p_{i,j} \sum_{m \geq 1}^{ } \prob_k(X_{m - 1} = i, T_k \geq m) \\
             &= \sum_{i \in S}^{ } p_{i,j} \sum_{r \geq 1}^{ } \prob_k(X_{r} = i, T_k \geq r + 1) \\
      \intertext{consider two cases: if \(i \neq k\), the term when \(r = 0\) is \(0\) and \(T_k \geq r + 1\) if and only if \(T_k \geq r\). If \(i = k\), the term when \(r = 0\) is \(1\) and all the other terms are zero. Thus}
             &= \sum_{i \in S}^{ }p_{i,j} \rho_i
    \end{align*}
  \item Pivot off the fact that \(\rho_k = 1\). Since \(\rho = \rho P\), we have \(\rho = \rho P^r\) for \(r \geq 1\). Thus
    \[
      \rho_i \geq \rho_k p_{k, i}(m), \rho_k \geq \rho_i p_{i, k}(n)
    \]
    By irreducibility, there exists \(m, n \geq 1\) with \(p_{k, i}(m), p_{i, k}(m) > 0\) so
    \[
      0 < p_{k, i}(m) \leq \rho_i \leq \frac{1}{p_{i, k}(n)} < \infty
    \]
  \end{enumerate}
\end{proof}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Let \(k\) be positive recurrent, hence \(\mu_k < \infty\). Then \(\pi_i := \rho_i/\mu_ik\) is an invariant distribution.
  \item Let \(\pi\) be an invariant distribution. Claim \(\pi_i > 0\) for all \(i \in S\):
    \begin{proof}
      Since \(\pi = \pi p\), we have \(\pi = \pi p^n\) for \(n \geq 0\) and hence
      \[
        \pi_i = \sum_j \pi_j p_{j,i}(n) \geq \pi_k p_{k,i}(n)
      \]
      for \(k \in S\). Since \(\sum_i \pi_i = 1\) we may pick \(k \in S\) with \(\pi_k > 0\). By irreducibility there exists \(n \geq 0\) such tat \(p_{k, i}(n) > 0\). Hence \(\pi_i > 0\).

      Suppose every state is transient, since \(\pi = \pi p^n\),
      \[
        \pi_j = \sum_i \pi_i p_{i, j}(n)
      \]
      taking limit as \(n \to \infty\),
      \[
        \sum_i \pi_i 0 = 0
      \]
      absurd.
      \begin{proof}[Proof of the limiting process]
        \begin{align*}
          0 \leq \sum_i \pi_i p_{i, j}(n) &= \sum_{i \in F} \cdots + \sum_i \notin F \cdots \text{ where } F \subseteq S, |F| < \infty \\
          &\leq \sum_{i \in F} p_{i, j}(n) + \sum_{i \notin F} \pi_i \text{ by boundedness} \\
                                          &\to 0 + \sum_{i \notin F} \pi_i \text{ as } n \to \infty \\
          &\to 0 \text{ as } F \to S^-
        \end{align*}
      \end{proof}
      Thus every state is recurrent.
    \end{proof}
    \begin{align*}
      \pi_i \mu_i &= \sum_{n = 1}^{\infty} \prob_i(T_i \geq n) \prob(X_0 = i) \\
                  &= \sum_{n = 1}^{\infty} \prob(X_0 = i, T_i \geq n)
                    %TBF
    \end{align*}
    Using stationarity,
    \begin{align*}
      \pi_i \mu_i &= \pi_i + \sum_{n = 2}^{\infty} (a_{n - 2} - a_{n - 1}) \text{ where } a_r = \prob(X_0 \neq i, \dots, X_r \neq i) \\
                  &= \pi_i + a_0 - \lim_{m \to \infty} a_m \\
                  &= \pi_i + (1 - \pi_i) - \underbrace{\prob(T_i = \infty)}_{= 0 \text{ by recurrence}} \\
                  &= 1
    \end{align*}
    Since \(\pi_i\mu_i = 1\),
    \[
      \mu_i = \frac{1}{\pi_i} < \infty
    \]
    since \(\pi_i > 0\). Thus \(i\) is positive recurrent.
  \end{enumerate}
\end{proof}

\section{Convergence to Equilibrium}

\begin{theorem}
  Consider an irreducible aperiodic, positive recurrent Markov chain. For \(i, j \in S\), \(p_{i, j}(n) \to \pi_j\) as \(n \to \infty\) where \(\pi\) is the unique invariant distribution.
\end{theorem}

Ergodic theorem

\begin{proof}
  ``coupling'' is the main idea.

  Let \(X = (X_n), Y = (Y_n)\) be independent Markov chains with the appropriate common invariant distribution. Let \(Z = (Z_n = (X_n, Y_n))_{n \geq 0}\). Then \(Z\) is a Markov chain with state space \(S \times S\) and transition matrix
  \[
    p_{ij, kl} = p_{i, k}p_{j, l}
  \]
  Fix \(s \in S\), let
  \[
    T = \inf \{n \geq 1: Z_n = (s, s)\}.
  \]
  Since \(X\) and \(Y\) have invariant distribution \(\pi\), \(Z\) has invariant distribution \(v_{ij} = \pi_i\pi_j\) since
  \[
    \sum_{i,j } v_{ij}p_{ij, kl} = \sum_i \pi_i p_{i, k} \sum_j \pi_j p_{j, l} = \pi_k \pi_l = v_{kl}
  \]
  Hence \(Z\) is positive recurrent and \(\prob(T < \infty) = 1\).

  We are still lacking one thing: \(Z\) is irreducible. This has something to do with aperiodicity.

  A digression about number theory: if \(D\) is a finite subset of non-negative integers with \(\gcd(D) = 1\), there exists \(N\) such that for \(n > N\) and expression
  \[
    n = \sum_{d \in D} \alpha_d d
  \]
  with \(\alpha_d \in \{0, 1, 2, \dots\}\).

  Since \(X\) is aperiodic, we deduce that \(p_{i,i}(n) > 0\) for all large \(n\). Thus
  \[
    p_{ij, ij}(n) = p_{i, i}(n)p_{j, j}(n) > 0
  \]
  for all large \(n\). Therefore \(Z\) is aperiodic.

  Similarly \(Z\) is irreducible (oops, check the book!)
\end{proof}

\begin{align*}
  p_{i, k}(n) &= \prob_i(X_n = k) \\
              &= \prob_{ij}(X_n = k) \\
              &= \sum_{t = 1}^{\infty} \prob_{ij}(X_n = k | T = t)\prob_{ij}(T = t) \\
              &= \sum_{t \leq n}^{ } \prob_{ij}(X_n = k | T = t) \prob_{ij}(T = k) + \sum_{t > n}^{ } \prob_{ij}(T = t) \\
              %&= \prob_S(X_{n - t} = k) = \prob_s(Y_{n - t} = k)
              &= \sum_{t \leq n}^{ } \prob_{ij}(Y_n = k, T = k) + \prob_{ij}(T > n) \\
              &\leq p_{j, k}(n) + \prob_{ij}(T > n)
\end{align*}
so
\[
  |p_{i, k}(n) - p_{j, k}(n)| \leq \prob_{ij}(T > n) \to 0
\]
as \(n \to \infty\). This says that if it converges they converge the the same value. Now we are just one line from the final result:
\[
  \pi_k - p_{ij}(n) = \sum_{i}^{ } \pi(p_{i,k}(n) - p_{j, k}(n)) \to 0
\]
by bounded convergence theorem.

This is an extremely elegant proof and it took a long time before this proof was found.

We are left with one final bit

\begin{theorem}
  Let \(V_i(n) = \sum_{k = 1}^n \V 1(x_k = i)\) be the total number of visits to \(i\) up to time \(n\). If the chain is irreducible and positive recurrent then
  \[
    \frac{V_i(n)}{n} \Rightarrow \frac{1}{\mu_i}
  \]
  as \(n \to \infty\), where \(\Rightarrow\) means weak convergence, i.e.
  \[
    \prob \left( \frac{V_i(n)}{n} \leq a\frac{1}{\mu_i} \right) \to
    \begin{cases}
      0 & a < 1 \\
      1 & a > 1
    \end{cases}
  \]
\end{theorem}

We are not going prove this. Renewal theorem.

\begin{remark}
  Let \(u_i\) be a typical interval length between successive visits to \(i\). \(V_i(n) \geq x\) if and only if \(\sum_{m = 1}^x u_i(m) \leq n\) where the \(u_i(m)\) are iid copies of \(u_i\).
\end{remark}

\section{Time Reversal}

some ruminations bout physics: in real life time reversal is possible but extremely unlikely. The typical explanation is entropy.

Let \(X = (X_n: n = 0, 1, \cdots, N)\) be an irreducible and positive recurrent Markov chain with transition matrix \(P\) and invariant distribution \(\pi\). Let \(Y_n = X_{N - n}\), so \(Y = (Y_0, \cdots, Y_N) = (X_n, \cdots, X_0)\), the reverse of \(X\).

We have to make some assumptions for our reversed chain to make sense: assume that \(X_0\) has distribution \(\pi\).

\begin{theorem}
  \(Y\) is an irreducible Markov chain with transition matrix
  \[
    \hat p_{i,j} = \frac{\pi_j}{\pi_i} p_{j, i}
  \]
  and invariant distribution \(\pi\).
\end{theorem}

\begin{proof}
  First check that \(\hat P = (\hat p_{i,j})\) is a stochastic matrix: the entries are non-negative and
  \[
    \sum_j \hat p_{i,j} = \sum_j \frac{\pi_j}{\pi_i} p_{j, i} = \frac{1}{\pi_i} \sum_j \pi_j p_{j,i} = 1.
  \]

  Claim \(\pi = \pi \hat P\):
  \begin{align*}
    \sum_i \pi_i \hat p_{i,j} &= \sum_i \pi_j p_{j, i} = \pi_j
  \end{align*}

  Now to prove it is a Markov chain,
  \begin{align*}
    \prob(Y_0 = i_0, \dots, Y_n = i_n) &= \prob(X_{N -n} = i_n, \dots, X_N = i_0) \\
                                       &= \pi_{i_n} p_{i_n, i_{n - 1}} \cdots p_{i_1, i_0} \\
                                       &= \pi_{i_{n - 1}} \hat p_{i_{n + 1}, i_n} p_{i_{n - 1}, i_{n - 1}} \cdots, p_{i_1, i_n} \\
                                       &= \pi_{i_0} \hat p_{i_0, i_1} \cdots \hat p_{i_{n - 1}, i_n}
  \end{align*}
  Hence \(Y\) has the stated properties.
\end{proof}

We call mY the \emph{time-reversal} of \(X\) and we say \(X\) is \emph{reversible} if \(Y\) and \(X\) have the same transition probabilities. By (*) (the equation is the statement of the theorem), \(X\) is recursive if and only if
\[
  \pi_i p_{i,j} = \pi_j p_{j, i}
\]
for all \(i, j \in S\). This is the detailed balance equation.

More generally, we say a transition matrix \(P\) and a distribution \(\lambda\) are in \emph{detailed balance} if
\[
  \lambda_i p_{i, j} = \lambda_j p_{j, i}
\]
for all \(i, j \in S\). An irreducible chain \(X\) with invariant distribution \(\pi\) is called \emph{reversible in equilibrium} if its \(P\) is in detailed balance with \(\pi\).

Equation such as \(\pi = \pi P\) can be difficult and may depend on some special structure on \(P\). On the other hand the detailed balance equation is almost trivial.

\begin{proposition}
  If \(\pi\) is a distribution satisfying
  \[
    \pi_i p_{i, j} = \pi_j p_{j, i}
  \]
  for all \(i, j \in S\) and \(S\) is irreducible, then \(\pi\) is the only invariant distribution of the chain and the chain is reversible in equilibrium
\end{proposition}

\begin{proof}
  Let \(\pi\) be a distribution satisfying the hypothesis. Then
  \[
    \sum_i \pi_j p_{j, i} = \sum_j \pi_i p_{i, j} = \pi_i
  \]
  as \((\pi P)_i = \pi_i\). Therefore \(\pi = \pi P\).
\end{proof}

\begin{eg}[Birth-death with retaining barrier]
  Try the detailed balance equation
  \[
    \pi_{i - 1} p_{i - 1} = \pi_i q_i
  \]
  so
  \begin{equation}
    \pi_i = \frac{p_{i - 1}}{q_i} \frac{p_{i - 2}}{q_{i - 1}} \cdots \frac{p_0}{q_1} \pi_0 = \rho_i \pi_0
  \end{equation}

  \[
    \sum_i \pi_i = \pi_0 \sum_i \rho_i
  \]
  If \(S = \sum_i \rho_i\) satisfies \(S < \infty\) then \(\pi_i = \rho_i/S\) is an invariant distribution and if \(S = \infty\) there is no invariant distribution.
\end{eg}

\section{Random Walk on a Graph}

A finite graph consists of vertices and edges and is denoted \(G = (V, E)\). We discuss graphs are simple (in which there are no parallel edges and loops) and connected. If \((u, v) \in E\) then \(v\) is called an \emph{neighbour} of \(u\). The \emph{degree} of \(u\), \(d(u)\), is the number of its neighbours.

A random walk on \(G\) is a Markov chain with state space \(V\) and transition probability
\[
  p_{u, n} =
  \begin{cases}
    0 & \text{ if \(v\) is not a neighbour of \(u\)} \\
    \frac{1}{d(u)} & \text{if \(v\) is a neighbour of \(u\)}
  \end{cases}
\]
This is irreducible if and only if \(G\) is connected, which we assume henceforth.

As always, the natural question to ask is if there is an invariant distribution. Try to solve
\[
  \cdot p_{u, v} = \cdot p_{v, u}
\]
for \((u, v) \in E\). We try to find ``things'' to multiply for the above relation to hold. The obvious choice is \(\pi_u = d(u)\). But we have to normalise it since
\[
  \sum_u d(u) = 2|E|
\]
Then \(\pi_u = \frac{d(u)}{2|E|}\) satsifies the above detailed balance equation, and hence is the unique invariant distribution.

\begin{eg}[Erratic Knight]
  A knight performs independent legal knight moves about a \(8 \times 8\) chessboard. This is a Markov chain on the state space \(S\), the smallest square on hte board. (Exercise: show this is irreducible). The question is: what is its invariant distribution?

  The answer is simple:
  \[
    \pi_i = \frac{\text{No. of legal moves from square}}{336}
  \]
\end{eg}

\begin{ex}[Erratic Bishop]
  There are two types of bishops, depending on the colour of the intial checkerboard. Consequently there are two commutative classes.
\end{ex}

\iffalse
\appendix

\section{Resources}

Reading list: Probability, an introduction Grimmett, Welsh, 2nd edition, Chapter 12

%webpage: www.statslab.cam.ac.uk/~grg/
\fi
\end{document}