\documentclass[sigconf]{acmart}

\settopmatter{printacmref=true}

\fancyhead{}

\usepackage{balance}

\def\BibTeX{{\rm B\kern-.05em{\sc i\kern-.025em b}\kern-.08emT\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}

\copyrightyear{2020}
\acmYear{2020}
\setcopyright{rightsretained}
\acmConference[SPAA '20]{Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures}{July 15--17, 2020}{Virtual Event, USA}
\acmBooktitle{Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA '20), July 15--17, 2020, Virtual Event, USA}
\acmPrice{15.00}
\acmISBN{978-1-4503-6935-0/20/07}
\acmDOI{10.1145/3350755.3400234}
%\acmSubmissionID{123-A56-BU3}

\newcommand{\dec}{\operatorname{dec}}
\newcommand{\poly}{\operatorname{poly}}
\newcommand{\polylog}{\operatorname{polylog}}
\newcommand{\github}{\url{github.com/awestover/Parallel-Partition}}
\newcommand{\defn}[1]{{\textit{\textbf{\boldmath #1}}}}
\renewcommand{\paragraph}[1]{\vspace{0.09in}\noindent{\bf \boldmath #1.}}
\usepackage{amsmath}
\usepackage{dsfont}
\def\E{\operatorname{\mathbb{E}}}
\usepackage{amssymb}
\usepackage{amsthm}

\newtheorem{thm}{Theorem}[section]
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{clm}[thm]{Claim}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{conj}[thm]{Conjecture}
\theoremstyle{remark}
\newtheorem{rem}[thm]{Remark}
\newtheorem{ex}[thm]{Example}

\newtheorem{theorem}{Theorem}[section]
\newtheorem{definition}[thm]{Definition}
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{proposition}[thm]{Proposition}
\newtheorem{claim}[thm]{Claim}
\newtheorem{corollary}[thm]{Corollary}
\newtheorem{conjecture}[thm]{Conjecture}
\theoremstyle{remark}
\newtheorem{remark}[thm]{Remark}
\newtheorem{example}[thm]{Example}
\newtheorem{observation}[thm]{Observation}

\begin{document}

\fancyhead{}

\title{Brief Announcement: \\Cache-Efficient Parallel-Partition Algorithms \\using Exclusive-Read-and-Write Memory}

\author{William Kuszmaul}
\authornote{Supported by a Hertz Fellowship, a NSF GRFP Fellowship, and NSF grant 1533644.}
\affiliation{%
  \institution{Massachusetts Institute of Technology}
}
\email{kuszmaul@mit.edu}

\author{Alek Westover}
\authornote{Supported by MIT PRIMES.}
\affiliation{%
  \institution{Belmont High School}
}
\email{alek.westover@gmail.com}

\renewcommand{\shortauthors}{William Kuszmaul and Alek Westover}

\begin{abstract}
We present an in-place algorithm for the parallel-partition problem
with linear work and polylogarithmic span. The algorithm uses only
exclusive read/write shared variables and can be implemented using
parallel-for-loops without any additional concurrency considerations
(i.e., the algorithm is EREW). A key feature of the algorithm is that
it exhibits provably optimal cache behavior up to small-order factors.

We also present a second in-place EREW algorithm with work $O(n)$ and span
$O(\log n \cdot \log \log n)$, which is within an $O(\log\log n)$ factor of the
optimal span. By using this low-span algorithm as a subroutine within the
cache-friendly algorithm, we obtain a single EREW algorithm that combines their
theoretical guarantees: the algorithm achieves span $O(\log n \cdot \log \log
n)$ and exhibits optimal cache behavior. As an immediate consequence, we also
get an in-place EREW Quicksort algorithm with work $O(n \log n)$ and span
$O(\log^2 n \cdot \log \log n)$.

Whereas the standard EREW algorithm for parallel-partition is
memory-bandwidth bound on large numbers of cores, our cache-friendly
algorithm is able to achieve near-ideal scaling in practice by
avoiding the memory-bandwidth bottleneck. Our algorithm's performance
is comparable to that of the Blocked Strided Algorithm of Francis,
Pannan, Frias, and Petit, which is the previous state-of-the-art for
parallel EREW sorting algorithms, but which lacks theoretical
guarantees on its span and cache behavior.
\end{abstract}

\begin{CCSXML}
<ccs2012>
   <concept>
       <concept_id>10003752.10003809.10010170.10010171</concept_id>
       <concept_desc>Theory of computation~Shared memory algorithms</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
 </ccs2012>
\end{CCSXML}

\ccsdesc[500]{Theory of computation~Shared memory algorithms}

\keywords{Parallel Partition; EREW; in-place algorithms; cache-efficient}

\maketitle
\section{Introduction}

Given an array $A$ of length $n$ and a \defn{decider function} \\$\dec: A \to \{0, 1\}$
labelling the elements of $A$ as \defn{predecessors} (if $\dec(A[i])=0$) and
\defn{successors} (if $\dec(A[i])=1$), a \defn{partition} operation rearranges the
elements in an array so that predecessors occur before successors. In addition
to playing a central role in parallel Quicksort, the parallel-partition
operation is used as a primitive throughout parallel algorithms.\footnote{In
  several well-known textbooks and surveys on parallel algorithms
\cite{AcarBl16,Blelloch96}, for example, parallel partitions are implicitly
used extensively to perform what are referred to as \emph{filter} operations.}

A parallel algorithm can be measured by its \defn{work}, the time
needed to execute in serial, and its \defn{span}, the time to execute
on infinitely many processors. There is a well-known algorithm for
parallel partition on arrays of size $n$ with work $O(n)$ and span
$O(\log n)$ \cite{Blelloch96,AcarBl16}. Moreover, the algorithm uses
only exclusive read/write shared memory variables (i.e., it is an
\defn{EREW} algorithm). This eliminates the need for concurrency
mechanisms such as locks and atomic variables, and ensures good
behavior even if the time to access a location is a function of the
number of threads trying to access it (or its cache line)
concurrently. EREW algorithms also have the advantage that their
behavior is internally deterministic, meaning that
the behavior of the algorithm will not differ from run to run, which
makes test coverage, debugging, and reasoning about performance
substantially easier \cite{BlellochFi12}.

This parallel-partition algorithm suffers from using a large amount of
auxiliary memory, however. To the best of our knowledge, the only
known linear-work and $\operatorname{polylog}(n)$-span algorithms for
parallel partition that are in-place require the use of atomic
operations (e.g, fetch-and-add)
\cite{HeidelbergerNo90,AxtmannWi17,TsigasZh03}.
% This parallel-partition algorithm suffers from using a large amount of
% auxiliary memory, however, and thus performs poorly in practice.

\paragraph{Results}
We present an in-place EREW algorithm, that we call the \defn{Smoothed Striding
Algorithm}, for the parallel partition problem with work $O(n)$ and span
$\polylog(n)$ (Section \ref{sec:smoothing}). A key feature of this algorithm is
that it exhibits provably optimal cache behavior up to small-order factors.

We also present a second in-place EREW algorithm with work $O(n)$ and span
$O(\log n \cdot \log \log n)$, which is within an $O(\log\log n)$ factor of the
optimal span (Section \ref{sec:prefixbasedalg}). By using this low-span
algorithm as a subroutine within the Smoothed Striding algorithm, we are able to
obtain a single EREW algorithm that combines their theoretical guarantees: the
algorithm achieves span $O(\log n \cdot \log \log n)$ and optimal cache
behavior (Theorem \ref{thm:nicethm}). As an immediate consequence, we also get an in-place EREW Quicksort
algorithm with work $O(n \log n)$ and span $O(\log^2 n \cdot \log \log n)$.

Whereas the standard EREW algorithm for parallel partitioning is
memory-bandwidth bound on large numbers of cores, our cache-friendly
algorithm is able to achieve near-ideal scaling in practice by being in-place.
% by avoiding the memory-bandwidth bottleneck.

The full analysis of the algorithms, along with the empirical evaluations are available in the extended paper \cite{ARXIV}.

\section{Preliminaries}\label{secprelim}

\paragraph{Workflow Model} We consider a simple language-based model of parallelism in which algorithms achieve parallelism through the use of \defn{parallel-for-loops} (see, e.g.,
\cite{Blelloch96,AcarBl16}); function calls within the inner loop
then allow for more complicated parallel structures (e.g., recursion). Our algorithms can also be implemented in the less restrictive PRAM model \cite{Blelloch96, AcarBl16}.

\paragraph{Memory Model}
Memory is \defn{exclusive-read} and \defn{exclusive-write} (i.e. we are in the
EREW model). That is, no two threads are ever permitted to attempt to read or
write to the same variable concurrently. Note that threads are not in lockstep
(i.e., they may progress at arbitrary different speeds), and thus the EREW
model requires algorithms to be data-race free in order to avoid the
possibility of non-exclusive data accesses.

In an \defn{in-place} algorithm, each thread is given $O(\polylog n)$
memory upon creation that is deallocated when the thread dies. This
memory can be shared with the thread's children. However, the depth of
the parent-child tree is not permitted to exceed $O(\polylog n)$.

\paragraph{Modeling Cache Misses}
We treat memory as consisting of fixed-size cache lines, each of some
size $b$. Each processor is assumed to have a small cache of
$\operatorname{polylog}{n}$ cache lines. A cache miss occurs on a
processor when the line being accessed is not currently in cache, in
which case some other line is evicted from cache to make room for the
new entry. Each cache is managed with the optimal off-line policy
(furthest-in-the-future).

% a LRU (Least Recently Used)
% eviction policy.

% However, we may assume for the sake of analysis that each cache is managed by
% the optimal off-line eviction strategy OPT (i.e. Furthest in the Future). This
% is because, up to resource augmentation, LRU eviction is $(1 + 1/\polylog
% n)$-competitive with OPT. Formally this is due to the following theorem by
% Sleator and Tarjan:
% \begin{theorem}[Resource Augmentation Theorem \cite{SleatorTa85}]
%   LRU operating on a cache of size $K\cdot M$ for some $K>1$ will incur at most
%   $1+\frac{1}{K-1}$ times the number of times cache misses of OPT operating on
%   a cache of size $M$, for the same series of memory accesses.
%   \label{thm:augmentation}
% \end{theorem}
% Because each processor's cache is managed by OPT (without loss of
% generality), we can assume that each processor \defn{pins} certain
% small arrays to cache (i.e., the elements of those arrays are never
% evicted).

\paragraph{The (Standard) Linear-Space Parallel Partition} The linear-space
implementation of parallel partition consists of two phases
\cite{Blelloch96,AcarBl16}:

\noindent\emph{The Parallel-Prefix Phase: }In this phase, the algorithm first
creates an array $D$ whose $i$-th element $D[i] = \dec(A[i])$ indicates whether
$A[i]$ is a predecessor ($\dec(A[i])=0$ if $A[i]$ is a predecessor and
$\dec(A[i]) = 1$ if $A[i]$ is a successor). Then the
algorithm constructs an array $S$ whose $i$-th element $S[i] = \sum_{j = 1}^i
D[i]$ is the number of predecessors in the first $i$ elements of $A$. The
transformation from $D$ to $S$ is called a \defn{parallel prefix sum} and can
be performed with $O(n)$ work and $O(\log n)$ span using a simple recursive
algorithm: (1) First construct an array $D'$ of size $n / 2$ with $D'[i] = D[2i
- 1] + D[2i]$; (2) Recursively construct a parallel prefix sum $S'$ of $D'$;
(3) Build $S$ by setting each $S[i] = S'[\lfloor i / 2 \rfloor] + A[i]$ for odd
$i$ and $S[i] = S'[i / 2]$ for even $i$.

\noindent\emph{The Reordering Phase: }In this phase, the algorithm constructs
an output-array $C$ by placing each predecessor $A[i] \in A$ in position $S[i]$
of $C$. If there are $t$ predecessors in $A$, then the first $t$ elements of
$C$ will now contain those $t$ predecessors in the same order that they appear
in $A$. The algorithm then places each successor $A[i] \in A$ in position $t +
i - S[i]$. Since $i - S[i]$ is the number of successors in the first $i$
elements of $A$, this places the successors in $C$ in the same order that they
appear in $A$. Finally, the algorithm copies $C$ into $A$, completing the
parallel partition.

Both phases can be implemented with $O(n)$ work and $O(\log n)$
span. However, the algorithm uses multiple auxiliary arrays of size $n$.
Reducing the extra space below $o(n)$ has remained open until now, even when
the number of threads is fixed.

\section{An In-Place Partition Algorithm \\with Span $O(\log n \log \log
n)$}\label{sec:prefixbasedalg} We now outline the key algorithmic ideas needed
to make the classic Parallel Partition algorithm in-place with span $O(\log n
\log \log n)$. Although it is in-place, and hence more cache-efficient than the
standard out-of-place parallel partition algorithm, the algorithm described
here still exhibits poor cache behavior, and performs poorly in practice;
however, using this algorithm as a subroutine on a small subproblem in the
Smoothed Striding Algorithm will allow the Smoothed Striding Algorithm to
achieve span $O(\log n \log \log n)$ without sacrificing its good cache behavior.

\paragraph{Algorithm Outline}
Prior to beginning the algorithm, the first implicit step of the
algorithm is to count the number of predecessors in the array in
order to determine whether the majority of elements are either
predecessors or successors. We assume without loss of generality that the total
number of successors in $A$ exceeds the number of predecessors since otherwise
their roles can simply be swapped in the algorithm. Further, we assume for
simplicity that the elements of $A$ are distinct; this assumption can be removed.

Consider how to remove the auxiliary array $C$ from the Reordering
Phase. If one attempts to simply swap in parallel each predecessor
$A[i]$ with the element in position $j = S[i]$ of $A$, then the swaps
will almost certainly conflict. Indeed, $A[j]$ may also be a
predecessor that needs to be swapped with $A[S[j]]$. Continuing like
this, there may be an arbitrarily long list of dependencies on the
swaps.

To combat this, we begin the algorithm with a Preprocessing Phase in
which $A$ is rearranged so that every prefix is
\defn{successor-heavy}, meaning that for all $t$, the first $t$
elements contain at least $\frac{t}{4}$ successors. Then we compute
the prefix-sum array $S$, and begin the Reordering Phase. Using the
fact that the prefixes of $A$ are successor-heavy, the reordering can
now be performed in place as follows: (1) We begin by recursively
reordering the prefix $P$ of $A$ consisting of the first $4/5 \cdot n$
elements so that the predecessors appear before the successors; (2)
Then we simply swap each predecessor $A[i]$ in the final $1/5 \cdot n$
elements with the corresponding element $S[A[i]]$. The fact that the
prefix $P$ is successor-heavy ensures that, after step (1), the final
$\frac{1}{5} \cdot n$ elements of (the reordered) $P$ are successors.
This implies in step (2) that for each of the swaps between predecessors $A[i]$
in the final $1/5 \cdot n$ elements and earlier positions $S[A[i]]$, the latter
element will be in the prefix $P$. In other words, the swaps are now conflict
free.

Next, consider how to remove the array $S$ from the Parallel-Prefix
Phase. At face value, this would seem quite difficult since the
reordering phase relies heavily on $S$. Our solution is to
\emph{implicitly} store the value of every $O(\log n)$-th element of
$S$ in the ordering of the elements of $A$. That is, we break $A$ into
blocks of size $O(\log n)$, and use the order of the elements in each
block to encode an entry of $S$. (If the elements are not all
  distinct, then a slightly more sophisticated encoding is necessary.)
Moreover, we modify the algorithm for building $S$ to only construct
every $O(\log n)$-th element. The new parallel-prefix sum performs
$O(n / \log n)$ arithmetic operations on values that are implicitly
encoded in blocks; since each such operation requires $O(\log n)$
work, the total work remains linear.

Analysis of the algorithm yields the following theorem:
\begin{theorem}
  \label{thminplace}
  There exists an in-place algorithm using exclusive-read-write
  variables that performs parallel-partition with work $O(n)$ and span
  $O(\log n \cdot \log \log n)$.
\end{theorem}

\section{A Cache Efficient In-Place Parallel Partition Algorithm}\label{sec:smoothing}

\paragraph{The Strided Algorithm \cite{FrancisPa92}}
Our Smoothed Striding Algorithm borrows several structural ideas from a
previous algorithm of Francis and Pannan \cite{FrancisPa92}, which we call the
\defn{Strided Algorithm}\footnote{The original algorithm of Francis and Pannan
  \cite{FrancisPa92} does not consider the cache-line size $b$. Frias and Petit
later introduced the parameter $b$ \cite{Frias08} and showed that by setting
$b$ appropriately, one obtains an algorithm whose empirical performance is
close to the state-of-the-art.}. The Strided Algorithm is designed to behave
well on random arrays $A$, achieving span $\tilde{O}(n^{2/3})$ and exhibiting
only $n/b + \tilde{O}(n^{2/3} / b)$  cache misses on such inputs. On worst-case
inputs, however, the Strided Algorithm has span $\Omega(n)$ and incurs $n/b +
\Omega(n/b)$ cache misses. Our algorithm, the Smoothed Striding Algorithm,
builds on the Strided Algorithm by randomly perturbing the internal structure
of the Strided Algorithm; in doing so, we are able to provide provable
performance guarantees for arbitrary inputs and add a previously impossible
recursion step.
The original Strided Algorithm consists of two steps: \\
\emph{The Partial Partition Step.}
Let $g \in \mathbb{N}$ be a
parameter, and assume for simplicity that $gb \mid n$. Partition the
array $A$ into $\frac{n}{gb}$ chunks $C_1, \ldots, C_{n / gb}$,
each consisting of $g$ cache lines of size $b$.
For $i \in \{1, 2, \ldots, g\}$, define
$P_i$ to consist of the $i$-th cache line from each of the
chunks $C_1, \ldots, C_{n / gb}$. One can think of the $P_i$'s
as forming a strided partition of array $A$, since
consecutive cache lines in $P_i$ are always separated by a fixed
stride of $g - 1$ other cache lines.
The first step of the algorithm is to perform an in-place serial
partition on each of the $P_i$s, rearranging the elements within the
$P_i$ so that the predecessors come first. This step requires work
$\Theta(n)$ and span $\Theta(n/g)$.
\emph{The Serial Cleanup Step. }
For each $P_i$, define the \defn{splitting position} $v_i$ to be the position
in $A$ of the first successor in (the already partitioned) $P_i$. Define
$v_{\text{min}} = \min\{v_1, \ldots, v_{g}\}$ and define $v_{\text{max}} =
\max\{v_1, \ldots, v_{g}\}$. Then the second step of the algorithm is to
perform a serial partition on the sub-array $A[v_{\text{min}}],\ldots,
A[v_{\text{max}}-1]$. This completes the full partition of $A$.

\paragraph{The Smoothed Striding Algorithm}
To obtain an algorithm with provable guarantees for all inputs $A$, we
randomly perturb the internal structure of each of the $P_i$'s. Define
$U_1, \ldots, U_{g}$ (which play a role analogous to $P_1,
\ldots, P_g$ in the Strided Algorithm) so that each $U_i$ contains a
randomly selected cache line from each of $C_1, \ldots, C_{n /
  gb}$ (rather than containing the $i$-th cache line of each
$C_j$). This ensures that the number of predecessors in each $U_i$ is
a sum of independent random variables with values in $\{0, 1, \ldots,
n/g\}$.

By Hoeffding's Inequality, with high probability in $n$, the number of
predecessors in each $U_i$ is tightly concentrated around $\frac{\mu
  n}{g}$, where $\mu$ is the fraction of elements in $A$ that are
predecessors. It follows that, if we perform in-place partitions of
each $U_i$ in parallel, and then define $v_i$ to be the position in
$A$ of the first successor in (the already partitioned) $U_i$, then
the difference between $v_{\text{min}} = \min_i v_i$ and
$v_{\text{max}} = \max_i v_i$ will be small (regardless of the input array
$A$!).

Rather than partitioning $A[v_{\text{min}}],\ldots,
A[v_{\text{max}}-1]$ in serial, the Smoothed Striding Algorithm simply
recurses on the subarray. Such a recursion would not have been
productive for the original Strided Algorithm because the strided
partition $P_1', \ldots, P_g'$ used in the recursive subproblem would
satisfy $P_1' \subseteq P_1, \ldots, P_g' \subseteq P_g$ and thus each
$P_i'$ is already partitioned. That is, in the original Strided
Algorithm, the problem that we would recurse on is a worst-case input
for the algorithm in the sense that the partial partition step makes
no progress.

The main challenge in designing the Smoothed Striding Algorithm
becomes the construction of $U_1, \ldots, U_{g}$ without
violating the in-place nature of the algorithm. A natural approach
might be to store for each $U_i, C_j$ the index of the cache
line in $C_j$ that $U_i$ contains. This would require the storage of
$\Theta(n / b)$ numbers as metadata, however, preventing the algorithm
from being in-place. To save space, the key insight is to select a
random offset $X_j \in \{1, 2, \ldots, g\}$ within each $C_j$, and
then to assign the $(X_j + i \pmod g) + 1$-th cache line of $C_j$ to
$U_i$ for $i \in \{1, 2, \ldots, g\}$. This allows for us to construct
the $U_i$'s using only $O(n/(gb))$ machine words
storing the metadata $X_1, \ldots, X_{n / gb}$. By setting $g$ to
be relatively large, so that $\frac{n}{gb} \le
\operatorname{polylog}(n)$, we obtain a cache-efficient in-place algorithm.

The recursive structure of the Smoothed Striding Algorithm allows for
the algorithm to achieve polylogarithmic span. As an alternative to
recursing, one can also use the in-place algorithm from Section \ref{sec:prefixbasedalg}
 in order to partition the subarray. Doing so results in the following theorem:
\begin{theorem}
  \label{thm:nicethm}
  There exists an in-place EREW parallel-partition algorithm with
  work $O(n)$ that, with high probability in $n$, has span $O(\log n \log \log
  n)$ and incurs fewer than $(n+o(n))/b$ cache misses.
\end{theorem}

%   (since the
% algorithm from the Appendix has span only \\ $O(\log n \log \log
% n)$), while still incurring only $n (1 + o(1))$ cache misses (since
% the cache-inefficient algorithm from the Appendix is only used
% on a small subarray of $A$). We analyze both the recursive version of
% the Smoothed Striding Algorithm, and the version which uses as a final
% step the algorithm from the Appendix; one significant advantage
% of the recursive version is that it is simple to implement in
% practice.

% \paragraph{Formal Algorithm Description} Let $b < n$ be the size of a cache line, let $A$ be an input array of size
% $n$, and let $g$ be a parameter. (One should think of $g$ as being
% relatively large, satisfying $\frac{n}{bg} \le
% \operatorname{polylog}(n)$.)  We assume for simplicity that that $n$
% is divisible by $gb$, and we define $s = \frac{n}{gb}$.
% \footnote{This assumption can be made without loss of generality by treating $A$ as
% an array of size $n' = n + {(gb - n \pmod {gb})}$, and then treating
% the final $gb - n \pmod {gb}$ elements of the array as being
% successors (which consequently the algorithm needs not explicitly
% access). Note that the extra $n' - n$ elements are completely virtual, meaning they do not physically exist or reside in memory.
% }

% In the \defn{Partial Partition Step} the algorithm partitions the
% cache lines of $A$ into $g$ sets $U_1, \ldots, U_{g}$ of size
% $s = \frac{n}{gb}$ and then performs a serial partition on each $U_i$
% in parallel over the $U_i$'s. To determine the sets
% $U_1, \ldots, U_{g}$, the algorithm uses as metadata an array
% $X = X[1], \ldots, X[s]$, where each $X[i] \in \{1, \ldots,
% g\}$.

% Formally, the Partial Partition Step performs the following procedure:
% \begin{itemize}
% \item Set each of $X[1], \ldots, X[s]$ to be uniformly random and
%   independently selected elements of $\{1, 2, \ldots, g\}$. For each $i \in
%   \{1, 2, \ldots, g\}$, $j \in \{1, 2, \ldots, s\}$, define
%   $$G_i(j) = (X[j] + i \pmod g) + (j - 1)g + 1.$$ Using this
%   terminology, we define each $U_i$ for $i \in \{1, \ldots, g\}$ to
%   contain the $G_i(j)$-th cache line of $A$ for each $j \in \{1, 2,
%   \ldots, s\}$. That is, $G_i(j)$ denotes the index of the $j$-th
%   cache line from array $A$ contained in $U_i$.

%   Note that, to compute the index of the $j$-th cache line in $U_i$,
%   one needs only the value of $X[j]$. Thus the only metadata needed by
%   the algorithm to determine $U_1, \ldots, U_g$ is the array
%   $X$. If $|X| = s = \frac{n}{gb} \le \operatorname{polylog}(n)$, then
%   the algorithm is in place.

% \item The algorithm performs an in-place (serial) partition on each
%   $U_i$ (and performs these partitions in parallel with one
%   another). In doing so, the algorithm, also collects
%   $v_{\text{min}}=\min_i{v_i}$, $v_{\text{max}}=\max_i{v_i}$, where
%   each $v_i$ with $i \in \{1, \ldots, g\}$ is defined to be the index
%   of the first successor in $A$ (or $n$ if no such successor
%   exists).\footnote{One can calculate $v_{\text{min}}$ and
%     $v_{\text{max}}$ without explicitly storing each of $v_1, \ldots,
%     v_{g}$ as follows. Rather than using a standard $g$-way parallel
%     for-loop to partition each of $U_1, \ldots, U_{g}$, one can
%     manually implement the parallel for-loop using a recursive
%     divide-and-conquer approach. Each recursive call in the
%     divide-and-conquer can then simply collect the maximum and minimum
%     $v_i$ for the $U_i$'s that are partitioned within that recursive
%     call. This adds $O(\log n)$ to the total span of the Partial
%     Partition Step, which does not affect the overall span
%     asymptotically. %% In practice, this can also be implemented using
%     %% CilkPlus Reducers (or OpenMP Reductions) \cite{FrigoLe09}, though
%     %% empirically we have found explicitly implementing the
%     %% divide-and-conquer structure to be worthwhile for performance.
%   }

%   The array $A$ is now partially partitioned, i.e. $A[i]$ is a
%   predecessor for all $i \le v_{\text{min}}$, and $A[i]$ is a successor
%   for all $i > v_{\text{max}}$.
% \end{itemize}

% The second step of the Smoothed Striding Algorithm is to complete the
% partitioning of $A[v_{\text{min}} + 1], \ldots, A[v_{\text{max}}]$. This can be done
% in one of two ways: The \defn{Recursive Smoothed Striding Algorithm}
% partitions $A[v_{\text{min}} + 1], \ldots, A[v_{\text{max}}]$ recursively using the
% same algorithm (and resorts to a serial base case when the subproblem
% is small enough that $g \le O(1)$); the \defn{Hybrid Smoothed Striding
%   Algorithm} partitions $A[v_{\text{min}} + 1], \ldots, A[v_{\text{max}}]$ using the
% in-place algorithm given in Theorem \ref{thminplace} with span $O(\log
% n \log \log n)$. In general, the Hybrid algorithm yields better
% theoretical guarantees on span than the recursive version; on the
% other hand, the recursive version has the advantage that it is
% simple to implement as fully in-place, and still achieves
% polylogarithmic span.

% % Detailed pseudocode for the Recursive Smoothed Striding Algorithm can
% % be found in Appendix \ref{sec:pseudocode}.

% %% note: g > s should hold
% \paragraph{Algorithm Analysis}
% We prove the following proposition regarding the Partial Partition Step of the
% Smoothed Striding Algorithm.
% \begin{proposition}
%   \label{prop:generalResult}
%   Let $\epsilon \in (0, 1/2)$ and $\delta \in (0, 1/2)$ such that
%   $\epsilon \ge \frac{1}{\poly(n)}$ and $\delta \ge
%   \frac{1}{\polylog(n)}$. Suppose $s > \frac{\ln
%     (n/\epsilon)}{\delta^2}$. Finally, suppose that each processor has
%   a cache of size at least $s + c$ for a sufficiently large constant
%   $c$.

%   Then the Partial-Partition Algorithm achieves work $O(n)$; achieves
%   span $O\paren*{b \cdot s}$; incurs $\frac{s+n}{b} + O(1)$ cache
%   misses; and guarantees with probability $1 - \epsilon$ that
%   $$v_{\text{max}}-v_{\text{min}} < 4 n \delta.$$
% \end{proposition}

% Using this we can analyze the hybrid and recursive smoothed striding algorithms.

% We will assume that both the hybrid and the recursive algorithms use $\epsilon
% = 1/n^c$ for $c$ of our choice (i.e. with high probability in $n$). Moreover,
% the Recursive Smoothed Striding Algorithm continues to use the same value of
% $\epsilon$ within recursive subproblems (i.e., the $\epsilon$ is chosen based
% on the size of the first subproblem in the recursion), so that the entire
% algorithm succeeds with high probability in $n$.

% For both algorithms, the choice of $\delta$ results in a tradeoff between cache
% misses and span. For the Recursive algorithm, we allow for $\delta$ to be
% chosen arbitrarily at the top level of recursion, and then fix $\delta  =
% \Theta(1)$ to be a sufficiently small constant at all levels of recursion after
% the first; this guarantees that we at least halve the size of the problem
% between recursive iterations\footnote{In general, setting $\delta = 1/8$ will
% result in the problem size being halved. However, this relies on the assumption
% that $gb \mid n$, which is only without loss of generality by allowing for the
% size of subproblems to be sometimes artificially increased by a small amount
% (i.e., a factor of $1 + gb / n = 1 + 1/s$). One can handle this issue by
% decreasing $\delta$ to, say, $1/16$.}. Optimizing $\delta$ further (after the
% first level of recursion) would only affect the number of undesired cache
% misses by a constant factor.

% Formally we prove the following performance results for these algorithms:
% \begin{corollary}
%   \label{cor:fullPartition}
% Suppose $b \le o(\log \log n)$. Then the Hybrid Smoothed Striding Algorithm using $\delta = \Theta\big(\sqrt{b/\log\log n}\big)$, achieves work $O(n)$, and with high probability in $n$, achieves span $O(\log n \log\log n)$ and incurs fewer than $(n+o(n))/b$ cache misses.\\\\
% \end{corollary}

% \begin{corollary}
%   \label{cor:groupedPartitionAlg}
%   With high probability in $n$, the Recursive Smoothed Striding Algorithm using parameter $\delta=1/\sqrt{\log n}$:
%   achieves work $O(n)$, attains span $O(b\log^2 n)$, and incurs $n/b \cdot (1 + O(1 / \sqrt{\log n}))$ cache misses.
% \end{corollary}

% \section{Performance Comparisons}\label{secexp}
% \begin{figure}[h]
%   \includegraphics[width=\linewidth]{graph_performance.png}
% % \CILKtable
% % \caption{We compare the performance of various partition implementations, relative to the Libc serial baseline, on random arrays of size $2^{30}$.
% %  The $x$-axis plots the number of worker threads being used, and the $y$-axis
% %  plots the multiplicative speedup over the serial baseline. Each time
% %  (including the serial baseline) is averaged over five trials.}
% %  \label{tablecilk}
% \end{figure}

\bibliographystyle{ACM-Reference-Format}
\bibliography{paper}

\end{document}


%%%%%%%%%%%
%% Home work template for Graduate School
%% Author : Thamme Gowda N.
%% Originally from  https://github.com/thammegowda/hw-tex-templ
%%%%%%%%%%%%%%

\documentclass[a4paper,doc,notimes]{article}
\usepackage[a4paper,margin=1in,footskip=0.25in]{geometry}
%%\documentclass[tikz]{standalone}
\usepackage{tikz}
%Required by APA6 package
\usepackage[normalem]{ulem}
\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage{adjustbox}

%Oft-used, oft-abused
\usepackage{afterpage}
\usepackage{booktabs}
\usepackage{caption}
\usepackage{censor}
\usepackage{color}
\usepackage{csquotes}
\usepackage{enumitem}
\usepackage{float}
\usepackage{hyperref}
\usepackage{lmodern}
%\usepackage{media9}
\usepackage{multirow}
\usepackage{outlines}
\usepackage{pdfpages}
\usepackage{placeins}
\usepackage{soul}
\usepackage{tabularx}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage{xcolor}
\usepackage{mathtools}
\usepackage{tabularx}
\graphicspath{ {images/}}

\usepackage{sectsty}
\sectionfont{\fontsize{14}{12}\selectfont}

\setenumerate[1]{label=\Roman*.}
\setenumerate[2]{label=\Alph*.}
\setenumerate[3]{label=\roman*.}
\setenumerate[4]{label=\alph*.}

% break page where ever is required
\allowdisplaybreaks
%\renewcommand{\thesubsection}{\thesection.\alph{subsection}}
\renewcommand{\thesubsubsection}{\thesubsection.\alph{subsubsection}}
\usepackage{titling}
\setlength{\droptitle}{-9em}
\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}
\usepackage{listings}

\title{\noindent  \textbf{ USC CSCI 567 HOMEWORK 6 SOLUTIONS} }
\author{\textbf{ThammeGowda Narayanaswamy} \\
EMail: tnarayan@usc.edu  USCID: 2074-6694-39}

%% \date{} %% NO date

\begin{document}
\maketitle

\section{Principal Component Analysis}
\subsection{ Derivation of Second Principal Component}
\subsubsection{}
Given :
The cost function
\begin{align*}
 J = & \frac{1}{N} \sum_{i=1}^{N}(x_i - p_{i1}e_{1} - p_{i2}e_{2})^T (x_i - p_{i1}e_{1} - p_{i2}e_{2})  & \\
   = &  \frac{1}{N} \sum_{i=1}^{N} x_i^Tx_i - 2p_{i1}e_1^Tx_i - 2p_{i2}e_2^Tx_i + p_{i1}^2e_1^Te_1 + 2 p_{i1}p_{i2}e_1^Te_2 + p_{i2}^2e_2^Te_2 & \\
   =&  \frac{1}{N} \sum_{i=1}^{N} x_i^Tx_i - 2p_{i1}e_1^Tx_i - 2p_{i2}e_2^Tx_i + p_{i1}^2 + p_{i2}^2  &   \because  e_1^Te_1 = 1, e_2^Te_2=1, e_1^Te_2 = 0   \numberthis \label{e:reducedcost}
\end{align*}
By differentiation \ref{e:reducedcost} with respect to $p_{i2}$, we get
\begin{align*}
 \frac{\partial J}{\partial p_{i2}} =  & 0 & \\
 \implies \frac{\partial}{\partial p_{i2}}  \frac{1}{N} \sum_{i=1}^{N} x_i^Tx_i - 2p_{i1}e_1^Tx_i - 2p_{i2}e_2^Tx_i + p_{i1}^2 + p_{i2}^2 = & 0 & \\
 \frac{1}{N}  \frac{\partial}{\partial p_{i2}}  [ x_i^Tx_i - 2p_{i1}e_1^Tx_i - 2p_{i2}e_2^Tx_i + p_{i1}^2 + p_{i2}^2 ] = & 0 & \because \text{other terms in sum are constant w.r.t a } p_{i2} \\
 - 2e_2^Tx_i + 2 p_{i2} = & 0 \\
\implies  p_{i2} = & e_2^Tx_i \\
\end{align*}

\subsubsection{}
\begin{align*}
 \tilde{J}  = -e_2^TSe_2 + \lambda_2(e_2^Te_2-1)+\lambda_{12}(e_2^Te_1-0) & \\
\implies  \text{First order derivative w.r.t } e_2 & \\
 \frac{\partial \tilde{J}}{\partial e_2}  = -2Se_2 + 2 \lambda_2 e_2 + \lambda_{12}e_1 = & 0 \\
(S -  \lambda_2 I) e_2 = & \frac{1}{2} \lambda_{12}e_1 \\
e_2 = & \frac{1}{2} (S -  \lambda_2 I)^{-1}  \lambda_{12}e_1 \\
\end{align*}

\subsection{A Real Example}
Given : \\
$ m = 3 $ \\
$ n = 100 $ \\
$ S =  \begin{bmatrix}
91.43    & 171.92 & 297.99 \\
171.92   & 373.92 & 545.21 \\
297.99   & 545.21 & 1297.26
\end{bmatrix} $

\subsubsection{}
By factorizing the matrix $S$ using singular value decomposition (SVD) method, we get \\
$
S = U \Sigma V^T =
 \begin{bmatrix}
	-0.218 & -0.247 & -0.944 \\
	-0.414 & -0.853 & 0.318 \\
	-0.884 & 0.461 & 0.084
\end{bmatrix}
\times
 \begin{bmatrix}
	1626.526  & 0          &  0 \\
	0.            & 128.986 &   0 \\
	0.            &  0           &  7.097 \\
\end{bmatrix}
\times
\begin{bmatrix}
	-0.218 & -0.414 & -0.884 \\
	-0.247 & -0.853 & 0.461 \\
	-0.944 & 0.318  & 0.084 \\
\end{bmatrix} $ \\
Where $U$ is an orthonormal matrix formed by eigen vectors, $\Sigma$ is a diagonal matrix having eigen values, and $V^T = U^T = U^{-1}$ for a symmetric matrix $S$.

Thus, the eigen values are: $\lambda_1=1626.526, \lambda_2=128.986, \lambda_3=7.097 $ \\
Eigen vectors (corresponding to eigen values in the same order) are \\
$
	\begin{bmatrix} -0.218 \\	-0.414 \\ 	-0.884 \\ \end{bmatrix} ,
	\begin{bmatrix} -0.247  \\	-0.853 \\ 	0.461 \\ \end{bmatrix} ,
	\begin{bmatrix} -0.944  \\	0.318  \\ 	0.084 \\ \end{bmatrix}
$

\begin{itemize}
	\item Note: The calculations of SVD method has been practiced as per the lecture from MIT 18.06SC Linear Algebra, Fall 2011 https://www.youtube.com/watch?v=cOUTpqlX-Xs .
	\item A computer program was used to perform the above calculations http://www.bluebit.gr/matrix-calculator/
\end{itemize}

\subsubsection{}
Yes, the eigen vector corresponding to eigen value $ \lambda_3=7.097$,  i.e. $\begin{bmatrix} -0.944  \\	0.318  \\ 	0.084 \\ \end{bmatrix} $ \textit{may be} omitted.
Reason: The eigenvalue $\lambda_3 $ is much smaller compared to the other eigen values. Since the eign value represents the variation along the component (the associated eigen vector), smaller eign value $\implies$ the less variance $\implies$ less information captured by that component.
\subsubsection{}
My interpretation for the eigenvectors that captures most information are as follows:
\begin{itemize}
	\item Eigen vectors forms the core of principal component analysis. All the eign vectors are sorted based on the descending order of eigen values. Then the top $k$ eigenvectors are used for projecting the data into newer space which has lesser number of dimensions
	\item Eigen values indicates the variance of data point when they are projected along the associated eigen vector. Higher variance is always preferred.
	\item The higher variance in the data set helps in easier statistical analysis, since (1) we can distinguish data points clearly when they are spread across, eg: classification. (2) we can reconstruct the original dimensions without loosing much information
\end{itemize}

\section{Hidden Markov Model}
\renewcommand{\thesubsection}{\thesection.\alph{subsection}}

Given: \\
The initial state probabilities: \indent $\pi_1 = P(X_1=s_1) = 0.6$   \indent $\pi_2 = P(X_1=s_2) = 0.4$ \\
Transition probabilities : \indent
\begin{tabular}{|c|c|c|} \hline
	          & $s_1$ & $s_2$ \\ \hline
	$s_1$ & 0.7 & 0.3 \\
	$s_1$ & 0.4 & 0.6 \\ \hline
\end{tabular} \\
Observation probabilities:
\begin{tabular}{|c|c|c|c|c|} \hline
	         &   A   & C   & G & T \\ \hline
	$s_1$ & 0.4 & 0.2 & 0.3 & 0.1 \\
	$s_1$ & 0.2 & 0.4 & 0.1 & 0.3 \\ \hline
\end{tabular} \\

\subsection{Probability of Observed sequence}
\begin{tabularx}{\textwidth} {|l | l | X | X |} \hline
  t  & $O_t$  & $\alpha_t(s_1)$    & $\alpha_t(s_2)$ \\ \hline
  1  &  A
  & $\alpha_1(1) = P(A|X_1=s_1) \times P(X_1=s_1)  \newline = 0.6 \times 0.4 \newline = 0.24$
  & $\alpha_1(2) = P(A|X_1=s_2) \times P(X_1=s_2)  \newline  = 0.4 \times 0.2 \newline =  0.08$  \\ \hline
  2  &  C
   &  $\alpha_2(1) = P(C | X_2=s_1) [a_{11}\alpha_1(1) + a_{21}\alpha_1(2)] \newline = 0.2 [0.7 \times 0.24 + 0.4 \times 0.08] \newline = 0.04 $
   &  $\alpha_2(2) = P(C | X_2=s_1) [a_{12}\alpha_1(1) + a_{22}\alpha_1(2)] \newline = 0.4 [0.3 \times 0.24 + 0.6 \times 0.08] \newline = 0.048$  \\ \hline
  3  &  C
	   & $\alpha_3(1) = 0.2[0.7 \times 0.04 + 0.4 \times 0.048]  \newline = 0.00944 $
	  & $\alpha_3(2) = 0.4[0.3 \times 0.04 + 0.6 \times 0.048]  \newline 0.01632 $ \\ \hline
  4  &  G
  &  $\alpha_4(1) = 0.3[0.7 \times 0.00944 + 0.4 \times 0.01632]  \newline = 0.0039408 $
  &  $\alpha_4(2) = 0.1[0.3 \times 0.00944 + 0.6 \times 0.01632]  \newline = 0.0012624 $ \\ \hline
  5  &  T
    &  $\alpha_5(1) = 0.1[0.7 \times 0.0039408 + 0.4 \times 0.0012624 ]  \newline = 0.000326352 $
  &  $\alpha_5(2) = 0.3[0.3 \times 0.0039408 + 0.6 \times 0.0012624]  \newline = 0.000581904 $ \\ \hline
  6  &  A
  & $\alpha_6(1)= \newline 0.4[0.7\times0.00032635 + 0.4\times0.0005819]  \newline = 0.000184482$
  & $\alpha_6(2)= \newline 0.2[0.3\times0.00032635+0.6\times0.0005819]  \newline = 0.000089409 $ \\ \hline
\end{tabularx} \\
\begin{align*}
& P(O=ACCGTA | \Theta) = \alpha_6(1) + \alpha_6(2) = 0.000184482 +  0.000089409 = 0.000273891
\end{align*}
\subsection{Filtering}
\begin{align*}
& P(X6=S_1 | O=ACCGTA; \Theta) = \frac{\alpha_6(1) \beta_6(1)}{ \alpha_6(1) \beta_6(1) + \alpha_6(2) \beta_6(2) } \\
& \text{We know that, for the final time stamp $t=6$, } \beta_6(1) = \beta_6(2) = 1 \\
& P(X6=S_1 | O=ACCGTA; \Theta) = \frac{ 0.000184482 }{ 0.000184482  + 0.000089409 }  = 0.6735599 \\
& P(X6=S_2 | O=ACCGTA; \Theta) = \frac{ 0.000089409 }{ 0.000184482  + 0.000089409 }  = 0.32644 \\
\end{align*}
\subsection{Smoothing}
\begin{tabularx}{\textwidth} {|l | l | X | X |} \hline
	t  & $O_t$  & $\beta_t(s_1)$    & $\beta_t(s_2)$ \\ \hline
	6  &  A  & $\beta_6(1) = 1$  & $\beta_6(2) = 1$\\ \hline
	5  &  T
	& \small $\beta_5(1) = \newline a_{11} P(A|X_6=s_1) \beta_6(1) + a_{12} P(A|X_6=s_2) \beta_6(2)
	\newline = 0.7 \times 0.4  \times 1 + 0.3 \times 0.2 \times 1 \newline = 0.34$
	& \small $\beta_5(2) = \newline a_{21} P(A|X_6=s_1) \beta_6(1) + a_{22} P(A|X_6=s_2) \beta_6(2)
	\newline =  0.4 \times 0.4 \times 1 + 0.6 \times 0.2 \times 1 \newline = 0.28$  \\ \hline
	4  &  G
	& \small $\beta_4(1) = \newline a_{11} P(T|X_5=s_1) \beta_5(1) + a_{12} P(T|X_5=s_2) \beta_(2)
	\newline = 0.7 \times 0.1 \times 0.34 + 0.3 \times 0.3 \times 0.28 \newline = 0.049$
	& \small $\beta_4(2) = \newline a_{21} P(T|X_6=s_1) \beta_6(1) + a_{22} P(T|X_6=s_2) \beta_6(2)
\newline = 0.4 \times 0.1 \times 0.34 + 0.6 \times 0.3 \times 0.28  \newline = 0.064$  \\ \hline
\end{tabularx} \\
\begin{align*}
 P(X4=s_1 | O=ACCGTA; \Theta) = & \frac{\alpha_4(1) \beta_4(1)}{ \alpha_4(1) \beta_4(1) + \alpha_4(2) \beta_4(2) } \\
   = & \frac{0.0039408 \times 0.049 }{0.0039408 \times 0.049  + 0.0012624 \times 0.064 } \\
   = & 0.1928995 \\
  P(X4=s_2 | O=ACCGTA; \Theta) = & \frac{ \alpha_4(2) \beta_4(2)}{ \alpha_4(1) \beta_4(1) + \alpha_4(2) \beta_4(2) } \\
   = & \frac{ 0.0012624 \times 0.064 }{0.0039408 \times 0.049  + 0.0012624 \times 0.064 } \\
   = & 0.8071
\end{align*}

\subsection{Most likely explanation}
We know that the recursive definition of Viterbi algorithm for maximum likehood path in HMM is given by: \\

\begin{equation}
	\delta_t(j) = \max_i \delta_{t-1} (i) a_{ij} P(o_t|X_t=s_j)
\end{equation}
\begin{align*}
\delta_1(1) & = 0.6 \times 0.4 = 0.24  \\
\delta_1(2) &  =  0.4 \times 0.2  = 0.08   \\
\delta_2(1) & =  \max \{ \delta_1(1) a_{11},  \delta_1(2) a_{21}  \} \times P(C|X_t=s_1) \\
               & =\max \{0.24 \times 0.7 , 0.08 \times 0.4 \} \times 0.2 = 0.0336 \\
\delta_2(2) & = \max \{ \delta_1(1) a_{12},  \delta_1(2) a_{22}  \} \times P(C|X_t=s_2) \\
			& =\max \{0.24 \times 0.3 , 0.08 \times 0.6 \} \times 0.4 = 0.0288 \\
\delta_3(1) & =  \max \{ \delta_2(1) a_{11},  \delta_2(2) a_{21}  \} \times P(C|X_t=s_1) \\
& =\max \{0.0336 \times 0.7 , 0.0288  \times 0.4 \} \times 0.2 = 0.004704 \\
\delta_3(2) & =  \max \{ \delta_2(1) a_{12},  \delta_2(2) a_{22}  \} \times P(C|X_t=s_2) \\
& =\max \{0.0336 \times 0.3 , 0.0288  \times 0.6 \} \times 0.4 = 0.003456 \\
\delta_4(1) & =  \max \{ \delta_3(1) a_{11},  \delta_3(2) a_{21}  \} \times P(G|X_t=s_1) \\
& =\max \{0.004704 \times 0.7 , 0.003456  \times 0.4 \} \times 0.2 = 0.00065856 \\
\delta_4(2) & =  \max \{ \delta_3(1) a_{12},  \delta_3(2) a_{22}  \} \times P(G|X_t=s_2) \\
& =\max \{0.004704 \times 0.3 , 0.003456  \times 0.6 \} \times 0.4 =0.00082944 \\
\delta_5(1) & =  \max \{ \delta_4(1) a_{11},  \delta_4(2) a_{21}  \} \times P(T|X_t=s_1) \\
& =\max \{0.00065856 \times 0.7 , 0.00082944  \times 0.4 \} \times 0.1 = 0.0000460992 \\
\delta_5(2) & =  \max \{ \delta_4(1) a_{12},  \delta_4(2) a_{22}  \} \times P(T|X_t=s_2) \\
& =\max \{0.00065856 \times 0.3 ,0.00082944  \times 0.6 \} \times 0.3 =0.0001492992 \\
\delta_6(1) & =  \max \{ \delta_5(1) a_{11},  \delta_5(2) a_{21}  \} \times P(A|X_t=s_1) \\
& =\max \{0.0000460992 \times 0.7 , 0.0001492992 \times 0.4 \} \times 0.4 =0.000023887872 \\
\delta_6(2) & =  \max \{ \delta_5(1) a_{12},  \delta_5(2) a_{22}  \} \times P(A|X_t=s_2) \\
& =\max \{0.0000460992 \times 0.3, 0.0001492992 \times 0.6 \} \times 0.2 =0.000017915904 \\
\end{align*}
So, the most likely explanation for the states that produced the observations \textbf{ACCGTA = $s_1s_1s_1s_2s_2s_1$}

\subsection{Prediction}
\begin{align*}
P(O_7|O) & = \sum_{j\in1,2} P(O_7, X_7=s_j) \\
& =  \sum_{j\in1,2} P(O_7| X_7=s_j) P(X_7=s_j, O) \\
& =  \sum_{j\in1,2} P(O_7| X_7=s_j) \times \sum_{k\in 1,2} P(X_7=s_j | X_6=s_k)  P(X_6=s_k, O) \\
& =  P(O_7| X_7=s_1) \times \sum_{k\in 1,2} P(X_7=s_1 | X_6=s_k)  P(X_6=s_k, O)  \\
& \indent + P(O_7| X_7=s_2) \times \sum_{k\in 1,2} P(X_7=s_2 | X_6=s_k)  P(X_6=s_k, O) \\
& = P(O_7| X_7=s_1) [ P(X_7=s_1 | X_6=s_1)  P(X_6=s_1, O) + P(X_7=s_1 | X_6=s_2)  P(X_6=s_2, O) ] \\
& \indent + P(O_7| X_7=s_2) [P(X_7=s_2 | X_6=s_1)  P(X_6=s_1, O) +  P(X_7=s_2 | X_6=s_2)  P(X_6=s_2, O)] \\
& = P(O_7| X_7=s_1) [ 0.7  \times 0.6735599 + 0.4 \times  0.32644]  + P(O_7| X_7=s_2) [0.3 \times 0.6735599  +  0.6  \times  0.32644)] \\
P(O_7|O)  & = 0.60206793 \times P(O_7| X_7=s_1)  + 0.39793197 \times P(O_7| X_7=s_2) \numberthis
\end{align*}
\begin{align*}
 P(O_7=A | O) &=  0.60206793 \times 0.4 + 0.39793197 \times 0.2 = 0.320413566  \\
 P(O_7=C | O) &=  0.60206793 \times 0.2 + 0.39793197 \times 0.4 = 0.279586374  \\
 P(O_7=G | O) &=  0.60206793 \times 0.3 + 0.39793197 \times 0.1 = 0.220413576  \\
 P(O_7=T | O) &=  0.60206793 \times 0.1 + 0.39793197 \times 0.3 = 0.179586384
\end{align*}
So, our prediction for $O_7$, the next observation, is \textbf{'A'}.
\end{document}


\input{../../_common/preamble}

\title{Introduction to time series modelling}

\begin{document}

\maketitle

\begin{frame}{Contents}
    \tableofcontents[hideallsubsections]
\end{frame}

\section{Statistics for time series}

\begin{frame}{Time series}
    \begin{center}
        Any data that change \alert{over time}
    \end{center}
    \vfill\pause
    \begin{block}{Seasonality}
        \begin{itemize}
            \item Cyclic pattern(s) repeated over time
            \item E.g.\ peak of sales in December
        \end{itemize}
    \end{block}
    \vfill
    \begin{block}{Trend}
        \begin{itemize}
            \item Change in `baseline' levels over time
            \item E.g.\ linear increase in sales over last 5 years
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Rolling (or moving) statistics}
    Each observation is replaced with some statistic (e.g.\ mean) of $k$
    consecutive time points:\vspace{-1ex}
    \begin{itemize}
        \item $k$ preceding points
        \item $k / 2$ points prior to and following a given time point
    \end{itemize}
    \vfill
    \begin{block}{Usage}
        \begin{itemize}
            \item Reduce influence of outliers
            \item Smooth time series to identify patterns
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Exponentially weighted averages}
    \begin{itemize}
        \item Rolling statistics weigh the $k$ time points equally
        \item Often, points closer in time are more important
        \item[$\rightarrow$] Weighting
    \end{itemize}
    \vfill\pause
    \begin{block}{Exponential weighting}
        \begin{itemize}
            \item $\text{EWMA}_{1} = y_{1}$
            \item $\text{EWMA}_{t} = \alpha\,y_{t} + \left( 1 - \alpha \right) \text{EWMA}_{t-1}$, $t > 1$
            \item[$\rightarrow$] $\alpha$ controls the \alert{decay}
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Expanding statistics}
    Each observation is replaced with some statistic (e.g.\ sum) of all points
    prior to the given time point
    \vfill
    \begin{block}{Usage}
        \begin{itemize}
            \item Visualise cumulative distribution over time
            \item Identify trends
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Autocorrelation}
    Correlation of the time series with itself at different lags:\vspace{-1ex}
    \begin{itemize}
        \item At lag 1, dependency on `yesterday'
        \item At lag 7, dependency on `last week'
        \item At lag 30, dependency on `last month'\ldots
    \end{itemize}
    \vfill
    \begin{block}{Usage}
        \begin{itemize}
            \item Identify trends
            \item Identify period of seasonal cycles
        \end{itemize}
    \end{block}
\end{frame}

\section{Forecasting}

\begin{frame}{Forecasting}
    \begin{block}{Prediction}
        \begin{itemize}
            \item Value of $\vec{y}$ given values for the predictors $\mat{X}$
            \item Does not depend on time (or temporal effect is negligible)
        \end{itemize}
    \end{block}
    \vfill\pause
    \begin{block}{Forecasting}
        \begin{itemize}
            \item Value of $\vec{y}$ given \alert{previous values} of $\vec{y}$
            \item Captures autocorrelation to `project forward'
            \item (Some models can also incorporate predictors)
        \end{itemize}
    \end{block}
\end{frame}

\begin{frame}{Stationarity}
    Many models require time series to be \alert{stationary}:\vspace{-1ex}
    \begin{itemize}
        \item Mean and variance constant over time
        \item[$\rightarrow$] Seasonality and trend must be removed
    \end{itemize}
    \vfill\pause
    \begin{block}{Solutions}
        \begin{itemize}
            \item Detrending (estimate and subtract `baseline')
            \item Differencing (predict change or `change in changes')
        \end{itemize}
    \end{block}
\end{frame}

\section{Time series models}

\begin{frame}{ARMA models}
    \begin{columns}[t]
        \begin{column}{0.5\textwidth}
            \begin{center}
                \textbf{A}uto\textbf{R}egressive
            \end{center}
            \begin{itemize}
                \item $y_{t}$ depends on $y_{t-1}$, \ldots
                \item Regression on past values
                \item Captures (slow) changes \\
                      in trend
            \end{itemize}
        \end{column}
        \begin{column}{0.5\textwidth}
            \begin{center}
                \textbf{M}oving \textbf{A}verage
            \end{center}
            \begin{itemize}
                \item $y_{t}$ depends on $\epsilon_{t-1}$, \ldots
                \item Smoothing of past errors
                \item Captures sudden changes \\
                      (e.g.\ spikes)
            \end{itemize}
        \end{column}
    \end{columns}
    \[
        y_{t} = \alpha +
                \beta_{1} \, y_{t-1} + \ldots + \beta_{p} \, y_{t-p} +
                \gamma_{1} \, \epsilon_{t-1} + \ldots + \gamma_{q} \, \epsilon_{t-q}
    \]
\end{frame}

\begin{frame}{ARIMA models}
    \vspace{1em}
    \begin{block}{First\hyp{}order differences $y_{t} - y_{t-1}$}
        \begin{itemize}
            \item Predict change
            \item Corresponds to velocity in physics
        \end{itemize}
    \end{block}
    \vfill
    \begin{block}{Second\hyp{}order differences $\left( y_{t} - y_{t-1} \right) - \left( y_{t-1} - y_{t-2} \right)$}
        \begin{itemize}
            \item Predict `change in changes'
            \item Corresponds to acceleration in physics
        \end{itemize}
    \end{block}
\end{frame}

\end{document}


\documentclass[12pt]{report}

% ----------------------------------------------------------------------
% Packages {Begin}
% ----------------------------------------------------------------------
\usepackage[Lenny]{fncychap}
\usepackage{color}
\usepackage{bm}
\usepackage{amsmath}
\definecolor{MyRed}{rgb}{0.90,0.00,0.00}
\setcounter{chapter}{3}
\renewcommand{\chaptername}{Exercise}

\begin{document}
\chapter{Random Sampling}

\textcolor{MyRed}{
\textbf{
    \Large
    Deadline: October 19, 2016.
}
}\\\\
\textbf{Note:\\}For calculation problem, please give details of the derivation. Answers will receive only half of the points.

\begin{enumerate}

%1
    \item Reading.
    \begin{enumerate}
        \item Lecture notes 4.
        \item Chapter 5 of the book "Statistical Inference".
    \end{enumerate}

%2
    \item Let $X_1, \ldots, X_n$ be a random sample of size $n$ from a $N(0, \sigma^2)$ population. Prove that
        \[
            \frac{(n-1)S^2}{\sigma^2} = \sum_{i=1}^n\Big(\frac{X_i-\bar{X}}{\sigma}\Big)^2
        \]
        has a $\chi^2$ distribution with $n-1$ degrees of freedom.

%3
    \item Let $X_1, \ldots, X_n$ be a random sample of size $n$ from a $N(0, \sigma^2)$ population. Prove that $\bar{X}$ and $S^2$ are independent random variables.

%4
    \item Let $X_1, \ldots, X_n$ be a random sample of size $n$ from a $N(\mu, 1)$ population. Define random variables $Y_1,\ldots,Y_n$ as
        \[
            Y_i= \left \{
            \begin{array}{ll}
              1 & \mbox{if } X_i > \mu \\
              0 & \mbox{if } X_i \leq    \mu
            \end{array}
            \right.
            .
        \]
        Derive the distribution of $\sum_{i=1}^nY_i$
%5
      \item Two samples $X_1,\ldots,X_m$ and $Y_1,\ldots,Y_n$ are obtained from two independent normal population $N(\mu_X, \sigma_X^2)$ and $N(\mu_Y, \sigma_Y^2)$, respectively, and the condition $   \sigma_X=\sigma_Y=\sigma$ holds.
        \begin{enumerate}
        \item Show that
            \[
                \frac{
                    (\bar{X} - \bar{Y}) - (\mu_X - \mu_Y)
                }{
                    \sigma \sqrt{\frac{1}{m} + \frac{1}{n}}
                }
            \]
            has a standard normal distribution.

        \item Show that
            \[
                \frac{
                    (m+n-2) S_p^2
                }
                {
                    \sigma^2
                }
            \]
            has a $\chi^2$ distribution with $m+n-2$ degrees of freedom, where
            \[
                S_p^2 = \frac{
                    (m-1) S_X^2 + (n-1) S_Y^2
                }
                {
                    m+n-2
                }
            \]
            is the pooled variance estimate.

        \item With the above two conclusion, show that
            \[
                \frac{
                    (\bar{X} - \bar{Y}) - (\mu_X - \mu_Y)
                }{
                    S_p \sqrt{\frac{1}{m} + \frac{1}{n}}
                }
            \]
            has a student's \emph{t} distribution with $m+n-2$ degrees of freedom.

        \end{enumerate}
%6
 \item Let $X$ be a random variable with student $t$ distribution with $p$ degrees of freedom.
    \begin{enumerate}
    	\item Derive the mean and variance of $X$.
    	\item Prove that when $p$ tends to infinite, $X$ converges in distribution to standard normal. That is,
        	\[
        	    \lim_{p \rightarrow \infty}f(x|p) = \frac{1}{\sqrt{2\pi}}e^{-x^2/2}, \mbox{  for any} -\infty < x < \infty
        	\]
    	\item Find the distribution of $X^2$
    \end{enumerate}

\end{enumerate}
\end{document}


\documentclass{article}
\usepackage[section]{placeins}
\usepackage{graphicx}
\usepackage{wrapfig}

\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=cyan,
    pdftitle={Overleaf Example},
    pdfpagemode=FullScreen,
    }

\author{Yaghoub Shahmari}
\title{Report - Problem Set no 3}
\date{\today}
\graphicspath{ {../Figs/} }

\begin{document}
    \maketitle
    \section*{Problem 1}
    \textbf{Basic description:}

    We want to simulate the Ballistic Deposition with relaxation.
    So, we will generate some random integers in the surface length range.
    Then, We put The random generated number as the index, and drop the paricle.
    but we have to check the neighbors of the chosen point of the surface.
    If there was a hole in the neighbors, Then the particle will move to that position.
    There is two way to store the data. We can have a 2D Matrix as our deposition layers or have a 1D vector that saves only the height of layers.
    I use the first method for Graphic view and the second to show the $W_{(t)}$ and $H_{(t)}$ in terms of time.

    \textbf{The results:}

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.5]{/Q1/Graphic}
        \label{fig:1.1}
        \caption{Ballistic Deposition with relaxation:
        For one hundred time steps, one thousand particles per each time step will drop on a surface of two hundred lengths.
        For every ten steps of time, The color of the visualization will change.}
    \end{figure}

    \pagebreak

    Now we're going to see the statistics of Deposition.
    To have a better view of the dynamic,
    we show our w plot on logarithmic scales.
    For each surface length, we split the time range into exponential periods.
    And for every considered condition, the drop rate of particles is one particle per time unit.

    We created every plot from ensembles of 1000 runs,
    and all of them have error bars
    (some of them have very small errorbar because of accuracy).

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.283]{/Q1/H-t(L=50)}
        \label{fig:1.2}
        \includegraphics[scale = 0.283]{/Q1/W-t(L=50)}
        \label{fig:1.3}
        \caption{Ballistic Deposition with relaxation for $L=50$}
    \end{figure}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.283]{/Q1/H-t(L=100)}
        \label{fig:1.4}
        \includegraphics[scale = 0.283]{/Q1/W-t(L=100)}
        \label{fig:1.5}
        \caption{Ballistic Deposition with relaxation for $L=100$}
    \end{figure}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.283]{/Q1/H-t(L=200)}
        \label{fig:1.6}
        \includegraphics[scale = 0.283]{/Q1/W-t(L=200)}
        \label{fig:1.7}
        \caption{Ballistic Deposition with relaxation for $L=200$}
    \end{figure}
    \pagebreak
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.283]{/Q1/H-t(L=300)}
        \label{fig:1.8}
        \includegraphics[scale = 0.283]{/Q1/W-t(L=300)}
        \label{fig:1.9}
        \caption{Ballistic Deposition with relaxation for $L=300$}
    \end{figure}

    The results satisfied our expectations.
    As we expected, the amount of $W_{(t)}$ grows linearly (in log-log plot),
    and after a while, becomes a horizontal line.
    The $\beta$ is equal to the initial line slope.
    So, for the following lengths, we have:

    \begin{equation}
        \centering
        L = 50, t_{s} \approx 6, t_{s} \approx L^z \Rightarrow z \approx 0.458,\
        \beta \approx 0.19, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.09
    \end{equation}
    \begin{equation}
        \centering
        L = 100, t_{s} \approx 8, t_{s} \approx L^z \Rightarrow z \approx 0.452,\
        \beta \approx 0.2, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.09
    \end{equation}
    \begin{equation}
        \centering
        L = 200, t_{s} \approx 11, t_{s} \approx L^z \Rightarrow z \approx 0.435,\
        \beta \approx 0.2, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.09
    \end{equation}
    \begin{equation}
        \centering
        L = 300, t_{s} \approx 12, t_{s} \approx L^z \Rightarrow z \approx 0.437,\
        \beta \approx 0.21, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.09
    \end{equation}

    The H(t) is also as we wanted to be, it has a linear behavior,
    and the amount of the line slope follows:

    \begin{equation}
        \centering
        line \ slope = drop \ rate \div L
    \end{equation}

    \section*{Problem 2}
    \textbf{Basic description:}

    In this question, we want to do the same things we have done in the previous question,
    but we have a little change. The particle will drop on the chosen spot,
    not in the neighbors, but also the particle is not going to place in the deepest place.
    The height will be the maximum height of neighbors and the midpoint.
    The run numbers and other parameters are the same as the previous question parameters.

    \textbf{The results:}

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.4]{/Q2/Graphic}
        \label{fig:2.1}
        \caption{Ballistic Deposition :
        For one hundred time steps, one thousand particles per each time step will drop on a surface of two hundred lengths.
        For every ten steps of time, The color of the visualization will change.}
    \end{figure}

    Now we're going to see the statistics of the Deposition.
    As before, to have a better view of the dynamic,
    we show our w plot on logarithmic scales,
    and for every considered condition,
    the drop rate of particles is one particle per time unit.

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.27]{/Q2/H-t(L=50)}
        \label{fig:2.2}
        \includegraphics[scale = 0.27]{/Q2/W-t(L=50)}
        \label{fig:2.3}
        \caption{Ballistic Deposition for $L=50$}
    \end{figure}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.27]{/Q2/H-t(L=100)}
        \label{fig:2.4}
        \includegraphics[scale = 0.27]{/Q2/W-t(L=100)}
        \label{fig:2.5}
        \caption{Ballistic Deposition for $L=100$}
    \end{figure}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.27]{/Q2/H-t(L=200)}
        \label{fig:2.6}
        \includegraphics[scale = 0.27]{/Q2/W-t(L=200)}
        \label{fig:2.7}
        \caption{Ballistic Deposition for $L=200$}
    \end{figure}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.27]{/Q2/H-t(L=300)}
        \label{fig:2.8}
        \includegraphics[scale = 0.27]{/Q2/W-t(L=300)}
        \label{fig:2.9}
        \caption{Ballistic Deposition for $L=300$}
    \end{figure}

    \pagebreak

    The results of the statistic section are as they should be as before.
    Until the saturated, we have an oblique line.
    After that, we have a horizontal line.
    The line slope follows the rules we knew.

    Now we are going to calculate the requested variables:

    \begin{equation}
        \centering
        L = 50, t_{s} \approx 3, t_{s} \approx L^z \Rightarrow z \approx 0.281,\
        \beta \approx 0.59, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.166
    \end{equation}
    \begin{equation}
        \centering
        L = 100, t_{s} \approx 4, t_{s} \approx L^z \Rightarrow z \approx 0.301,\
        \beta \approx 0.58, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.175
    \end{equation}
    \begin{equation}
        \centering
        L = 200, t_{s} \approx 5, t_{s} \approx L^z \Rightarrow z \approx 0.304,\
        \beta \approx 0.55, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.167
    \end{equation}
    \begin{equation}
        \centering
        L = 300, t_{s} \approx 6, t_{s} \approx L^z \Rightarrow z \approx 0.314,\
        \beta \approx 0.53, L^{\beta \dot z} = L^{\alpha} \Rightarrow \alpha \approx 0.166
    \end{equation}

    The H(t) is also as we wanted to be, it has a linear behavior,
    and the amount of the line slope follows:

    \begin{equation}
        \centering
        line \ slope = drop \ rate \div L
    \end{equation}

    \pagebreak
    \section*{Problem 3}
    \textbf{Basic description:}

    For this question, we need to change the previous code a little.
    The only change we need to make is considering a condition.
    Only two types of particles can be dropped and caught.
    The particles that stick to the midpoint and those stick to the other particles.

    The parameters are as following:

    \textbullet \ The length of the surface is 300, and we will deposit 50000 particles.

    \textbf{The results:}

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.5]{/Q3/Graphic}
        \label{fig:3.1}
        \caption{Competitive Ballistic Deposition :
        50000 particles will drop on a surface of three hundred lengths.}
    \end{figure}

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.5]{/Q3/Q3-L(t)}
        \label{fig:3.2}
        \caption{Competitive Ballistic Deposition for $L=300$.}
    \end{figure}

    As it is clear, the transverse width of the shrub grows linearly,
    until it becomes equal to the surface length.

    \pagebreak
    \section*{Problem 4}
    \textbf{Basic description:}

    Here we want to simulate the percolation of a binary defined network.
    In that case, we create a random binary network using the python networkx.
    Then we will check if there is a path between every node on the sides of the network.
    If we find a route, we will consider the percolation successful. And we check that for a range of P.
    So here is the result:

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.5]{/Q4/Q4}
        \label{fig:4.1}
        \caption{The results of percolation for a range of P for 1000 runs.}
    \end{figure}

    \section*{Problem 5}
    \textbf{Basic description:}

    The coloring algorithm deployed as the question wanted.
    My program can find percolation in a binary network and return the result as 0 or 1.
    For some critical P (or at least near to critical point),
    I drew an animation of stages of the dynamic of percolation for colors.
    The results are available in the Q5 folder in the Figs folder.

    \section*{Problem 6}
    \textbf{Basic description:}

    For large networks, the coloring algorithm is not fast enough.
    So I deployed the H-K algorithm for a network of sizes 100 and 200.
    So I exported the result of percolation of P in a range many times and showed the statistic results as following plots:

    \textbf{The results:}
    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.4]{/Q6/10x10}
        \label{fig:6.1}
        \includegraphics[scale = 0.4]{/Q6/100x100}
        \label{fig:6.2}
        \includegraphics[scale = 0.4]{/Q6/200x200}
        \label{fig:6.3}
        \caption{Percolation result for $L=50, 100, 200$}
    \end{figure}

    \pagebreak
    \section*{Problem 7}
    \textbf{Basic description:}
    There is only one small change in the previous question in detecting percolation to achieve the answer for this question. We just need to check if a random point is in the connector cluster.

    \begin{figure}[!htb]
        \centering
        \includegraphics[scale = 0.4]{/Q7/10x10}
        \label{fig:7.1}
        \includegraphics[scale = 0.4]{/Q7/100x100-10000}
        \label{fig:7.2}
        \includegraphics[scale = 0.4]{/Q7/200x200-10000}
        \label{fig:7.3}
        \caption{$Q_{\infty}$ for $L=50, 100, 200$}
    \end{figure}

    \centering
    \textbf{The whole data I gathered is in \href{https://github.com/shahmari/ComputationalPhysics-Fall2021/tree/main/ProblemSet3/Data}{this link}}
\end{document}


\documentclass{article}
\usepackage{hyperref}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{color,graphicx}

\begin{document}
\title{Libsvm using GPU}
\author{Anirudhan J. Rajagopalan \\ N18824115 \\ ajr619}
\maketitle

% write about lib svm format (svm light)
% two possible places for optimizations.
% Calculating the gram matrix
% Porting the Sequential Minimal Optimization to GPU.

% Structures in libsvm
% svm_problem
% svm_node

% lru cache for all the rows processed in the kernel
% svm.cpp dot function (line no. 294). Modifying the dot function is difficult
% as the internal datastructures has to be changed.

\newpage

\section{Abstract}
This project is an attempt to implement the Support Vector Machine~\cite{cortes1995} classifier using GPUs.
We tried two approaches:
The first one is to try to port the Sequential Minimal Optimization part to the GPU\@.
The second one is to implement the Gram matrix~\cite{wiki:gram_matrix} calculation to GPUs.
We end up using the second approach and we show show performance improvements over the conventional approach used by libsvm.
\section{Introduction}
The most popular implementation of SVM is libsvm~\cite{CC01a}.
It is used in almost all the traditional machine learning applications.
Since it projects data into a higher dimensional space, we were able to solve problems that are not easily classifiable in the lower dimensional space.

Porting a SVM to GPUs will help us use this classifier easily in deep learning models too.
Though deep learning already uses various non-linearity to support higher dimensional data, the use of SVM might help us learn the underlying representation without using deeper layers in a deep learning model.

In this project, we explore various ways to parallelize the SVM problem and implement it in GPU\@.

\section{Background Information}

SVM is a linear classifier and is mainly successful due to its use of Hilbert space.
SVM uses kernel functions which are functions in Reproducing Kernel Hilbert Space (RKHS).
Implementation wise, these kernel functions can be as simple as computing a dot product between two set of features.o
For the purpose of this project, we will be worrying about these simple RKHS functions.
The idea is that, a non-linear lower dimensional data might be separable in a higher dimensional space.

Once, the higher dimensional features are calculated, SVM uses this higher dimensional features to find a classifier that has maximal distance from any of the points from the dataset.

The other part of SVM is the Quadratic Programming part.
This part involves actually finding the plane that classifies the points in the dataset while also having a maximum margin from both the data points.

Given l examples $(\bar{x}_{1}, y_{1}), (\bar{x}_{2}, y_{2}), \ldots ,(\bar{x}_{l}, y_{l}), $ with $\bar{x}_{i} \in \mathbb{R}^{n}$ and $y_{i} \in \{-1, 1\} $ $\forall{i} $ where the regularization is controlled by $C$.

\begin{align}
    \min_{f \in H} C\sum_{i = 1}^{l} V(y_{i}, f(\bar{x}_{i})) + \frac{1}{2} {\vert\vert f \vert\vert}_{k}^{2}
\end{align}

Here V is the loss function which is typically Hinge loss.
Usage of Hinge loss or any approximation of Hing loss such as Huber-Hinge loss helps us find a maximal margin plane that classifies the samples.

Typically we solve for the dual of this actual problem, which can be given as

\begin{align}
    \max_{\alpha \in \mathbb{R}^{l}} \sum_{i = 1}^{l} \alpha_{i} - \frac{1}{2} \alpha^{T}K\alpha
\end{align}

Subject to: $\sum_{i=1}^{l} y_{i}\alpha_{i} = 0$ and $0 \le \alpha_{i} \le C, $ $ i = 1,\ldots,l $ where $K_{ij} = y_{i}y{j}k(\bar{x}{i}, \bar{x}_{j})$ is a kernel function. Solving this requires quadratic programming approaches which gives us the classification function.

\begin{align}
    f(x) = \sum_{i=1}^{l} y_{i}\alpha_{i}k(\bar{x}, \bar{x}_{i}) + b
\end{align}

With all the theory mentioned as briefly as possible, we can proceed to find a way to implement this interesting problem using GPUs.

\section{Literature Survey}
There are a couple of previous work in this area.  We mainly use
\begin{enumerate}
    \item Parallel Multiclass Classification Using SVMs on GPUs by Herrero-Lopez, Sergio and Williams, John R. and Sanchez, Abel~\cite{Herrero-Lopez:2010:PMC:1735688.1735692}.
    \item GPU acceleration for support vector machines by Athanasopoulos, Andreas and Dimou, Anastasios and Mezaris, Vasileios and Kompatsiaris, Ioannis~\cite{athanasopoulos2011gpu}
\end{enumerate}

There are chapters in Nvidia GPU Gems book which also describe a way to parallelize the SVM implementation.  It is almost similar to the method published by Herrero-Lopez.

Our implementation of SVM is going to be loosely based on libsvm's implementation.  As such, it became important to understand libsvm's codebase.

Libsvm expects data in svm-light format.  Each line of the input file has the structure defined in Fig~\ref{fig:svmlight}

\begin{figure}[ht!]
  \centering
  \includegraphics[width=1\textwidth]{images/svmlight}
  \caption{Svm light format.\label{fig:svmlight}}
\end{figure}

Libsvm uses internal structures such as \textit{svm\_node} and \textit{svm\_problem} to read the input file and store it internally.
After reading the input file, it calculates the Kernel matrix which is then used for finding the maximum margin classifier.

An input file consists of samples.
Each samples consists of target and features.

Each and every feature is stored in a svm\_node with index and value.
All these are stored in an array per sample.
This structure doesn't easily translate to vector representation and hence plugging in the GPU based gram matrix calculation or SMO implementation.

\section{Proposed solution}
We propose two different approaches for implementing SVMs using GPU\@.

\begin{enumerate}
\item Implement the Sequential Minimal Optimization method for solving the dual problem using GPUs and OpenMpi.
\item Implement the Gram matrix calculation using GPU and use libsvm to solve the actual SMO problem.
\end{enumerate}

\section{Experimental Setup}
We use cuda2 and cuda5 machines for all the experiments.
The cuda machines got surprisingly slow in the last few days and I resorted to using NYU HPC machines with Titan GPU for later experiments.
We also use MNIST~\cite{lecun1998mnist} dataset for all the experiments.

MNIST dataset contains a total of 60,000 samples for test and 10,000 samples for test.
We track the time, accuracy of test after training on 600, 1200, 6000, 15000, 30000 and 60000 samples.

I started out with implementing the SMO in GPU following the reference implementation at~\cite{dpsvm}.
The reference implementation uses OpenMPI, CUDA, Thrust, and BLAS (cuBLAS, cBLAS, and BLAS) libraries.
Even though I had the reference implementation, understanding and reimplementing the reference implementation proved to be a difficult task.
After working on the reference implementation for two days, I started to search for alternate methods to speedup the SVM classifier.

Libsvm computes the kernel function and caches them using a LRU cache.
The LRU cache has a fixed maximum size.
So, if we have a huge amount of data, then the number of rows from the kernel matrix that can be cached will be less.
Whenever an entry has to be created in the cache, a new row of the kernel has to be calculated and pushed into the cache.

\section{Experimental Result \& Analysis}

We implemented matrix multiplication for generating the gram matrix similar to the implementation given in~\cite{matrixmuleg}.
We were able to get the same accuracy for both implementations.

Since the implementation I am using was cuda blas implementation, I cound't find a way to optimize it further.
Also, we can observe that there are huge performance improvement in case of larger datasets.

\begin{figure}[ht!]
  \centering
  \includegraphics[width=1\textwidth]{images/gpuvscpu}
  \caption{Mnist training gpu vs cpu performance.\label{fig:gpuvscpu}}
\end{figure}

\section{Conclusions}

\bibliographystyle{plain}
\bibliography{references}

\end{document}


\chapter{Introduction}

\section{Motivation}
Every year, dozens of new programming languages are proposed and each time this
happens,  the inventors of the new language have to answer the same question:
There are hundreds of programming languages already, why do we need yet another one?
Of course, the answer is always the same.  It consists of an economical argument, a
theological argument and a practical argument. For the
convenience of the reader, let us brievely review these arguments as they read for \setlx.
\begin{enumerate}
\item Nothing less than the prosperity and welfare of the entire universe is at
      stake and \setlx\ provides the means to save it.
\item Programming in \setlx\ is the only way to guarantee redemption from
      the eternal hell fire that awaits those not programming in \setlx.
\item Programming in \setlx\ is fun!
\end{enumerate}
The economical argument has already been discussed at length by Adams \cite{adams:1980},
therefore we don't have to repeat it here.   We deeply
regret the fact that the philosophical background of the average computer scientist does not permit
them to follow advanced theological discussions.  Therefore, we have to refrain from giving a
detailed proof of the second claim.  Nevertheless, we hope the examples given in this
tutorial will convince the reader of the truth of the third claim.
  One of the reasons for this is that \setlx\ programs are both very concise and
readable.  This often makes it possible to fit the implementation of complex algorithms in \setlx\ on a
single slide because the \setlx\ program is no longer than the pseudocode that is usually used to
present algorithms in lectures.  The benefit of this is that instead of pseudocode, students have a
running program that they can test and modify. This and the conciseness of \textsc{SetlX} programs was one of
the reasons for the first author to adopt \setlx\ as a programming language in his various lectures on computer
science and mathematics: It is often feasible to write a complete \setlx\ program in a few lines
onto the blackboard, since \setlx\ programs are nearly as compact as mathematical formul\ae.

\setlx\ is well suited to implement complex algorithms. This is achieved because SetlX
provides a number of sophisticated builtin data types that enable the user to code at a very high
abstraction level.
These data types are sets, lists, first-order terms, and functions.
As sets are implemented as ordered binary trees, sets of pairs can represent
\href{https://en.wikipedia.org/wiki/Map_(mathematics)}{mappings} as well as
\href{https://en.wikipedia.org/wiki/Priority_queue}{priority queues}.  This enables very neat implementations
of a number of graph theoretical algorithms.  As \textsc{SetlX} is implemented in \textsl{Java} and
the \emph{Java virtual machine} is quite fast, various tests have shown that \textsc{SetlX} beats
\textsl{Python} in terms of performance.

The purpose of this tutorial is to introduce the most important features of \setlx\ and to
show, how the use of the above mentioned data types leads to programs that are both shorter
and clearer than the corresponding programs in other programming languages.  This was the
prime motivation of the first author to develop \setlx:  It turns out that \setlx\ is very
convenient as a tool to present algorithms at a high abstraction level in a class room.
Furthermore, \setlx\ makes the abstract concepts of set theory tangible for students of
computer science.

\section{Overview}
The remainder
of this tutorial is structured as follows:
\begin{enumerate}
\item In the second chapter, we discuss the data types available in \setlx.
\item The third chapter provides the control structures.
\item The fourth chapter deals with \emph{regular expressions}.
\item The fifth chapter discusses \emph{functional programming} and \emph{closures}.
\item The sixth chapter discusses the \texttt{try-}\texttt{catch} and \texttt{throw}
      mechanism and demonstrates the use of \emph{backtracking}.
\item The seventh chapter discusses how vectors and matrices can be used and demonstrates the linear
      algebra package that is provided by \textsc{SetlX}.
\item The eighth chapter discusses the statistical functions provided by \textsc{SetlX}.
\item The ninth chapter presents the functions that can be used for plotting.
\item Chapter number ten introduces classes and demonstrates that \textsc{SetlX}
      supports an object-oriented programming style.
\item The final chapter lists and explains all predefined functions.
\item The appendix discusses the graphical primitives provided by \textsc{SetlX}.
      These primitives can be used, for example, to animate algorithms.
\end{enumerate}
This tutorial is not meant as an introduction to programming.  It assumes that the reader
has had some preliminary exposure to programming and has already written a few programs in
either \texttt{C}, \textsl{Java}, or a similar language.

\section{Downloading}
The current distribution of  \setlx\ can be downloaded from
\\[0.2cm]
\hspace*{1.3cm}
\href{http://randoom.org/Software/SetlX}{\texttt{http://randoom.org/Software/SetlX}}.
\\[0.2cm]
\setlx\ is
written in \textsl{Java} and is therefore supported on a number of different operating
systems.  Currently, \setlx\ is supported on \textsl{Linux}, \textsl{OS X},
\textsl{Microsoft Windows}, and
\href{https://play.google.com/store/apps/details?id=org.randoom.setlxUI.android}{\textsl{Android}}.
The websites given above explains how to install the language on various platforms.
The distribution contains the \textsl{Java} code and a development guide that gives an
overview of the implementation. For those
interested in peeking under the hood, the source code of \textsc{SetlX}
is available at
\\[0.2cm]
\hspace*{1.3cm}
\href{https://github.com/herrmanntom/setlX}{\texttt{https://github.com/herrmanntom/setlX}}.

\subsection*{Disclaimer}
The development of \setlx\ is an ongoing project.  Therefore some of the material presented in
this tutorial might be out of date, while certain aspects of the language won't be
covered.  The current version of this tutorial is not intended to be a reference manual.
The idea is rather to provide the reader with an introduction that is sufficient to get started.

\subsection*{Encouragement}
The authors would be grateful for any kind of feedback.  They can be contacted via
email as follows:
\begin{tabbing}
\qquad \= Karl Stroetmann: \qquad \= \href{mailto:karl.stroetmann@dhbw-mannheim.de}{\texttt{karl.stroetmann@dhbw-mannheim.de}} \\[0.2cm]
       \> Tom Herrmann:           \> \href{mailto:setlx@randoom.org}{\texttt{setlx@randoom.org}}
\end{tabbing}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "tutorial"
%%% End: