\documentclass[english, aspectratio=169]{beamer}
% english is for the language used in standard texts (figures, tables etc)
% aspectratio of 16:9 or set it for more old school to 4:3 (without the ':')

% ---------------------------------------------------------------------------- %
% Load base preamble
% ---------------------------------------------------------------------------- %

% Import from absolute path
% \usepackage{import}
% \import{C:/GitHub/LaTeX-Preamble_and_Examples/preamble/}{beamer.tex}

% Import from a relative path
\usepackage{import}
\subimport{../preamble/}{beamer.tex}

% ---------------------------------------------------------------------------- %
% Local macros
% ---------------------------------------------------------------------------- %

% It is quite common that you are going to write the same thing multiple times.
% As part of the preamble I have predefined some macros for what I keep on
% having to use, and while I endorse you send a pull request with more, here you
% can add the ones you only need for this one document.

% This one is used in the files imported using \input.
\newcommand{\Gmul}{\ensuremath{\calG_{\mathrm{mul}}}}

% ----------------------------------------------------------------------------------------
% TITLEPAGE
% ----------------------------------------------------------------------------------------
\title{Reverse engineerable \LaTeX examples:}
\subtitle{Beamer presentations}

\author{\textbf{Steffan Christ S\o lvsten}}

\institute{Aarhus University}

\date{\today}

% ---------------------------------------------------------------------------- %
% Content
% ---------------------------------------------------------------------------- %
\begin{document}

\titleframe

\section{Basic slides}
\begin{frame}
  This is a very simple slide, which contains some math symbols such as
  $\sigma$, and also a math equation such as
  \begin{equation*}
    \sum_{i=1}^n i = \frac{n(n+1)}{2}
  \end{equation*}
  Here is a theorem with a proof

  \begin{theorem}[Euler]
    If $n$ and $a$ are coprime positive integers, then $a^{\phi(n)} \equiv 1
    \mod n$
  \end{theorem}
  \begin{proof}
    Left to the reader.
  \end{proof}
\end{frame}

\begin{frame}
  It is quite common to have your slide divided into two or more parts. This is
  done using columns.

  \begin{columns}
    \begin{column}{0.5\textwidth}
      Here is some text in the left column
    \end{column}
    \begin{column}{0.5\textwidth}
      Here is some text in the right column
    \end{column}
  \end{columns}
\end{frame}

\section{Code}
\begin{frame}[fragile]
  You can have code snippets on your slides just the same way you do in the
  documents. You only have to mark the frame \emph{fragile}, such that overflows
  do not immediately break the compilation.

  \begin{lstlisting}[language=coq, caption={Exercise on proving strong induction
      in \emph{Software Foundations - Volume 1}}]
  Theorem strong_induction :
    forall P : nat -> Prop,
    (forall n : nat, (forall m : nat, m < n -> P m) -> P n) ->
    forall n : nat, P n.
  Proof.
    intros P IH_strong n.
    assert (H : forall k, k <= n -> P k).
    { ... }
    now apply H.
  Qed.
  \end{lstlisting}
\end{frame}

\section{Figures, Tables, and Graphics}
\begin{frame}
  \frametitle{Tables}
  Tables are just as simple as you are used to.

  \begin{table}
    \centering
    \begin{tabular}{c|c}
      A cell & Another one
      \\ \hline
      $\gamma$ & $\beta$
    \end{tabular}

    \caption{A table}
    \label{tab:label}
  \end{table}
\end{frame}

\begin{frame}
  \frametitle{Graphics}
  Figures are also exactly as easy. That means, you can actually have the whole
  figure in another file, that you can include in both an article and in a
  presentation!

  \begin{figure}
    \centering
    \input{figures/polynomia.tex}

    \caption{The label for the figure.}
    \label{fig:label}
  \end{figure}
\end{frame}

\section{Blank frames}

\begin{frame}
  A completely blank frame is such an underrated tool at bringing full attention
  to you and what you are saying. Specifically for this I have made the
  \lstinline{\\blankframe} command.
\end{frame}

\blankframe

\section{Animations}
\subsection{Simple use of pause and onslide}
\begin{frame}
  \frametitle{pause command}
  To animate slides it is luckily not necessary to copy paste all of the code
  and change one thing after the other! Otherwise Bærbak would definitely have
  been very sad.

  The simplest animations you need is slowly revealing the slide, such as the
  bullet points below. This is done using the \lstinline{\\pause}
  command.

  \begin{itemize}
  \item An item, that is shown at the very beginning.

    \pause

  \item An item, that is first shown on the ``next slide''
  \end{itemize}

  \pause

  It of course also works for enumeration
  \begin{enumerate}
  \item<+-> To not make the \LaTeX\ too heavy on commands the items support a
    shorthand for the onslide notation on the next slide
    onslide command further explained on the next slide
  \item<+-> Finally, if you need to compile handouts without animations, just add
    \lstinline{handout} in the \lstinline{documentclass} above
  \end{enumerate}
\end{frame}

\begin{frame}
  \frametitle{onslide command}

  To do more complicated animations you want to trigger parts of the slide at
  different times. You do this using the \lstinline{\\onslide<a-b>} where $a$,
  and $b$ are optional slide numbers.

  \onslide<2->{This paragraph is shown on the second step and forwards}

  \onslide<2-3>{This paragraph is shown on the second and third steps of the slide}

  \onslide<1-2>{This paragraph is shown on the first two steps of the slides}

  \onslide<4>{This paragraph is shown on the fourth (last) step of the slides}
\end{frame}

\subsection{Tikz animations}
\begin{frame}
  \frametitle{Animated Tikz}

  \begin{columns}
    \begin{column}{0.5\linewidth}
      Here is a simple example using the \lstinline{\\onslide} command to
      trigger different parts of the slide at different times.
    \end{column}
    \begin{column}{0.5\linewidth}
      \begin{tikzpicture}[thick]
        \node[state, label=left:$\rightarrow$] (q0) {$q_0$};
        \node[state, above left=of q0] (q1) {$q_1$};
        \node[state, below left=1cm and 0.2cm of q0] (q2) {$q_2$};
        \node[state, below right=1cm and 0.2cm of q0] (q3) {$q_3$};
        \node[state, accepting, above right=of q0] (q4) {$q_4$};

        % path 1
        \draw[->]
        (q0) edge[bend left] node[below left] {$a$} (q1)
        (q1) edge[bend left] node[above right] {$a$} (q0)
        (q0) edge node[below right] {$b$} (q4)
        ;

        % path 1
        \draw[->]
        (q0) edge[bend right] node[left] {$a$} (q2)
        (q2) edge[bend right] node[below] {$a$} (q3)
        (q3) edge[bend right] node[right] {$b$} (q0)
        ;

        \onslide<2->{
          \draw[->, cyan]
          (q0) edge[bend left] (q1)
          (q1) edge[bend left] (q0)
          (q0) edge (q4)
          ;
          \node[state, accepting, above right=of q0, cyan] (q4) {$q_4$};
        }
      \end{tikzpicture}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Animated Tikz}

  \begin{columns}
    \begin{column}{0.5\linewidth}
      Here is a much more complicated example together with an animation in
      sync below.

      \begin{align*}
        \{ q_0 \} &\onslide<2-5>{\rightarrow \{ q_1, q_2 \}}
        \\ &\onslide<3-5>{\rightarrow \{ q_0, q_3 \}}
        \\ &\onslide<4-5>{\rightarrow \{ q_0, q_4 \}}
      \end{align*}

      \onslide<4-5>{Are any of the final states accepting?}
      \onslide<5-5>{Yes, $q_4$ is!}
    \end{column}
    \begin{column}{0.5\linewidth}
      \begin{tikzpicture}[thick]
        \node[state, label=left:$\rightarrow$] (q0) {$q_0$};
        \node[state, above left=of q0] (q1) {$q_1$};
        \node[state, below left=1cm and 0.2cm of q0] (q2) {$q_2$};
        \node[state, below right=1cm and 0.2cm of q0] (q3) {$q_3$};
        \node[state, accepting, above right=of q0] (q4) {$q_4$};

        % path 1
        \draw[->]
          (q0) edge[bend left] node[below left] {$a$} (q1)
          (q1) edge[bend left] node[above right] {$a$} (q0)
          (q0) edge node[below right] {$b$} (q4)
        ;

        % path 1
        \draw[->]
          (q0) edge[bend right] node[left] {$a$} (q2)
          (q2) edge[bend right] node[below] {$a$} (q3)
          (q3) edge[bend right] node[right] {$b$} (q0)
        ;

        % animation
        \onslide<1-1>{
          \node[state, label=left:$\rightarrow$, cyan] {$q_0$};
        }

        \onslide<2-2>{
          \draw[->, cyan]
            (q0) edge[bend left] node[below left] {$a$} (q1)
            (q0) edge[bend right] node[left] {$a$} (q2)
          ;
          \node[state, above left=of q0, cyan] {$q_1$};
          \node[state, below left=1cm and 0.2cm of q0, cyan] {$q_2$};
        }

        \onslide<3-3>{
          \draw[->, cyan]
            (q1) edge[bend left] node[above right] {$a$} (q0)
            (q2) edge[bend right] node[below] {$a$} (q3)
          ;
          \node[state, label=left:$\rightarrow$, cyan] {$q_0$};
          \node[state, below right=1cm and 0.2cm of q0, cyan] {$q_3$};
        }

        \onslide<4-4>{
          \draw[->, cyan]
            (q0) edge node[below right] {$b$} (q4)
            (q3) edge[bend right] node[right] {$b$} (q0)
          ;
          \onslide<4-5>
          \node[state, cyan] {$q_0$};
          \node[state, accepting, above right=of q0, cyan] {$q_4$};
        }
      \end{tikzpicture}
    \end{column}
  \end{columns}
\end{frame}

\subsection{Animating code}
\begin{frame}[fragile]
  By escaping from lstlisting out to \LaTeX\ using the
  \texttt{*@}\dots\texttt{@*} macro you can also slowly reveal code. This could
  be useful for showing a \emph{Coq} proof as below
  \begin{lstlisting}[language=coq]
  Theorem plus_n_O : forall n:nat, n = n + 0.
  Proof.*@\pause@*
    induction n as [| n' IHn'].*@\pause@*
    - (* n = 0 *)
      reflexivity.*@\pause@*
    - (* n = S n' *)
      simpl. rewrite <- IHn'. reflexivity.*@\pause@*
  Qed.
  \end{lstlisting}
\end{frame}

\begin{frame}[plain,noframenumbering]{}

  {\Large \textbf{Steffan Christ Sølvsten}}
  {\hrule width0.5\linewidth}

  \vspace{5pt}

  \begin{itemize}
  \item[\faIcon{envelope}] \mailto{soelvsten@cs.au.dk}
  \item[\faIcon{twitter}] \href{https://www.twitter.com/ssoelvsten}{@ssoelvsten}
  \end{itemize}
\end{frame}

\end{document}


\documentclass[a4paper,12pt]{article}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{color}
\usepackage{ulem}
\usepackage{multicol}
\usepackage{pifont}
\usepackage{amsthm}
\usepackage{tabularx}
\usepackage[linesnumbered,lined,boxed,commentsnumbered,ruled,vlined]{algorithm2e}
\usepackage[utf8]{inputenc}
\usepackage{xcolor}

\theoremstyle{definition}
\newtheorem{example}{Example}[section]
\newtheorem{definition}{Definition}
\newtheorem{remark}{Remark}

\newcommand{\cmark}{\ding{51}}
\newcommand{\xmark}{\ding{55}}
\newcommand{\forcond}{$i=0$ \KwTo $n$}

\title{\textbf{Design Analysis of Algorithms}}
\author{Keymaker}
\date{August 2021}

\definecolor{classicpink}{rgb}{0.95,0.88,0.90}
%\pagecolor{classicpink}

\pagecolor{black}
\color{white}

\begin{document}

    \maketitle

    \section{What is an algorithm?}

    \underline{Finite} set of steps to solve a problem is called an algorithm. And there shouldn't be any ambiguity, it should follow proper conventions and notations.

    \begin{enumerate}
        \item Analysis is process of comparing two algorithms with respect to time, space, etc.
        \item Type of analysis:
        \begin{itemize}
            \item \textbf{Priory}: It is the analysis before the execution of an algorithm. Note, that priory analysis is independent of the hardware.
            \item \textbf{Posterior}: It is the analysis after the execution of the algorithm. Also, note that, posterior analysis is dependent on a particular hardware.
        \end{itemize}
    \end{enumerate}

    \subsection{Examples}

    \noindent Algorithm for adding two numbers:
    \begin{enumerate}
        \itemsep 0mm
        \item Start
        \item Read A and B
        \item Sum of A and B
        \item Display the result
        \item End
    \end{enumerate}

    \noindent Algorithm for calculating an exponent of a number:
    \begin{enumerate}
        \itemsep 0mm
        \item Start
        \item Read A and B
        \item Power of A to the power B
        \item Display the result
        \item End
    \end{enumerate}

    \noindent
    Priory analysis mostly uses asymptotic notations such as $\Theta$ called ``Big Theta", $\Omega$ called ``Big Omega" and $O$ called ``Big Oh''

    \section*{Asymptotic Notations}

    Asymptotic notations are called mathematical way of representing the time complexity. We need a proper mathematical notation to say --- represent the number of times a statement is executed or, the number of times a function call occurs or, the frequency of calls or, perhaps the magnitude, all of which needs a notation that'll represent such property.

    \section{The Big Oh notation}

    \textbf{Big Oh} (with a capital letter O, not a zero), also called Landau's symbol, is a symbolism used in complexity theory, computer science, and mathematics to describe the asymptotic behavior of functions. Basically, it tells you how fast a function grows or declines.

    \subsection{Definitions}

    A formal definition would be, suppose $f(x)$ and $g(x)$ are two functions defined on some subset of real numbers. We write

    \begin{center}
        $f(x) = O(g(x))$
    \end{center}

    \noindent
    (or $f(x) = O(g(x))$ for $x \rightarrow \infty$ to be more precise) if and only if there exist constants $N$ and $C$ such that

    \begin{center}
        $|f(x)| \leq C \cdot |g(x)| \forall x > N$
    \end{center}

    \noindent
    Informally, the Big Oh notation represents the upper bound of the algorithm or, problem. That is, how much time \emph{at most} will an algorithm take to complete a task. It's the worst case of an algorithm. So, $f(x)$ can be represented in $O(g(x))$ such that

    \begin{center}
        $|f(x)| \leq C \cdot |g(x)| : (C > 0 \wedge x > K) : (K > 0 \wedge x \rightarrow \infty)$
    \end{center}

    \subsection{Example}

    \noindent
    Let $f(x) = 2x^2 + x$ Now, we need to represent $f(x)$ in terms of order i.e.,
    $|2x^2 + x| \leq C \cdot |g(x)|$ where the unknown value (i.e., the value of $g(x)$) is the
    value you'd have to choose out of the expression. Now, recall what the definition of Big Oh was i.e.,
    we need to pick out the most dominating term out of $f(x)$.

    \noindent
    In this case, that is, $x^2$.
    And, as for the value of $C$ we can try taking it as 2 first so, the inequality will be something like:

    \begin{center}
        $|f(x)| \leq C \cdot |g(x)| \implies |2x^2 + x| \leq 2 \cdot |x^2|$
    \end{center}

    \noindent
    which clearly is a contradiction as:

    \begin{center}
        $\forall x \in \mathbb{Z^+} \therefore |f(x)| \nleq 2 \cdot |g(x)|$.
    \end{center}

    \noindent
    Now, we would need to look for another value which satisfies this $|f(x)| \leq C \cdot |g(x)|$ inequality.
    And, the value immediately after 2 is 3. So, now let, $C = 3$ and now we see that the inequality holds:

    \begin{center}
        $|2x^2 + x| \leq 3 \cdot |g(x)| \implies |2x^2 + x| \leq 3 \cdot |x^2|$
    \end{center}

    \noindent
    So finally, for all the values of inputs and $C$ value should be 3,
    $\forall x\ |\ x \geq 1 \wedge x \leq x^2 \wedge x \geq 1$ the original equation will always hold.

    \section{Big Omega}

    Similar to Big Oh notation, \textbf{Big Omega} ($\Omega$) function is used in computer science to describe the
    performance or complexity of an algorithm. If a running time is (f(x)), then for large enough n, the running
    time is at least $Kf(x)$ for some constant k. Here’s how to think of a running time that is $\Omega$(f(x)).

    We say that the running time is ``Big-$\Omega$ of $f(x)$." We use Big-$\Omega$ notation for asymptotic lower bounds,
    since it bounds the growth of the running time from below for large enough input sizes.

    \subsection{Definitions}

    Informally, this asymptotic notation defines the lower bound of an algorithm.
    That is, this algorithm represents that how much time at least is taken by the
    algorithm to execute into completion.
    It is the best case of the algorithm.

    A formal definition would be, suppose $f(x)$ and $g(x)$ are two functions defined on some subset of real numbers.
    We write

    \begin{center}
        $f(x) = \Omega(g(x))$
    \end{center}

    \noindent
    (or $f(x) = \Omega(g(x))$ for $x \rightarrow \infty$ to be more precise) if and only if there exist constants $N$ and $C$ such that

    \begin{center}
        $|f(x)| \geq C \cdot |g(x)|$ for all $x > N$
    \end{center}

    \subsection{Example}

    Again lets take that same example i.e., let $f(x) = 2x^2 + x$. So, we can write:

    \begin{center}
        $|f(x)| \geq C \cdot |g(x)| \implies |2x^2 + x| \geq C \cdot |n^2|$
    \end{center}

    \noindent
    Now, the reason behind taking $g(x) = x^2$ is because, $\Omega(g(x))$ dictates that the greatest lower bound should be
    taken which is the exact opposite of Big Oh which takes the least upper bound.
    Hence, now again take $C$ value as 2:

    \begin{center}
        $|f(x)| \geq 2 \cdot |g(x)| \implies |2x^2 + x| \geq \cdot |x^2|$
    \end{center}

    \noindent
    And, this time the inequality actually holds.
    Once again note that you can have $g(x)$ equal to anything like in this case $x^2$ or, $x$ but, always remember
    to take the next step carefully i.e., choose the value of $C$ carefully as not to break the inequality.

    \section{Big Theta}

    We say a function $T(x)$ is $\Theta(f(x))$ if it is both Big-Oh of $(f(x))$ and Big-Omega of $(f(x))$.
    A very easy and quick way to find the $\Theta$ is to take the function $T(x)$. Ignore all leading constant factors.
    Ignore all terms apart from the highest order term.
    This is the theta of $T(x)$.

    Additionally, Big-Theta is commonly denoted by $\Theta$, is an Asymptotic Notation to denote the average case analysis
    of an algorithm.
    The theta notation defines exact asymptotic behavior and bounds a function from above and below.

    \subsection{Definition}

    A more formal definition would be let $f(x)$ and $g(x)$ be two functions such that $f(x) = \Theta(g(x))$ if and only
    if there are three constants such that $C_1$, $C_2$ and $x_0$ such that:

    \begin{center}
        $C_1 \cdot |g(x)| \leq |f(x)| \leq C_2 \cdot |g(x)\ |\ \forall x \geq x_0 \implies f(x) = \Theta(g(n))$
    \end{center}

    \noindent
    If $f(x)$ is non-negative, we can simplify the last condition to:

    \begin{center}
        $0 \leq C_1 \cdot g(x) \leq C_2 \cdot g(n) \forall x \geq x_0 \implies f(x) = \Theta(g(n))$
    \end{center}

    \noindent
    Therefore, as $x$ increases, $f(n)$ grows at the same rate as $g(n)$.
    In other words, $g(n)$ is an asymptotically tight bound on $f(n)$.

    \subsection{Example}

    Let, $x^2 + 5x + 7 = \Theta(x^2)$ where $f(x) = x^2 + 5x + 7$, also, $g(x) = x^2$
    \noindent
    when $x \geq 1$, $x^2 + 5x + 7 \geq x^2 + 5x^2 + 7x^2 \geq 13x^2$
    \noindent
    when $x \geq 0$, $x^2 \geq x^2 + 5x + 7$
    \noindent
    thus, when $x \geq 1$, $1x^2 \leq x^2 + 5x + 7 \leq 13x^2$
    \noindent
    Thus, we have shown that
    $x^2 + 5x + 7 = \Theta(x^2)$ (by definition of Big-$\Theta$, with $x_0 = 1$, $C_1 = 1$, $C_2 = 13$.)

    \section{Little Oh}

    It is almost similar to their counterparts i.e. for Little Oh we take a look at its counterpart, Big Oh, recall:

    \begin{center}
        $f(x) = O(g(x)) \Leftrightarrow \exists\:N,C : |f(x)| \leq C|g(x)|\ \forall x > N$
    \end{center}

    \subsection{Definition}

    Now seeing that take a note of the $\leq$ part, we define that little oh notation is used to
    describe an upper bound that cannot be tight.
    In other words, loose upper bound of $f(x)$.

    Let $f(x)$ and $g(x)$ are the functions that map positive reals.
    We can say that the function $f(x)$ is $o(g(x))$ if for any real positive constant $C$,
    there exists an integer constant $x_0 \leq 1$ such that $f(x) > 0$.

    Also in terms of mathematical notations, the definition will be:

    \begin{center}
        $f(x) = O(g(x)) \Leftrightarrow \exists\:N,C : |f(x)| < C|g(x)|\ \forall x > N$
    \end{center}

    \subsection{Mathematical~Relation}

    Now using the ``Definition" section as basis we can conclude that:

    \begin{center}
        $\lim_{x \rightarrow \infty}{\frac{f(x)}{g(x)}} = 0$
    \end{center}

    \subsection{Example}

    If $f(x) = x^2$ and $g(x) = x^3$ then check whether $f(x) = o(g(x))$ or not.

    \parindent 8mm
    $= \lim_{x \rightarrow \infty}{\frac{f(x)}{g(x)}}$

    \parindent 8mm
    $= \lim_{x \rightarrow \infty}{\frac{x^2}{x^3}}$

    \parindent 8mm
    $= \lim_{x \rightarrow \infty}{\frac{1}{\infty}}$

    \parindent 8mm
    $= 0$

    \noindent
    The result is $0$, and it satisfies the equation mentioned above. So we can say that $f(x) = o(g(x))$.

    \section{Little Omega}

    Let $f(x)$ and $g(x)$ be functions that map positive integers to positive real numbers.
    We say that $f(x) = \omega(g(x))$ or $f(x) \in \omega(g(x))$ if for any real constant $C > 0$,
    there exists an integer constant $x_0 \geq 1$ such that $f(x) > C \cdot g(x) \geq 0$ for every integer $x \geq x_0$.

    \subsection{Definition}

    $f(x)$ has a higher growth rate than $g(x)$ so main difference between Big Omega ($\Omega$) and little
    omega ($\omega$) lies in their definitions.
    In the case of Big Omega $f(x) = \Omega(g(x))$ and the bound is $0 \leq C \cdot g(x) \leq f(x)$,
    but in case of little omega, it is true for $0 \leq C \cdot g(x) < f(x)$.

    The relationship between Big Omega ($\Omega$) and Little Omega ($\omega$) is similar to that of Big-O and Little-o
    except that now we are looking at the lower bounds.
    Little Omega ($\omega$) is a rough estimate of the order of the growth whereas Big Omega ($\Omega$) may
    represent exact order of growth.
    We use $\omega$ notation to denote a lower bound that is not asymptotically tight.
    And:

    \begin{center}
        $f(x) \in \omega(g(x)) \Leftrightarrow g(x) \in o(f(x))$
    \end{center}

    \subsection{Mathematical Relation}

    Now using the ``Definition'' section as basis we can conclude that
    if $f(x) \in \omega(g(x))$ then:

    \begin{center}
        $\lim_{x \rightarrow \infty}{\frac{f(x)}{g(x)}} = \infty$
    \end{center}

    \subsection{Example}

    Prove that $4x + 6 \in \omega(1)$

    The little omega($\omega$) running time can be proven by applying limit formula given below.
    if $\lim_{x \rightarrow \infty}{\frac{f(x)}{g(x)}} = \infty$ then functions $f(x)$ is $\omega(g(x))$
    here, we have functions $f(x) = 4x+6$ and $g(x) = 1$

    \begin{center}
        $\lim_{x \rightarrow \infty}{\frac{f(x)}{g(x)}} = \infty$\\
        $\lim_{x \rightarrow \infty}{\frac{4n + 6}{1}} = \infty$
    \end{center}

    And, also for any $C$ we can get $x_0$ for this inequality $0 \leq c \cdot g(x) < f(x),\ 0 \leq C \cdot 1 < 4n + 6$
    Hence proved.

    \section{Asymptotic Properties}

    First a bit of recapitulation, we know that:

    \begin{equation}
        Big(O); f(n) \leq C \cdot g(n)
        \label{eq:bigoh}
    \end{equation}
    \begin{equation}
        Big(\Omega); f(n) \geq C \cdot g(n)
        \label{eq:bigom}
    \end{equation}
    \begin{equation}
        Big(\Theta);\: C_1 \cdot g(n) \leq f(n) \leq C_2 \cdot g(n)
        \label{eq:bigth}
    \end{equation}

    \noindent
    Here, for simplicity just let $a = f(n)$ and $b = C \cdot g(n)$ now, therefore we can re-write (\ref{eq:bigoh}), (\ref{eq:bigom}) and (\ref{eq:bigth}) as:

    Big($O$); $a \leq b$

    Big($\Omega$); $a \geq b$

    Big($\Theta$); $b_1 \leq a \leq b_2$

    or, simply $f(n) = C \cdot g(n)$

    i.e., $a = b$

    \noindent
    And, follow these same steps for little(o) and little($\omega$).

    \section{Asymptotic Properties}

    This section will note the Reflexive, Symmetric and Transitive properties of the asymptotic notations that
    was introduced before.
    Such as the big oh, omega and theta notations and also their counterparts such as little oh and omega.

    \begin{table}[!hbt]
        \centering
        \caption{Reflexive, Symmetric and Transitive properties of Asymptotes}
        \label{tb:asmp}
        % alignments: c - center, l - left, r - right
        \begin{tabular}{|c|c|c|c|c|}
            \hline \textbf{Serial} & \textbf{Variant}        & \textbf{Reflexive} & \textbf{Symmetric} & \textbf{Transitive} \\
            \hline 1               & Big Oh ($O$)            & \cmark             & \xmark             & \cmark              \\
            \hline 2               & Big Omega ($\Omega$)    & \cmark             & \xmark             & \cmark              \\
            \hline 3               & Big Theta ($\Theta$)    & \cmark             & \cmark             & \cmark              \\
            \hline 4               & Little Oh ($o$)         & \xmark             & \xmark             & \cmark              \\
            \hline 5               & Little Omega ($\omega$) & \xmark             & \xmark             & \cmark              \\
            \hline
        \end{tabular}
    \end{table}

    \subsection{Brief Recapitulation}

    \begin{definition}[Reflexive Property]
        If you look in a mirror, what do you see? Your reflection! You are seeing an image of yourself.
        You could look at the reflexive property of equality as when a number looks across an equal sign and
        sees a mirror image of itself!
        Reflexive pretty much means something relating to itself.
    \end{definition}

    \subsection*{Further Explanation}

    The reflexive property of equality simply states that a value is equal to itself.
    Further, this property states that for all real numbers, $x = x$.
    What is a real number, though?

    Real numbers include all the numbers on a number line.
    They include rational numbers and irrational numbers.
    A rational number is any number that can be written as a fraction.
    An irrational number, on the other hand, is a real number that cannot be written as a simple fraction.
    Square roots would be in this category. In fact, real numbers pretty much entail every number possible except
    for negative square roots because they are imaginary numbers.

    Therefore, the reflexive property of equality pretty much covers most values and numbers.
    Again, it states simply that any value or number is equal to itself.

    \begin{definition}[Symmetric Property]
        The symmetric property of equality states that if two variables $a$ and $b$ exist, and $a = b$, then $b = a$.
        The symmetric property of equality is one of the equivalence properties of equality.
    \end{definition}

    \subsection*{Further Explanation}

    The symmetric property of equality allows individuals to manipulate an equation by flipping the statements on
    each side of the equals sign.
    For example, in algebra, this means that the equations $11 = 2x + 5$ and $2x + 5 = 11$ are equivalent.

    One useful application of the symmetric property of equality is that reorganizing equations makes it easier
    to solve systems of equations. For example, if one had two linear equations, $11 = 2x + 5$ and $-2x + 3y = 6$,
    the symmetric property of equality allows the equations to be ordered so that they can be added together.
    Thanks to the symmetric property of equality, $11 = 2x + 5$ can be transformed to say $2x + 5 = 11$.
    One can add the two equations together to solve for $y$: $(-2x + 3y = 6) + (2x + 5 = 11)$.
    These two equations sum to $3y + 5 = 17$.

    This simplifies to $3y = 12$, or $y = 4$.
    Then by substituting in for $y, -2x + 3(4) = 6$; this yields $-2x + 12 = 6, or -2x = -6, or x = 3$.
    So the solution to the linear system is $x = 3, y = 4$.

    \begin{definition}[Transitive Property]
        The transitive property is also known as the transitive property of equality.
        It states that if two values are equal, and either of those two values is equal to a third value, that all
        the values must be equal.
        This can be expressed as follows, where $a$, $b$, and $c$, are variables that represent the same number:

        \begin{center}
            $if (a = b \wedge b = c) \rightarrow (a = c)$
        \end{center}
    \end{definition}

    \begin{example}
        If a = b, b = c, and c = 2, what are the values of a and b?
    \end{example}

    \noindent
    \textbf{Solution.}
    Given that, $c = 2$, and $b = c$, so plugging 2 in for $c$, we know that $b = 2$. We can then plug 2 into the first
    equation, $a = b$, to find that $a = 2$ as well.
    Therefore, $a = b = c = 2$.

    \section{Complexity Comparison}

    Following is a precedence chart for common time complexities.
    Also read greatness increases from top to bottom order:

    \begin{enumerate}
        \item $O(c)$
        \item $O(\log \log n)$
        \item $O(\log n)$
        \item $O(n^\frac{1}{2})$
        \item $O(n)$
        \item $O(n\log n)$
        \item $O(n^2)$
        \item $O(n^3)$
        \item $O(n^k)$
        \item $O(2^n)$
        \item $O(n^n)$
        \item $O(2^{2^n})$
    \end{enumerate}

    \section{Recurrence Relation}\label{lb:recurrence_main}

    In terms of computer science, a recurrence relation is a method where a function or, a class method calls itself
    over and over again.
    Hence, the term \textit{recurrence} relation.

    \begin{definition}[Recurrence]
        A new occurrence of something that happened or appeared before; a repeated occurrence.
    \end{definition}

    \subsection{Binary Search}

    This concept has been explained with the help of binary search.

    \begin{definition}[name of the definition]
        In computer science, binary search, also known as half-interval search, logarithmic search, or binary chop,
        is a search algorithm that finds the position of a target value within a \textit{sorted} array.
        Binary search compares the target value to the middle element of the array.
        If they are not equal, the half in which the target cannot lie is eliminated and the search continues on the
        remaining half, again taking the middle element to compare to the target value, and repeating this until
        the target value is found.
        If the search ends with the remaining half being empty, the target is not in the array.
    \end{definition}

    \begin{remark}
        Binary search runs in logarithmic time in the worst case, making $O(\log n)$ comparisons, where $n$, n is the
        number of elements in the array.
        Binary search is faster than linear search except for small arrays.
        However, the array must be sorted first to be able to apply binary search.
    \end{remark}

    \subsection*{Algorithm}

    Binary search works on sorted arrays.
    Binary search begins by comparing an element in the middle of the array with the target value.
    If the target value matches the element, its position in the array is returned.
    If the target value is less than the element, the search continues in the lower half of the array.
    If the target value is greater than the element, the search continues in the upper half of the array.
    By doing this, the algorithm eliminates the half in which the target value cannot lie in each iteration.

    \subsection*{Pseudocode}

    \begin{algorithm}[H]
        \SetKwFunction{FRecurs}{BS}
        \SetKwProg{Fn}{Function}{:}{}
        \Fn(\tcc*[h]{code for binary search}){\FRecurs{a, i, j, x}}{
            \tcc{i \& j are indices or, commonly high \& low}
            $mid \leftarrow (i + f) / 2$\\
            \If(\tcc*[h]{return the index}){$a[mid] == x$}{
                return $mid$
            }\Else{
                \uIf{$a[mid] > x$}{
                    BS(a, i, mid - 1, x)
                }\Else{
                    BS(a, mid + 1, j, x)
                }
            }
        }
    \end{algorithm}

    \subsection*{Test Cases}
    Lets, take the key as 10 in a list of sorted natural numbers, such that ~[10,~20,~30,~40,~50,~60,~70].
    Therefore,here the length of the list is 7 and the first and last indices are 1 and 7.

    \begin{enumerate}
        \item So, the code will calculate the ``mid" variable's value i.e. $\frac{(i + j)}{2} = \frac{1 + 7}{2} = 4$.
        So, this part means we are dividing a problem into two smaller sub-problems or, in this case we're
        dividing a list into two smaller lists.
        \item Now, the code will move onto the \textit{if block} and check if the middle element is the value we want
        or not i.e., a[mid] is 40 but, we want to know where 10 is so, this isn't the right index.
        \item The match is false now the code will be moved to the \textit{else block} then, the nested
        if block will be executed.
        \item Now, the nested if block will check if a[mid] is greater than the value to be found or, not i.e., if 40
        is greater than 10 or not.
        Which is true and hence the code will now start a recursive call and the list gets sliced in half so the
        current value of the list will be [10, 20, 30].
        And, the arguments that will be passed are BS([10,~20,~30,~40,~50,~60,~70], 1, 3, 10)
        \item Firstly, the new i and j value will be 1 and 3.
        Again, the mid-value will be recalculated i.e., $\frac{(1 + 3)}{2} = \frac{4}{2} = 2$.
        \item Then as before the if statement will again check for a[mid] is 10 or, not i.e., false as a[mid] is
        actually 20.
        So, it will move onto the else block.
        \item In the else block the code will move onto the nested if block first and will check if a[mid] is greater
        than x or not i.e., 20 is greater than 10 or, not. Which is true, and yet again another recursive call
        will occur with the list sliced in half i.e. [10].
        \item Now, the arguments that will be passed are\\ BS([10,~20,~30,~40,~50,~60,~70], 1, 1, 10) i.e., i and j
        will 1 and 1.
        And, yet again, the mid-value will be calculated first i.e.,$\frac{(1 + 1)}{2} = \frac{2}{2} = 1$.
        \item Finally, the if block will be executed i.e., is a[mid] is equal to x or not; true as mid[1] is 10 and
        x is 10 therefore the value has been found!
    \end{enumerate}

    \subsection*{Recurrence}\label{lb:recurrence}

    \noindent
    This is the main topic of this section i.e. for binary search the recurrence relation will be
    $T(n) = T(\frac{n}{2}) + C$

    \noindent
    Why? Because the list gets sliced into two parts upon each iteration or, more precisely, on each recursive call.
    Also, note that the $C$ signifies the statements that are being done in constant time like the calculation
    of mid-value for instance.

    \section{Substitution Method}

    \noindent
    Consider the following recurrence relation equation which was derived in the previous section's
    subsection \ref{lb:recurrence} i.e., also called Binary Search Algorithm:

    \begin{equation*}
        \label{eq:recurrence_main}
        T(n) = \begin{Bmatrix}
                   T(\frac{n}{2}) + C & if\ n > 1 \\
                   1                  & if\ n = 1
        \end{Bmatrix}
    \end{equation*}

    \noindent
    Now, for this: $T(n) = T(\frac{n}{2}) + C$ we have to check whether the value of $n$ is decreasing or,
    not i.e. substitute ($n = \frac{n}{2}$) and we'll have $T(\frac{n}{2}) = T(\frac{n}{4}) + C$ and so on until
    it reaches the end like:

    \begin{equation}
        \label{eq:unsubtituted}
        T(n) = T\left(\frac{n}{2}\right) + C
    \end{equation}

    \begin{equation}
        \label{eq:firstsubstitution}
        T\left(\frac{n}{2}\right) = T\left(\frac{n}{4}\right) + C
    \end{equation}

    \begin{equation}
        \label{eq:secondsubstitution}
        T\left(\frac{n}{4}\right) = T\left(\frac{n}{8}\right) + C
    \end{equation}

    \subsection*{Backward Substitution}

    \noindent
    This is a part of substitution method.
    Now, we need to substitute the values that we've acquired from continuous substitution backwards i.e., apply
    (\ref{eq:secondsubstitution}) equation to (\ref{eq:firstsubstitution}), we have:

    \begin{align}
        \label{eqs:firstsubstituted}
        T\left(\frac{n}{2}\right) & = \left\{T\left(\frac{n}{8}\right) + C\right\} + C \\
        & = T\left(\frac{n}{8}\right) + 2C
    \end{align}

    \noindent
    Similarly, substituting equation (\ref{eqs:firstsubstituted}) to (\ref{eq:unsubtituted}) we have:

    \begin{align}
        T(n) & = \left\{T\left(\frac{n}{8}\right) + 2C\right\} + C \\
        & = T\left(\frac{n}{8}\right) + 3C                    \\
        & = T\left(\frac{n}{2^3}\right) + 3C
    \end{align}

    \noindent
    Hence, from these tests we can infer a pattern of sorts i.e. the power of 2 is increasing by 1 upon each
    iteration and the co-efficient of $C$ is also increases as same as the power of 2 i.e.:

    \begin{align*}
        T(n)                       & = T\left(\frac{n}{2}\right) + C  \\
        T\left(\frac{n}{2}\right)  & = T\left(\frac{n}{4}\right) + C  \\
        T\left(\frac{n}{4}\right)  & = T\left(\frac{n}{8}\right) + C  \\
        T\left(\frac{n}{8}\right)  & = T\left(\frac{n}{16}\right) + C \\
        T\left(\frac{n}{16}\right) & = T\left(\frac{n}{32}\right) + C
    \end{align*}

    \noindent
    Or, a general term such as:

    \begin{center}
        $T\left\{\frac{n}{\left(\frac{K}{2}\right)}\right\} = T\left(\frac{n}{2^K}\right) + K \cdot C$
    \end{center}

    \noindent
    Now, we need to write the termination~equation\footnote{Also known as base case, base condition, termination
    condition, anchor case and anchor condition}, that is, we need to think of a condition when the value of $T(n)$
    will be $T(1)$ in the Right Hand Part. After a bit of thinking you probably would've noticed that $2^K$ needs to
    be the same as $n$ in order for it to be one i.e.

    \begin{center}
        Let, $2^K = n$\\
        $\therefore T\left\{\frac{n}{\left(\frac{K}{2}\right)}\right\} = T(\frac{n}{n}) +
        K \cdot C \implies T\left\{\frac{n}{\left(\frac{K}{2}\right)}\right\} = T(1) + K \cdot C$
    \end{center}

    \noindent
    Now, we need to write the Time Complexity of the algorithm which should be in terms of $n$ by convention:

    \begin{center}
        Given that: $n = 2^K$\\
        Taking $\log$ on both sides: $\log n = \log 2^K$\\
        As, $\log a^b = b\log a \therefore \log n = K\log 2$
        Finally, $K = \log \left(n - 2\right)$
    \end{center}

    \noindent
    Now, according to (\ref{eq:recurrence_main}) we have $T(1) = 1 \therefore T(1) = 1 + K \cdot C$ and finally,
    we have:

    \begin{align*}
        T(1) & = 1 + K \cdot C                               \\
        & = 1 + \log \left(n - 2\right) \cdot C              \\
        & = 1 + C \cdot \left(\frac{\log n}{\log 2}\right)   \\
        & = 1 + \frac{C}{\log 2} \cdot \log n                \\
        & = \log n\ \ [\text{As, constants aren't dominant}]
    \end{align*}

    \subsection*{Illustrative Example}

    Consider another example to get a better understanding of finding the time complexity using recurrence relation
    by substitution and back substitution method:

    \begin{equation*}
        \label{eq:illus_eg}
        T(n) = \begin{Bmatrix}
                   1                    & if\ n = 1 \\
                   n \cdot T(n - 1) + C & if\ n > 1
        \end{Bmatrix}
    \end{equation*}

    \noindent
    Same as before, we'll just proceed through a few steps and get a feel of them.
    Note, that we will omit the constant $C$ and its coefficients for simplicity:

    \begin{equation}
        T(n) = n \cdot T(n - 1)
        \label{eq:illus_uns}
    \end{equation}

    \begin{equation}
        T(n - 1) = (n - 1) \cdot T(n - 2)
        \label{eq:illus_firstsubs}
    \end{equation}

    \begin{equation}
        T(n - 2) = (n - 2) \cdot T(n - 3)
        \label{eq:illus_secondsubs}
    \end{equation}

    \noindent
    Now next is backwards substitution i.e., first we substitute (\ref{eq:illus_secondsubs})
    to (\ref{eq:illus_firstsubs}):

    \begin{equation}
        T(n - 1) = (n - 1) \cdot (n - 2) \cdot T(n - 3)
        \label{eq:bk_firstsubs}
    \end{equation}

    \noindent
    Now, the next step is for us to substitute (\ref{eq:bk_firstsubs}) to (\ref{eq:illus_uns}) and we will have:

    \begin{equation}
        T(n) = n \cdot (n - 1) \cdot (n - 2) \cdot T(n - 3)
        \label{eq:bk_unsubs}
    \end{equation}

    \noindent
    From (\ref{eq:bk_unsubs}) equation we can infer a pattern that will help us in finding the time complexity
    i.e. substituting up to $n - 1$ steps, we have:

    \begin{align*}
        T(n) & = n \cdot (n - 1) \cdot (n - 2) \cdot (n - 3) \cdots T(\left\{n - (n - 1)\right\})                                                               \\
        & = n \cdot (n - 1) \cdot (n - 2) \cdot (n - 3) \cdots T(1)                                                                                \\
        & = n \cdot (n - 1) \cdot (n - 2) \cdot (n - 3) \cdots 1                                                                                \\
        & = n \cdot n\left(1 - \frac{1}{n}\right) \cdot n\left(1 - \frac{2}{n}\right) \cdot n\left(1 - \frac{3}{n}\right) \cdots n\left(\frac{1}{n}\right) \\
        & = n^n \cdot \left(1 - \frac{1}{n}\right) \cdot \left(1 - \frac{2}{n}\right) \cdot \left(1 - \frac{3}{n}\right) \cdots \left(\frac{1}{n}\right)   \\
        & = n^n
    \end{align*}

    Therefore, $O(n^n)$ i.e., factorial time is the time complexity of the algorithm as, it's the most dominant term.
    Note that we are substituting up to $n - 1$ steps because it is the only way the $T(1)$ form will appear.

    \section{Master Theorem}

    Previously, on the recurrence section (\ref{lb:recurrence_main}) we saw how to evaluate the time complexities
    of algorithms.
    Now, building up on that, you should be made aware that the recurrence relation will work for each and every
    algorithm but, it has a drawback i.e., it is slow or, more precisely, there are a lot of mathematical
    computations involved in finding the time complexities.

    Now, this problem is eased (\textit{somewhat}) by another theorem called the "Master Theorem".
    Now, this theorem will only be applicable for a fixed targeted type of algorithms and that is in this form:

    \begin{equation*}
        T(n) = a \cdot T\left(\frac{n}{b}\right) + f(n) : a \geq 1,b > 1
    \end{equation*}

    \subsection{Applicability}

    \noindent
    Now, to make it even more clearer we will see expressions in which the Master Theorem will apply and another
    in which it won't apply:

    \begin{equation}
        \label{eq:master_right1}
        T(n) = 8 \cdot T\left(\frac{n}{2}\right) + n^2
    \end{equation}

    \noindent
    For the above equation (\ref{eq:master_right1}) the theorem will apply as all of its conditions are satisfied
    i.e. a is 8 which is greater than 1 and $b = 2$ that is also greater than 1 and $f(n) = n^2$
    also the equation in the right form.

    \begin{equation}
        \label{eq:master_right2}
        T(n) = T\left(\frac{n}{2}\right) + C
    \end{equation}

    \noindent
    Same case for sample equation (\ref{eq:master_right2}) i.e., it can be applied to master theorem.

    \begin{equation}
        \label{eq:master_wrong}
        T(n) = T(n - 1) + 1
    \end{equation}

    \noindent
    No, for this sample equation (\ref{eq:master_wrong}) will clearly not apply as $b = 1$ but, it is required
    that $b > 1$ but, this can be solved with substitution method.
    Anyways, now, to actually get the time complexity using the Master Theorem, we'd need to use this expression:

    \begin{equation*}
        T(n) = n^{\log_b a} \cdot U(n)
    \end{equation*}

    \noindent
    Here, $U(n)$ depends on $h(n)$ where $h(n)$ is:

    \begin{equation*}
        h(n) = \frac{f(n)}{n^{\log_b a}}
    \end{equation*}

    \begin{table}[!hbt]
        \centering
        \caption{Relation between $U(n)$ and $h(n)$}
        \label{tab:hnu}
        \begin{tabular}{|c|c|}
            \hline \textbf{h(n)}            & \textbf{U(n)}                    \\
            \hline $n^r, r > 0$             & $O(n^r)$                         \\
            \hline $n^r, r < 0$             & $O(1)$                           \\
            \hline $(\log_2 n)^i, i \geq 0$ & $\frac{(\log_2 n)^{i+1}}{i + 1}$ \\
            \hline
        \end{tabular}
    \end{table}

    \subsection{Illustrative Examples}

    \noindent
    The best way to solidify your knowledge is to learn with examples that is again consider the
    equation (\ref{eq:master_right1}) where we have the given values such as $a = 8, b = 2, f(n) = n^2$
    and $a \geq 1, b > 1$ are true.
    Therefore:

    \begin{align*}
        T(n) & = n^{\log_2 8} \cdot U(n) \\
        & = n^3 \cdot U(n)
    \end{align*}

    \noindent
    Now, for $h(n)$ we have $f(n) = n^2, n^{\log_b a} = n^3$ that is:

    \begin{align*}
        h(n) & = \frac{n^2}{n^3} \\
        & = \frac{1}{n}     \\
        & = n^{-1}
    \end{align*}

    \noindent
    Therefore the value of $r$ is -1 which is less than zero and hence the value of $U(n)$ will be $O(1)$ according
    to the (\ref{tab:hnu}).
    This concludes the Second case for finding out $U(n)$'s value by utilising table {\ref{tab:hnu}}.
    As, for the First case, no example will be shown as it is pretty straightforward
    (i.e., it's very similar to the Second one).

    Now, to check whether the Third variant, the following sample example will be used:

    \begin{equation}
        \label{eq:thirdmaster}
        T(n) = T\left(\frac{n}{2}\right) + C
    \end{equation}

    \noindent
    In equation (\ref{eq:thirdmaster}), we have $a = 1,b = 2,f(n) = C$ which both conforms to the
    Master Theorem's conditions.
    And, therefore:

    \begin{align*}
        T(n) & = n^{\log_2 1} \cdot U(n)         \\
        & = n^{0 \cdot \log_2 2} \cdot U(n) \\
        & = n^0 \cdot U(n)                  \\
        & = 1 \cdot U(n)                    \\
        & = U(n)
    \end{align*}

    \noindent
    Same as the previous example i.e., now, we need to calculate $h(n)$ also, we have $f(n) = C, n^{\log_b a} = 1$
    hence:

    \begin{align*}
        h(n) & = \frac{C}{1} \\
        & = C
    \end{align*}

    \noindent
    Now, according to table (\ref{tab:hnu}) we have $i = 0$ as $h(n) = C$ can also be written as,
    $(\log_2 n)^0 \cdot C$.
    And, as the Third variant's conditions are all satisfied i.e., $i \geq 0$ holds true.
    Therefore:

    \begin{align*}
        h(n) & = C                                                      \\
        & = \frac{(\log_2 n)^{0+1}}{0 + 1} \cdot C                 \\
        & = (\log_2 n) \cdot C                                     \\
        & = \log_2 n\ \ \text{[After removing the dominant terms]}
    \end{align*}

    \noindent
    Hence, the time complexity is $O(\log n)$

    \section{Divide and Conquer}

    Divide~and~Conquer~in~Computer~Science\footnote{
        Divide and conquer originates from a strategy in politics which goes by a similar term called
        "Divide and Rule" in simple Layman's terms it can be described as breaking up larger concentrations
        of power into smaller pieces that individually have less power than the one implementing the strategy.
    }
    means breaking up a larger problem into smaller sub-problems that are similar to the original problem,
    recursively solves the sub-problems, and finally combines the solutions to the sub-problems to solve
    the original problem.
    Because divide-and-conquer solves sub-problems recursively, each sub-problem must be smaller than the
    original problem, and there must be a base case for sub-problems.
    You should think of a divide-and-conquer algorithm as having three parts:

    \subsection{A Rough Idea}

    \begin{enumerate}
        \item \textbf{Divide} the problem into a number of sub-problems that are smaller instances of the same problem.
        \item \textbf{Conquer} the sub-problems by solving them recursively. If they are small enough,
        solve the sub-problems as base cases.
        \item \textbf{Combine} the solutions to the sub-problems into the solution for the original problem.
    \end{enumerate}

    You can easily remember the steps of a divide-and-conquer algorithm as divide, conquer, combine.
    Here's how to view one step, assuming that each divide step creates two sub-problems
    (though some divide-and-conquer algorithms create more than two)

    \subsection{Illustrative Pseudocode}

    Below is a pseudocode to give you a rough idea about how Divide and Conquer works.

    \begin{algorithm}[H]
        \SetKwFunction{FRecurs}{DAC}
        \SetKwProg{Fn}{Function}{:}{}
        \Fn(\tcc*[h]{general DAC code}){\FRecurs{P}}{
            \tcc{Let P be any problem}
            \If(\tcc*[h]{solve right here if P is small}){$Small(P)$}{
                return $S(P)$
            }\Else{
                divide P into $P_1, P_2, P_3, \cdots, P_k$\\
                Apply $DAC(P_1), DAC(P_2), DAC(P_3), \cdots, DAC(P_k)$\\
                Combine ($DAC(P_1), DAC(P_2), DAC(P_3), \cdots, DAC(P_k)$)
            }
        }
    \end{algorithm}

    \noindent
    And some of the well known algorithms which apply this concept are as follows:

    \begin{enumerate}
        \item Binary Search
        \item Find Minimum
        \item Find Maximum
        \item Quick Sort
        \item Merge Sort
        \item Strassen's Matrix Multiplication
    \end{enumerate}

    \section{Analysis of Quick Sort}

    Like merge sort, quicksort uses divide-and-conquer, and so it's a recursive algorithm.
    The way that quicksort uses divide-and-conquer is a little different from how merge sort does.
    In merge sort, the divide step does hardly anything, and all the real work happens in the combine step.
    Quicksort is the opposite: all the real work happens in the divide step. In fact, the combine step in
    quicksort does absolutely nothing.

    Here is how quicksort uses divide-and-conquer.
    As with merge sort, think of sorting a \texttt{sub-array array[p...r]}, where initially the
    sub-array is \texttt{array[0...n-1]}.

    \subsection{Algorithm}

    The following is the algorithm of quicksort.

    \begin{enumerate}
        \item
        \textbf{Divide} by choosing any element in the sub-array \texttt{array[p...r]}.
        Call this element the \textbf{pivot}.

        Rearrange the elements in \texttt{array[p...r]} so that all elements in\\ \texttt{array[p...r]} that are less
        than or equal to the pivot are to its left and all elements that are greater than the pivot are to its right.
        We call this procedure \textbf{partitioning}.
        At this point, it doesn't matter what order the elements to the left of the pivot are in relation to each
        other, and the same holds for the elements to the right of the pivot.
        We just care that each element is somewhere on the correct side of the pivot.

        As a matter of practice, we'll always choose the rightmost element in the sub-array, \texttt{array[r]},
        as the pivot.
        So, for example, if the sub-array consists of \texttt{[9, 7, 5, 11, 12, 2, 14, 3, 10, 6]}, then we choose 6
        as the pivot.
        After partitioning, the sub-array might look like \texttt{[5, 2, 3, 6, 12, 7, 14, 9, 10, 11]}.
        Let q be the index of where the pivot ends up.
        \item
        \textbf{Conquer} by recursively sorting the sub-arrays \texttt{array[p...q-1]}
        (all elements to the left of the pivot, which must be less than or equal to the pivot)
        and \texttt{array[q+1...r]} (all elements to the right of the pivot, which must be greater than the pivot).
        \item
        \textbf{Combine} by doing nothing. Once the conquer step recursively sorts, we are done.
        Why? All elements to the left of the pivot, in\\ \texttt{array[p...q-1]}, are less than or equal to the pivot
        and are sorted, and all elements to the right of the pivot, in \texttt{array[q+1..r]}, are greater than the
        pivot and are sorted.
        The elements in \texttt{array[p..r]} can't help but be sorted!
        Think about our example.
        After recursively sorting the sub-arrays to the left and right of the pivot, the sub-array to the left of
        the pivot is \texttt{[2, 3, 5]}, and the sub-array to the right of the pivot is
        \texttt{[7, 9, 10, 11, 12, 14]}.
        So, the sub-array has \texttt{[2, 3, 5]}, followed by 6, followed
        by \texttt{[7, 9, 10, 11, 12, 14]}. The sub-array is sorted.
    \end{enumerate}

    \subsection{Illustrative Example}

    Let, \texttt{A = [35, 50, 15, 25, 80, 20, 90, 45]}.
    So, lets start right away and assign \texttt{v = 35} where \texttt{v} is the pivot variable.
    Also, let there be two pointers i.e. \texttt{P and Q} where \texttt{P} is pointing to 50 and
    \texttt{Q} is pointing to 45.
    And, \texttt{P} should rightward in the given list; it will stop its movement when its value is \textbf{greater}
    than the pivot variable.
    Similarly, for \texttt{Q} it will be the opposite i.e., it'll move in a leftward direction, and it'll stop when
    its value is \textbf{lesser} than the pivot value.

    Note that, the pivot element merely filters the list i.e., sorts the small elements to the left and bigger
    elements to the right and \texttt{P and Q} simply assists them; \texttt{P} pointer assists in sorting the
    \textbf{bigger} elements and \texttt{Q} in \textbf{smaller} or, \textbf{equal}.

    \subsection*{How to stop the loop?}

    Also, it would be a good idea to update the list and add an infinity symbol to make this less confusing.
    That is, \texttt{[35, 50, 15, 25, 80, 20, 90, 45,..., $+\infty$]}. An example of this would be:

    \begin{center}
        \texttt{[35, 5, 4, 3, 2, 1]}
    \end{center}

    \noindent
    Here, the pivot element is 35, therefore the \texttt{P} pointer will check the element as:

    \begin{enumerate}
        \item Is 5 greater than 35? No.
        \item Move rightward.
        \item Is 4 greater than 35? No.
        \item Move rightward.
        \item Is 3 greater than 35? No.
        \item Move rightward.
        \item Is 2 greater than 35? No.
        \item Move rightward.
        \item Is 1 greater than 35? No.
        \item Move rightward.
        \item The loop still continues.
    \end{enumerate}

    \noindent
    Now, the loop needs to be stopped. And, we need to somehow add a value that is greater than the pivot.
    Therefore, we take $+\infty$ and $35 < +\infty \therefore$ the pointer \texttt{P} finally stops.

    You may be wondering that why don't we add a $-\infty$ to the start of the list? That's because the \texttt{Q}
    pointer is flowing leftwards and it check if it is smaller or, \texttt{equal} to the pivot element.
    Which is bound to hit.

    \subsection*{Algorithmic Steps}

    Following is a step-by-step run of the algorithm.

    \begin{enumerate}
        \item Start
        \item Let,\\ \texttt{A = [35, 50, 15, 25, 80, 20, 90, 45,..., $+\infty$],\\ v = 35, P = 50} and \texttt{Q = 45}
        \item Now, is $P > v$ or, $50 > 35$? True.
        \item \texttt{P} will stop.
        \item Now, is $Q \leq v$ or, $45 \leq 35$? False.
        \item Set \texttt{Q = 90}. That is, move leftward.
        \item Now, is $Q \leq v$ or, $90 \leq 35$? False.
        \item Move leftward, \texttt{Q = 20}.
        \item Now, is $Q \leq v$ or, $20 \leq 35$? True.
        \item Stop \texttt{Q}
        \item Now, we need to check if \texttt{P} and \texttt{Q} have crossed each other or not.
        Which is false; they haven't crossed each other.
        \item Then swap \texttt{P} and \texttt{Q} i.e.,\\ \texttt{A = [35, 20, 15, 25, 80, 50, 90, 45,..., $+\infty$]}

        \item Now, is $P > v$ or, $20 > 35$? False.
        \item Set \texttt{P = 15}. That is move rightward.
        \item Now, is $P > v$ or, $15 > 35$? False.
        \item Set \texttt{P = 25}. That is move rightward.
        \item Now, is $P > v$ or, $25 > 35$? False.
        \item Set \texttt{P = 80}. That is move rightward.
        \item Now, is $P > v$ or, $80 > 35$? True.
        \item \texttt{P} will stop.
        \item Now, is $Q \leq v$ or, $50 \leq 35$? False.
        \item Move leftward, \texttt{Q = 80}.
        \item Now, is $Q \leq v$ or, $80 \leq 35$? False.
        \item Move leftward, \texttt{Q = 25}.
        \item Now, is $Q \leq v$ or, $25 \leq 35$? True.
        \item Now, \texttt{Q} will stop.
        \item Now, we need to check if \texttt{P} and \texttt{Q} have crossed each other or not.
        Which is true; they have crossed each other.
        \item For this predicament you'd need to swap the pivot element and \texttt{Q} i.e. \texttt{v = 25, Q = 35},\\
        A = [25, 20, 15, 35, 80, 50, 90, 45,..., $+\infty$].
        This concludes the first pass\footnote{Change of the pivot indicates the completion of a pass}.
        \item As of now the pivot element is in the right position as all of the small values from it are in its
        right and the bigger values are on the left. That is:\\ \texttt{Left: [25, 20, 15,..., $+\infty$]}\\
        \texttt{Right: [80, 50, 90, 45,..., $+\infty$]}

        \item Working on the left part first (you can do any first).
        \begin{enumerate}
            \item Now, \texttt{A = [25, 20, 15,..., $+\infty$]} and the next pivot element is 25 i.e. \texttt{v = 25}.
            And, \texttt{P = 20, Q = 15}.
            \item Now, is $P > v$ or, $20 > 25$? False.
            \item Set \texttt{P = 20}. That is move rightward.
            \item Now, is $P > v$ or, $15 > 25$? False.
            \item Set \texttt{P = $+\infty$}. That is \texttt{P} will stop.

            \item Now, is $Q \leq v$ or, $15 \leq 25$? True.
            \item Now, \texttt{Q} will stop.
            \item Now, we need to check if \texttt{P} and \texttt{Q} have crossed each other or not.
            Which is true; they have crossed each other.
            \item Swap pivot element with the \texttt{Q} i.e. \texttt{A = [15, 20, 25]}
            \item Sub-array is now sorted.
        \end{enumerate}
        \item Now, we shall work on the right part.
        \begin{enumerate}
            \item Now, \texttt{A = [80, 50, 90, 45,..., $+\infty$]} and the next pivot element is 80 i.e.
            \texttt{v = 80}.
            And, \texttt{P = 50, Q = 45}.
            \item Now, is $P > v$ or, $50 > 80$? False.
            \item Set \texttt{P = 90}.
            That is move rightward.
            \item Now, is $P > v$ or, $90 > 80$? True.
            \item Now, we need to check if \texttt{P} and \texttt{Q} have crossed each other or not.
            Which is false; they have not crossed each other.
            \item Now, swap \texttt{P} and \texttt{Q}, we have \texttt{A = [80, 50, 45, 90,..., $+\infty$]}
            \item Now, is $Q \leq v$ or, $90 \leq 80$? False.
            \item Move leftward, \texttt{Q = 45}.
            \item Now, is $Q \leq v$ or, $45 \leq 80$? True.
            \item Now, \texttt{Q} will stop.
            \item Now, we need to check if \texttt{P} and \texttt{Q} have crossed each other or not.
            Which is true; they have crossed each other.
            \item Swap pivot element with the \texttt{Q} i.e. \texttt{A = [45, 50, 80, 90,..., $+\infty$]}
            \item Sub-array is sorted.
        \end{enumerate}

        \item Combining left part and right part \\
        \texttt{[15, 20, 25] + [45, 50, 80, 90]} we have:
        \begin{center}
            \texttt{[15, 20, 25, 45, 50, 80, 90]}
        \end{center}
        \item The array is finally sorted.
        \item End
    \end{enumerate}

    \subsection{Complexity Analysis}

    Everything depends on the pivot element i.e., you need to make a note of where the position of the pivot
    element will be after the first pass.

    If the pivot element is at the middle position of the list then the problem gets divided into
    $\frac{n}{2}$ and $\frac{n}{2} - 1$ and $\frac{n}{2}$ from the left and $\frac{n}{2}$ from the right
    will be passed, the middle element check only takes O(1) and don't forget the time taken for the first p
    ass i.e. O(n).
    So, the recurrence relation will look like $T(n) = T\left(\frac{n}{2}\right) + T\left(\frac{n}{2}\right) + n + C$
    which can be further simplified as, $T(n) = 2 \cdot \left(\frac{n}{2}\right) + n$ this is the final recurrence
    relation for the average case.

    This recurrence relation is a Master Theorem candidate therefore, the time complexity will be: $O(n\log n)$

    \subsection*{Worst Case}

    The worst case occurs in multiple cases one of which is when the list is already sorted.
    For instance, take the same list: \texttt{[15, 20, 25, 45, 50, 80, 90]}.
    Here, the list is sorted but the algorithm doesn't know that it is sorted.
    So, it will commence the sorting as usual:

    \begin{enumerate}
        \item Start
        \item Let, \texttt{P = 20, Q = 90, v = 15}
        \item $P > v$? True. Stop.
        \item $Q \leq v$? False.
        \item Move \texttt{Q} leftwards.
        \item \texttt{Q} never becomes true and stops at pivot.
        \item Change the pivot to the element after pivot
        \item Repeat from step 2
        \item End
    \end{enumerate}

    Here, the first pass takes $n$ amount of time and rest takes $T(n - 1)$.
    Therefore, the recurrence relation will be $T(n) = T(n - 1) + n$ which can be solved by substitution method.
    That is, the time complexity will be $O(n^2)$.

    \section{Analysis of Merge Sort}

\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
    \DeclareGraphicsExtensions{.png, .jpeg}
\usepackage{caption}
\usepackage[top=1in, bottom=1in, left=1in, right=1in]{geometry}

\title{STAT 775: Machine Learning \\ HW 01}
\author{Terence Henriod}
\date{\today}

\begin{document}

\clearpage            % All
\maketitle            % this,
\thispagestyle{empty} % removes the page number from the title page

\begin{abstract}
Exploration of K-Nearest Neighbor (k-NN), Linear Regression, and associated topics.
\end{abstract}

\newpage
\section{Exercise 2.1 (2 points)}
Suppose each of $K$-classes has an associated target $t_{k}$, which is a vector of all zeros, except a one in the $k^{th}$ position. Show that classifying to the largest element of $\hat{y}$ amounts to choosing the closest target, $min_{k} \|t_{k} - \hat{y}\|$, if the elements of $\hat{y}$ sum to one.\\

\textit{Solution}:\\
Let us consider the difference between the distance between $\hat{y}$ and the target $t_{m}$ corresponding to $\hat{y}$'s greatest member value, $\hat{y}_{m}$, call it $\hat{y}_{m}$, and the distance between $\hat{y}$ and any other target $t_{n}$.

First, let us consider $\hat{y}_{m}$ and $\hat{y}_{l}$. By definition, $\hat{y}_{m} \ge \hat{y}_{l}$ for any $l$.
Second, since the signs of these differences will be the same, let us just consider both distances to be positive. This will let us consider distance values ${d : d \in [0, 1]}$, and, given the monotonicity of $x$ and $x^{2}$ in the range $[0, \infty)$, we can safely consider the Euclidean distance squared ($\|x\|^{2}$), in place of normal Euclidean distance.

Next, we compare the difference of the distances described above:
\begin{multline*}
 \|\hat{y} - t_{m}\|^{2} - \|\hat{y} - t_{n}\|^{2} =\\
   \sqrt{ (\hat{y}_{m} - 1)^{2} + (\hat{y}_{n} - 0)^{2} + \sum_{i, i \neq m, n}{(\hat{y}_{i} - 0)^{2}} }^{2} -
    \sqrt{ (\hat{y}_{m} - 0)^{2} + (\hat{y}_{n} - 1)^{2} + \sum_{i, i \neq m, n}{(\hat{y}_{i} - 0)^{2}} }^{2}
\end{multline*}
\begin{align*}
   &=  (\hat{y}_{m} - 1)^{2} + \hat{y}_{n}^{2}       + \sum_{i, i \neq m, n}{(\hat{y}_{i} - 0)^{2}} -
       \hat{y}_{m}^{2}       - (\hat{y}_{n} - 1)^{2} - \sum_{i, i \neq m, n}{(\hat{y}_{i} - 0)^{2}}\\
   &=  (\hat{y}_{m} - 1)^{2} + \hat{y}_{n}^{2} - \hat{y}_{m}^{2} - (\hat{y}_{n} - 1)^{2}\\
   &=  \hat{y}_{m}^{2} - 2\hat{y}_{m} + 1 - \hat{y}_{m}^{2} - \hat{y}_{n}^{2} + 2\hat{y}_{n} - 1 + \hat{y}_{n}^{2}\\
   &=  -2\hat{y}_{m} + 2\hat{y}_{n}\\
   &=  2(\hat{y}_{n} - \hat{y}_{m})\\
   &\le 0 \text{ since }\hat{y}_{m} \ge \hat{y}_{l}
\end{align*}

Thus, the distance between $\hat{y}$ and $t_{m}$ must be less than or equal to that of the distance between $\hat{y}$ and $t_{n}$. Therefore, classifying to the largest element of $\hat{y}$ amounts to choosing the closest target, $min_{k} \|t_{k} - \hat{y}\|$.\\
%%%%%%%%%
\newpage
\section{Exercise 2.3 (4 points)}
Derive equation (2.24). ($d(p, N) = (1 - \frac{1}{2}^{1/N})^{1/p}$)\\

\textit{Solution}:\\
This question refers to our consideration of a $p$-dimensional unit ``ball" centered about the origin and the median distance from the origin to the closest of any of $N$ points uniformly distributed in the ball. For the sake of alleviating confusion, I believe that ``uniformly distributed" means that the points are distributed where each coordinate of the point follows a $U(-1, 1)$ distribution, NOT that the points are distributed to be spaced evenly from one another.

Let us denote the median distance from the origin to the closest of any of the $N$ points as $d$ (short for $d(p, N)$). Let us call any of the $N$ points as $a_{i}$. For simplicity, let the notation $\|\cdot\|$ mean the Euclidean distance between a point and the origin.

By the definition of the median, half of the $x_{i}$s should be further from the origin than $d$. Since the points are distributed indepently, we can say that:
$$\frac{1}{2} = \prod_{i=1}^{N}{ P(\|a_{i}\| > d) }$$

We can also say that the likelihood of a point being randomly assigned to a certain region of the ball is the same as the fraction of the volume of the whole ball that region occupies. So we can estimate the probability that a point is further from the origin than $d$ (via its complement) by:
\begin{align*}
  P(\|a_{i}\| > d) &= 1 - P(\|a_{i}\| \le d)\\
                   &= 1 - \frac{kd^{p}}{k(1)^{p}}\\
                   &= 1 - \frac{kd^{p}}{k}\\
                   &= 1 - d^{p}
\end{align*}
where $k$ is the constant used in determining the volume of hypershperes (for $p=2$ (a circle), $k=\pi$; for $p=3$ (a sphere), $k=\frac{4}{3}\pi$; and so on...). Also, recall that the ball is a unit sphere with a radius of $1$.

Since $P(\|a_{i}\| > d) = 1 - d^{p}$, we have:
$$\frac{1}{2} = \prod_{i=1}^{N}{ 1 - d^{p} }$$

and solving for $d$, we get:
\begin{align*}
  \prod_{i=1}^{N}{ 1 - d^{p} } &= \frac{1}{2}\\
  (1 - d^{p})^{N}              &= \frac{1}{2}\\
  1 - d^{p}                    &= (\frac{1}{2})^{\frac{1}{N}}\\
  d^{p}                        &= 1 - (\frac{1}{2})^{\frac{1}{N}}\\
  d(p, N)                      &= (1 - (\frac{1}{2})^{\frac{1}{N}})^{\frac{1}{p}}
\end{align*}
And so the derivation is complete.
%%%%%%%%%%%%%%%
\newpage
\section{Exercise 2.4 (4 points)}
The edge effect problem discussed on page 23 is not peculiar to uniform sampling from bounded domains. Consider inputs drawn from a spherical multinormal distribution $X \sim N(0, \textbf{I}_{p})$. The squared distance from any sample point to the origin has a $\chi^{2}_{p}$ distribution with mean $p$. Consider a prediction point $x_{0}$ drawn from this distribution, and let $a = \frac{x_{0}}{\|x_{0}\|}$ be an associated unit vector. Let $z_{i} = a^{T} x_{i}$ be the projection of each of the training points on this direction.
Show that the $z_{i}$ are distributed $N(0, 1)$ with expected squared distance from the origin $1$, while the target point has expected squared distance $p$ from the origin.
Hence for $p = 10$, a randomly drawn test point is about $3.1$ standard deviations from the origin, while all the training points are on average one standard deviation along direction $a$. So most prediction points see themselves as lying on the edge of the training set.\\

\textit{Solution}:\\
Since $z_{i}$ are the projections of the $x_{i}$ onto $a$, the $z_i$ are linear combinations of variables that follow an $N(0, \textbf{1})$ distribution, so the $z_{i}$ themselves are normal with $0$ mean and a variance of
$$Var(z_{i}) = \|a^{T}\|^{2}Var(x_{i}) = 1 * Var(x_{i}) = 1$$
since $a$ is a unit vector and the $x_{i}$ have variance of $1$ by the problem statement.
%%%%%%%%%%%%%%%%
\newpage
\section{Exercise 2.8 (10 points)}
Compare the classification performance of linear regression and $k$-nearest neighbor classification on the \texttt{zipcode} data. In particular, consider only the $2$’s and $3$’s, and $k = 1, 3, 5, 7$ and $15$. Show both the training and test error for each choice. The \texttt{zipcode} data are available from the book website\hfill\\
\texttt{www-stat.stanford.edu/ElemStatLearn}.\\

\textit{Solution}:\\
It would appear that k-NN is the better classifier. It only performed marginally worse than linear regression on the training data (which is not of practical significance anyway), but on the novel test data, k-NN consistently performed better than the regression - and by a wider margin.

  \begin{figure}[h]
    \centering
    \includegraphics[width=.9\linewidth]{error_plot}
    \caption{A graphical comparison of the errors of the Linear Regression and the k-NN classification methods.}
    \label{fig:errors}
  \end{figure}

The R code used is as follows:
\begin{verbatim}
##
# ESL Exercise 2.8
#
##

######################
# some initial setup #
######################
working.directory = "~/Desktop/STAT775/HW01/Exercise_02_08"
training.file.name = "zip.test"
test.file.name = "zip.train"

setwd(working.directory)

require(ggplot2)
require(reshape)

####################
# Data Preparation #
####################
read.and.filter.data <- function(file.name) {
  #
  #
  # Args:
  #
  # Returns:
  #
  data = read.table(
    file.name,
    header = F,
    row.name = NULL
  )
  data = subset(data, V1 == 2 | V1 == 3)
  return(data)
}

separate.classes <- function(x) {
  if (x == 3) {
    return(1)
  } else {
    return(-1)
  }
}

zip.train <- read.and.filter.data(training.file.name)
training.data <- data.matrix(zip.train[, -1])
training.targets <- data.matrix(zip.train[, 1])
for (i in 1:length(training.targets)) {
  training.targets[i] <- separate.classes(training.targets[i])
}

zip.test <- read.and.filter.data(test.file.name)
test.data <- data.matrix(zip.train[, -1])
test.targets <- data.matrix(zip.train[, 1])
for (i in 1:length(test.targets)) {
  test.targets[i] <- separate.classes(test.targets[i])
}

k.range <- 1:15  # simpler than doing just 1, 3, 5, 7, 15
determine.category <- function(x) {
  if(x >= 0) {
    return(1)
  } else {
    return(-1)
  }
}

#####################
# Linear Regression #
#####################
least.squares <- function(x, y) {
  xt.x <- t(x) %*% x
  xt.x.inv <- solve(xt.x)
  full.mat <- xt.x.inv %*% t(x)
  regression.weights <- full.mat %*% y
  return(regression.weights)
}

linear.predict <- function(inputs, betas) {
  # sometimes the formula is listed as x^t * b, but our 'inputs' data already has each
  # test vector in a row when we pass it as a matrix (for this script)
  result <- inputs %*% betas
  result <- lapply(result, function(x) {return(determine.category(x))})
  return(result)
}

digit.model.weights <- least.squares(x = training.data, y = training.targets)

linear.training.predictions <- list()
for(i in 1:length(training.targets)) {
  linear.training.predictions[i] <- linear.predict(inputs = training.data[i, ], betas = digit.model.weights)
}
linear.test.predictions <- list()
for(i in 1:length(test.targets)) {
  linear.test.predictions[i] <- linear.predict(inputs = test.data[i, ], betas = digit.model.weights)
}

########
# K-NN #
########
euclidean.distance <- function(x, y) {
  diff <- x - y
  squares <- diff^2
  return(sqrt(sum(squares)))
}

my.knn.predict <- function(input.point, training.points, training.labels, k) {
  distances = NULL
  for (i in 1:length(training.points[, 1])) {
    distances[i] <- euclidean.distance(input.point, training.points[i, ])
  }

  category <- training.labels
  results <- data.frame(distances, category)
  results <- results[order(-distances), ]
  results <- results$category[1:k]

  return(determine.category(mean(results)))
}

knn.training.predictions <- matrix(0, length(k.range), length(training.targets))
knn.test.predictions <- matrix(0, length(k.range), length(test.targets))
for (k in k.range) {
  for (j in 1:length(training.targets)) {
    knn.training.predictions[k, j] <- my.knn.predict(
      input.point = training.data[j, ],
      training.points = training.data,
      training.labels = training.targets,
      k = k
    )
  }
}
for (k in k.range) {
  for (j in 1:length(test.targets)) {
    knn.test.predictions[k, j] <- my.knn.predict(
      input.point = test.data[j, ],
      training.points = training.data,
      training.labels = training.targets,
      k = k
    )
  }
}

######################
# Error Computations #
######################
compute.error.rate <- function(targets, predictions) {
  incorrect.predictions = 0
  for (i in 1:length(targets)) {
    if (targets[1] != predictions[i]) {
      incorrect.predictions <- (incorrect.predictions + 1)
    }
  }
  return(incorrect.predictions / length(targets))
}

linear.training.error.rates <- compute.error.rate(training.targets, linear.training.predictions)
linear.training.error.rates <- rep(linear.training.error.rates, 15)
linear.test.error.rates <- compute.error.rate(test.targets, linear.test.predictions)
linear.test.error.rates <- rep(linear.test.error.rates, 15)

knn.training.error.rates <- NULL
for (k in k.range) {
  knn.training.error.rates[k] <- compute.error.rate(training.targets, knn.training.predictions[k, ])
}
knn.test.error.rates <- NULL
for (k in k.range) {
  knn.test.error.rates[k] <- compute.error.rate(test.targets, knn.test.predictions[k, ])
}

errors.for.plotting <- data.frame(
  "K" = 1:15,
  "Linear (Training)" = linear.training.error.rates,
  "Linear (Test)" = linear.test.error.rates,
  "KNN (Training)" = knn.training.error.rates,
  "KNN (Test)" = knn.test.error.rates
)

############
# Plotting #
############

plot.data <- melt(errors.for.plotting, id = "K", variable_name = "Method")
ggplot(data = plot.data, aes(x = K, y = value, color = Method)) +
  geom_line() +
  ggtitle("A comparison of Linear Regression and k-NN Approaches") +
  xlab("K") +
  ylab("Error Rate") +
  ylim(0, 0.05) +
  xlim(0, 16) +
  scale_color_hue(
    name = "Method (Data)",
    labels = c(
      "Linear Regression (Training)",
      "Linear Regression (Test)",
      "k-NN (Training)",
      "k-NN (Test)"
    )
  )
ggsave('error_plot.png')
\end{verbatim}

\end{document}