% intro_to_algo.tex
\documentclass[fontsize=12pt,paper=a4]{book}
\usepackage[utf8]{inputenc}

\usepackage{ifluatex}
% fonts for smallcaps
\ifluatex
	\usepackage{fontspec}
	% \setmainfont{Baskerville}
	% \newfontfamily\smallcaps{Baskerville Small Caps SSi}
\else
	% \usepackage{librebaskerville}
\fi

%\usepackage[T1]{fontenc}
\usepackage{graphicx}

% packages for math functions
\usepackage{amsmath}
\usepackage{amssymb}

\usepackage{xcolor}
\usepackage{listings}

% packages for algorithm
\usepackage{algorithm}
\usepackage[compatible]{algpseudocode}
\usepackage{algpascal}
% \usepackage[linesnumbered,ruled]{algorithm2e}

% define variables for algorithm

\newcommand{\Break}{\textbf{break}}
\newcommand{\RETURN}{\textbf{return}}
\newcommand{\To}{\ \textbf{to}\ }
\newcommand{\down}{\ \textbf{down}\ }
\renewcommand{\Comment}[1]{// #1}
\newcommand{\Lcomment}[1]{\Statex /* #1 */}

%\renewcommand{\thealgorithm}{}

% algorithm settings

% Set formats for each heading level

\begin{document}
\title{Introduction to Algorithms - Solutions \\
 \large 3rd Edition}
\author{Purbayan Chowdhury}

\maketitle

\tableofcontents

\listofalgorithms

\chapter{The Role of Algorithms in Computing}

\chapter{Getting Started}

\section{Insertion Sort}
\begin{enumerate}
 \item[Ex 2.1-1]
       Using Figure 2.2 as a model, illustrate the operation of Insertion-Sort on the array $A = (31, 41, 59, 26, 41, 58)$.
 \item[A.]

 \item[Ex 2.1-2]
       Rewrite the Insertion-Sort procedure to sort into non-increasing instead of non-decreasing order.
 \item[A.]
       \alglanguage{pseudocode}
       \begin{algorithm}
        \caption{Non-increasing Insertion Sort}
        \begin{algorithmic}[1]
         \Procedure{InsertionSort}{$A$}
         \For {$j \gets 1 \To A.length-1$}
         \State $key \gets A[j]$
         \Comment{Insert A[j] into the sorted sequence A[1..j-1]}
         \State $i \gets j-1$
         \While{$i \ge 0 \And A[i] > key$}
         \State $A[i+ 1] \gets A[i]$
         \State $i := i-1$
         \EndWhile
         \State $A[i+ 1] \gets key$
         \EndFor
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[Ex 2.1-3]
       Considering the \textbf{\textit{searching problem}}:\\
       \textbf{Input}: A sequence of $n$ numbers $A = (a_1, a_2, \dots, a_n)$ and a value $v$. \\
       \textbf{Output}: An index $i$ such that $v = A[i]$ or the special value \textsc{NIL} if $v$ does not appear in $A$. \\
       Write pseudocode for \textbf{\textit{linear search}}, which scans through the sequence, looking for $v$. Using a loop invariant, prove that your algorithm is correct. Make sure that your loop invariant fulfills the three necessary properties.
 \item[A.]
       \begin{algorithm}
        \caption{Linear Search}
        \begin{algorithmic}[1]
         \Procedure{LinearSearch}{$A$,$v$}
         \State $i \gets NIL$
         \For{$j \gets 0 \To A.length-1$}
         \If{$A[j]=v$}
         \State $i \gets j$
         \State \Break
         \EndIf
         \EndFor
         \State \Return $i$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[Ex 2.1-4]
       Consider the problem of adding two n-bit binary integers, stored in two n-element arrays $A$ and $B$. The sum of the two integers should be stored in binary form in an $(n + 1)$-element array $C$ . State the problem formally and write pseudocode for
       adding the two integers.
 \item[A.]
       \begin{algorithm}
        \caption{$n$-bit Binary Addition}
        \begin{algorithmic}[1]
         \Procedure{BinaryAddition}{$A$,$B$}
         \State $carry \gets 0$
         \For{$i \gets n-1 \To 0$}
         \State $C[i+1] \gets (A[i] + B[i]+carry) (\bmod 2)$
         \If{$A[i] + B[i] + carry \ge 2$}
         \State $carry \gets 1$
         \Else
         \State $carry \gets 0$
         \EndIf
         \EndFor
         \State $C[0] \gets carry$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}
\end{enumerate}

\section{Analyzing algorithms}
\begin{enumerate}
 \item[Ex 2.2-1]
       Express the function $n^3/1000 - 100n^2 -100n + 3$ in terms of $\Theta$-notation.

 \item[Ex 2.2-2]
       Consider sorting $n$ numbers stored in array $A$ by first finding the smallest element of $A$ and exchanging it with the element in $A[1]$. Then find the second smallest element of $A$, and exchange it with $A[2]$. Continue in this manner for the first $n-1$ elements of $A$. Write pseudocode for this algorithm, which is known as \textbf{\textit{selection sort}}. What loop invariant does this algorithm maintain? Why does it need to run for only the first $n-1$ elements, rather than for all n elements? Give the best-case and worst-case running times of selection sort in $\Theta$-notation.
 \item[A.]
       \alglanguage{pseudocode}
       \begin{algorithm}
        \caption{Selection Sort}
        \begin{algorithmic}[1]
         \Procedure{SelectionSort}{$A$}
         \For{$i \gets 0 \To n-1$}
         \State $min \gets i$
         \For{$j \gets i+1 \To n$}
         \Lcomment{Find the index of the ith smallest element}
         \If{$A[j] < A[min]$}
         \State $min \gets j$
         \EndIf
         \EndFor
         \EndFor
         \State Swap $A[min]$ and $A[i]$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[Ex 2.2-3]
       Consider linear search again. How many elements of the input sequence need to be checked on the average, assuming that the element being searched for is equally likely to be any element in the array? How about in the worst case? What are the average-case and worst-case running times of linear search in $\Theta$-notation? Justify your answers.

 \item[Ex 2.2-4]
       How can we modify almost any algorithm to have a good best-case running time?

\end{enumerate}

\section{Designing algorithms}
\begin{enumerate}
 \item[Ex 2.3.1]
       Illustrate the operation of merge sort on the array A=(3, 41, 52, 26, 38, 57, 9, 49).

 \item[Ex 2.3.2]
       Rewrite the Merge procedure so that it does not use sentinels, instead stopping once either array $L$ or $R$ has had all its elements copied back to $A$ and then copying the remainder of the other array back into $A$.
 \item[A.]
       \alglanguage{pseudocode}
       \begin{algorithm}
        \caption{Merge Sort}
        \begin{algorithmic}[1]
         \Procedure{Merge}{$A$, $p$, $q$, $r$}
         \State $n1 \gets q-p+1$
         \State $n2 \gets r-q$
         \State let $L[1,\dots,n1]$ and $R[1,\dots,n2]$ be new arrays
         \For{$i \gets 0 \To n1-1$}
         \State $L[i] \gets A[p+i]$
         \EndFor
         \For{$j \gets 0 \To n2-1$}
         \State $R[j] \gets A[q+j+i]$
         \EndFor
         \State $i \gets 0$
         \State $j \gets 0$
         \State $k \gets p$
         \While{$i \neq n1 \And j \neq n2$}
         \If{$L[i] \leq R[j]$}
         \State $A[k] \gets L[i]$
         \State $i \gets i+1$
         \Else
         \State $A[k] \gets R[j]$
         \State $j \gets j+1$
         \EndIf
         \State $k \gets k+1$
         \EndWhile
         \If{$i = n1$}
         \For{$m \gets j \To n2-1$}
         \State $A[k] \gets R[m]$
         \State $k \gets k+1$
         \EndFor
         \EndIf
         \If{$j = j1$}
         \For{$m \gets j \To n1-1$}
         \State $A[k] \gets L[m]$
         \State $k \gets k+1$
         \EndFor
         \EndIf
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[Ex 2.3-3]
       Use mathematical induction to show that when $n$ is an exact power of 2, the solution of the recurrence\\
       \[ T(n) = \begin{cases} 2         & \quad \text{if $n$ = 2}                \\
         2T(n/2)+n & \quad \text{if $n$ = $2^k$, for $k>1$}\end{cases} \] \\
       is $T(n) = n \lg n$.

 \item[Ex 2.3-4]
       We can express insertion sort as a recursive procedure as follows. In order to sort $A[1 \dots n]$, we recursively sort $A[1 \dots n-1]$ and then insert $A[n]$ into the sorted array $A[1 \dots n-1]$. Write a recurrence for the running time of this recursive version of insertion sort.
 \item[A.]
       Let $T(n)$ be running time for insertion sort on an array of size $n$.\\
       \[ T(n) = \begin{cases} \Theta(1)     & \quad \text{ if $n \leq c$} \\
         T(n-1) + I(n) & \quad \text{otherwise}
        \end{cases} \]
       where $I(n)$ denotes the amount of time taken to insert $A[n]$ into the sorted array $A[1 \dots n-1]$. Since we have to shift as many as $n-1$ elements once we find the correct place to insert $A[n]$, we have $I(n) = \Theta(n)$.

 \item[Ex 2.3-5]
       If the sequence $A$ is sorted, we can check the midpoint of the sequence against $v$ and eliminate half of the sequence from further consideration. The binary search algorithm repeats this procedure, halving the size of the remaining portion of the sequence each time. Write pseudocode, either iterative or recursive, for binary search. Argue that the worst-case running time of binary search is $\Theta(\lg n)$.
 \item[A.]
       \begin{algorithm}
        \caption{Recursive Binary Search}
        \begin{algorithmic}[1]
         \Procedure{RecBinSearch}{$A$, $a$, $b$, $v$}
         \If{$a>b$}
         \State \Return \textsc{NIL}
         \EndIf
         \State $m \gets \lfloor \frac{a+b}{2} \rfloor$
         \If{$A[m] = v$}
         \State \Return $m$
         \EndIf
         \If{$A[m] < v$}
         \State \Return \Call{RecBinSearch}{$a$, $m$, $v$}
         \EndIf
         \State \Return \Call{RecBinSearch}{$m+1$, $b$, $v$}
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

       After the initial of \Call{RecBinSearch}{$A$,$0$,$n$,$v$}, each call results a constant number of operations and a call to a problem instance where $b-a$ is a factor of $\frac{1}{2}$. So the recurrence relation satisfies $T(n) = T(n/2) + c$. So, $T(n) \in \Theta(\lg(n))$.

 \item[Ex 2.3-6]
       Observe that the while loop of lines 5-7 of the INSERTION-SORT procedure in Section 2.1 uses a linear search to scan (backward) through the sorted subarray $A[1 \dots j-1]$. Can we use a binary search instead to improve the overall worst-case running time of insertion sort to $\Theta(n \lg n)$?

 \item[Ex 2.3-7]
       Describe a $\Theta(n \lg n)$-time algorithm that, given a set $S$ of $n$ integers and another integer $x$, determines whether or not there exist two elements in $S$ whose sum is exactly $x$.
 \item[A.]
       Use Merge Sort to sort the array $S$ in time $\Theta(n \lg n)$.\\
       \begin{algorithm}
        \caption{Find Sum}
        \begin{algorithmic}[1]
         \Procedure{FindSum}{$S$,$x$}
         \State $i \gets 0$
         \State $j \gets n$
         \While{$i<j$}
         \If{$S[i] + S[j] = x$}
         \State \Return $true$
         \EndIf
         \EndWhile
         \If{$S[i] + S[j] < x$}
         \State $i \gets i+1$
         \EndIf
         \If{$S[i] + S[j] > x$}
         \State $j \gets j-1$
         \EndIf
         \State \Return $false$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}
\end{enumerate}

\subsection*{Problems}
\begin{enumerate}
 \item[2-1]
       \textbf{\textit{Insertion sort on small arrays in merge sort}}\\
       Although merge sort runs in $\Theta(n \lg n)$ worst-case time and insertion sort runs in $\Theta(n^2)$ worst-case time, the constant factors in insertion sort can make it faster in practice for small problem sizes on many machines. Thus, it makes sense to coarsen the leaves of the recursion by using insertion sort within merge sort when subproblems become sufficiently small. Consider a modification to merge sort in which $n/k$ sublists of length $k$ are sorted using insertion sort and then merged using the standard merging mechanism, where $k$ is a value to be determined.
       \begin{enumerate}
        \item Show that insertion sort can sort the $n/k$ sublists, each of length $k$, in $\Theta(nk)$ worst-case time.
        \item[A.]
              Time for insertion sort to sort a single list of length $k$ is $\Theta(k^2)$, so $n/k$ of them will take $\Theta(\frac{n}{k} k^2)$ = $\Theta(nk)$.

        \item Show how to merge the sublists in $\Theta(n \lg(n/k))$ worst-case time.
        \item[A.]
              Provided coarseness $k$, we can start usual merging procedure starting at the level in which array has a size at most $k$. So the depth of merge recursion tree is $\lg(n)-\lg(k)$ = $lg(n/k)$. Each level of merging is $cn$, so the total merging takes $\Theta(n \lg(n/k))$.

        \item Given that the modified algorithm runs in $\Theta(nk + n \lg(n/k))$ worst-case time, what is the largest value of $k$ as a function of $n$ for which the modified algorithm has the same running time as standard merge sort, in terms of $\Theta$-notation?
        \item[A.]
              Considering $k$ as a function of $n$, $k(n)\in O(\log(n))$, gives the same asymptotics and for any constant choice of $k$, the asymptotics are the same.

        \item How should we choose $k$ in practice?
        \item[A.]
              We optimize the expression to get $c_1n - n(c_2)$ = 0 where $c_1$ and $c_2$ are coefficients of $nk$ and $n \lg(n/k)$. A constant choice of $k$ is optimal, in particular.
       \end{enumerate}

 \item[2-2]
       \textbf{\textit{Correctness of bubblesort}}\\
       Bubblesort is a popular, but inefficient, sorting algorithm. It works by repeatedly swapping adjacent elements that are out of order.

       \begin{enumerate}
        \item Let $A'$ denote the output of BUBBLESORT(A). To prove that BUBBLESORT is correct, we need to prove that it terminates and that
              \begin{align}
               A'[1] \leq A'[2] \leq \dots \leq A'[n],		\label{2.3}
              \end{align}
              where $n=A.length$. In order to show that BUBBLESORT actually sorts, what else do we need to prove?\\
              The next two parts will prove inequality (2.3).
        \item[A.]
              We need to prove that A 0 contains the same elements as A, which is easily seen to be true because the only modification we make to A is swapping its elements, so the resulting array must contain a rearrangement of the elements in the original array.

        \item State precisely a loop invariant for the for loop in lines 2–4, and prove that this loop invariant holds. Your proof should use the structure of the loop invariant proof presented in this chapter.
        \item[A.]
              At the start of each iteration, the position of the smallest element of $A[i \dots n]$ is at most $j$, it’s true prior to first iteration where the position of any element is at most $A.length$. To see that each iteration maintains the loop invariant, suppose that $j = k$ and the position of the smallest element of $A[i \dots n]$ is at most $k$, then we compare $A[k]$ to $A[k-1]$. If $A[k] < A[k-1]$ then $A[k-1]$ is not the smallest element of $A[i \dots n]$, so when we swap $A[k]$ and $A[k-1]$ we know that the smallest element of $A[i \dots n]$ must occur in the first $k-1$ positions of the subarray, the maintaining the invariant. On the other hand, if $A[k] \geq A[k-1]$ then the smallest element can’t be $A[k]$. Since we do nothing, we conclude that the smallest element has position at most $k-1$. Upon termination, the smallest element of $A[i \dots n]$ is in position $i$.

        \item Using the termination condition of the loop invariant proved in part(b), state a loop invariant for the for loop in lines 1–4 that will allow you to prove in-equality(2.3). Your proof should use the structure of the loop invariant proof presented in this chapter.
        \item[A.]
              At the start of each iteration the subarray $A[1..i-1]$ contains the $i-1$ smallest elements of $A$ in sorted order. Prior to the first iteration $i = 1$, and the first 0 elements of $A$ are trivially sorted. To see that each iteration maintains the loop invariant, fixing $i$ and suppose that $A[1 \dots i - 1]$ contains the $i - 1$ smallest elements of $A$ in sorted order. Then we run the loop in lines 2 through 4. We showed in part b that when this loop terminates, the smallest element of $A[i \dots n]$ is in position $i$. Since the $i-1$ smallest elements of $A$ are already in $A[1 \dots i - 1]$, $A[i]$ must be the $i$th smallest element of $A$. Therefore $A[1 \dots i]$ contains the $i$ smallest elements of $A$ in sorted order, maintaining the loop invariant. Upon termination, $A[1 \dots n]$ contains the $n$ elements of $A$ in sorted order as desired.

        \item What is the worst-case running time of bubblesort? How does it compare to the running time of insertion sort?
        \item[A.]
              The $i$th iteration of the for loop of lines 1 through 4 will cause $n-i$ iterations of the for loop of lines 2 through 4, each with constant time execution so the worst-case running time is $\Theta(n^2)$.
              This is the same as insertion sort; however bubble sort also has best-case running time $\Theta(n^2)$ whereas insertion sort has best-case running time $\Theta(n)$.
       \end{enumerate}

 \item[\textbf{2-3}]
       \textbf{\textit{Correctness of Horner's rule}}\\
       The following code fragment implements Horner's rule for evaluating a polynomial\\
       \begin{equation*}
        \begin{split}
         P(x) & = \sum_{k=0}^{n} a_k x^k \\
         & = a_0 + x (a_1 + x(a_2 + \dots + x(a_{n-1} + xa_n)\dots))
        \end{split}
       \end{equation*}
       given the coefficients $a_0, a_1, \dots ,a_n$  and a value for $x$:
       \alglanguage{pseudocode}
       \begin{algorithmic}
        \State $y \gets 0$
        \For{$i=n$ down to $0$}
        \State $y_i = a_i + x . y$
        \EndFor
       \end{algorithmic}

       \begin{enumerate}
        \item In terms of $\Theta$-notation, what is the running time of this code fragment for Horner’s rule?
        \item[A.]
              Assuming the arithmetic function is executed in constant time, then since the loop is being executed $n$ times, it has runtime $\Theta(n)$.

        \item Write pseudocode to implement the naive polynomial-evaluation algorithm that computes each term of the polynomial from scratch. What is the running time of this algorithm? How does it compare to Horner’s rule?
        \item[A.]
              \alglanguage{pseudocode}
              \begin{algorithmic}
               \State $y \gets 0$
               \For{$i \gets 0$ to $n$}
               \State $y_i \gets 1$
               \For{$j \gets 1$ to $i$}
               \State $y_i \gets y_i * x$
               \EndFor
               \State $y = y + a_i . y_i$
               \EndFor
              \end{algorithmic}
              The code has runtime $\Theta(n^2)$ as it has two nested for loops each running in linear time. It’s slower than Horner’s rule.

        \item Consider the following loop invariant:\\
              At the start of each iteration of the for loop of lines 2–3,
              \begin{equation*}
               y = \sum_{k=0}^{n-(i+1)} a_{k+i+1} x^k
              \end{equation*}
              Interpret a summation with no terms as equalling 0. Following the structure of the loop invariant proof presented in this chapter, use this loop invariant to show that, at termination, $\sum_{k=0}^{n} a_k x^k$.
        \item[A.]
              Initially $i=n$, so the upper bound of the summation is -1, so the sum evaluates to 0, which is the value of $y$. Assume that it is true for an $i$, then
              \begin{equation*}
               \begin{split}
                y & = a_i + x \sum_{k=0}^{n (i+1)} a_{k+i+1} x^k \\
                & = a_i + x \sum_{k=1}^{n-i} a_{k+i} x^{k-1} \\
                & = \sum_{k=0}^{n-i} a_{k+i} x^{k}
               \end{split}
              \end{equation*}

        \item Conclude by arguing that the given code fragment correctly evaluates a polynomial characterized by the coefficients $a_0, a_1, \dots , a_n$.
        \item[A.]
              As stated in the previous problem, we evaluated the algorithm $\sum_{k=0}^{n} a_k x^k$ and the value of the polynomial evaluated at $x$.
       \end{enumerate}

 \item[\textbf{2-4}]
       \textbf{\textit{Inversions}}\\
       Let $A[1 \dots n]$ be an array of n distinct numbers. If $i < j$ and $A[i] > A[j]$, then the pair($i$, $j$) is called an inversion of $A$.
       \begin{enumerate}
        \item List the five inversions of the array $(2, 3, 8, 6, 1)$.
        \item[A.]
              The five inversions are (2, 1), (3, 1), (8, 6), (8, 1), and (6, 1).

        \item What array with elements from the set ${1, 2,\dots,n}$ has the most inversions? How many does it have?
        \item[A.]
              The n-element array with the most inversions is $(n, n-1, \dots , 2, 1)$. It has $n-1+n-2+...+2+1 = n(n-1)/2$ inversions.

        \item What is the relationship between the running time of insertion sort and the number of inversions in the input array? Justify your answer.
        \item[A.]
              The running time is a constant times the no of inversions. Let $I(i)$ denote the number of $j < i$ such that $A[j] > A[i]$, and $\sum_{n}^{i=1} I(i)$ equals the number of inversions in $A$. Considering the while loop in the insertion sort algorithm, the loop will execute once for each element of A which has index less than j is larger than $A[j]$. Thus, it will execute $I(j)$ times. We reach the while loop once for each iteration in the for loop, so the no of constant time steps of insertion sort is  $\sum_{n}^{i=1} I(i)$ times of the inversion number of $A$.

        \item Give an algorithm that determines the number of inversions in any permutation on $n$ elements in $\Theta(n \lg n)$ worst-case time. (Hint: Modify merge sort)
        \item[A.]
              \alglanguage{pseudocode}
              \begin{algorithm}
               \caption{Inversions}
               \begin{algorithmic}
                \Procedure{Inversions}{$A$, $p$, $r$}
                \If{$p<r$}
                \State $q \gets \lfloor (p+r)/2 \rfloor $
                \State $left \gets$ \Call{Inversions}{$A$, $p$, $q$}
                \State $right \gets$ \Call{Inversions}{$A$, $q+1$, $r$}
                \State $inv \gets$ \Call{CountInversions}{$A$, $p$, $q$, $r$}$ + left + right$
                \State \Return $inv$
                \EndIf
                \State \Return 0
                \EndProcedure
               \end{algorithmic}
              \end{algorithm}

              \begin{algorithm}
               \caption{Count no of inversions}
               \begin{algorithmic}
                \Procedure{CountInversions}{$A$, $p$, $q$}
                \State $inv \gets 0$
                \State $n1 \gets q-p+1$
                \State $n2 \gets r-q$
                \State let $L[1,\dots , n1]$ and $R[1,\dots , n2]$ be new arrays
                \For{$i \gets 0 \To n1 - 1$}
                \State $L[i] \gets A[p+i]$
                \EndFor
                \For{$j \gets 0 \To n2 - 1$}
                \State $R[j] \gets A[q+j+1]$
                \EndFor
                \State $i \gets 0$
                \State $j \gets 0$
                \State $k \gets p$
                \While{$i \neq n1 \And j \neq n2$}
                \If{$L[i] \leq R[j]$}
                \State $A[k] \gets L[i]$
                \State $i \gets i+1$
                \Else
                \State $inv \gets inv+j$
                \Lcomment{This keeps track of the number of inversions between the left and right arrays}
                \State $j \gets j+1$
                \EndIf
                \State $k \gets k+1$;
                \EndWhile
                \If{$i = n1$}
                \For{$m \gets j \To n2-1$}
                \State $A[k] \gets R[m]$
                \State $k \gets k+1$
                \EndFor
                \EndIf
                \If{$j = n2$}
                \For{$m \gets i \To n1-1$}
                \State $A[k] \gets L[m]$
                \State $inv \gets inv+n2$
                \Lcomment{Tracks inversions once we have exhausted the right array. At this point, every element of the right array contributes an inversion}
                \State $k \gets k+1$
                \EndFor
                \EndIf
                \State \Return $inv$
                \EndProcedure
               \end{algorithmic}
              \end{algorithm}
       \end{enumerate}
\end{enumerate}

\chapter{Growth of Functions}

\section{Asymptotic notation}

\begin{enumerate}
 \item[Ex 3.1-1]
       Let $f(n)$ and $g(n)$ be asymptotically nonnegative functions. Using the basic definition of $\Theta$-notation, prove that max($f(n).g(n)$) = $\Theta(f(n)+g(n))$.
 \item[A.]

 \item[Ex 3.1-2]
       Show that for any real constants $a$ and $b$, where $b$ > 0,\\
       $(n+a)^b = \Theta(n^b)$
 \item[A.]
       Let $c = 2^b$ and $n_0 \geq 2a$, then for all $n \geq n_0$, we have $(n+a)^b \leq (2n)^b = cn^b$, so $(n+a)^b = O(n^b)$. Let $n_0 \geq \frac{-a}{1-1/2^{1/b}}$ and $c = 1/2$, then $n \geq n_0 \geq \frac{-a}{1-1/2^{1/b}}$ if and only if $n-2^{1/b} \geq -a$, also $n+a \geq (1/2)^{a/b}n$, also $(n+a)^b \geq cn^b$. Therefore $(n+a)^b = \Omega(n^b)$. By Theorem 3.1, $(n+a)^b = \Theta(n^b)$.

 \item[Ex 3.1-3]
       Explain why the statement, “The running time of algorithm $A$ is at least $O(n^2)$,” is meaningless.
 \item[A.]

 \item[Ex 3.1-4]
       Is $2^{n+1} = O(n^2)$? Is $2^{2n} = O(n^2)$?
 \item[A.]
       $2^{n+1} \geq 2 . 2^n$ for all $n \geq 0$, so $2^{n+1} = O(2^n)$, but, $2^{2n}$ is not $O(2^n)$. If there would exist $n_0$ and $c$ such that $n \geq n_0$ implies $2^n . 2^n = 2^{2n} \leq c2^n,$ so $2^n \leq c$ for $\ n \geq n_0$, which is clearly impossible since $c$ is a constant.

 \item[Ex 3.1-5]
       Prove Theorem 3.1.
 \item[A.]
       Suppose $f(n) \in \Theta(g(n))$, then $\exists c_1, c_2, n_0, \forall n \geq n_0, 0 \leq c_1g(n) \leq f(n) \leq c_2g(n)$. We have $c_1g(n) \leq f(n)(f(n) \in \Omega(g(n)))$ and $f(n) \leq c_2g(n)(f(n) \in O(g(n)))$.\\
       Suppose that we had $\exists n_1, c_1, \forall n \geq n_1, c_1g(n) \leq f(n)$ and $\exists n_2,c_2,\forall n \geq n_2, f(n) \leq c_2g(n)$. Putting these together, letting $n_0 = max(n_1,n_2)$, we have $\forall n \geq n_0$, $c_1g(n) \leq f(n) \leq c_2g(n)$.

 \item[Ex 3.1-6]
       Prove that the running time of an algorithm is $\Theta(g(n))$ if and only if its worst-case running time is $O(g(n))$ and its best-case running time is $\Omega(g(n))$.

 \item[Ex 3.1-7]
       Prove that $o(g(n)) \cap \omega(g(n))$ is the empty set.

 \item[Ex 3.1-8]
       We can extend our notation to the case of two parameters $n$ and $m$ that can go to infinity independently at different rates. For a given function $g(n, m)$, we denote by $O(g(n, m))$ the set of functions
       \begin{equation*}
        \begin{split}
         O(g(n, m)) = \{f(n,m) : & \text{ there exist positive constants} \\
         & \text{ such that } 0 \leq f(n,m) \leq cg(n,m) \\
         & \text{ for all } n \geq n_0 \text{ or } m \geq m_0 \}.
        \end{split}
       \end{equation*}
       Give corresponding definitions for $\Omega(g(n,m))$ and $\Theta(g(n,m))$.
\end{enumerate}

\section{Standard notations and common functions}

\begin{enumerate}
 \item[\textbf{Ex 3.2-1}]
       Show that if $f(n)$ and $g(n)$ are monotonically increasing functions, then so are the functions $f(n) + g(n)$ and $f(g(n))$, and if $f(n)$ and $g(n)$ are in addition nonnegative, then $f(n) . g(n)$ is monotonically increasing.

 \item[\textbf{Ex 3.2-2}]
       Prove the equation (3.16).

 \item[\textbf{Ex 3.2-3}]
       Prove the equation (3.19). Also prove that $n! = \omega(2^n)$ and $n! = o(n^n)$.

 \item[\textbf{Ex 3.2-4}]
       Is the function $\lceil \lg n \rceil !$ polynomially bounded? Is the function $\lceil \lg \lg n \rceil !$ polynomially bounded?

 \item[\textbf{Ex 3.2-5}]
       Which is asymptotically larger: $\lg(\lg^* n)$ or $\lg^*(\lg n)$?

 \item[\textbf{Ex 3.2-6}]
       Show that the golden ratio $\phi$ and its conjugate $\hat{\phi}$ both satisfy the equation $x^2 = x + 1$.

 \item[\textbf{Ex 3.2-7}]
       Prove by induction that the $i$th Fibonacci number satisfies the equality \\
       $F_i = \frac{\phi^i - \hat{\phi^i}}{\sqrt{5}}$ \\
       where $\phi$ is the golden ratio and $\hat{\phi}$ is its conjugate.

 \item[\textbf{Ex 3.2-8}]
       Show that $k \ln k = \Theta(n)$ implies $k=\Theta (n/\ln n)$.
\end{enumerate}

\subsection*{Problems}
\begin{enumerate}
 \item[\textbf{Prob 3-1}]
       \textbf{\textit{Asymptotic behavior of polynomials}}\\
       Let \\
       $p(n) = \sum_{i=0}^{d} a_in^i$,\\
       where $a_d=0$, be a degree-$d$ polynomial in $n$, and let $k$ be a constant. Use the definitions of the asymptotic notations to prove the following properties.
       \begin{enumerate}
        \item If $k \geq d$, then $p(n) = O(n^k)$.
        \item If $k \leq d$, then $p(n) = \Omega(n^k)$.
        \item If $k = d$, then $p(n) = \Theta(n^k)$.
        \item If $k > d$, then $p(n) = o(n^k)$.
        \item If $k < d$, then $p(n) = \omega(n^k)$.
       \end{enumerate}

 \item[\textbf{Prob 3-2}]
       \textbf{\textit{Relative asymptotic growths}}\\
       Indicate, for each pair of expressions $(A, B)$ in the table below, whether $A$ is $O$, $o$, $\Omega$, $\omega$, or $\Theta$ of $B$. Assume that $k \geq 1$, $\epsilon \> 0$, and $c>1$ are constants. Your answer should be in the form of the table with “yes” or “no” written in each box.
       \begin{tabular}{c c | p{2em} | p{2em} | p{2em} | p{2em} | p{2em} |}
        $A$         & $B$          & $O$ & $o$ & $\Omega$ & $\omega$ & $\Theta$ \\
        \hline
        $\lg^k n$   & $n^\epsilon$ &     &     &          &          &          \\
        \hline
        $n^k n$     & $c^n$        &     &     &          &          &          \\
        \hline
        $\sqrt{n}$  & $n^{\sin n}$ &     &     &          &          &          \\
        \hline
        $2^n$       & $2^{n/2}$    &     &     &          &          &          \\
        \hline
        $n^{\lg c}$ & $c^{\lg n}$  &     &     &          &          &          \\
        \hline
        $\lg(n!) n$ & $\lg(n^n)$   &     &     &          &          &          \\
        \hline
       \end{tabular}

 \item[\textbf{Prob 3-3}]
       \textbf{\textit{Ordering by asymptotic growth rates}}
       \begin{enumerate}
        \item Rank the following functions by order of growth; that is, find an arrangement $g_1, g_2, \dots , g_30$ of the functions satisfying $g_1 = \Omega (g_2), g_2 D \Omega (g_3), \dots,g_29 = \Omega (g_30)$s. Partition your list into equivalence classes such that functions $f(n)$ and $g(n)$ are in the same class if and only if $f(n) = \Theta(g(n))$.
              \begin{tabular}{c c c c c c}
               $\lg(\lg^* n)$    & $2^{lg^* n}$         & $(\sqrt{2})^{\lg n}$ & $n^2$           & $n!$      & $(\lg n)!$     \\
               $(\frac{3}{2})^n$ & $n^3$                & $\lg^2 n$            & $\lg(n!)$       & $2^{2^n}$ & $n^{1/\lg n}$  \\
               $\ln \ln n$       & $\lg^* n$            & $n . 2^n$            & $n^{\lg \lg n}$ & $\ln n$   & $1$            \\
               $2^{\lg n}$       & $(\lg n)^{\lg n}$    & $e^n$                & $4^{\lg n}$     & $(n+1)!$  & $\sqrt{\lg n}$ \\
               $(\lg^*(\lg n)$   & $2^{\sqrt{2 \lg n}}$ & $n$                  & $2^n$           & $n \lg n$ & $2^{2^{n+1}}$  \\
              \end{tabular}
        \item Give an example of a single nonnegative function $f(n)$ such that for all functions $g_i(n)$ in part (a), $f(n)$ is neither $O(g_i(n))$ nor $\Theta(g_i(n))$.
       \end{enumerate}

 \item[\textbf{Prob 3-4}]
       \textbf{\textit{Asymptotic notation properties}}\\
       Let $f(n)$ and $g(n)$ be asymptotically positive functions. Prove or disprove each of the following conjectures.
       \begin{enumerate}
        \item $f(n) = O(g(n))$ implies $g(n) = O(f(n))$
        \item $f(n) + g(n) = \Theta(min(f(n), g(n)))$
        \item $f(n) = O(g(n))$ implies $(\lg(f(n)))$, where $\lg(g(n)) \geq 1$ and $f(n) \geq 1$ for all sufficiently large $n$.
        \item $f(n) = O(g(n))$ implies $2^{f(n)} = O(2^{g(n)})$.
        \item $f(n) = O((f(n))^2)$.
        \item $f(n) = O(g(n))$ implies $g(n) = \Omega(f(n))$.
        \item $f(n) = \Theta((f(n/2))$
        \item $f(n) + o(f(n)) = \Theta((f(n))$
       \end{enumerate}

 \item[\textbf{Prob 3-5}]
       \textbf{\textit{Asymptotic notation properties}}\\
       Some authors define $\Omega$ in a slightly different way than we do; let’s use $\Omega^\infty$ (read “omega infinity”) for this alternative definition. We say that $f(n) = \Omega^\infty (g(n))$ if there exists a positive constant $c$ such that $f(n) \geq cg(n) \geq 0$ for infinitely many integers $n$.
       \begin{enumerate}
        \item Show that for any two functions $f(n)$ and $g(n)$ that are asymptotically nonnegative, either $f(n) = O(g(n))$ or $f(n) = \Omega^\infty(g(n))$ or both, whereas this is not true if we use $\Omega$ in place of $\Omega^\infty$.
        \item Describe the potential advantages and disadvantages of using $\Omega^\infty$ instead of $\Omega$ to characterize the running times of programs.\\
              Some authors also define $O$ in a slightly different manner; let’s use $O'$ for the alternative definition. We say that $f(n) = O'(g(n))$ if and only if $\lvert f(n) \rvert = O(g(n))$.
        \item What happens to each direction of the “if and only if” in Theorem 3.1 if we substitute $O'$ for $O$ but still use $\Omega$?\\
              Some authors define $\tilde{O}$ (read “soft-oh”) to mean $O$ with logarithmic factors ignored:
              \begin{equation*}
               \begin{split}
                \tilde{O}g(n) = \{f(n) : & \text{ there exist positive constants  } c, k \text{ and } n_0 \text{ such that }\\
                & \ 0\leq f(n) \leq cg(n)\lg^k(n) \text{ for all } n \geq n_0 \}
               \end{split}
              \end{equation*}
        \item Define $\tilde{\Omega}$ and $\tilde{\Theta}$ in a similar manner. Prove the corresponding analog to Theorem 3.1.
       \end{enumerate}

 \item[\textbf{Prob 3-6}]
       \textbf{\textit{Iterated functions}}\\
       We can apply the iteration operator* used in the lg* function to any monotonically increasing function $f(n)$ over the reals. For a given constant $c \in \mathbb{R}$, we define the iterated function $f_c^*$ by\\
       \[ f_c^*(n) = \min\{ i \geq 0\ :\ f^{(i)} \leq c\},\] \\
       which need not be well defined in all cases. In other words, the quantity $f_c^*(n)$ is the number of iterated applications of the function $f$ required to reduce its argument down to $c$ or less.\\
       For each of the following functions $f(n)$ and constants $c$, give as tight a bound as possible on $f_c^*(n)$. \\
       \begin{tabular}{c c | c |}
        $f(n)$     & $c$ & $f_c^*(n)$                        \\
        \hline
        $n-1$      & $0$ & $\lceil n \rceil$                 \\
        \hline
        $\lg n$    & $1$ & $\lg^* n$                         \\
        \hline
        $n/2$      & $1$ & $\lceil \lg(n) \rceil$            \\
        \hline
        $n/2$      & $2$ & $\lceil \lg(n) \rceil - 1$        \\
        \hline
        $\sqrt{n}$ & $2$ & $\lg \lg n$                       \\
        \hline
        $\sqrt{n}$ & $1$ & $undefined$                       \\
        \hline
        $n^{1/3}$  & $2$ & $\lg_3 \lg_2 (n)$                 \\
        \hline
        $n/\lg n$  & $2$ & $\Omega(\frac{\lg n}{\lg \lg n})$ \\
        \hline
       \end{tabular}

\end{enumerate}

\chapter{Divide-and-Conquer}

\section{The maximum-subarray problem}
\begin{enumerate}
 \item[\textbf{Ex 4.1-1}]
       What does FIND-MAXIMUM-SUBARRAY return when all elements of $A$ are negative?
 \item[A.]
       It will return the least negative one. As each of the cross sums are computed, the most positive one must have the shortest possible length. The algorithm doesn’t consider zero sub-arrays, so it must have length 1.

 \item[\textbf{Ex 4.1-2}]
       Write pseudocode for the brute-force method of solving the maximum subarray problem. Your procedure should run in $\Theta(n^2)$ time.
 \item[A.]
       \alglanguage{pseudocode}
       \begin{algorithm}
        \caption{Brute-force Max Subarray}
        \begin{algorithmic}
         \Procedure{MaxSubarray}{A}
         \State $left \gets 0$
         \State $right \gets 0$
         \State $max \gets A[0]$
         \Comment{Increment left end of subarray}
         \For{$i \gets 0 \To n-1$}
         \State $curSum \gets 0$
         \Comment{Increment right end of subarray}
         \For{$j \gets i \To n-1$}
         \State $curSum \gets curSum + A[j]$
         \If{$curSum > max$}
         \State $max = curSum$
         \State $left = i$
         \State $right = j$
         \EndIf
         \EndFor
         \EndFor
         \State \Return $i$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[\textbf{Ex 4.1-3}]
       Implement both the brute-force and recursive algorithms for the maximum-subarray problem on your own computer. What problem size $n_0$ gives the crossover point at which the recursive algorithm beats the brute-force algorithm? Then, change the base case of the recursive algorithm to use the brute-force algorithm whenever the problem size is less than $n_0$ . Does that change the crossover point?

 \item[\textbf{Ex 4.1-4}]
       Suppose we change the definition of the maximum subarray problem to allow the result to be an empty subarray, where the sum of the values of an empty subarray is 0. How would you change any of the algorithms that do not allow empty subarrays to permit an empty subarray to be the result?
 \item[A.]
       Do a linear scan of the input array to see if it contains any positive entries. If it does, run the algorithm as usual otherwise return the empty subarray with sum 0 and terminate the algorithm.

 \item[\textbf{Ex 4.1-5}]
       Use the following ideas to develop a nonrecursive, linear-time algorithm for the maximum subarray problem. Start at the left end of the array, and progress toward the right, keeping track of the maximum subarray seen so far. Knowing a maximum subarray of $A[1 \dots j]$, extend the answer to find a maximum subarray ending at index $j+1$ by using the following observation: a maximum subarray of $A[1 \dots j+1]$ is either a maximum subarray of $A[1 \dots j]$ or a subarray $A[i \dots j+1]$, for some $1 \leq i \leq j+1$. Determine a maximum subarray of the form $A[i \dots j+1]$ in constant time based on knowing a maximum subarray ending at index $j$.
\end{enumerate}

\section{Strassen’s algorithm for matrix multiplication}
\begin{enumerate}
 \item[\textbf{Ex 4.2-1}]
       Use Strassen's algorithm to compute the matrix product\\
       \[
        \begin{pmatrix}	1 & 3 \\ 7 & 5 \end{pmatrix}
        \begin{pmatrix}	1 & 3 \\ 7 & 5 \end{pmatrix}\\
       \]
       Show your work.
 \item[A.]
       $S_1$ = 8 - 2 = 6\\
       $S_2$ = 1 + 3 = 4\\
       $S_3$ = 7 + 5 = 12\\
       $S_4$ = 4 - 6 = -2\\
       $S_5$ = 1 + 5 = 6\\
       $S_6$ = 6 + 2 = 8\\
       $S_7$ = 3 - 5 = -2\\
       $S_8$ = 4 + 2 = 6\\
       $S_9$ = 1 - 7 = -6\\
       $S_{10}$ = 6 + 8 = 14

       $P_1$ = 6, $P_2$ = 8, $P_3$ = 72,
       $P_4$ = -10, $P_5$ = 48, $P_6$ = -12,
       $P_7$ = -84\\

       $C_{11}$ = 48 - 10 - 8 - 12 = 18 \\
       $C_{12}$ = 6 + 8 = 14 \\
       $C_{21}$ = 72 - 10 = 62 \\
       $C_{22}$ = 48 + 6 - 72 + 84 = 96 \\
       So, we get the final result:
       \[
        \begin{pmatrix} 18 & 14 \\ 62 & 96 \end{pmatrix}
       \]

 \item[\textbf{Ex 4.2-2}]
       Write pseudocode for Strassen’s algorithm.
 \item[A.]
       \alglanguage{pseudocode}
       \begin{algorithm}
        \caption{Strassen's Algorithm}
        \begin{algorithmic}
         \Procedure{Strassen}{A, B}
         \Lcomment{Let $A[i \dots j][k \dots m]$ denote the submatrix of $A$ consisting of rows $i$ through $j$ and columns $k$ through $m$.}

         \If{$A.length = 1$}
         \Return $A[0] . B[0]$
         \EndIf
         \Comment{Let $C$ be a new $n \times n$ matrix}
         \State $A11 \gets A[0 \dots n/2-1][0 \dots n/2-1]$
         \State $A12 \gets A[0 \dots n/2-1][n/2 \dots n-1]$
         \State $A21 \gets A[n/2 \dots n-1][0 \dots n/2-1]$
         \State $A22 \gets A[n/2 \dots n-1][n/2 \dots n-1]$
         \State $S1 = B12 - B22$
         \State $S2 = A11 + A12$
         \State $S3 = A21 + A22$
         \State $S4 = B21 - B11$
         \State $S5 = A11 + A22$
         \State $S6 = B11 + B22$
         \State $S7 = A12 - A22$
         \State $S8 = B21 + B22$
         \State $S9 = A11 - A21$
         \State $S10 = B11 + B12$
         \State $P1 = \Call{Strassen}{A11, S1}$
         \State $P2 = \Call{Strassen}{S2, B22}$
         \State $P3 = \Call{Strassen}{S3, B11}$
         \State $P4 = \Call{Strassen}{A22, S4}$
         \State $P5 = \Call{Strassen}{S5, S6}$
         \State $P6 = \Call{Strassen}{S7, S8}$
         \State $P7 = \Call{Strassen}{S9, S10}$
         \State $C[0 \dots n/2-1][0 \dots n/2-1] = P5 + P4 - P2 + P6$
         \State $C[0 \dots n/2-1][n/2 \dots n-1] = P1 + P2$
         \State $C[n/2 \dots n-1][0 \dots n/2-1] = P3 + P4$
         \State $C[n/2 \dots n-1][n/2 \dots n-1] = P5 + P1 - P3 - P7$
         \State \Return $C$
         \EndProcedure
        \end{algorithmic}
       \end{algorithm}

 \item[\textbf{Ex 4.2-3}]
       How would you modify Strassen’s algorithm to multiply $n \times n$ matrices in which $n$ is not an exact power of 2? Show that the resulting algorithm runs in time $\Theta(n^{\lg 7})$.
 \item[A.]
       We can pad out the input matrices to the next largest powers of 2 and run the given algorithm. This will at most double the value of n because each power of 2 is off from each other by a factor of 2. So, this will have runtime \\
       $m^{\lg 7} \leq (2n)^{\lg 7} = 7n^{\lg 7} \in O(n^{\lg 7})$ and $m^{\lg 7} \geq n^{\lg 7} \in \Omega(n^{\lg 7})$ \\
       Putting these together, we get the runtime is $\Theta(n^{\lg 7})$.

 \item[\textbf{Ex 4.2-4}]
       What is the largest $k$ such that if you can multiply $3 \times 3$ matrices using $k$ multiplications (not assuming commutativity of multiplication), then you can multiply $n \times n$ matrices in time $o(n^{\lg 7})$? What would the running time of this algorithm be?
 \item[A.]
       Assume that $n=3^m$ for some $m$, then using block matrix multiplication, we obtain the recursive running time $T(n) = kT(n/3) + O(1)$. Using the Master theorem, we need the largest integer $k$ such that $\log_3 k < \lg 7$. This is given by $k=21$.

 \item[\textbf{Ex 4.2-5}]
       V. Pan has discovered a way of multiplying $68 \times 68$ matrices using 132,464 multiplications, a way of multiplying $70 \times 70$ matrices using 143,640 multiplications, and a way of multiplying $72 \times 72$ matrices using 155,424 multiplications. Which method yields the best asymptotic running time when used in a divide-and-conquer matrix-multiplication algorithm? How does it compare to Strassen’s algorithm?

 \item[\textbf{Ex 4.2-6}]
       How quickly can you multiply a $k n \times n$ matrix by an $n \times k n$ matrix, using Strassen’s algorithm as a subroutine? Answer the same question with the order of the input matrices reversed.

 \item[\textbf{Ex 4.2-7}]
       Show how to multiply the complex numbers $a + bi$ and $c + di$ using only three multiplications of real numbers. The algorithm should take $a$, $b$, $c$, and $d$ as input and produce the real component $ac - bd$ and the imaginary component $ad + bc$ separately.
 \item[A.]
       We can see that the final result should be
       \[ (a+bi) (c+di) = ac-bd+(cb + ad)i \]
       We will be multiplying \\
       \[ P1 = (a+b)c = ac + bc,\ P2 = b(c+d) = bc + bd,\ P3 = (a-b)d \]
       We can get real part by taking $P1-P2$ and imaginary part by taking $P2+P3$.
\end{enumerate}

\section{The substitution method for solving recurrences}
\begin{enumerate}
 \item[\textbf{Ex 4.3-1}]
       Show that the solution of $T(n) = T(n-1)+n$ is $O(n^2)$.
 \item[A.]
       Assume by induction, $T(n) \leq cn^2$, where $c$ is taken to be max(1, $T(1)$), then
       \[ T(n) = T(n-1) + n \leq c(n-1)^2 + n = cn^2 + (1-2c)n + 1 \leq cn^2 + 2 - 2c \leq cn^2 \]
       By inductive hypothesis from the first inequality, the second from the fact that $n \geq 1$ and $1-2c<0$ and from the fact that $c \geq 1$.

 \item[\textbf{Ex 4.3-2}]
       Show that the solution of $T(n) = T(\lceil n/2 \rceil) + 1$ is $O(\lg n)$.
 \item[A.]
       Assume $T(n) \leq 3\log n - 1$, which implies $T(n) = O(\lg n)$.
       \begin{equation*}
        \begin{split}
         T(n) & = T(\lceil n/2 \rceil) + 1 \\
         & \leq 3 \lg(\lceil n/2 \rceil) -1 + 1 \\
         & \leq 3 \lg(3n/4)\\
         & = 3\lg n + 3\lg(3/4)\\
         & \leq 3\lg n + \lg(1/2)\\
         & = 3\lg n - 1
        \end{split}
       \end{equation*}

 \item[\textbf{Ex 4.3-3}]
       We saw that the solution of $T(n) = 2T(\lfloor n/2 \rfloor) + n$ is $O(n \lg n)$. Show that the solution of this recurrence is also $\Omega(n \lg n)$. Conclude that the solution is $\Theta(n \lg n)$.

 \item[\textbf{Ex 4.3-4}]
       Show that by making a different inductive hypothesis, we can overcome the difficulty with the boundary condition $T(1) = 1$ for recurrence (4.19) without adjusting the boundary conditions for the inductive proof.

 \item[\textbf{Ex 4.3-5}]
       Show that $\Theta(n \lg n)$ is the solution to the “exact” recurrence (4.3) for merge sort.

 \item[\textbf{Ex 4.3-6}]
       Show that the solution to $T(n) = 2T(\lceil n/2 \rceil + 17) + n$ is $O(n \lg n)$.

 \item[\textbf{Ex 4.3-7}]
       Using the master method in Section 4.5, you can show that the solution to the recurrence $T(n) = 4T(n/3)+n$ is $T(n) = \Theta(n^{\log_3 4})$. Show that a substitution proof with the assumption $T(n) \leq cn^{\log_3 4}$ fails. Then show how to subtract off a lower-order term to make a substitution proof work.

 \item[\textbf{Ex 4.3-8}]
       Using the master method in Section 4.5, you can show that the solution to the recurrence $T(n) = 4T(n/3)+n^2$ is $T(n) = \Theta(n^2)$. Show that a substitution proof with the assumption $T(n) \leq cn^2$ fails. Then show how to subtract off a lower-order term to make a substitution proof work.
 \item[A.]

 \item[\textbf{Ex 4.3-9}]
       Solve the recurrence $T(n) = 3T(n) + \log n$ by making a change of variables. Your solution should be asymptotically tight. Do not worry about whether values are integral.
 \item[A.]
       Consider $n$ of the form $2k$ and then the recurrence becomes
       \[T(2^k) = 3T(2^{k/2}) + k\]
       We define $S(k) = T(2^k)$, so $S(k) = 3S(k/2) + k$ \\
       We use the inductive hypothesis $S(k) \leq (S(1)+2)k^{\lg 3} - 2k$
       \[ S(k) = 3S(k/2) + k \leq 3 (S(1) +2) (k/2)^{\lg 3} - 3k + k = (S(1) + 2)k^{\lg 3} - 2k \]
       as desired. Similarly, we also show that $S(k) \geq (S(1) + 2)k^{\lg 3} - 2k$
       \[ S(k) = 3S(k/2) + k \geq (S(1) + 2)k^{\lg 3} - 2k\]
       So, we get $S(k) = (S(1) + 2)k^{\lg 3} - 2k$ and putting this into $T$, $T(2^k) = (T(2) + 2)k^{\lg 3} - 2k$. So, $T(n) = (T(2) + 2)(\lg n)^{\lg 3} - 2\lg(n)$.
\end{enumerate}

\section{The recursion-tree method for solving recurrences}
\begin{enumerate}
 \item[\textbf{Ex 4.4-1}]
       Use a recursion tree to determine a good asymptotic upper bound on the recurrence $T(n) = 3T(\lceil n/2 \rceil) + n$. Use the substitution method to verify your answer.

 \item[\textbf{Ex 4.4-2}]
       Use a recursion tree to determine a good asymptotic upper bound on the recurrence $T(n) = T(n/2) + n^2$. Use the substitution method to verify your answer.

 \item[\textbf{Ex 4.4-3}]
       Use a recursion tree to determine a good asymptotic upper bound on the recurrence $T(n) = 4T(n/2+2) + n$. Use the substitution method to verify your answer.

 \item[\textbf{Ex 4.4-4}]
       Use a recursion tree to determine a good asymptotic upper bound on the recurrence $T(n) = 2T(n-1) + 1$. Use the substitution method to verify your answer.

 \item[\textbf{Ex 4.4-5}]
       Use a recursion tree to determine a good asymptotic upper bound on the recurrence $T(n) = T(n-1) + T(n/2) + n$. Use the substitution method to verify your answer.

 \item[\textbf{Ex 4.4-6}]
       Argue that the solution to the recurrence $T(n) = T(n/3) + T(2n/3) + cn$, where $c$ is a constant, $\Omega(n \lg n)$ by appealing to a recursion tree.

 \item[\textbf{Ex 4.4-7}]
       Draw the recursion tree for $T(n) = 4T(\lceil n/2 \rceil) + cn$, where $c$ is a constant, and provide a tight asymptotic bound on its solution. Verify your bound by the substitution method.

 \item[\textbf{Ex 4.4-8}]
       Use a recursion tree to give an asymptotically tight solution to the recurrence $T(n) = T(n-a) + T(a) + cn$, where $a \leq 1$ and $c> 0$ are constants.

 \item[\textbf{Ex 4.4-9}]
       Use a recursion tree to give an asymptotically tight solution to the recurrence $T(n) = T(\alpha n) + T((1 - \alpha)n) + cn$, where $\alpha$ is a constant in the range $0< \alpha <1$ and $c > 0$ is also a constant.
\end{enumerate}

\section{The master method for solving recurrences}
\begin{enumerate}
 \item[\textbf{Ex 4.5-1}]
       Use the master method to give tight asymptotic bounds for the following recurrences.
       \begin{enumerate}
        \item $\mathbf{T(n) = 2T(n/4) + 1}$
        \item[A.]
              $a=2$, $b=4$, $n^{\log_b a}=n^{1/2}$ , $f(n)=1=O(n^{1/2})$ \\
              By rule 1, $T(n) = \Theta(\sqrt{n})$

        \item $\mathbf{T(n) = 2T(n/4) + \sqrt{n}}$
        \item[A.]
              $a=2$, $b=4$, $n^{\log_b a}=n^{1/2}$ , $f(n)=n^{1/2}=\Theta(n^{1/2})$\\						By rule 2, $T(n) = \Theta(\sqrt{n} \lg n)$

        \item $\mathbf{T(n) = 2T(n/4) + n}$
        \item[A.]
              $a=2$, $b=4$, $n^{\log_b a}=n^{1/2}$ , $f(n)=n^{1}=\Omega(n^{1/2})$\\
              By rule 3, $T(n) = \Theta(n)$

        \item $\mathbf{T(n) = 2T(n/4) + n^2}$
        \item[A.]
              $a=2$, $b=4$, $n^{\log_b a}=n^{1/2}$ , $f(n)=n^{2}=\Omega(n^{1/2})$\\
              By rule 3, $T(n) = \Theta(n^2)$
       \end{enumerate}

 \item[\textbf{Ex 4.5-2}]
       What is the largest integer value of a for which Professor Caesar’s algorithm would be asymptotically faster than Strassen’s algorithm?

 \item[\textbf{Ex 4.5-3}]
       Use the master method to show that the solution to the binary-search recurrence $\mathbf{T(n) = T(n/2)+ \Theta(1)}$ is $\mathbf{T(n)= \Theta(\lg n)}$.

 \item[\textbf{Ex 4.5-4}]
       Can the master method be applied to the recurrence $T(n) = 4T(n/2)+n^2 \lg n$? Why or why not? Give an asymptotic upper bound for this recurrence.

 \item[\textbf{Ex 4.5-5}]
       Consider the regularity condition $af(n/b) \leq cf(n)$ for some constant $c < 1$ , which is part of case 3 of the master theorem. Give an example of constants $a \geq 1$ and $b > 1$ and a function $f(n)$ that satisfies all the conditions in case 3 of the master theorem except the regularity condition.

\end{enumerate}

\section{Proof of the master theorem}
\begin{enumerate}
 \item[\textbf{Ex 4.6-1}]
       Give a simple and exact expression for $n_j$ in equation (4.27) for the case in which $b$ is a positive integer instead of an arbitrary real number.

 \item[\textbf{Ex 4.6-2}]
       Show that if $f(n) = \Theta(n^{\log_b a} \lg^k n)$, where $k \geq 0$, then the master recurrence has solution $T(n) = \Theta(n^{\log_b a} \lg^{k+1} n)$. For simplicity, confine your analysis to exact powers of $b$.

 \item[\textbf{Ex 4.6-3}]
       Show that case 3 of the master theorem is overstated, in the sense that the regularity condition $af(n/b) \leq cf(n)$ for some constant $c < 1$ implies that there exists a constant $\epsilon > 0$ such that $f(n) = \Omega(n^{\log_b a+ \epsilon})$.

\end{enumerate}

\subsection*{Problems}
\begin{enumerate}
 \item[\textbf{4-1}]
       \textbf{\textit{Recurrence examples}}\\
       Give asymptotic upper and lower bounds for $T(n)$ in each of the following recurrences. Assume that $T(n)$ is constant for $n \leq 2$. Make your bounds as tight as possible, and justify your answers.
       \begin{enumerate}
        \item $T(n) = 2T(n/2) + n^4$
        \item $T(n) = T(7n/10) + n$
        \item $T(n) = 16T(n/4) + n^2$
        \item $T(n) = 7T(n/3) + n^2$
        \item $T(n) = 7T(n/2) + n^2$
        \item $T(n) = 2T(n/4) + \sqrt{n}$
        \item $T(n) = T(n-2) + n^2$
       \end{enumerate}

 \item[\textbf{4-2}]
       \textbf{\textit{Parameter-passing costs}}\\
       We assume that parameter passing during procedure calls takes constant time, even if an N -element array is being passed. This assumption is valid in most systems because a pointer to the array is passed, not the array itself. This problem examines the implications of three parameter-passing strategies:\\
       1. An array is passed by pointer. Time = $\Theta(1)$.\\
       2. An array is passed by copying. Time = $\Theta(N)$, where $N$ is the size of the array.
       3. An array is passed by copying only the subrange that might be accessed by the
       called procedure. Time = $\Theta(q - p + 1)$ if the subarray $A[p \dots q]$ is passed.
       \begin{enumerate}
        \item Consider the recursive binary search algorithm for finding a number in a sorted array (see Exercise 2.3-5). Give recurrences for the worst-case running times
              of binary search when arrays are passed using each of the three methods above, and give good upper bounds on the solutions of the recurrences. Let N be the size of the original problem and n be the size of a subproblem.
        \item Redo part(a) for the MERGE-SORT algorithm from Section 2.3.1.
       \end{enumerate}

 \item[\textbf{4-3}]
       \textbf{\textit{More recurrence examples}}\\
       Give asymptotic upper and lower bounds for $T(n)$ in each of the following recurrences. Assume that $T(n)$ is constant for sufficiently small $n$. Make your bounds
       as tight as possible, and justify your answers.
       \begin{enumerate}
        \item $T(n) = 4T(n/3) + n \lg n$.
        \item $T(n) = 3T(n/3) + n / \lg n$.
        \item $T(n) = 4T(n/2) + n^2 \sqrt{n}$.
        \item $T(n) = 3T(n/3-2) + n/2$.
        \item $T(n) = 2T(n/2) + n / \lg n$.
        \item $T(n) = T(n/2) + T(n/4) + T(n/8) + n$.
        \item $T(n) = T(n-1) + 1 / n$.
        \item $T(n) = T(n-1) + \lg n$.
        \item $T(n) = T(n-2) + 1 / \lg n$.
        \item $T(n) = \sqrt{n} T(\sqrt{n}) + n$.
       \end{enumerate}

 \item[\textbf{4-4}]
       \textbf{\textit{Fibonacci numbers}}\\
       This problem develops properties of the Fibonacci numbers, which are defined by recurrence (3.22). We shall use the technique of generating functions to solve the Fibonacci recurrence. Define the generating function (or formal power series) F as
       \begin{equation*}
        \begin{split}
         F(z) & = \sum_{i=0}^\infty F_iz^i\\
         & = 0 + z + z^2 + 3z^4 + 5z^5 + 8z^6 + 13z^7 + 21z^8 + ... ,
        \end{split}
       \end{equation*}
       where $F_i$ is the $i$th Fibonacci number.
       \begin{enumerate}
        \item Show that $F(z) = z + zF(z) + z^2F(z)$.
        \item Show that
              \begin{equation*}
               \begin{split}
                F(z) & = \frac{z}{1-z-z^2} \\
                & = \frac{z}{(1-\phi z)(1-\hat{\phi} z)}\\
                & = \frac{1}{\sqrt{5}} \left( \frac{1}{1-\phi z} - \frac{1}{1-\hat{\phi} z} \right),
               \end{split}
              \end{equation*}
              where $\phi = \frac{1+\sqrt{5}}{2} = 1.61803 \dots$\\
              and\\
              where $\hat{\phi} = \frac{1-\sqrt{5}}{2} = -0.61803 \dots$
        \item Show that
              \[ F(z) = \sum_{i=0}^{\infty} \frac{1}{\sqrt{5}} (\phi^i-\hat{\phi^i}) z^i . \]
        \item Use part(c) to prove that $F_i = \phi^i/\sqrt{5}$ for $i>0$, rounded to the nearest integer. ($\hat{\phi} < 1$)
       \end{enumerate}

 \item[\textbf{4-5}]
       \textbf{\textit{Chip testing}}\\
       Professor Diogenes has n supposedly identical integrated-circuit chips that in principle are capable of testing each other. The professor's test jig accommodates two
       chips at a time. When the jig is loaded, each chip tests the other and reports whether
       it is good or bad. A good chip always reports accurately whether the other chip is
       good or bad, but the professor cannot trust the answer of a bad chip. Thus, the four
       possible outcomes of a test are as follows: \\
       \begin{tabular}{c c c}
        Chip $A$ says & Chip $B$ says & Conclusion                     \\
        \hline
        B is good     & A is good     & both are good, or both are bad \\
        B is good     & A is bad      & at least one is bad            \\
        B is bad      & A is good     & at least one is bad            \\
        B is bad      & A is bad      & at least one is bad
       \end{tabular}
       \begin{enumerate}
        \item Show that if more than $n/2$ chips are bad, the professor cannot necessarily determine which chips are good using any strategy based on this kind of pairwise test. Assume that the bad chips can conspire to fool the professor.
        \item Consider the problem of finding a single good chip from among n chips, assuming that more than $n/2$ of the chips are good. Show that $\lceil n/2 \rceil$ pairwise tests are sufficient to reduce the problem to one of nearly half the size.
        \item Show that the good chips can be identified with $\Theta(n)$ pairwise tests, assuming that more than $n/2$ of the chips are good. Give and solve the recurrence that describes the number of tests.
       \end{enumerate}

 \item[\textbf{4-6}]
       \textbf{\textit{Monge arrays}}\\
       An $m \times n$ array $A$ of real numbers is a Monge array if for all $i, j, k,$ and $l$ such that $1 \leq i \leq k \leq m$ and $1 \leq j \leq l \leq n$, we have\\
       \[ A[i,j] + A[k,l] \leq A[i,l] + A[k,j]. \]
       In other words, whenever we pick two rows and two columns of a Monge array and
       consider the four elements at the intersections of the rows and the columns, the sum
       of the upper-left and lower-right elements is less than or equal to the sum of the
       lower-left and upper-right elements. For example, the following array is Monge:\\
       \begin{tabular}{c c c c c}
        10 & 17 & 13 & 28 & 23 \\
        17 & 22 & 16 & 29 & 23 \\
        24 & 28 & 22 & 34 & 24 \\
        11 & 13 & 6  & 17 & 7  \\
        45 & 44 & 32 & 37 & 23 \\
        36 & 33 & 19 & 21 & 6  \\
        75 & 66 & 51 & 53 & 34 \\
       \end{tabular}
       \begin{enumerate}
        \item Prove that an array is Monge if and only if for all $i = 1, 2, \dots, m-1$ and $j = 1, 2, \dots, n-1$, we have \\
              \[A[i, j] + A[i + 1, j + 1] \leq A[i, j + 1] + A[i + 1, j].\]
              (Hint: For the “if” part, use induction separately on rows and columns.)
        \item The following array is not Monge. Change one element in order to make it Monge. (Hint: Use part (a).) \\
              \begin{tabular}{c c c c}
               37 & 23 & 22 & 32 \\
               21 & 6  & 7  & 10 \\
               53 & 34 & 30 & 31 \\
               32 & 13 & 9  & 6  \\
               43 & 21 & 15 & 8  \\
              \end{tabular}
        \item Let $f(i)$ be the index of the column containing the leftmost minimum element of row i. Prove that $f(1) \leq f(2) \leq \dots \leq f(m)$ for any $m \times n$ Monge array.
        \item Here is a description of a divide-and-conquer algorithm that computes the leftmost minimum element in each row of an $m \times n$ Monge array A:\\
              Construct a submatrix $A'$ of $A$ consisting of the even-numbered rows of $A$. Recursively determine the leftmost minimum for each row of $A'$. Then compute the leftmost minimum in the odd-numbered rows of $A$.\\
              Explain how to compute the leftmost minimum in the odd-numbered rows of $A$
              (given that the leftmost minimum of the even-numbered rows is known) in $O(m + n)$ time.
        \item Write the recurrence describing the running time of the algorithm described in part(d). Show that its solution is $O(m + n \log m)$.
       \end{enumerate}

\end{enumerate}

\chapter{Probabilistic Analysis and Randomized Algorithms}

\section{The hiring problem}
\begin{enumerate}
 \item[\textbf{Ex 5.1-1}]
       Show that the assumption that we are always able to determine which candidate is best, in line 4 of procedure HIRE-ASSISTANT, implies that we know a total order on the ranks of the candidates.

 \item[\textbf{Ex 5.1-2}]
       Describe an implementation of the procedure RANDOM(a, b) that only makes calls to RANDOM(0, 1). What is the expected running time of your procedure, as a function of a and b?

 \item[\textbf{Ex 5.1-3}]
       Suppose that you want to output 0 with probability 1/2 and 1 with probability 1/2. At your disposal is a procedure BIASED-RANDOM, that outputs either 0 or 1. It outputs 1 with some probability $p$ and 0 with probability $1 - p$, where $0 < p < 1$, but you do not know what $p$ is. Give an algorithm that uses BIASED-RANDOM as a subroutine, and returns an unbiased answer, returning 0 with probability 1/2 and 1 with probability 1/2. What is the expected running time of your algorithm as a function of $p$?

\end{enumerate}

\section{Indicator random variables}
\begin{enumerate}
 \item[\textbf{Ex 5.2-1}]
       In HIRE-ASSISTANT, assuming that the candidates are presented in a random order, what is the probability that you hire exactly one time? What is the probability that you hire exactly $n$ times?

 \item[\textbf{Ex 5.2-2}]
       In HIRE-ASSISTANT, assuming that the candidates are presented in a random order, what is the probability that you hire exactly twice?

 \item[\textbf{Ex 5.2-3}]
       Use indicator random variables to compute the expected value of the sum of $n$ dice.

 \item[\textbf{Ex 5.2-4}]
       Use indicator random variables to solve the following problem, which is known as the hat-check problem. Each of n customers gives a hat to a hat-check person at a restaurant. The hat-check person gives the hats back to the customers in a random order. What is the expected number of customers who get back their own hat?

 \item[\textbf{Ex 5.2-5}]
       Let $A[1 \dots n]$ be an array of n distinct numbers. If $i < j$ and $A[i] > A[j]$, then the pair $(i, j)$ is called an inversion of $A$. (See Problem 2-4 for more on inversions.) Suppose that the elements of $A$ form a uniform random permutation of ($1, 2, \dots , n$). Use indicator random variables to compute the expected number of inversions.

\end{enumerate}

\section{Randomized algorithms}
\begin{enumerate}
 \item[\textbf{Ex 5.3-1}]
       Professor Marceau objects to the loop invariant used in the proof of Lemma 5.5. He questions whether it is true prior to the first iteration. He reasons that we could just as easily declare that an empty subarray contains no 0-permutations. Therefore, the probability that an empty subarray contains a 0-permutation should be 0, thus invalidating the loop invariant prior to the first iteration. Rewrite the procedure RANDOMIZE-IN-PLACE so that its associated loop invariant applies to a nonempty subarray prior to the first iteration, and modify the proof of Lemma 5.5 for your procedure.

 \item[\textbf{Ex 5.3-2}]
       Professor Kelp decides to write a procedure that produces at random any permutation besides the identity permutation. He proposes the following procedure:\\
       \alglanguage{pseudocode}
       \begin{algorithmic}
        \Procedure{Permute-Without-Identity}{A}
        \State $n \gets A.length$\;
        \For{$i=1 \To n-1$}
        \State swap $A[i]$ with $A[\Call{Random}{i+1, n}]$
        \EndFor
        \EndProcedure
       \end{algorithmic}
       Does this code do what Professor Kelp intends?

 \item[\textbf{Ex 5.3-3}]
       Suppose that instead of swapping element $A[i]$ with a random element from the subarray $A[i \dots n]$, we swapped it with a random element from anywhere in the array:
       \begin{algorithmic}
        \Procedure{Permute-With-All}{A}
        \State $n \gets A.length$
        \For{$i \gets i \To n$}
        \State swap $A[i]$ with $A[\Call{Random}{1,n}]$
        \EndFor
        \EndProcedure
       \end{algorithmic}
       Does this code produce a uniform random permutation? Why or why not?

 \item[\textbf{Ex 5.3-4}]
       Professor Armstrong suggests the following procedure for generating a uniform random permutation:
       \begin{algorithmic}
        \Procedure{Permute-By-Cyclic}{A}
        \State $n \gets A.length$
        \State let $B[1 \dots n]$ be a new array
        \State $offset \gets RANDOM(1,n)$
        \For{$i \gets 1$ \To $n$}
        \State $dest \gets i + offset$
        \If{$dest>n$}
        \State $dest \gets dest - n$
        \EndIf
        \State $B[dest] \gets A[i]$
        \EndFor
        \State \Return $B$\;
        \EndProcedure
       \end{algorithmic}
       Show that each element $A[i]$ has a $1/n$ probability of winding up in any particular position in $B$. Then show that Professor Armstrong is mistaken by showing that the resulting permutation is not uniformly random.

 \item[\textbf{Ex 5.3-5}]
       Prove that in the array $P$ in procedure PERMUTE-BY-SORTING, the probability that all elements are unique is at least $1-1/n$.

 \item[\textbf{Ex 5.3-6}]
       Explain how to implement the algorithm PERMUTE-BY-SORTING to handle the case in which two or more priorities are identical. That is, your algorithm should produce a uniform random permutation, even if two or more priorities are identical.
\end{enumerate}

\section{Probabilistic analysis and further uses of indicator random variables}

\subsection*{Problems}

\chapter{Heapsort}

\section{Heaps}

\section{Maintaining the heap property}

\section{Building a heap}

\section{The heapsort algorithm}

\section{Priority queues}

\subsection*{Problems}

\chapter{Quicksort}

\section{Description of quicksort}

\section{Performance of quicksort}

\section{A randomized version of quicksort}

\section{Analysis of quicksort}

\subsection*{Problems}

\chapter{Sorting in Linear Time}

\section{Lower bounds for sorting}

\section{Counting sort}

\section{Radix sort}

\section{Bucket sort}

\subsection*{Problems}

\chapter{Medians and Order Statistics}

\section{Minimum and maximum}

\section{Selection in expected linear time}

\section{Selection in worst-case linear time}

\subsection*{Problems}

\chapter{Elementary Data Structures}

\section{Stacks and queues}

\section{Linked lists}

\section{Implementing pointers and objects}

\section{Representing rooted trees}

\subsection*{Problems}

\end{document}


\documentclass{article}

% if you need to pass options to natbib, use, e.g.:
% \PassOptionsToPackage{numbers, compress}{natbib}
% before loading nips_2017
%
% to avoid loading the natbib package, add option nonatbib:
% \usepackage[nonatbib]{nips_2017}

% to compile a camera-ready version, add the [final] option, e.g.:
% \usepackage[final]{nips_2017}

\usepackage[final]{nips_2018}

\usepackage{placeins}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
\usepackage{hyperref}       % hyperlinks
\usepackage{url}            % simple URL typesetting
\usepackage{booktabs}       % professional-quality tables
\usepackage{amsfonts}       % blackboard math symbols
\usepackage{nicefrac}       % compact symbols for 1/2, etc.
\usepackage{microtype}      % microtypography
\usepackage{graphicx}

\title{COMP 4211 - Machine Learning Programming Assignment 1 Report}

\author{%
	Cheng Chi Fung \\
	\texttt{cfchengac@connect.ust.hk} \\
}

\begin{document}

\maketitle

\begin{abstract}
  In this assignment, we used linear regression, logistic regression and single layer neural network to perform the regression and classification on three binary classification and regression data sets.

\end{abstract}

\section{Linear Regression}

\subsection{Experienment Model Settings}

For linear regression, we used the built-in models   \textbf{LinearRegression} in skitlearn to perform the linear regression on three regression datasets .This model fits a linear model with coefficients  	$w = (w_1,..., w_p)$  to minimize the residual sum of squares between the observed responses in the dataset, and the responses predicted by the linear approximation. This model has \textbf{no hyperparameter} to set.

\begin{table}[htb]
	\caption{Experiments Datasets Settings}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $13191$ & $4397$ \\
		Finance & $2754$ & $918$ \\
		Orbits & $9642$ & $3215$ \\
		\bottomrule
	\end{tabular}
\end{table}

\subsection{Experiment Results}

The following are the experiment results which using the above model setting on fifa, finanace and orbits regression datasets.

\begin{table}[htb]
	\caption{Experiments Results of Linear Regression on Three Regression Datasets}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{Coefficient of Determination ($R^2$ score)}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.51944$ & $0.51223$  \\
		Finance & $0.42637$  & $0.28766$ \\
		Orbits & $0.61671$  & $0.61016$  \\
		\bottomrule
	\end{tabular}
\end{table}

From the experiment results, we found out that excepts the \textbf{finance} datasets, the $R^2$ score of the training set and test set for the other twos are almost the same. And, for the finance datasets, the $R^2$ score of the test set ($0.42637$) are a much higher than the training set ($0.28766$). It means that this model  \textbf{overfit} the training data of the finance datasets.

\pagebreak

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{fifa_lg.png}
  \caption{Squared Error Histogram on test set of orbits data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{finance_lg.png}
  \caption{Squared Error Histogram on test set of finance data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{orbit_lg.png}
  \caption{Squared Error Histogram on test set of orbit data set}
\end{figure}

From the square loss histogram of test sets, we found out that the mean squared error of the most of the data points are around zero. It means that this model perform quite well for most of the data points. However, for the finance datasets, there are some data points that have the sqaured error far away from zero ($23.77581$) which means that there might be some \textbf{outliers} that far away from the curve.

\pagebreak

\section{Logistic Regression}

\subsection{Experienmental Model Settings}

To perform the logistic regression with gradient-descent algorithm by minimizing the cross-entropy loss on the three classification datasets, we used the built-in model \textbf{SGDClassifier} in skitlearn. This model implements regularized linear models with \textbf{stochastic gradient descent} (SGD) learning.

To get better training results, we first performed the \textbf{hyperparameters tunning} by using \textbf{GridSearchCV} in skitlearn to obtain best hyperparameters parameters on the training set of each datasets. The following are the settings for parameters tuning.

\begin{table}[htb]
	\caption{Model Settings for Hyperparameters Tuning (Tuned hyperparameters *)}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Name     &  Parameter Setttings	\\
		\midrule
		max\_iter 		  & 	$5000$  \\
		learning\_rate 		  & 	optimal  \\
		loss 		  & 	log  \\
		tol      	          & 	  $0.000000001$        \\
		alpha* 		  & 	$0.1, 0.01,...,1^{-9}, 1^{-10}$    \\
		cv		  &   $5$     \\
		\bottomrule
	\end{tabular}
\end{table}

\begin{table}[htb]
	\caption{Best Hyperparameters after Hyperparameters Tuning}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Dataset     &  alpha	\\
		\midrule
		fifa & $1^{-5}$  \\
		finance & $0.001$        \\
		orbits & $1^{-6}$ \\
		\bottomrule
	\end{tabular}
\end{table}

\subsection{Experiment Results}

The following are the experiment results which using the same setting as the hyperparameter tuning except the \textbf{alpha} using the best results after hyperparameters tuning on fifa, finanace and orbits classification datasets.

\begin{table}[htb]
	\caption{Experiments Results of Logistic Regression on Three Classification Datasets}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{Classification Accuracy}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.82533$ & $0.81669$  \\
		Finance & $0.80355$  & $0.79084$ \\
		Orbits & $0.98610$  & $0.98755$  \\
		\bottomrule
	\end{tabular}
	\hspace{5mm}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{AUC Score}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.93828$ & $0.93449$  \\
		Finance & $0.90664$  & $0.89185$ \\
		Orbits & $0.99995$  & $0.99992$  \\
		\bottomrule
	\end{tabular}
\end{table}

From the experiment results, we found out that the model perform \textbf{quite well} on three datasets which getting $>79\%$ on three datasets. However, its classification performace still \textbf{worse than single layer neural network} which will be disscussed in the next section. Furthermore, the better the AUC Score, the better the performace of the classifier. And from experiment results, we got a very high AUC score which mean that the model can classify the dataset very well.

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{fifa_lo_train.png}
  \includegraphics[scale=0.3]{fifa_lo_test.png}
  \caption{Confusion matrix of fifa data set}
\end{figure}

\FloatBarrier

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{finance_lo_train.png}
  \includegraphics[scale=0.3]{finance_lo_test.png}
  \caption{Confusion matrix of finance data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{orbits_lo_train.png}
  \includegraphics[scale=0.3]{orbits_lo_test.png}
  \caption{Confusion matrix of orbits data set}
\end{figure}

From the visulization of confusion matrix on three datasets, we found out that the model perform very well on both fifa and orbits dataset since there are deep purple color on the top right and the bottom left in the confusion matrix which means that it has \textbf{high true positive and true negative} rate during prediction. However, for the finance datasets, the bottom left purple color are not as deep as the other two which means that the prediction of this model on the finance datasets have a little bit higher \textbf{false postive} rate.

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{fifa_auc_train.png}
  \includegraphics[scale=0.3]{fifa_auc_test.png}
  \caption{ROC curve of fifa data set}
\end{figure}

\FloatBarrier

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{finance_auc_train.png}
  \includegraphics[scale=0.3]{finance_auc_test.png}
  \caption{ROC curve of finance data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{orbits_auc_train.png}
  \includegraphics[scale=0.3]{orbits_auc_test.png}
  \caption{ROC curve of orbits data set}
\end{figure}

From the visulization of ROC Curve, since the curve are smooth on three datasets. Therefore, this model \textbf{does not overfit} the dataset.

\pagebreak

\section{Single Layer Neural Network}

\subsection{Experienmental Model Settings}

To perform the classification with single layer neural network on the three classification datasets, we used the built-in model \textbf{MLPClassifier} in skitlearn. This model implements a multi-layer perceptron classifier and optimizes with the log-loss function using stochastic gradient descent.

To determine the number of \textbf{hidden units} should be used in the hidden layer, we evaludate the The generalization performance of the model with $i$ hidden units with \textbf{cross validation} (CV) using the utilities \textbf{cross\_val\_score} in skitlearn. The following are the model settings for cross validation and the results of that.

\begin{table}[htb]
	\caption{Model Settings for Cross Validation}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Name     &  Parameter Setttings	\\
		\midrule
		max\_iter & $5000$  \\
		learning\_rate & optimal  \\
		solver & sgd  \\
		tol & $0.000000001$        \\
		alpha & determined by using grid search for each $i$ hidden unit   \\
		cv & $5$     \\
		\bottomrule
	\end{tabular}
\end{table}

\begin{table}[htb]
	\caption{Performance of the Model with $i$ Hidden Units using Cross Validation}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		$i$ hidden units    &  Score (Average Classification Accuracy)	\\
		\cmidrule{1-2}
		 &  Fifa, Finance, Orbits	\\
		\midrule
		1 & $0.65346$, $0.67176$, $0.99398$ \\
		2 & $0.86142$, $0.68424$, $0.90243$ \\
		3 & $0.86028$, $0.80501$, $0.99429$ \\
		4 & $0.86119$, $0.80647$, $0.99367$ \\
		5 & $0.86089$, $0.80464$, $0.99367$ \\
		6 & $0.85869$, $0.80900$, $0.99336$ \\
		7 & $0.85975$, $0.80247$, $0.99398$ \\
		8 & $0.86020$, $0.80501$, $0.99387$ \\
		9 & $0.86134$, $0.80429$, $0.99419$ \\
		10 & $0.86088$, $0.80320$, $0.99263$ \\
		\midrule
		Number of hidden units with highest score & $2 (0.86142)$, $6 (0.80900)$, $3 (0.99429)$ \\
		\bottomrule
	\end{tabular}
\end{table}

Furthermore, to \textbf{avoid overfitting}, we performed the parameters tuning on the \textbf{number of iterations} after getting the best number of hidden units with cross validation. The model settings for parameters tuning
used the same model settings as cross validation and the number hidden units using what we got from cross validation. We tuned the number of iteraion by returning the maximum iterations before loss of the test set start to decrease.

\begin{table}[htb]
	\caption{Model Settings for Parameter Tuning(Tuned hyperparameters *)}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Name     &  Parameter Setttings	\\
		\midrule
		max\_iter* & $1000,1500,2000,...,8000,8500$  \\
		learning\_rate & optimal  \\
		solver & sgd  \\
		tol & $0.000000001$        \\
		alpha & determined by using grid search before   \\
		hidden units & determined by using CV before     \\
		\bottomrule
	\end{tabular}
\end{table}

\pagebreak

\begin{table}[htb]
	\caption{Best Hyperparameters after Hyperparameters Tuning}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Dataset     &  max\_iter	\\
		\midrule
		fifa & $1000$  \\
		finance & $4500$        \\
		orbits & $8000$ \\
		\bottomrule
	\end{tabular}
\end{table}

After hyperparameters tuning on number of iterations, we obtain the best \textbf{max\_iteration} before overfitting to the training set. Finally, we got the following model settings for our final training.

\begin{table}[h]
	\caption{Model Settings of Using Single Layer Nerual Network }
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		Name     &  Parameter Setttings	\\
		\cmidrule{1-2}
		 &  Fifa, Finance, Orbits	\\
		\midrule
		max\_iter & $1000$, $1000$, $1500$  \\
		learning\_rate & optimal \\
		solver & sgd \\
		tol & $0.000000001$ \\
		alpha & $0.0001$, $1^{-9}$, $0.01$   \\
		number of hidden units & $2$, $6$, $3$    \\
		\bottomrule
	\end{tabular}
\end{table}

\subsection{Experienment Results}

The following are the experiment results which using the above model setting on fifa, finanace and orbits classification datasets.

\begin{table}[htb]
	\caption{Experiments Results of Logistic Regression on Three Classification Datasets}
	\label{sample-table}
	\centering

		\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{Classification Accuracy}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.93828$ & $0.93449$  \\
		Finance & $0.82933$  & $0.80718$ \\
		Orbits & $0.99533$  & $0.99502$  \\
		\bottomrule
	\end{tabular}

		\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{Loss}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.31560$ & $0.32098$  \\
		Finance & $0.38444$  & $0.39380$ \\
		Orbits & $0.07369$  & $0.05241$  \\
		\bottomrule
	\end{tabular}
	\hspace{5mm}
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-3}
		\multicolumn{3}{c}{AUC Score}\\
		\midrule
		Dataset & Training set & Test set \\
		\midrule
		Fifa & $0.94828$ & $0.93449$  \\
		Finance & $0.91933$  & $0.89718$ \\
		Orbits & $0.99997$  & $0.99995$  \\
		\bottomrule
	\end{tabular}

\end{table}

From the experiment results, we found out that the single layer neural network \textbf{perform better than logistic regression} which has higher classification accuracy on three datasets. And the loss of the training set and test set are similiar which means that the model are \textbf{not overfit} to the training data set. And for the AUC score, single layer neural network perform slightly better than logistic.

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{auc_fifa_train.png}
  \includegraphics[scale=0.3]{auc_fifa_test.png}
  \caption{ROC curve of fifa data set}
\end{figure}

\FloatBarrier

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{auc_finance_train.png}
  \includegraphics[scale=0.3]{auc_finance_test.png}
  \caption{ROC curve of finance data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{orbits_auc_train.png}
  \includegraphics[scale=0.3]{orbits_auc_test.png}
  \caption{ROC curve of orbits data set}
\end{figure}

From the visulization of ROC Curve, since the curve are smooth on three datasets. Therefore, this model \textbf{does not overfit} the dataset.

\pagebreak

The following are the visualization of the peroformace during training. (Y axis = loss / X axis = iterations)

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{loss_fifa.png}
  \caption{Loss curve of train set of fifa data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{loss_finance.png}
  \caption{Loss curve of train set of finance data set}
\end{figure}

\begin{figure}[h]
  \centering
  \includegraphics[scale=0.3]{loss_orbits.png}
  \caption{Loss curve of train set of orbits data set}
\end{figure}

\pagebreak

\section{Computing Environment and Time Used}

\subsection{Computing Environment}

\begin{table}[htb]
	\caption{Computing Environment}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		    &  Environment Settings	\\
		\midrule
		CPU & AMD Ryzen™ 7 2700X   \\
		GPU & GTX 1080Ti        \\
		RAM & 16GB \\
		STORAGE & 1TB M2 NVME SSD \\
		OS & Ubuntu 16.04 \\
		IDE & Pycharm \\
		Python Version & Python 3.5.2 \\
		Libraries Used & skitlearn, numpy, matplotlib.pyplot \\
		\bottomrule
	\end{tabular}
\end{table}

\subsection{Time Used}

\begin{table}[htb]
	\caption{Total Running Time For Each Tasks}
	\label{sample-table}
	\centering
	\begin{tabular}{lll}
		\toprule
		\cmidrule{1-2}
		    &  Wall Time	(seconds)\\
		\midrule
		Linear Regression & $0.470023$ \\
		Logistic Regression & $10.86583$ \\
		Single Layer Neural Network & $8376.50412$ \\
		\bottomrule
	\end{tabular}
\end{table}

\end{document}