\newif\ifshowsolutions
\showsolutionstrue
\input{./preamble}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% HEADER
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chead{
  {\vbox{
      \vspace{2mm}
      \large
      Machine Learning \& Data Mining \hfill
      Caltech CS/CNS/EE 155 \hfill \\[1pt]
      Set 4\hfill
      January $27^\text{th}$, 2021 \\
    }
  }
}

\begin{document}
\pagestyle{fancy}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% POLICIES
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Policies}
\begin{itemize}
  \item Due 9pm PST Wednesday Feb 3rd, via Gradescope.
  \item You are free to collaborate on all of the problems, subject to the collaboration policy stated in the syllabus.
  \item In this course, we will be using Google Colab for code submissions. You will need a Google account.
  \item We ask that you use Python 3 (set that as your Colab runtime's Python version) and sklearn version 0.22 (should be the default version for Python 3 in Colab) for your code, and that you comment your code such that the TAs can follow along and run it without any issues.
\end{itemize}

\section*{Submission Instructions}
\begin{itemize}
   \item You are highly encouraged to use the submission template
   \item Submit your report as a single .pdf file to Gradescope (entry code N8XV6Z), under "Problem Set 4".
   \item In the report, \textbf{include any images generated by your code} along with your answers to the questions.
   \item Submit your code by \textbf{sharing a link in your report} to your Google Colab notebook for each problem (see naming instructions below). Make sure to set sharing permissions to at least "Anyone with the link can view". \textbf{Links that can not be run by TAs will not be counted as turned in.} Check your links in an incognito window before submitting to be sure.
   \item For instructions specifically pertaining to the Gradescope submission process, see \url{https://www.gradescope.com/get_started#student-submission}.

\end{itemize}

\section*{Google Colab Instructions}

For each notebook in the course gitub \url{https://github.com/lakigigar/Caltech-CS155-2021}, you need to save a copy to your drive.

\begin{enumerate}
  \item Open the github preview of the notebook, and click the icon to open the colab preview.
  \item On the colab preview, go to File $\rightarrow$ Save a copy in Drive.
  \item Edit your file name to “lastname_firstname_originaltitle”, e.g.”wei_evelyn_set4-prob2.ipynb”
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% PROBLEM 1
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newpage
\section{Deep Learning Principles [35 Points]}
\materials{lectures on deep learning}

 For problems A and B, we'll be utilizing the \href{http://playground.tensorflow.org/}{Tensorflow Playground} to visualize/fit a neural network.

\begin{problem}[5]
  Backpropagation and Weight Initialization Part 1
\end{problem}

Fit the neural network at \href{http://playground.tensorflow.org/#activation=relu&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.65409&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false}{this link} for about 250 iterations, and then do the same for the neural network at  \href{http://playground.tensorflow.org//\#activation=relu&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.6&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=true&hideText=false}{this link}.  Both networks have the same architecture and use ReLU activations.  The only difference between the two is how the layer weights were initialized -- you can examine the layer weights by hovering over the edges between neurons.

Give a mathematical justification, based on what you know about the backpropagation algorithm and the ReLU function, for the difference in the performance of the two networks.

Note: MacOS users may have trouble opening the links if you use Preview's or Safari's pdf readers. Please use chrome's pdf reader to open the pdf file and click the links.
\begin{subsolution}

\end{subsolution}

\begin{problem}[5]
  Backpropagation and Weight Initialization Part 2
\end{problem}
Reset the two demos from part i (there is a reset button to the left of the ``Run'' button), change the activation functions of the neurons to sigmoid instead of ReLU, and train each of them for 4000 iterations.

Explain the differences in the models learned, and the speed at which they were learned, from those of part i in terms of the backpropagation algorithm and the sigmoid function.

\begin{subsolution}

\end{subsolution}

\problem \textbf{[10 Points]}

When training any model using SGD, it's important to shuffle your data to avoid correlated samples. To illustrate one reason for this that is particularly important for ReLU networks, consider a dataset of 1000 points, 500 of which have positive (+1) labels, and 500 of which have negative (-1) labels. What happens if we train a fully-connected network with ReLU activations using SGD, looping through all the negative examples before any of the positive examples? (Hint: this is called the ``dying ReLU'' problem.)

\begin{solution}

\end{solution}

\problem Approximating Functions Part 1 \textbf{[7 Points]}

Draw or describe a fully-connected network with ReLU units that implements the OR function on two 0/1-valued inputs,  $x_1$ and $x_2$.  Your networks should contain the minimum number of hidden units possible.  The OR function $\text{OR}(x_1, x_2)$ is defined as:
\begin{gather*}
\text{OR}(1, 0) \geq 1 \\
\text{OR}(0, 1) \geq 1 \\
\text{OR}(1, 1) \geq 1 \\
\text{OR}(0, 0) = 0
\end{gather*}

Your network need only produce the correct output when $x_1 \in \{0, 1\}$ and $x_2 \in \{0, 1\}$ (as described in the examples above).

\begin{subsolution}

\end{subsolution}

\problem Approximating Functions Part 2 \textbf{[8 Points]}

What is the minimum number of fully-connected layers (with ReLU units) needed to implement an XOR of two 0/1-valued inputs $x_1, x_2$? Recall that the XOR function is defined as:
\begin{gather*}
\text{XOR}(1, 0) \geq 1 \\
\text{XOR}(0, 1) \geq 1 \\
\text{XOR}(0, 0) = \text{XOR}(1, 1) = 0
\end{gather*}

For the purposes of this problem, we say that a network $f$ computes the XOR function if $f(x_1, x_2) = \text{XOR}(x_1, x_2)$ when $x_1 \in \{0, 1\}$ and $x_2 \in \{0, 1\}$ (as described in the examples above).

Explain why a network with fewer layers than the number you specified cannot compute XOR.

\begin{subsolution}

\end{subsolution}

% problem 2
\newpage
\section{Depth vs Width on the MNIST Dataset  [25 Points]}

\textit{Relevant Materials: Lectures on Deep Learning}

MNIST is a classic dataset in computer vision. It consists of images of handwritten digits (0 - 9) and the correct digit classification. In this problem you will implement a deep network using PyTorch to classify MNIST digits. Specifically, you will explore what it really means for a network to be "deep", and how depth vs. width impacts the classification accuracy of a model. You will be allowed at most $N$ hidden units, and will be expected to design and implement a deep network that meets some performance baseline on the MNIST dataset.

\medskip

\problem \textbf{The Data} \textbf{[3 Points]}

Load the MNIST dataset using torchvision; see the problem 2 sample code for how.

Image inputs in PyTorch are generally 3D tensors with the shape (no. of channels, height, width). Examine the input data. What are the height and width of the images? What do the values in each array index represent?  How many images are in the training set? How many are in the testing set? You can use the \textbf{imshow} function in matplotlib if you'd like to see the actual pictures (see the sample code).

\begin{subsolution}

\end{subsolution}

 \problem \textbf{Modeling Part 1} \textbf{[10 Points]}

 Using PyTorch's "Sequential" model class, build a deep network to classify the handwritten digits. You may \textbf{only} use the following layers:

 \begin{itemize}
  \item \textbf{Linear:} A fully-connected layer
  \item \textbf{ReLU (activation):} Sets negative inputs to 0
  \item \textbf{Softmax (activation):} Rescales input so that it can be interpreted as a (discrete) probability distribution.
  \item \textbf{Dropout:} Takes some probability and at every iteration sets weights to zero at random with that probability (effectively regularization)
  \item \textbf{Flatten:} Flattens any tensor into a single vector
\end{itemize}

A sample network with 20 hidden units is in the sample code file. (Note: activations, Dropout, and your last Linear layer do not count toward your hidden unit count, because the final layer is ``observed'' and not \emph{hidden}.)

Use categorical cross entropy as your loss function. There are also a number of optimizers  you can use (an optimizer is just a fancier version of SGD), and feel free to play around with them, but RMSprop and Adam are the most popular and will probably work best. You also should find the batch size and number of epochs that give you the best results (default is batch size = 32, epochs=10).

Look at the sample code to see how to train your model. PyTorch should make it very easy to tinker with your network architecture.

\textbf{Your task}: Using at most 100 hidden units, build a network using only the allowed layers that achieves test accuracy of at least 0.975. You may use multiple of each layer as long as the hidden units are within the limit. Include the code of your model as well as the best test accuracy that it achieved in your report.

\begin{solution}

 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

 \problem \textbf{Modeling Part 2} \textbf{[6 EC Points]}

 Repeat problem C, except that now you may use 200 hidden units and must build a model with at least 2 hidden layers that achieves test accuracy of at least 0.98.

 \textit{Hint}: Adjusting regularization via dropout probability can affect the final performance significantly. Keep in mind that there is a tradeoff between bias and variance. Also make sure you are calling model.eval() before evaluation. It shouldn't require trying many combinations to achieve the desired accuracy.

 \begin{solution}
 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

  \problem \textbf{Modeling Part 3} \textbf{[6 EC Points]}

 Repeat problem C, except that now you may use 1000 hidden units and must build a model with at least 3 hidden layers that achieves test accuracy of at least 0.983.

  \begin{solution}
 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

 \newpage
 % problem 3
 \section{Convolutional Neural Networks  [40 Points]}
 \textit{Relevant Materials: Lecture on CNNs}

 \problem Zero Padding \textbf{[5 Points]}

 Consider a convolutional network in which we perform a convolution over each $8 \times 8$ patch of a $20 \times 20$ input image. It is common to zero-pad input images to allow for convolutions past the edges of the images. An example of zero-padding is shown below:

\begin{center}
  \includegraphics[width=.49\textwidth]{Plots/ConvolutionExample.png}
\end{center}
\begin{small}
Figure: A convolution being applied to a $2 \times 2$ patch (the red square) of a $3 \times 3$ image that has been zero-padded to allow convolutions past the edges of the image.
\end{small}

What is one benefit and one drawback to this zero-padding scheme (in contrast to an approach in which we only perform convolutions over patches entirely contained within an image)?

\begin{solution}

\end{solution}

\subsection{5 x 5 Convolutions}

Consider a single convolutional layer, where your input is a $32 \times 32$ pixel, RGB image. In other words, the input is a $32 \times 32 \times 3$ tensor. Your convolution has:

\begin{itemize}
\item Size: $5 \times 5 \times 3$
\item Filters: 8
\item Stride: 1
\item No zero-padding
\end{itemize}

\problem[2] What is the number of parameters (weights) in this layer, including a bias term?

\begin{subsolution}

\end{subsolution}

\problem[3] What is the shape of the output tensor?

\begin{subsolution}

\end{subsolution}

 \subsection{Max/Average Pooling}

Pooling is a downsampling technique for reducing the dimensionality of a layer's output. Pooling iterates across patches of an image similarly to a convolution, but pooling and convolutional layers compute their outputs differently: given a pooling layer $B$ with preceding layer $A$, the output of $B$ is some function (such as the max or average functions) applied to patches of $A$'s output.

Below is an example of max-pooling on a 2-D input space with a $2\times 2$ filter (the max function is applied to $2\times 2$ patches of the input) and a stride of 2 (so that the sampled patches do not overlap):

\begin{center}
  \includegraphics[width=.49\textwidth]{Plots/MaxPool.png}
\end{center}

Average pooling is similar except that you would take the average of each patch as its output instead of the maximum.

Consider the following 4 matrices:
$$
\begin{bmatrix}
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    0 & 0 & 0 & 0
\end{bmatrix},
%
\begin{bmatrix}
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 0 & 0 & 0
\end{bmatrix},
%
\begin{bmatrix}
  0 & 0 & 0 & 0 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1
\end{bmatrix},
%
\begin{bmatrix}
    0 & 0 & 0 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0
\end{bmatrix}
$$
\problem[3]

Apply $2 \times 2$ average pooling with a stride of 2 to each of the above images.

\begin{subsolution}

\end{subsolution}

\problem[3]

Apply $2 \times 2$ max pooling with a stride of 2 to each of the above images.

\begin{subsolution}

\end{subsolution}

\problem[4]

Consider a scenario in which we wish to classify a dataset of images of various animals, taken at various angles/locations and containing small amounts of noise (e.g. some pixels may be missing). Why might pooling be advantageous given these distortions in our dataset?

\begin{subsolution}

\end{subsolution}

\subsection{PyTorch implementation}
\problem[20 EC]

Using PyTorch ``Sequential'' model class as you did in 2C, build a deep \emph{convolutional}
network to classify the handwritten digits in MNIST. You are now allowed to use
the following layers (but \textbf{only} the following):

 \begin{itemize}
  \item \textbf{Linear:} A fully-connected layer
  \begin{itemize}
    \item In convolutional networks, Linear (also called dense) layers are typically used to knit together higher-level feature representations.
    \item Particularly useful to map the 2D features resulting from the last convolutional layer to categories for classification (like the 1000 categories of ImageNet or the 10 categories of MNIST).
    \item Inefficient use of parameters and often overkill: for $A$ input activations and $B$ output activations, number of parameters needed scales as $O(AB)$.
  \end{itemize}
  \item \textbf{Conv2d:} A 2-dimensional convolutional layer
  \begin{itemize}
    \item The bread and butter of convolutional networks, conv layers impose a translational-invariance prior on a fully-connected network. By sliding filters across the image to form another image, conv layers perform ``coarse-graining'' of the image.
    \item Networking several convolutional layers in succession helps the convolutional network knit together more abstract representations of the input. As you go higher in a convolutional network, activations represent pixels, then edges, colors, and finally objects.
    \item More efficient use of parameters. For $N$ filters of $K \times K$ size on an input of size $L \times L$, the number of parameters needed scales as $O(N K^2)$. When $N, K$ are small, this can often beat the $O(L^4)$ scaling of a Linear layer applied to the $L^2$ pixels in the image.

  \end{itemize}
  \item \textbf{MaxPool2d:} A 2-dimensional max-pooling layer
  \begin{itemize}
    \item Another way of performing ``coarse-graining'' of images, max-pool layers are another way of ignoring finer-grained details by only considering maximum activations over small patches of the input.
    \item Drastically reduces the input size. Useful for reducing the number of parameters in your model.
    \item Typically used immediately following a series of convolutional-activation layers.
  \end{itemize}
  \item \textbf{BatchNorm2d:} Performs batch normalization (Ioffe and Szegedy, 2014). Normalizes the activations of previous layer to standard normal (mean 0, standard deviation 1).
  \begin{itemize}
    \item Accelerates convergence and improves performance of model, especially when saturating nonlinearities (sigmoid) are used.
    \item Makes model less sensitive to higher learning rates and initialization, and also acts as a form of regularization.
    \item Typically used immediately before nonlinearity (Activation) layers.
  \end{itemize}
  \item \textbf{Dropout:} Takes some probability and at every iteration sets weights to zero at random with that probability
  \begin{itemize}
    \item An effective form of regularization. During training, randomly selecting activations to shut off forces network to build in redundancies in the feature representation, so it does not rely on any single activation to perform classification.
  \end{itemize}
  \item \textbf{ReLU (activation):} Sets negative inputs to 0
  \item \textbf{Softmax (activation):} Rescales input so that it can be interpreted as a (discrete) probability distribution.
  \item \textbf{Flatten:} Flattens any tensor into a single vector (required in order to pass a 2D tensor output from a convolutional layer as input into Linear layers)
\end{itemize}

\textbf{Your tasks.} Build a network with only the allowed layers that achieves \textbf{test accuracy of at least 0.985}. You are required to use categorical cross entropy as your loss function and to train for 10 epochs with a batch size of 32. Note: your model must have fewer than 1 million parameters, as measured by the method given in the sample code.
Everything else can change: optimizer (RMSProp, Adam, ???), initial learning rates, dropout probabilities, layerwise regularizer strengths, etc. You are not required to use all of the layers, but \emph{you must have
at least one dropout layer and one batch normalization layer in your final model}. Try to figure out the best possible architecture and hyperparameters given these building blocks!

In order to design your model, you should train your model for 1 epoch (batch size 32) and look at the final \textbf{test accuracy}  after training. This should take no more than 10 minutes, and should give you an immediate sense for how fast your network converges and how good it is.

Set the probabilities of your
dropout layers to 10 equally-spaced values $p \in [0, 1]$, train for 1 epoch, and report the final model accuracies for each.

You can perform all of your hyperparameter validation in this way: vary your parameters and train for an epoch. After you're satisfied with the model design, you should train your model for the full 10 epochs.

\textbf{In your submission.} Turn in the code of your model, the test accuracy for the 10 dropout probabilities $p \in [0, 1]$, and the
final test accuracy when your model
is trained for 10 epochs. We should have everything needed to reproduce your results.

Discuss what you found to be the most effective strategies in designing a convolutional network. Which regularization method was most effective (dropout, layerwise regularization, batch norm)?

Do you foresee any problem with this way of validating our hyperparameters? If so, why?

\emph{Hints:}

\begin{itemize}
  \item You are provided with a sample network that achieves a high accuracy. Starting with this network, modify some of the regularization parameters (layerwise regularization strength, dropout probabilities) to see if you can maximize the test accuracy. You can also add layers or modify layers (e.g.
  changing the convolutional kernel sizes, number of filters, stride, dilation, etc.) so long as the total number of parameters remains under the cap of 1 million.
  \item You may want to read up on successful convolutional architectures, and emulate some of their design principles. Please cite any idea you use that is not your own.
  \item To better understand the function of each layer, check the PyTorch documentation.
  \item Linear layers take in single vector inputs (ex: \emph{(784, )}) but Conv2D layers
  take in tensor inputs (ex: \emph{(28, 28, 1)}): width, height, and channels. Using the transformation \verb$transforms.ToTensor()$ when loading the dataset will reshape the training/test $X$ to a 4-dimensional tensor (ex: \emph{(num_examples, width, height, channels)}) and normalize values. For the MNIST dataset, \emph{channels=1}. Typical color images have 3 color channels, 1 for each color in RGB.
  \item Other useful CNN design principles:
  \begin{itemize}
    \item CNNs perform well with many stacked convolutional layers, which develop increasingly large-scale representations of the input image.
    \item Dropout ensures that the learned representations are robust to some amount of noise.
    \item Batch norm is done after a convolutional or dense layer and immediately prior to an activation/nonlinearity layer.
    \item Max-pooling is typically done after a series of convolutions, in order to gradually reduce the size of the representation.
    \item Finally, the learned representation is passed into a dense layer (or two), and then filtered down to the final softmax layer.
  \end{itemize}
\end{itemize}
\begin{solution}

\end{solution}

%\problem Extra credit \textbf{[5 bonus points possible]}

%In the sample code, we have provided a notebook to visualize the final results of your trained model, e.g. what images maximally activate certain neurons, the feature representations learned by successively-higher neurons, etc. We have also provided code that can visualize the training / validation accuracy of your model as a function of epochs trained.

%Discuss the features your model learned. How will the visualizations help you design better convolutional networks? Next, investigate the behavior of the training accuracy over time. What causes this behavior?

\end{document}
\newif\ifshowsolutions
\showsolutionstrue
\input{./preamble}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% HEADER
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chead{
  {\vbox{
      \vspace{2mm}
      \large
      Machine Learning \& Data Mining \hfill
      Caltech CS/CNS/EE 155 \hfill \\[1pt]
      Set 4\hfill
      January $27^\text{th}$, 2021 \\
    }
  }
}

\begin{document}
\pagestyle{fancy}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% POLICIES
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Policies}
\begin{itemize}
  \item Due 9pm PST Wednesday Feb 3rd, via Gradescope.
  \item You are free to collaborate on all of the problems, subject to the collaboration policy stated in the syllabus.
  \item In this course, we will be using Google Colab for code submissions. You will need a Google account.
  \item We ask that you use Python 3 (set that as your Colab runtime's Python version) and sklearn version 0.22 (should be the default version for Python 3 in Colab) for your code, and that you comment your code such that the TAs can follow along and run it without any issues.
\end{itemize}

\section*{Submission Instructions}
\begin{itemize}
   \item You are highly encouraged to use the submission template
   \item Submit your report as a single .pdf file to Gradescope (entry code N8XV6Z), under "Problem Set 4".
   \item In the report, \textbf{include any images generated by your code} along with your answers to the questions.
   \item Submit your code by \textbf{sharing a link in your report} to your Google Colab notebook for each problem (see naming instructions below). Make sure to set sharing permissions to at least "Anyone with the link can view". \textbf{Links that can not be run by TAs will not be counted as turned in.} Check your links in an incognito window before submitting to be sure.
   \item For instructions specifically pertaining to the Gradescope submission process, see \url{https://www.gradescope.com/get_started#student-submission}.

\end{itemize}

\section*{Google Colab Instructions}

For each notebook in the course gitub \url{https://github.com/lakigigar/Caltech-CS155-2021}, you need to save a copy to your drive.

\begin{enumerate}
  \item Open the github preview of the notebook, and click the icon to open the colab preview.
  \item On the colab preview, go to File $\rightarrow$ Save a copy in Drive.
  \item Edit your file name to “lastname_firstname_originaltitle”, e.g.”wei_evelyn_set4-prob2.ipynb”
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% PROBLEM 1
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newpage
\section{Deep Learning Principles [35 Points]}
\materials{lectures on deep learning}

 For problems A and B, we'll be utilizing the \href{http://playground.tensorflow.org/}{Tensorflow Playground} to visualize/fit a neural network.

\begin{problem}[5]
  Backpropagation and Weight Initialization Part 1
\end{problem}

Fit the neural network at \href{http://playground.tensorflow.org/#activation=relu&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.65409&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false}{this link} for about 250 iterations, and then do the same for the neural network at  \href{http://playground.tensorflow.org//\#activation=relu&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.6&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=true&hideText=false}{this link}.  Both networks have the same architecture and use ReLU activations.  The only difference between the two is how the layer weights were initialized -- you can examine the layer weights by hovering over the edges between neurons.

Give a mathematical justification, based on what you know about the backpropagation algorithm and the ReLU function, for the difference in the performance of the two networks.

\begin{subsolution}

\end{subsolution}

\begin{problem}[5]
  Backpropagation and Weight Initialization Part 2
\end{problem}
Reset the two demos from part i (there is a reset button to the left of the ``Run'' button), change the activation functions of the neurons to sigmoid instead of ReLU, and train each of them for 4000 iterations.

Explain the differences in the models learned, and the speed at which they were learned, from those of part i in terms of the backpropagation algorithm and the sigmoid function.

\begin{subsolution}

\end{subsolution}

\problem \textbf{[10 Points]}

When training any model using SGD, it's important to shuffle your data to avoid correlated samples. To illustrate one reason for this that is particularly important for ReLU networks, consider a dataset of 1000 points, 500 of which have positive (+1) labels, and 500 of which have negative (-1) labels. What happens if we train a fully-connected network with ReLU activations using SGD, looping through all the negative examples before any of the positive examples? (Hint: this is called the ``dying ReLU'' problem.)

\begin{solution}

\end{solution}

\problem Approximating Functions Part 1 \textbf{[7 Points]}

Draw or describe a fully-connected network with ReLU units that implements the OR function on two 0/1-valued inputs,  $x_1$ and $x_2$.  Your networks should contain the minimum number of hidden units possible.  The OR function $\text{OR}(x_1, x_2)$ is defined as:
\begin{gather*}
\text{OR}(1, 0) \geq 1 \\
\text{OR}(0, 1) \geq 1 \\
\text{OR}(1, 1) \geq 1 \\
\text{OR}(0, 0) = 0
\end{gather*}

Your network need only produce the correct output when $x_1 \in \{0, 1\}$ and $x_2 \in \{0, 1\}$ (as described in the examples above).

\begin{subsolution}

\end{subsolution}

\problem Approximating Functions Part 2 \textbf{[8 Points]}

What is the minimum number of fully-connected layers (with ReLU units) needed to implement an XOR of two 0/1-valued inputs $x_1, x_2$? Recall that the XOR function is defined as:
\begin{gather*}
\text{XOR}(1, 0) \geq 1 \\
\text{XOR}(0, 1) \geq 1 \\
\text{XOR}(0, 0) = \text{XOR}(1, 1) = 0
\end{gather*}

For the purposes of this problem, we say that a network $f$ computes the XOR function if $f(x_1, x_2) = \text{XOR}(x_1, x_2)$ when $x_1 \in \{0, 1\}$ and $x_2 \in \{0, 1\}$ (as described in the examples above).

Explain why a network with fewer layers than the number you specified cannot compute XOR.

\begin{subsolution}

\end{subsolution}

% problem 2
\newpage
\section{Depth vs Width on the MNIST Dataset  [25 Points]}

\textit{Relevant Materials: Lectures on Deep Learning}

MNIST is a classic dataset in computer vision. It consists of images of handwritten digits (0 - 9) and the correct digit classification. In this problem you will implement a deep network using PyTorch to classify MNIST digits. Specifically, you will explore what it really means for a network to be "deep", and how depth vs. width impacts the classification accuracy of a model. You will be allowed at most $N$ hidden units, and will be expected to design and implement a deep network that meets some performance baseline on the MNIST dataset.

\medskip

\problem \textbf{The Data} \textbf{[3 Points]}

Load the MNIST dataset using torchvision; see the problem 2 sample code for how.

Image inputs in PyTorch are generally 3D tensors with the shape (no. of channels, height, width). Examine the input data. What are the height and width of the images? What do the values in each array index represent?  How many images are in the training set? How many are in the testing set? You can use the \textbf{imshow} function in matplotlib if you'd like to see the actual pictures (see the sample code).

\begin{subsolution}

\end{subsolution}

 \problem \textbf{Modeling Part 1} \textbf{[10 Points]}

 Using PyTorch's "Sequential" model class, build a deep network to classify the handwritten digits. You may \textbf{only} use the following layers:

 \begin{itemize}
  \item \textbf{Linear:} A fully-connected layer
  \item \textbf{ReLU (activation):} Sets negative inputs to 0
  \item \textbf{Softmax (activation):} Rescales input so that it can be interpreted as a (discrete) probability distribution.
  \item \textbf{Dropout:} Takes some probability and at every iteration sets weights to zero at random with that probability (effectively regularization)
  \item \textbf{Flatten:} Flattens any tensor into a single vector
\end{itemize}

A sample network with 20 hidden units is in the sample code file. (Note: activations, Dropout, and your last Linear layer do not count toward your hidden unit count, because the final layer is ``observed'' and not \emph{hidden}.)

Use categorical cross entropy as your loss function. There are also a number of optimizers  you can use (an optimizer is just a fancier version of SGD), and feel free to play around with them, but RMSprop and Adam are the most popular and will probably work best. You also should find the batch size and number of epochs that give you the best results (default is batch size = 32, epochs=10).

Look at the sample code to see how to train your model. PyTorch should make it very easy to tinker with your network architecture.

\textbf{Your task}: Using at most 100 hidden units, build a network using only the allowed layers that achieves test accuracy of at least 0.975. You may use multiple of each layer as long as the hidden units are within the limit. Include the code of your model as well as the best test accuracy that it achieved in your report.

\begin{solution}

 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

 \problem \textbf{Modeling Part 2} \textbf{[6 EC Points]}

 Repeat problem C, except that now you may use 200 hidden units and must build a model with at least 2 hidden layers that achieves test accuracy of at least 0.98.

 \textit{Hint}: Adjusting regularization via dropout probability can affect the final performance significantly. Keep in mind that there is a tradeoff between bias and variance. Also make sure you are calling model.eval() before evaluation. It shouldn't require trying many combinations to achieve the desired accuracy.

 \begin{solution}
 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

  \problem \textbf{Modeling Part 3} \textbf{[6 EC Points]}

 Repeat problem C, except that now you may use 1000 hidden units and must build a model with at least 3 hidden layers that achieves test accuracy of at least 0.983.

  \begin{solution}
 \begin{lstlisting}[language=Python]
model = nn.Sequential(
# TO ADD
)
Test accuracy:
\end{lstlisting}
\end{solution}

 \newpage
 % problem 3
 \section{Convolutional Neural Networks  [40 Points]}
 \textit{Relevant Materials: Lecture on CNNs}

 \problem Zero Padding \textbf{[5 Points]}

 Consider a convolutional network in which we perform a convolution over each $8 \times 8$ patch of a $20 \times 20$ input image. It is common to zero-pad input images to allow for convolutions past the edges of the images. An example of zero-padding is shown below:

\begin{center}
  \includegraphics[width=.49\textwidth]{Plots/ConvolutionExample.png}
\end{center}
\begin{small}
Figure: A convolution being applied to a $2 \times 2$ patch (the red square) of a $3 \times 3$ image that has been zero-padded to allow convolutions past the edges of the image.
\end{small}

What is one benefit and one drawback to this zero-padding scheme (in contrast to an approach in which we only perform convolutions over patches entirely contained within an image)?

\begin{solution}

\end{solution}

\subsection{5 x 5 Convolutions}

Consider a single convolutional layer, where your input is a $32 \times 32$ pixel, RGB image. In other words, the input is a $32 \times 32 \times 3$ tensor. Your convolution has:

\begin{itemize}
\item Size: $5 \times 5 \times 3$
\item Filters: 8
\item Stride: 1
\item No zero-padding
\end{itemize}

\problem[2] What is the number of parameters (weights) in this layer, including a bias term?

\begin{subsolution}

\end{subsolution}

\problem[3] What is the shape of the output tensor?

\begin{subsolution}

\end{subsolution}

 \subsection{Max/Average Pooling}

Pooling is a downsampling technique for reducing the dimensionality of a layer's output. Pooling iterates across patches of an image similarly to a convolution, but pooling and convolutional layers compute their outputs differently: given a pooling layer $B$ with preceding layer $A$, the output of $B$ is some function (such as the max or average functions) applied to patches of $A$'s output.

Below is an example of max-pooling on a 2-D input space with a $2\times 2$ filter (the max function is applied to $2\times 2$ patches of the input) and a stride of 2 (so that the sampled patches do not overlap):

\begin{center}
  \includegraphics[width=.49\textwidth]{Plots/MaxPool.png}
\end{center}

Average pooling is similar except that you would take the average of each patch as its output instead of the maximum.

Consider the following 4 matrices:
$$
\begin{bmatrix}
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    0 & 0 & 0 & 0
\end{bmatrix},
%
\begin{bmatrix}
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 0 & 0 & 0
\end{bmatrix},
%
\begin{bmatrix}
  0 & 0 & 0 & 0 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1 \\
    0 & 1 & 1 & 1
\end{bmatrix},
%
\begin{bmatrix}
    0 & 0 & 0 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0 \\
    1 & 1 & 1 & 0
\end{bmatrix}
$$
\problem[3]

Apply $2 \times 2$ average pooling with a stride of 2 to each of the above images.

\begin{subsolution}

\end{subsolution}

\problem[3]

Apply $2 \times 2$ max pooling with a stride of 2 to each of the above images.

\begin{subsolution}

\end{subsolution}

\problem[4]

Consider a scenario in which we wish to classify a dataset of images of various animals, taken at various angles/locations and containing small amounts of noise (e.g. some pixels may be missing). Why might pooling be advantageous given these distortions in our dataset?

\begin{subsolution}

\end{subsolution}

\subsection{PyTorch implementation}
\problem[20 EC]

Using PyTorch ``Sequential'' model class as you did in 2C, build a deep \emph{convolutional}
network to classify the handwritten digits in MNIST. You are now allowed to use
the following layers (but \textbf{only} the following):

 \begin{itemize}
  \item \textbf{Linear:} A fully-connected layer
  \begin{itemize}
    \item In convolutional networks, Linear (also called dense) layers are typically used to knit together higher-level feature representations.
    \item Particularly useful to map the 2D features resulting from the last convolutional layer to categories for classification (like the 1000 categories of ImageNet or the 10 categories of MNIST).
    \item Inefficient use of parameters and often overkill: for $A$ input activations and $B$ output activations, number of parameters needed scales as $O(AB)$.
  \end{itemize}
  \item \textbf{Conv2d:} A 2-dimensional convolutional layer
  \begin{itemize}
    \item The bread and butter of convolutional networks, conv layers impose a translational-invariance prior on a fully-connected network. By sliding filters across the image to form another image, conv layers perform ``coarse-graining'' of the image.
    \item Networking several convolutional layers in succession helps the convolutional network knit together more abstract representations of the input. As you go higher in a convolutional network, activations represent pixels, then edges, colors, and finally objects.
    \item More efficient use of parameters. For $N$ filters of $K \times K$ size on an input of size $L \times L$, the number of parameters needed scales as $O(N K^2)$. When $N, K$ are small, this can often beat the $O(L^4)$ scaling of a Linear layer applied to the $L^2$ pixels in the image.

  \end{itemize}
  \item \textbf{MaxPool2d:} A 2-dimensional max-pooling layer
  \begin{itemize}
    \item Another way of performing ``coarse-graining'' of images, max-pool layers are another way of ignoring finer-grained details by only considering maximum activations over small patches of the input.
    \item Drastically reduces the input size. Useful for reducing the number of parameters in your model.
    \item Typically used immediately following a series of convolutional-activation layers.
  \end{itemize}
  \item \textbf{BatchNorm2d:} Performs batch normalization (Ioffe and Szegedy, 2014). Normalizes the activations of previous layer to standard normal (mean 0, standard deviation 1).
  \begin{itemize}
    \item Accelerates convergence and improves performance of model, especially when saturating nonlinearities (sigmoid) are used.
    \item Makes model less sensitive to higher learning rates and initialization, and also acts as a form of regularization.
    \item Typically used immediately before nonlinearity (Activation) layers.
  \end{itemize}
  \item \textbf{Dropout:} Takes some probability and at every iteration sets weights to zero at random with that probability
  \begin{itemize}
    \item An effective form of regularization. During training, randomly selecting activations to shut off forces network to build in redundancies in the feature representation, so it does not rely on any single activation to perform classification.
  \end{itemize}
  \item \textbf{ReLU (activation):} Sets negative inputs to 0
  \item \textbf{Softmax (activation):} Rescales input so that it can be interpreted as a (discrete) probability distribution.
  \item \textbf{Flatten:} Flattens any tensor into a single vector (required in order to pass a 2D tensor output from a convolutional layer as input into Linear layers)
\end{itemize}

\textbf{Your tasks.} Build a network with only the allowed layers that achieves \textbf{test accuracy of at least 0.985}. You are required to use categorical cross entropy as your loss function and to train for 10 epochs with a batch size of 32. Note: your model must have fewer than 1 million parameters, as measured by the method given in the sample code.
Everything else can change: optimizer (RMSProp, Adam, ???), initial learning rates, dropout probabilities, layerwise regularizer strengths, etc. You are not required to use all of the layers, but \emph{you must have
at least one dropout layer and one batch normalization layer in your final model}. Try to figure out the best possible architecture and hyperparameters given these building blocks!

In order to design your model, you should train your model for 1 epoch (batch size 32) and look at the final \textbf{test accuracy}  after training. This should take no more than 10 minutes, and should give you an immediate sense for how fast your network converges and how good it is.

Set the probabilities of your
dropout layers to 10 equally-spaced values $p \in [0, 1]$, train for 1 epoch, and report the final model accuracies for each.

You can perform all of your hyperparameter validation in this way: vary your parameters and train for an epoch. After you're satisfied with the model design, you should train your model for the full 10 epochs.

\textbf{In your submission.} Turn in the code of your model, the test accuracy for the 10 dropout probabilities $p \in [0, 1]$, and the
final test accuracy when your model
is trained for 10 epochs. We should have everything needed to reproduce your results.

Discuss what you found to be the most effective strategies in designing a convolutional network. Which regularization method was most effective (dropout, layerwise regularization, batch norm)?

Do you foresee any problem with this way of validating our hyperparameters? If so, why?

\emph{Hints:}

\begin{itemize}
  \item You are provided with a sample network that achieves a high accuracy. Starting with this network, modify some of the regularization parameters (layerwise regularization strength, dropout probabilities) to see if you can maximize the test accuracy. You can also add layers or modify layers (e.g.
  changing the convolutional kernel sizes, number of filters, stride, dilation, etc.) so long as the total number of parameters remains under the cap of 1 million.
  \item You may want to read up on successful convolutional architectures, and emulate some of their design principles. Please cite any idea you use that is not your own.
  \item To better understand the function of each layer, check the PyTorch documentation.
  \item Linear layers take in single vector inputs (ex: \emph{(784, )}) but Conv2D layers
  take in tensor inputs (ex: \emph{(28, 28, 1)}): width, height, and channels. Using the transformation \verb$transforms.ToTensor()$ when loading the dataset will reshape the training/test $X$ to a 4-dimensional tensor (ex: \emph{(num_examples, width, height, channels)}) and normalize values. For the MNIST dataset, \emph{channels=1}. Typical color images have 3 color channels, 1 for each color in RGB.
  \item Other useful CNN design principles:
  \begin{itemize}
    \item CNNs perform well with many stacked convolutional layers, which develop increasingly large-scale representations of the input image.
    \item Dropout ensures that the learned representations are robust to some amount of noise.
    \item Batch norm is done after a convolutional or dense layer and immediately prior to an activation/nonlinearity layer.
    \item Max-pooling is typically done after a series of convolutions, in order to gradually reduce the size of the representation.
    \item Finally, the learned representation is passed into a dense layer (or two), and then filtered down to the final softmax layer.
  \end{itemize}
\end{itemize}
\begin{solution}

\end{solution}

%\problem Extra credit \textbf{[5 bonus points possible]}

%In the sample code, we have provided a notebook to visualize the final results of your trained model, e.g. what images maximally activate certain neurons, the feature representations learned by successively-higher neurons, etc. We have also provided code that can visualize the training / validation accuracy of your model as a function of epochs trained.

%Discuss the features your model learned. How will the visualizations help you design better convolutional networks? Next, investigate the behavior of the training accuracy over time. What causes this behavior?

\end{document}


\documentclass[12pt,presentation]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{fixltx2e}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{float}
\usepackage{wrapfig}
\usepackage{soul}
\usepackage{textcomp}
\usepackage{marvosym}
\usepackage{wasysym}
\usepackage{latexsym}
\usepackage{amssymb}
\usepackage{hyperref}
\tolerance=1000
\usepackage[english]{babel} \usepackage{ae,aecompl}
\usepackage{mathpazo,courier,euler} \usepackage[scaled=.95]{helvet}
\usepackage{listings}
\lstset{language=Python, basicstyle=\ttfamily\bfseries,
commentstyle=\color{red}\itshape, stringstyle=\color{green},
showstringspaces=false, keywordstyle=\color{blue}\bfseries}
\providecommand{\alert}[1]{\textbf{#1}}

\title{SEES: Test Driven Development}
\author{FOSSEE}

\usetheme{Warsaw}\usecolortheme{default}\useoutertheme{infolines}\setbeamercovered{transparent}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\maketitle

\begin{frame}
\frametitle{Outline}
\setcounter{tocdepth}{3}
\tableofcontents
\end{frame}

\section{Introduction}

\begin{frame}
  \frametitle{Objectives}
  At the end of this section, you will be able to:
  \begin{itemize}
  \item Write your code using the TDD paradigm.
  \item Use doctests to test your Python code.
  \item Use unittests to test your Python code.
  \item Use the nose module to test your code.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{What is TDD?}
  The basic steps of TDD are roughly as follows --
  \begin{enumerate}
  \item Decide upon the feature to implement and the methodology of
    testing it.
  \item Write the tests for the feature decided upon.
  \item Just write enough code, so that the test can be run, but it fails.
  \item Improve the code, to just pass the test and at the same time
    passing all previous tests.
  \item Run the tests to see, that all of them run successfully.
  \item Refactor the code you've just written -- optimize the algorithm,
    remove duplication, add documentation, etc.
  \item Run the tests again, to see that all the tests still pass.
  \item Go back to 1.
  \end{enumerate}
\end{frame}

\section{First Test}

\begin{frame}[fragile]
  \frametitle{First Test -- GCD}
  \begin{itemize}
  \item simple program -- GCD of two numbers
  \item What are our code units?
    \begin{itemize}
    \item Only one function \texttt{gcd}
    \item Takes two numbers as arguments
    \item Returns one number, which is their GCD
    \end{itemize}
\begin{lstlisting}
c = gcd(44, 23)
\end{lstlisting}
  \item c will contain the GCD of the two numbers.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Test Cases}
  \begin{itemize}
  \item Important to have test cases and expected outputs even before
    writing the first test!
  \item $a=48$, $b=48$, $GCD=48$
  \item $a=44$, $b=19$, $GCD=1$
  \item Tests are just a series of assertions
  \item True or False, depending on expected and actual behavior
  \end{itemize}

\end{frame}

\begin{frame}[fragile]
  \frametitle{Test Cases -- Code}
\begin{lstlisting}
tc1 = gcd(48, 64)
if tc1 != 16:
    print "Failed for a=48, b=64. Expected 16. \
    Obtained %d instead." % tc1
    exit(1)

tc2 = gcd(44, 19)
if tc2 != 1:
    print "Failed for a=44, b=19. Expected 1. \
    Obtained %d instead." % tc2
    exit(1)

print "All tests passed!"
\end{lstlisting}
\begin{itemize}
\item The function \texttt{gcd} doesn't even exist!
\end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Stubs}
  \begin{itemize}
  \item First write a very minimal definition of \texttt{gcd}
    \begin{lstlisting}
def gcd(a, b):
    pass
    \end{lstlisting}
  \item Written just, so that the tests can run
  \item Obviously, the tests are going to fail
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{\texttt{gcd.py}}
\begin{lstlisting}
def gcd(a, b):
    pass

if __name__ == '__main__':
    tc1 = gcd(48, 64)
    if tc1 != 16:
        print "Failed for a=48 and b=64. \
        Expected 16. Obtained %d instead." % tc1
        exit(1)
    tc2 = gcd(44, 19)
    if tc2 != 1:
        print "Failed for a=44 and b=19. \
        Expected 1. Obtained %d instead." % tc2
        exit(1)
    print "All tests passed!"
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{First run}
\begin{lstlisting}
$ python gcd.py
Traceback (most recent call last):
  File "gcd.py", line 7, in <module>
  print "Failed for a=48 and b=64. Expected 16.
  Obtained %d instead." % tc1
TypeError: %d format:
a number is required, not NoneType
\end{lstlisting} %$

  \begin{itemize}
  \item We have our code unit stub, and a failing test.
  \item The next step is to write code, so that the test just passes.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Euclidean Algorithm}
  \begin{itemize}
  \item Modify the \texttt{gcd} stub function
  \item Then, run the script to see if the tests pass.
  \end{itemize}
\begin{lstlisting}
def gcd(a, b):
    if a == 0:
        return b
    while b != 0:
        if a > b:
            a = a - b
        else:
            b = b - a
    return a
\end{lstlisting}
\begin{lstlisting}
$ python gcd.py
All tests passed!
\end{lstlisting} %$
  \begin{itemize}
  \item \alert{Success!}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Euclidean Algorithm -- Modulo}
  \begin{itemize}
  \item Repeated subtraction can be replaced by a modulo
  \item modulo of \texttt{a\%b} is always less than b
  \item when \texttt{a < b}, \texttt{a\%b} equals \texttt{a}
  \item Combine these two observations, and modify the code
\begin{lstlisting}
def gcd(a, b):
    while b != 0:
        a, b = b, a % b
    return a
\end{lstlisting}
  \item Check that the tests pass again
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Euclidean Algorithm -- Recursive}
  \begin{itemize}
  \item Final improvement -- make \texttt{gcd} recursive
  \item More readable and easier to understand
\begin{lstlisting}
def gcd(a, b):
    if b == 0:
        return a
    return gcd(b, a%b)
\end{lstlisting}
  \item Check that the tests pass again
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Document \texttt{gcd}}
  \begin{itemize}
  \item Undocumented function is as good as unusable
  \item Let's add a docstring \& We have our first test!
  \end{itemize}
\begin{lstlisting}
def gcd(a, b):
    """Returns the Greatest Common Divisor of the
    two integers passed as arguments.

    Args:
      a: an integer
      b: another integer

    Returns: Greatest Common Divisor of a and b
    """
    if b == 0:
        return a
    return gcd(b, a%b)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Persistent Test Cases}
  \begin{itemize}
  \item Tests should be pre-determined and written, before the code
  \item Test Data is repeatedly used; Hence, saved in persistent
    format
  \item Let's save data for GCD tests in a text file.
  \item The file shall have multiple lines of test data
  \item Each line corresponds to a single test case
  \item Each line consists of three comma separated values --
    \begin{itemize}
    \item First two coloumns are the integers for which the GCD has to be
      computed
    \item Third coloumn is the expected GCD to the preceding two
      numbers.
    \end{itemize}
  \item Let us call our data file \texttt{gcd\_testcases.dat}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Modify \texttt{gcd.py}}
\begin{lstlisting}
if __name__ == '__main__':
    for line in open('gcd_testcases.dat'):
        values = line.split(', ')
        a = int(values[0])
        b = int(values[1])
        g = int(values[2])

        tc = gcd(a, b)
        if tc != g:
            print "Failed for a=%d and b=%d.\
            Expected %d. Obtained %d instead."\
            % (a, b, g, tc)
            exit(1)

    print "All tests passed!"
\end{lstlisting}
\end{frame}

\section{Python Testing Frameworks}

\begin{frame}[fragile]
  \frametitle{Python Testing Frameworks}
  \begin{itemize}
  \item Testing frameworks essentially, ease the job of the user
  \item Python provides two frameworks for testing code
    \begin{itemize}
    \item \texttt{unittest} framework
    \item \texttt{doctest} module
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{\texttt{doctest} module}

\begin{frame}[fragile]
  \frametitle{doctest}
  \begin{itemize}
  \item Documentation always accompanies a well written piece of code
  \item Use \texttt{docstring} to document functions or modules
  \item Along with description and usage, examples can be added
  \item Interactive interpreter session inputs and outputs are
    copy-pasted
  \item \texttt{doctest} module picks up all such interactive examples
  \item Executes them and determines if the code runs, as documented
  \end{itemize}
  Let's use the \texttt{doctest} module for our \texttt{gcd} function
\end{frame}

\begin{frame}[fragile]
  \frametitle{doctest for \texttt{gcd.py}}
\begin{tiny}
\begin{lstlisting}
def gcd(a, b):
    """Returns the Greatest Common Divisor of the two integers
    passed as arguments.

    Args:
      a: an integer
      b: another integer

    Returns: Greatest Common Divisor of a and b

    >>> gcd(48, 64)
    16
    >>> gcd(44, 19)
    1
    """
    if b == 0:
        return a
    return gcd(b, a%b)
\end{lstlisting}
\end{tiny}
\begin{itemize}
\item We have added examples to the \texttt{docstring}
\item Now we need to tell the \texttt{doctest} module to execute
\end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{doctest for \texttt{gcd.py} \ldots}
\begin{lstlisting}
if __name__ == "__main__":
    import doctest
    doctest.testmod()
\end{lstlisting}
\begin{itemize}
\item \texttt{testmod} automatically picks all sample sessions
\item Executes them and compares output with documented output
\item It doesn't give any output, when all the results match
\item Complains only when one or more tests fail.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{\texttt{doctest} -- Execution}
  \begin{itemize}
  \item Run the doctests by running \texttt{gcd.py}
\begin{lstlisting}
$ python gcd.py
\end{lstlisting} %$
  \item All the tests pass, and doctest gives no output
  \item For a more detailed report we can run with -v argument
\begin{lstlisting}
$ python gcd.py -v
\end{lstlisting} %$
  \item If the output contains a blank line, use \texttt{<BLANKLINE>}
  \item To see a failing test case, replace \texttt{return a} with \texttt{b}
  \end{itemize}
\end{frame}

\subsection{\texttt{unittest} framework}

\begin{frame}[fragile]
  \frametitle{\texttt{unittest}}
  \begin{itemize}
  \item It won't be long, before we complain about the power of
    \texttt{doctest}
  \item \texttt{unittest} framework can efficiently automate tests
  \item Easily initialize code and data for executing the specific
    tests
  \item Cleanly shut them down once the tests are executed
  \item Easily aggregate tests into collections and improved reporting
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{\texttt{unittest}}
  \begin{itemize}
  \item It won't be long, before we complain about the power of
    \texttt{doctest}
  \item \texttt{unittest} framework can efficiently automate tests
  \item Easily initialize code and data for executing the specific
    tests
  \item Cleanly shut them down once the tests are executed
  \item Easily aggregate tests into collections and improved reporting
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{\texttt{unittest}ing \texttt{gcd.py}}
  \begin{itemize}
  \item Subclass the \texttt{TestCase} class in \texttt{unittest}
  \item Place all the test code as methods of this class
  \item Use the test cases present in \texttt{gcd\_testcases.dat}
  \item Place the code in \texttt{test\_gcd.py}
  \end{itemize}
\end{frame}

\begin{frame}[fragile,allowframebreaks]
  \frametitle{\texttt{test\_gcd.py}}
\small
\begin{lstlisting}
import gcd
import unittest

class TestGcdFunction(unittest.TestCase):

    def setUp(self):
        self.test_file = open('gcd_testcases.dat')
        self.test_cases = []
        for line in self.test_file:
            values = line.split(', ')
            a = int(values[0])
            b = int(values[1])
            g = int(values[2])

            self.test_cases.append([a, b, g])

    def test_gcd(self):
        for case in self.test_cases:
            a = case[0]
            b = case[1]
            g = case[2]
            self.assertEqual(gcd.gcd(a, b), g)

    def tearDown(self):
        self.test_file.close()
        del self.test_cases

if __name__ == '__main__':
    unittest.main()
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{\texttt{test\_gcd.py}}
  \begin{itemize}
  \item \texttt{setUp} -- we read all the test data and store it in a
    list
  \item \texttt{tearDown} -- delete the data to free up memory and
    close open file
  \item \texttt{test\_gcd} -- actual test code
  \item \texttt{assertEqual} -- compare actual result with expected one
  \item Write documentation for above code.
  \end{itemize}
\end{frame}

\section{\texttt{nose}}

\begin{frame}[fragile]
  \frametitle{\texttt{nose} tests}
  \begin{itemize}
  \item It is not easy to organize, choose and run tests scattered
    across multiple files.
  \item \texttt{nose} module aggregate these tests automatically
  \item Can aggregate \texttt{unittests} and \texttt{doctests}
  \item Allows us to pick and choose which tests to run
  \item Helps output the test-results and aggregate them in various
    formats
  \item Not part of the Python distribution itself
\begin{lstlisting}
$ easy_install nose
\end{lstlisting} %$
  \item Run the following command in the top level directory
\begin{lstlisting}
$ nosetests
\end{lstlisting} %$
  \end{itemize}
\end{frame}

\end{document}


\section{Failure Modes of Shafts}

\newthought{There are a number of shaft failure modes} and it is important to recognise the types of failure that can occur and be able to perform the subsequent analysis to mitigate this occurring within our design. It is up to you to do some research on the types of failure and to discuss the failures that might occur for your shaft. Whilst this section does not detail the types of failure, it does provide the calculations required to determine the stresses within the shaft so that you will be able to evaluate whether the shaft will survive the forces that it is being subjected to.

\subsection{Types of Failure}

\begin{framed}
    \vspace{1cm}
    \begin{center}
        \Large
        -- Own Research \& Lectures --
    \end{center}
    \vspace{1cm}
\end{framed}

\subsection{Calculating Shaft Stresses}

\marginnote{Direct Stress $(\sigma_d)$} The first is the direct stress acting through the cross-section of the shaft.
If the shaft diameter is not large enough then shaft will simply shear as if one were creating a cross-section through the shaft.
The Direct Stress is given by \cref{equ-direct-stress}.

\begin{equation}
    \sigma_d = \frac{F}{A}
    \label{equ-direct-stress}
\end{equation}

However, it is very rare for a shaft to fail through Direct Stress through the shafts cross-section. It is more likely that a shaft will fail due to the combined bending, hoop and torsional stresses acting on the outer surface of the shaft, which can be resolved using Mohr's Circle~\cite{clifford2012}.

Taking\marginnote{Mohr's circle}  a 2D element on the outer surface of the shaft, the bending, hoop and torsional stresses can be related as shown in \cref{fig-mohrs-circle}. Where $\sigma_x$ is the bending stress, $\sigma_y$ is the hoop stress and $\tau_{xy}$ is the torsional stress.

\begin{marginfigure}
  \center{}
  \begin{tikzpicture}[scale=2]

    \draw[] (0,0) -- (0,1) -- (1,1) -- (1,0) -- (0,0);

    \draw[->] (-0.1,0.5) -- (-0.4,0.5) node[left]{$\sigma_x$};
    \draw[->] (1.1,0.5) -- (1.4,0.5) node[right]{$\sigma_x$};
    \draw[->] (0.5,1.1) -- (0.5,1.4) node[above]{$\sigma_y$};
    \draw[->] (0.5,-0.1) -- (0.5,-0.4) node[below]{$\sigma_y$};

    \draw[{Straight Barb[left]}-] (0.0,-0.05) -- (1.0,-0.05) node[pos=0.0, below]{$\tau_{xy}$};
    \draw[-{Straight Barb[left]}] (0.0,1.05) -- (1.0,1.05) node[pos=1.0, above]{$\tau_{xy}$};

    \draw[{Straight Barb[right]}-] (-0.05,0.0) -- (-0.05,1.0) node[pos=0.0, left]{$\tau_{xy}$};
    \draw[-{Straight Barb[right]}] (1.05,0.0) -- (1.05,1.0) node[pos=1.0, right]{$\tau_{xy}$};

  \end{tikzpicture}
  \caption{2D mohr's circle}
  \label{fig-mohrs-circle}
\end{marginfigure}

Bending\marginnote{Bending Stress $(\sigma_x)$}  stress is the Direct Stress in the direction of the axis of the shaft and is due to the bending moment. This can be calculated using Equation~\ref{equ-bending-stress}.

\begin{equation}
  \sigma_{x} = \frac{My}{I}
  \label{equ-bending-stress}
\end{equation}

Where \(I\) is the Second Moment of Area:

\begin{equation}
  I = \frac{\pi r^4}{4}
\end{equation}

Hoop\marginnote{Hoop Stress \(\sigma_y\)} stress is the direct stress due to forces attempting to expand the shaft. This is only really the case when considering pressure vessels where the interior pressure of the gas or liquid is wanting to expand the vessel. In our cases, we can say that \(\sigma_y = 0\).

Torsional\marginnote{Torsional Stress \(\tau_{xy}\)} stress is the shear stress due to torsion (twisting) and can by calculated using Equation~(\ref{equ:torsional-stress}).

\begin{equation}
  \tau_{xy} = \frac{Tr}{J}
  \label{equ:torsional-stress}
\end{equation}

Where \(J\) is the Polar Moment of Area:

\begin{equation}
  J = \frac{\pi d^4}{32}
\end{equation}

These stress depend upon the orientation of the element. If another element is taken, twisted through an angle \(\phi\), as shown in Figure, then the Direct Stresses are \(\sigma_x\) and \(\sigma_X\) and the shear stress is \(\tau_{xy}\) = \(\tau_{XY}\). And these are normally different from the first set of stresses in \cref{fig-rotated-mohrs-circle}.

\begin{marginfigure}
  \center{}
  \begin{tikzpicture}[scale=2]

    \draw[rotate=45] (0,0) -- (0,1) -- (1,1) -- (1,0) -- (0,0);

    \draw[->, rotate=45] (-0.1,0.5) -- (-0.4,0.5) node[left]{$\sigma_X$};
    \draw[->, rotate=45] (1.1,0.5) -- (1.4,0.5) node[right]{$\sigma_X$};
    \draw[->, rotate=45] (0.5,1.1) -- (0.5,1.4) node[above]{$\sigma_Y$};
    \draw[->, rotate=45] (0.5,-0.1) -- (0.5,-0.4) node[below]{$\sigma_Y$};

    \draw[{Straight Barb[left]}-, rotate=45] (0.0,-0.05) -- (1.0,-0.05) node[pos=0.0, below]{$\tau_{XY}$};
    \draw[-{Straight Barb[left]}, rotate=45] (0.0,1.05) -- (1.0,1.05) node[pos=1.0, above]{$\tau_{XY}$};

    \draw[{Straight Barb[right]}-, rotate=45] (-0.05,0.0) -- (-0.05,1.0) node[pos=0.0, left]{$\tau_{XY}$};
    \draw[-{Straight Barb[right]}, rotate=45] (1.05,0.0) -- (1.05,1.0) node[pos=1.0, right]{$\tau_{XY}$};

  \end{tikzpicture}
  \caption{Mohr's circle rotated by \(\phi\)}
  \label{fig-rotated-mohrs-circle}
\end{marginfigure}

The two sets of stresses are related by formulae involving the sine and cosine of twice the angle, 2\(\phi\). Mohr's circle is a graphical way of representing these formulae. The construction of the circle is as follows and is shown in \cref{fig-m-circle}.

\begin{enumerate}
  \item Draw the line AB between points with coordinates: \(\text{A} = (\sigma_x, -\tau_{xy}) \) and \(\text{B} = (\sigma_y, \tau_{xy}) \)
  \item Draw the circle with AB as diameter
\end{enumerate}

\begin{figure}[h!]
  \center{}
  \begin{tikzpicture}[scale=0.75]

    \draw[] (-1,-5) -- (-1,5) node[pos=1.0, anchor=east] {$\tau_n$};
    \draw[] (-1,0) -- (12,0) node[pos=1.0, anchor=north] {$\sigma_n$};

    \draw[] (6,0) circle (4);

    \draw[] (2,0) -- (10,0)
    node[pos=0.0, anchor=south east] {$Q=(\sigma_2,0)$}
    node[pos=1.0, anchor=south west] {$P=(\sigma_1,0)$};

    \draw[] (6,-4) -- (6,4)
    node[pos=0.0, anchor=north] {$S=(0,-\tau_{\max} )$}
    node[pos=1.0, anchor=south] {$R=(0,\tau_{\max} )$};

    \draw[rotate around={25:(6,0)}] (2,0) -- (10,0)
    node[pos=0.0, anchor=north east] {$D=(\sigma_y,\tau_{YX})$}
    node[pos=1.0, anchor=south west] {$C=(\sigma_x,-\tau_{XY})$};

    \draw[rotate around={-40:(6,0)}] (2,0) -- (10,0)
    node[pos=0.0, anchor=south east] {$B=(\sigma_y,\tau_{yx})$}
    node[pos=1.0, anchor=north west] {$A=(\sigma_x,-\tau_{xy})$};

    \draw[] (7,0) arc (0:25:1);
    \draw[] (7,0) arc (0:-40:1) node[pos=0.2, anchor=north west] {$2\phi$};

  \end{tikzpicture}
  \caption{Mohr's circle relating stresses associated with elements with different orientations}
  \label{fig-m-circle}
\end{figure}

Then, to find the stresses related to the element rotated through angle \(\phi\), the diameter AB is rotated through angle \(2\phi\), giving diameter CD. The positions of C and D give the other stress with: \(\text{C} = (\sigma_X, -\tau_{XY}) \) and \(\text{D} = (\sigma_Y, \tau_{XY}) \).

Mohr's\marginnote{Principal Stresses} circle shows there are two special cases. The first is when the diameter AB is rotated so that it becomes horizontal. The corresponding element undergoes no shear stress, but is subject to two direct stresses (in perpendicular directions) which are \(\sigma_1\) and \(\sigma_2\). These are the principal direct stress and represent the maximum and minimum values of direct stress in any direction.

\begin{equation}
  \sigma_1 = \frac{1}{2}(\sigma_x+\sigma_y) + \sqrt{\left(\frac{1}{4}(\sigma_x-\sigma_y)^2\right)+\tau_{xy}^2}
\end{equation}

\begin{equation}
  \sigma_2 = \frac{1}{2}(\sigma_x+\sigma_y) - \sqrt{\left(\frac{1}{4}(\sigma_x-\sigma_y)^2\right)+\tau_{xy}^2}
\end{equation}

The\marginnote{Principal Shear Stresses} second special case is when the diameter AB is rotated so that it becomes vertical in circle. The corresponding element sees the same direct stresses (in the two perpendicular directions) and a shear stress. From \cref{fig-m-circle}, this common direct stress is the average of the principal stresses \(\frac{1}{2}(\sigma_1+\sigma_2)\). The shear stress is the largest possible shear stress in any direction, denoted (on \cref{fig-m-circle}) by \(\tau_{\max}\).

\begin{equation}
  \tau_{\max} = \sqrt{\left(\frac{1}{4}(\sigma_x-\sigma_y)^2\right)+\tau_{xy}^2}
  \label{equ:tau-max}
\end{equation}

In designing a component, there are limiting values placed on the direct and shear stresses (associated with any elements or directions). These may be based upon the yield tensile stress and the yield shear stress, or upon the ultimate tensile stress and ultimate shear stress.

So there are two conditions to satisfy as follows.

\begin{equation}
|\sigma_1|,|\sigma_2| \leq \text{max\ allowable\ direct\ stress}
\end{equation}

\begin{equation}
|\tau_{\max}| \leq \text{max\ allowable\ shear\ stress}
\end{equation}

The\marginnote{Assumptions for Shafts} first assumption is that only stresses on the outside of the shaft need be considered. This is where the axial direct stress and the shear stress are maximal.

The second assumption is that only the condition related to the maximum shear stress needs to be considered. This is because of the observation that shafts made of ductile materials tend to fail due to shear.

The third assumption is that the \acf{UTS} and the \acf{USS} for steel and roughly related by the relation:

\begin{equation}
  \text{USS steel} \simeq 0.75 \times \text{UTS steel}
\end{equation}

An alternative third assumption is that the \acf{TYS} and the \acf{SYS} for steel are roughly related by the relation:

\begin{equation}
  \text{SYS steel} \simeq 0.58 \times \text{TYS steel}
\end{equation}

These are empirical relations based upon typical values for steel as shown in \cref{tbl-uts-relationships} by~\citet{deutschman1975}~\cite{deutschman1975}.

\begin{table}
  \caption[Approximate relationships between shear and tensile stresses]{Approximate relationships between shear and tensile stresses~\citep{deutschman1975}}
  \label{tbl-uts-relationships}
  \center{}
  \small
  \begin{tabular}{p{0.2\textwidth} p{0.35\textwidth} p{0.35\textwidth}}
    \toprule
    Material & Ultimate strength relationship & Yield strength relationship \\
    \midrule
    steels & \(\text{USS } \simeq 0.75 \times \text{UTS}\) & \(\text{SYS} \simeq 0.58 \times \text{TYS}\) \\
    ductile iron & \(\text{USS } \simeq 0.9 \times \text{UTS}\) & \(\text{SYS} \simeq 0.75 \times \text{TYS}\) \\
    malleable iron & \(\text{USS } \simeq 1.0 \times \text{UTS}\) &  \\
    wrought iron & \(\text{USS } \simeq 0.83 \times \text{UTS}\) &  \\
    cast iron & \(\text{USS } \simeq 1.3 \times \text{UTS}\) &  \\
    aluminiums & \(\text{USS } \simeq 0.65 \times \text{UTS}\) & \(\text{SYS} \simeq 0.55 \times \text{TYS}\) \\
    \bottomrule
  \end{tabular}
\end{table}

\section{Failure Criteria}

The\marginnote{Note: to deal with these theories properly requires considering the stress in the third dimension, which we have been ignoring as it is zero for the cases considered here. While the failure criteria are completely general, to keep things simple the equations in this section are not necessarily valid for other general stress states.} yield strength of materials is measured experimentally, often using uniaxial tests. In this type of test, a sample is progressively loaded in tension, and its extension recorded, until it starts to deform plastically. The stress at this point is called the yield strength of the material. However, the combined stress state described above is more complicated than in a uniaxial test. It is not feasible to experimentally test for every combination of stresses that a material might encounter, so some theory is needed to relate the combined stress state described above to the uniaxial yield strength of the material. There are multiple theories available to explain this relationship, known as failure criteria. Here we will discuss the two simplest, the Tresca criteria and the von Mises stress.

The Tresca criterion is based on the maximum shear stress, and predicts that failure will occur when the maximum shear stress reaches some critical value:

\begin{equation}
  \tau_{\max} = \tau_{\text{failure}}
\end{equation}

The Tresca\marginnote{Tresca criterion} criterion is based on the maximum shear
stress, and predicts that failure will occur when the maximum shear stress
reaches some critical value:

\begin{equation}
\label{eq:2}
\tau_\mathrm{max} = \tau_\mathrm{failure}
\end{equation}

The critical value is calibrated to the yield strength $S_y$ measured in a
uniaxial test. In the uniaxial test at the onset of yield, $\sigma_x = S_y$ and
$\sigma_y = \tau_{xy} = 0$. From \cref{equ:tau-max},

\begin{equation}
\label{eq:4}
\tau_\mathrm{failure} = \sqrt{\left(\frac{\sigma_x-\sigma_y}{2}\right)^2+\tau_{xy}^2} = \sqrt{\left(\frac{S_y-0}{2}\right)^2+0} = \frac{S_y}{2}
\end{equation}

Therefore, the Tresca failure criterion (\cref{eq:2}) for shafts under bending
and torsion (\cref{equ:tau-max}) can be written as:

\begin{align}
\label{eq:5}
\frac{1}{2} \sqrt{\sigma_x^2+4\tau_{xy}^2} &= \frac{S_y}{2} \\
              \intertext{or simply}
  \label{eq:tresca-failure}
\sqrt{\sigma_x^2+4\tau_{xy}^2} &= S_y
\end{align}

The von Mises\marginnote{von Mises criterion} criterion is an alternative
theory. It has the advantage of being more accurate than the Tresca criterion
(which is always more conservative) and behaves in a more continuous manner when
the stress state is varying. The `von Mises stress' is defined as

\begin{equation}
\label{eq:7}
\sigma' = \left[ \frac{
    \left( \sigma_1 - \sigma_2 \right)^2 +
    \left( \sigma_2 - \sigma_3 \right)^2 +
    \left( \sigma_3 - \sigma_1 \right)^2
  }{2} \right]^{1/2}
\end{equation}

Here $\sigma_3$ is the principle stress in the third dimension, which can be
assumed to be zero for the shaft stresses we are considering here. The
expression can be simplified then to:

\begin{align}
\label{eq:9}
  \sigma' &= \left[ \frac{
  \left( \sigma_1 - \sigma_2 \right)^2 + \sigma_2^2 + \sigma_1^2
  }{2} \right]^{1/2} \\
&= \left[ \frac{\left( \sigma_1^2 + \sigma_2^2 - 2 \sigma_1\sigma_2 \right) + \sigma_2^2 + \sigma_1^2}{2} \right]^{1/2} \\
          &= \left[ \sigma_1^2 + \sigma_2^2 - \sigma_1\sigma_2 \right]^{1/2}
\end{align}

Substituting for $\sigma_1$ and $\sigma_2$ from \cref{equ:tau-max}, this can be written
as:

\begin{align}
\sigma' &= \frac{1}{2} \left[
          \left( \sigma_x + \sqrt{\sigma_x^2+4\tau_{xy}^2} \right)^2 +
          \left( \sigma_x - \sqrt{\sigma_x^2+4\tau_{xy}^2} \right)^2 -
          \left( \sigma_x + \sqrt{\sigma_x^2+4\tau_{xy}^2} \right)
          \left( \sigma_x - \sqrt{\sigma_x^2+4\tau_{xy}^2} \right)
          \right]^{1/2} \\
&= \frac{1}{2} \left[\sigma_x^2 + 3\left( \sigma_x^2+4\tau_{xy}^2 \right) \right]^{1/2} \\
  &= \sqrt{\sigma_x^2 + 3\tau_{xy}^2}
  \label{eq:8}
\end{align}

The von Mises failure criterion for a shaft under bending and torsion is
therefore:

\begin{equation}
\sqrt{\sigma_x^2 + 3\tau_{xy}^2} = S_y
\label{eq:10}
\end{equation}

Comparing this to \cref{eq:5}, you might be able to see that the Tresca
criterion will always predict failure at the same or lower stress than the von
Mises criterion. In the end, it is no more complicated to use \cref{eq:10} than
\cref{eq:5}, so the von Mises criterion is often a good approach to use.


\chapter{Introduction}\label{chap:introduction}
\par Ever since I was a child, I liked playing with Rubik's Cubes. I was told by my uncle that there is a lot of mathematics behind this fascinating toy, such as the Rubik's Cube can be viewed as a group. Now, I am at the age where I gathered enough skills to understand the complexities of the Rubik's Cube. In this thesis, I investigate how to link Rubik's Cube with cryptography, a subject I obsessed with since sophomore year of college.
\par We start with discussing what cryptography is and listing motivations for us to integrate Rubik's Cubes with encryption protocols. While presenting design ideas of the Rubik's Cube encryption, we consider limitations and make corresponding improvements. Then, under the formal definition of security, we test if our Rubik's Cube encryption is well designed, and tweak it to make the encryption protocol better under the definition of security. Finally, we illustrate some known structures of the Rubik's Cube groups and introduce a Rubik's Cube key exchange protocol which has a similar structure that the Diffie-Hellman key exchange protocol uses.

\section{What is cryptography}
\par The definition of cryptography\cite{cryptography} given in the Merriam-Webster dictionary is ``\textit{secret writing}''. This is historically precise, since in ancient times, people applied the idea of cryptography solely to enable secret communication. Most ancient cryptography schemes, as well as some modern ones look like some kind of puzzle. Yet creating and breaking the scheme would rely on how well you can manipulate the puzzle.
\par One basic scheme is the Substitution cipher, which replaces each unit of the plaintext to form the ciphertext. If we are working with English, then each unit of plaintext will be a letter.
\begin{figure}[ht]
    \centering
    \setlength\tabcolsep{3pt}
        \begin{tabular}{cccccccccccccccccccccccccc}
        A & B & C & D & E & F & G & H & I & J & K & L & M & N & O & P & Q & R & S & T & U & V & W & X & Y & Z \\
        $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ \\
        Z & E & B & R & A & S & C & D & F & G & H & I & J & K & L & M & N & O & P & Q & T & U & V & W & X & Y \\
    \end{tabular}
    \caption{Map letter to letter}\label{fig:replace-letter-by-letter}
\end{figure}
One trivial example could be substituting each letter to another unique letter as displayed in Figure~\ref{fig:replace-letter-by-letter}. If we encrypt the sentence \textit{``life is like a box of chocolates''} with the relations specified in Figure~\ref{fig:replace-letter-by-letter}, we get \textit{``ifsafpifhazelwlsbdlblizqap''} back as the ciphertext. Correspondences between plaintext and ciphertext can be very creative as long as the communicating parties hold the same information.
\begin{figure}[ht]
    \centering
    \setlength\tabcolsep{3pt}
    \resizebox{\linewidth}{!}{
        \begin{tabular}{cccccccccccccccccccccccccc}
            A & B & C & D & E & F & G & H & I & J & K & L & M & N & O & P & Q & R & S & T & U & V & W & X & Y & Z \\
            $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ & $\downarrow$ \\
            \faAsterisk{} & \faBank{} & \faCube{} & \faDiamond{} & \faEnvelope{} & \faFilm{} & \faGear{} & \faHashtag{} & \faImage{} & \faJoomla{} &\faKey{} & \faLeaf{} & \faMagnet{} & \faNavicon{} & \faOpenid{} & \faPaperPlane{} & \faQrcode{} & \faRandom{} & \faSave{} & \faTag{} &\faUser{} & \faVideoCamera{} & \faWifi{} & \faXing{} & \faYoutubePlay{} & \faTv{}
        \end{tabular}
    }
    \caption{Map letter to icon}\label{fig:replace-letter-by-icon}
\end{figure}
The ``creative'' example is shown in Figure~\ref{fig:replace-letter-by-icon}, where we are replacing letters by icons which are completely irrelevant. We can encrypt the same sentence \textit{``\;life is like a box of chocolates''} again and this time we get ``\faImage{} \faFilm{} \faSave{} \faAsterisk{} \faFilm{} \faPaperPlane{} \faImage{} \faFilm{} \faHashtag{} \faAsterisk{} \faTv{} \faEnvelope{} \faLeaf{} \faWifi{} \faLeaf{} \faSave{} \faBank{} \faDiamond{} \faLeaf{} \faBank{} \faLeaf{} \faImage{} \faTv{} \faQrcode{} \faAsterisk{} \faPaperPlane{}\;'' back.
\par Intuitively, we know the security of substitution cipher is based on the fact that the correspondence between plaintext and ciphertext is unknown to the attacker. Although we may find the second substitution to be more abstract, the brute force attack or frequency analysis will be equally effective on both.
\par Historically, the biggest motivation for people to research in this field, and most applications, were military related. Thus cryptography seemed far away from most people's daily life. However, cryptography has primarily changed since the invention of computers and the internet. It has merged deeply into our everyday life though you may not have noticed. Whenever you log in your email account or when you purchase an item from an online shop, you have undoubtedly used cryptography. While you are communicating with the web servers, your email account password or credit card number will first be encrypted and then sent through the networks. If that critical information were never encrypted, anyone who can monitor your network would be able to retrieve your private information.
\par While secure communication is still the major motivation of cryptography, the study of cryptography has developed other related branches that are equally important. Some of the noticeable branches are key exchanging protocols, message authentication code, hash functions and more. Later in this writing, we will discover more about key exchanging protocols as they are crucial components of symmetric encryptions. Message authentication code, usually shorten as MAC, is used to confirm that the message came from the stated sender and has not been modified. Hash function is one commonly used tool to build valid MACs.
\par Computers have also brought us ample computational powers. In modern times, a 30 dollars Raspberry Pi can perform an exhaustive attack on the substitution cipher on 26 English letters showed in our first example in the time it takes to make a cup of coffee. Thus, deviating from studying the art of clever puzzles, modern cryptography makes extensive use of mathematics, including fields such as abstract algebra, number theory, statistics, combinatorics, and computational complexities.
\par In a nutshell, cryptography has gone from the set of clever puzzles concerned with ensuring secret communication for the military to science that provides security for ordinary people across the globe. Modern cryptography heavily relies on mathematical theory and computer science practice. Cryptographic protocols are designed around some computational hardness assumptions, meaning that theoretically, it is possible for one to break such protocol, but it is infeasible to do so by any practical means.

\section{Rubik's Cube and cryptography}
% ---------------------- Figure for a solved Rubik's Cube ----------------------
\begin{wrapfigure}{r}{6cm}
    \centering
    \RubikFaceUpAll{Y}
    \RubikFaceFrontAll{G}
    \RubikFaceRightAll{O}
    \ShowCube{5cm}{0.9}{\DrawRubikCubeRU}
    \caption{Rubik's Cube}\label{fig:solved-cube}
\end{wrapfigure}
% ------------------------------------------------------------------------------
\par Rubik's Cube, as shown in Figure~\ref{fig:solved-cube}, is a three dimensional puzzle invented in 1974 by Hungarian professor Erno Rubik. The most common Rubik's Cube has six faces. The faces of the cube are covered by the following solid colors: white, red, blue, yellow, orange and green. For a solved cube, the white face is always opposite to the yellow face, red is opposite to orange and blue is opposite to green. As shown in Figure~\ref{fig:solved-cube}, the Rubik's Cube has three small cubes on each side. We say that this is a 3 by 3 by 3 cube, or that it is a cube with side length of 3. Clearly each face of the cube includes nine squares. In future discussions, we will call those squares \textit{cubies}.
\par After years of developing, Rubik's Cubes now exist in many forms. For the six faced cubes, their side lengths may vary from 2 to 13 or even larger number. However, since their structures are similar, they share a lot of properties in common. For example, regardless of the side length, all six faced cubes can be solved in an almost identical fashion.
\begin{figure}[ht]
    \centering
    \begin{minipage}{0.45\textwidth}
        \centering
        \includegraphics[width=.6\linewidth]{figures/introduction/pyraminx_cube.jpg}
        \caption{A solved pyraminx cube}\label{fig:pyraminx-cube}
    \end{minipage}
    \begin{minipage}{0.45\textwidth}
        \centering
        \includegraphics[width=.6\linewidth]{figures/introduction/pentagon_cube.jpg}
        \caption{A solved pentagon cube}\label{fig:pentagon-cube}
    \end{minipage}
\end{figure}
We will refer Rubik's Cube with side length $n$ as $C_n$. For instance, the normal 3 by 3 by 3 Rubik's Cube will be denoted as $C_3$. Also when $n$ is an odd number we will call $C_n$ an odd cube and when $n$ is an even number we will call $C_n$ an even cube. Further more, Rubik's Cubes also have other structures; ``cubes'' with four or twelve faces as listed in Figure~\ref{fig:pyraminx-cube} and Figure~\ref{fig:pentagon-cube} have also been made in production.
\par Why would we relate Rubik's Cubes and cryptography? As we mentioned in the last section, cryptographic protocols are designed around some hard problems, and Rubik's Cubes are known to be hard to solve. One may argue that this is not true since the best human players can solve a 3 by 3 by 3 cube in about 6 seconds. By carefully following the developed algorithms, anyone can solve a well shuffled Rubik's Cube within a couple of minutes. \par Suppose we play the Rubik's Cube differently; after shuffling the cube, we give it to a player without letting him/her observe it. Therefore, to solve the puzzle, the player has to guess and shuffle it randomly. Then, the player needs to confirm with a judge to see if the cube is solved. The player wins if the cube is solved, but if the cube is not solved, it will be reversed back to its beginning state and the player has to guess again. Under this setting, the Rubik's Cube will become much harder to solve because the player has to go through an enormous amount of possible cases.
\par This is essentially what we will do in Chapter~\ref{chap:encryption}. We place our plaintext on the cube and remove the colors from the faces and then shuffle the cube. Let us consider each different shuffling result of the Rubik's Cube as a state. The most common cube $C_3$ has over 43 quintillion different states. It will take decades for a powerful desktop to run through all of them. Hence it will almost be impossible for a human player to run through all possibilities and guess if the cube is correctly solved. Also, when the side length of a Rubik's Cube grows, the number of states the cube produces will grow significantly fast. Being such a powerful device, Rubik's Cubes can be used to design schemes that are safe against exhaustive search attacks. In addition, we can algebraically view Rubik's Cubes as groups. The nice properties of groups are widely used in cryptography, and we will see how to use the group structure of Rubik's Cube to define a key exchange protocol.

\section{Preliminaries}
\par First let's take a look at \textbf{\textit{Kerckhoffs's principle}}:
\begin{quote}
    \textit{The cipher method must not be required to be secret, and it must be able to fall into the hands of the enemy without affecting the security of the cipher method.}
\end{quote}
This principle tells us that the security of an encryption protocol does not rely on the encryption protocol procedures being unknown. One may wonder how do we obtain the security while the eavesdropper knows all the details. But there is one thing the eavesdropper does not know, which is the \textbf{\textit{key}}. In fact, the security of all classical encryption schemes depends on the key shared by the communicating parties being unknown by the eavesdropper. For our substitution cipher example, we consider the correspondence between plaintext and ciphertext as the key. Like we said, the security of substitution cipher depends on the key being secret. This scenario is known as the \textit{private-key} setting.
\par Formally, to create a private-key encryption scheme, we need to specify a message space $\mathcal{M}$. This message space $\mathcal{M}$ defines the set of ``legal'' messages. Taking the substitution cipher described in Figure~\ref{fig:replace-letter-by-letter} as an example, a ``legal'' message could be any combination of English letters without spaces or punctuations, regardless of if it has any actual meaning. In contrast, any message containing a Greek letter will make the message  ``illegal''. Along with defining the message space, we need three algorithms for a private-key encryption scheme, which are \textbf{Gen}, \textbf{Enc} and \textbf{Dec}:
\begin{itemize}
    \item \textbf{Gen} is the procedure for generating keys. It is a probabilistic algorithm that outputs a key $k$ with desired length according to some distribution.
    \item \textbf{Enc} is the procedure for encrypting the message. It takes a key $k$ and a message $m \in \mathcal{M}$ as inputs. Then it uses $k$ to encrypt $m$ and outputs ciphertext $c$. We denote this process by \textbf{Enc}$_k(m)$.
    \item \textbf{Dec} is the procedure for decrypting the message. It takes a key $k$ and a ciphertext $c$ as inputs. Then it uses $k$ to decrypt $c$ and outputs plaintext $m$. We denote this process by \textbf{Dec}$_k(c)$.
\end{itemize}
\par It is worth noting that the set of all possible keys output by the key-generation algorithm is called the key space and is denoted by $\mathcal{K}$. In most cases, \textbf{Gen} randomly picks one key from the key space uniformly.
\par With these algorithms, we can move on to construct the set of legal encryption schemes. One requirement an encryption scheme must satisfy is defined as the following.
\begin{definition} \textbf{The correctness of an encryption scheme} \\
    For any encryption scheme $\Pi = (\textbf{Enc}, \textbf{Gen}, \textbf{Dec})$ and for every $k$ generated by \textbf{Gen} and every message $m \in \mathcal{M}$, it must hold that \[\textbf{Dec}_k(\textbf{Enc}_k(m)) = m\]
\end{definition}
\par This simply means that we accurately decrypt any encrypted message when we hold the correct key. To provide an example of a valid encryption scheme, we simply define $\Pi = (\textbf{Enc}, \textbf{Gen}, \textbf{Dec})$, where \textbf{Enc}$_k(m) = m$ and \textbf{Dec}$_k(c) = c$. In words, both \textbf{Enc} and \textbf{Dec} do not modify the input string regardless of what the key $k$ is. So $\Pi$ is a valid encryption scheme as it satisfies the correctness of an encryption scheme. But obviously, $\Pi$ is not a secure scheme. As we explore more about using Rubik's Cube to perform encryption, we will also explore what it means for an encryption scheme to be secure.


         %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
         % Information sheet for the Matlab lab - Maths 6111 %
         %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentclass[10pt]{article}
\input ma_no_html_header

\usepackage{color}
\usepackage{hyperref}
\hypersetup{breaklinks=true,colorlinks=true}
%\input ma_header
\setlength{\parindent}{0pt}
\pagestyle{myheadings}
% \markright{
% \protect {\protect \epsfxsize=0.2 true cm \protect \epsffile {dolph.line.eps}}
% \it Maths 3018/6111 - Numerical methods \hfill}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\thispagestyle{empty}
\begin{center}
\textbf{\Large Maths 3018/6111 - Numerical Methods \\*[8mm]
Worksheet 5}\\*[.8cm]
\end{center}

\section*{Theory}

\begin{enumerate}
\item Explain when multistep methods such as Adams-Bashforth are
  useful and when multistage methods such as RK methods are better.
\item Compute the coefficients of the AB3 algorithm.
\item Explain the meaning of stability, consistency and convergence
  when applied to numerical methods for IVPs. State the theorem
  connecting these.
\item Using the stability polynomial and your results above, check the
  order of accuracy and the stability of the 3 step Adams-Bashforth
  method.
\end{enumerate}

\section*{Coding}

\begin{enumerate}
\item Apply the 2-step Adams-Bashforth method to the ODE from
  Worksheet 4,
  \begin{equation*}
    y' + 2 y = 2 - e^{-4 x}, \quad y(0) = 1.
  \end{equation*}
  Use the Euler or Euler predictor-corrector method to start. Again,
  find the value of $y(1)$ (analytic answer is $1 -
  (e^{-2}-e^{-4})/2$) and see how your method converges with
  resolution.
\item Apply the 2-step implicit Adams-Moulton method to the above ODE,
  using the 2-step Adams-Bashforth method as a predictor. Use the
  Euler or Euler predictor-corrector method to start. See how your
  method converges with resolution.

\end{enumerate}

\end{document}


% !TEX program = pdflatex
\documentclass{tufte-handout}

\title{\centering Course:  Course Name}
\author{I'm the author}

\date{\today} % without \date command, current date is supplied

%\geometry{showframe} % display margins for debugging page layout

\usepackage{graphicx} % allow embedded images
  \setkeys{Gin}{width=\linewidth,totalheight=\textheight,keepaspectratio}
\usepackage{amsmath}  % extended mathematics
\usepackage{booktabs} % book-quality tables
\usepackage{units}    % non-stacked fractions and better unit spacing
\usepackage{multicol} % multiple column layout facilities
\usepackage{lipsum}   % filler text
\usepackage{fancyvrb} % extended verbatim environments
  \fvset{fontsize=\normalsize}% default font size for fancy-verbatim environments

% Standardize command font styles and environments
\newcommand{\doccmd}[1]{\texttt{\textbackslash#1}}% command name -- adds backslash automatically
\newcommand{\docopt}[1]{\ensuremath{\langle}\textrm{\textit{#1}}\ensuremath{\rangle}}% optional command argument
\newcommand{\docarg}[1]{\textrm{\textit{#1}}}% (required) command argument
\newcommand{\docenv}[1]{\textsf{#1}}% environment name
\newcommand{\docpkg}[1]{\texttt{#1}}% package name
\newcommand{\doccls}[1]{\texttt{#1}}% document class name
\newcommand{\docclsopt}[1]{\texttt{#1}}% document class option name
\newenvironment{docspec}{\begin{quote}\noindent}{\end{quote}}% command specification environment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% add numbers to chapters, sections, subsections
\setcounter{secnumdepth}{2}
\usepackage{xcolor}
\definecolor{g1}{HTML}{077358}
\definecolor{g2}{HTML}{00b096}
% chapter format  %(if you use tufte-book class)
%\titleformat{\chapter}%
%{\huge\rmfamily\itshape\color{red}}% format applied to label+text
%{\llap{\colorbox{red}{\parbox{1.5cm}{\hfill\itshape\huge\color{white}\thechapter}}}}% label
%{2pt}% horizontal separation between label and title body
%{}% before the title body
%[]% after the title body

% section format
\titleformat{\section}%
{\normalfont\Large\itshape\color{g1}}% format applied to label+text
{\llap{\colorbox{g1}{\parbox{1.5cm}{\hfill\color{white}\thesection}}}}% label
{1em}% horizontal separation between label and title body
{}% before the title body
[]% after the title body

% subsection format
\titleformat{\subsection}%
{\normalfont\large\itshape\color{g2}}% format applied to label+text
{\llap{\colorbox{g2}{\parbox{1.5cm}{\hfill\color{white}\thesubsection}}}}% label
{1em}% horizontal separation between label and title body
{}% before the title body
[]% after the title body

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{color-tufte}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\maketitle% this prints the handout title, author, and date

\begin{abstract}
\noindent
A simple notes template. Inspired by Tufte-\LaTeX class and beautiful notes by \begin{verbatim*}
	https://github.com/abrandenberger/course-notes
\end{verbatim*}
\end{abstract}

%\printclassoptions

\section{Page Layout}\label{sec:page-layout}

\lipsum[1][1-8]\footnote[1]{Footnotes will appear on the margins}

\begin{definition}%%  [can be kept empty]
	Here's is the beautiful Schr\"odinger equation
	\[ i\hbar {\frac {\partial }{\partial t}}\Psi (x,t)=
	\left[-{\frac {\hbar ^{2}}{2m}}{\frac {\partial ^{2}}{\partial x^{2}}}+V(x,t)\right]\Psi (x,t)\]
\end{definition}

\subsection{Headings}\label{sec:headings}

\marginnote{\begin{proof}[Proof (Theorem 1.1)]

		\lipsum[1][1-3]\end{proof}}
\begin{theorem}%%  [can be kept empty]
	\lipsum[1][1-3] %% for dummy text
\end{theorem}

\begin{lemma}%%  [can be kept empty]
	\lipsum[1][1-3] %% for dummy text

\end{lemma}
\begin{proof}
	\lipsum[1][1-5]
\end{proof}

%\marginnote{\begin{proof}\lipsum[1][1-3]\end{proof}}

\begin{corollary}%%  [can be kept empty]
	\lipsum[1][1-3] %% for dummy text
\end{corollary}

\begin{proposition}
	\lipsum[1][1-3] %% for dummy text
\end{proposition}
\begin{problem}
	\lipsum[1][1-2]
\end{problem}

\begin{proof}
	\lipsum*[1]
\end{proof}

\end{document}