\documentclass{beamer}

% \bibliographystyle{abbrv}
\setbeamertemplate{bibliography item}{\insertbiblabel}

\renewcommand{\footnotesize}{\tiny}
\setbeamerfont{framesubtitle}{size=\small}
\usepackage{url}

\usetheme{Warsaw}

%slides numbers
\addtobeamertemplate{navigation symbols}{}{%
    \usebeamerfont{footline}%
    \usebeamercolor[fg]{footline}%
    \hspace{1em}%
    \insertframenumber/\inserttotalframenumber
}

\title{Paying attention to Attention}
\author{Piotr Mazurek}
\centering
\date{\today}
\begin{document}
\maketitle

\begin{frame}{Agenda}
\begin{enumerate}

\item Introduction to Attention

\item Introduction to Neural Networks

\item Basics of the Attention mechanism

\item Deep dive into Attention

\item Attention applications

\end{enumerate}
\end{frame}

\section{Introduction to attention}
\begin{frame}{Introduction to Attention}
\framesubtitle{Basic intuition}

\begin{figure}
    \includegraphics[width = .8\textwidth]{images/attention-intuition.png}
    \caption{We teach a neural network to focus on some part of the data\footnote{\url{https://www.youtube.com/watch?v=W2rWgXJBZhU&t=180s}}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Introduction to Attention}
\framesubtitle{Who, where and when?}

\begin{figure}
    \centering
    \includegraphics[width = .8\textwidth]{images/first-attention.png}
    \caption{Idea of the Attention first time mentioned (three times in the two consecutive lines, but still counts), Sept. 2014\cite{first-paper}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Introduction to Attention}
\framesubtitle{Who, where and when?}

\begin{figure}
    \centering
    \includegraphics[width = .8\textwidth]{images/attention-is-all-you-need.png}
    \caption{Attention is all You need paper, Dec. 2017\cite{attention-is-all-you-need}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Introduction to Attention}
\framesubtitle{Boom in Attention papers}

\begin{figure}
    \centering
    \includegraphics[width = .8\textwidth]{images/attention-arxiv.png}
    \caption{Attention - a hot research topic}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Introduction to Attention}
\framesubtitle{Better start paying attention now}

\begin{figure}
    \centering
    \includegraphics[width = .99\textwidth]{images/gated-attention.png}
    \caption{Not All Attention Is Needed, Dec. 2019}\cite{not-all-attention-is-neededr}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\section{Introduction to Neural Networks}
\begin{frame}{Neural Networks - basic introduction}

\begin{figure}
    \centering
    \includegraphics[width = .9\textwidth]{images/cat-neural-network.png}
    \caption{Not trained neural network\footnote{\url{https://www.fromthegenesis.com/artificial-neural-network-part-5/}}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Neural Networks}
\framesubtitle{Linear Algebra}
\begin{figure}
    \centering
    \includegraphics[width = .8\textwidth]{images/matrix.png}
    \caption{How matrices are used to compute values in neurons  \footnote{\url{https://www.jeremyjordan.me/intro-to-neural-networks/}}}
    % \label{fig:my_label}
\end{figure}{}
\end{frame}

\begin{frame}{Neural Networks}
\framesubtitle{Algebraic intuition}
\begin{figure}
    \centering
    \includegraphics[width = .7\textwidth]{images/linear-transformation.png}
    \caption{Linear Transformation \footnote{\url{https://www.geogebra.org/m/B9Nbwh7w}}}
    % \label{fig:my_label}
\end{figure}{}
\end{frame}

\begin{frame}{Mathematical Formalization}
\framesubtitle{Oversimplified}
        \begin{block}{Neural network in general}
        \begin{equation}
        f(x, \theta) = \hat{y}
        \end{equation}
        where:\\
        $x$: input data\\
        $\Theta$: model parameters\\
        $\hat{y}$: distribution of probability over set of classes
        \end{block}\pause

        \begin{block}{Example}
        \begin{equation}
        f(x, w, b) = \sigma (x^{T}w + b)
        \end{equation}
        \end{block}
\end{frame}

\begin{frame}{Loss Function}
\framesubtitle{How to find $\theta$}

        \begin{block}{How to find $\theta$}
        \begin{equation}
        \underset{\theta}{\text{minimize}} \: J(\theta)
        \end{equation}{}

        For example we can use SGD optimizer

        \end{block}\pause

        \begin{block}{Example loss function}
        \begin{equation}
        J(\theta)=\frac{1}{2} \mathbb{E}_{\mathbf{x}, \mathbf{y} \sim \hat{p}_{\mathrm{data}}}\|\boldsymbol{y}-f(\boldsymbol{x} ; \boldsymbol{\theta})\|^{2}
        \end{equation}
        \end{block}
\end{frame}

\begin{frame}{Latent space}
\framesubtitle{Auto-encoder}

\begin{figure}
    \centering
    \includegraphics[width = .7\textwidth]{images/auto-encoder.jpg}
    \caption{Example auto-encoder\footnote{\url{https://towardsdatascience.com/applied-deep-learning-part-3-autoencoders-1c083af4d798}}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Latent space}
\framesubtitle{The big picture}

\begin{columns}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Used to find the hidden representation of data

    \item By training model on a particular set of data we create some kind of general knowledge

    \end{itemize}
\end{column}
\begin{column}{0.5\textwidth}  %%<--- here
\begin{figure}
    \begin{center}
    \includegraphics[width = .7\textwidth]{images/latent-space.png}
    \caption{Finding information in chaos}
    \end{center}{}

    % \label{fig:my_label}
\end{figure}
\end{column}

\end{columns}

\end{frame}

\begin{frame}{Recurrent Neural Networks (RNNs)}
\framesubtitle{Oversimplified}

\begin{columns}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Used for sequential data. E.g for text analysis, video recognition etc.\pause

    \item Have so-called "local memory" or "state memory"\pause

    \item Computationally expensive to train\pause

    \item Problems with vanishing gradient\pause

    \end{itemize}
\end{column}
\begin{column}{0.5\textwidth}  %%<--- here
    \begin{figure}
        \centering
        \includegraphics[width=0.8\textwidth]{images/CRNN.png}
        \caption{Sequence of frames\footnote{\url{https://awesomeopensource.com/project/HHTseng/video-classification}}}
        % \label{fig:my_label}
    \end{figure}{}
\end{column}
\end{columns}

{}

\end{frame}

\begin{frame}{Recurrent Neural Networks (RNNs)}
\framesubtitle{Oversimplified}

\begin{figure}
    \centering
    \includegraphics[width = .7\textwidth]{images/rnn.png}
    \caption{Folded and unfolder RNN \footnote{\url{https://towardsdatascience.com/understanding-rnn-and-lstm-f7cdf6dfc14e}}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Types of RNN}
\framesubtitle{Vector to sequence model}
\begin{figure}
    \centering
    \includegraphics[width = .6\textwidth]{images/vec-2-sec.png}
    \caption{From an image (vector) generate a caption(sequence) \footnote{\url{https://www.youtube.com/watch?v=TQQlZhbC5ps&t=610s}}}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Types of RNN}
\framesubtitle{Sequence to vector model}
\begin{figure}
    \centering
    \includegraphics[width = .5\textwidth]{images/seq-2-vec.png}
    \caption{Take a review (sequence) and predict whether it is positive or negative (vector)\footnotemark[7]
    }
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Types of RNN}
\framesubtitle{Sequence to sequence (seq2seq) model}
\begin{figure}
    \centering
    \includegraphics[width = .6\textwidth]{images/seq-2-seq.png}
    \caption{Translate a sentence in Spanish (sequence) into a sentence in English (sequence)\footnotemark[7]
    }
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\section{Introduction to attention mechanism}
\begin{frame}{Attention basic concept - recap}
\begin{figure}
    \includegraphics[width = .8\textwidth]{images/attention-intuition.png}
    \caption{We teach a neural network to focus on the part of the data\footnotemark[1]}
    % \label{fig:my_label}
\end{figure}{}

\end{frame}

\begin{frame}{Attention basic framework}
\framesubtitle{EEAP}
\begin{center}
\begin{enumerate}
    \item Embed\pause

    \item Encode\pause

    \item Attend\pause

    \item Predict
\end{enumerate}{}
\end{center}{}
\end{frame}

\section{Deep dive into attention}

\begin{frame}{Pet problem to work with}
\framesubtitle{Encoder}
    \begin{figure}
    \includegraphics[width = .4\textwidth]{images/encoder-network.png}
    \caption{Our encoder model\footnote{\url{https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html}}}
    % \label{fig:my_label}
\end{figure}{}
\end{frame}

\begin{frame}{Pet problem to work with}
\framesubtitle{Attention}

\begin{columns}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item We calculate attention weights based on \textbf{decoder input} and a hidden state

    \item We multiply it with \textbf{encoder outputs} to create a weighted combination

    \item It will help the decoder to "find out" which part of encoder output is "responsible" for which part of decoder output

    \end{itemize}
\end{column}
\begin{column}{0.5\textwidth}  %%<--- here
\begin{figure}
\includegraphics[width = .9\textwidth]{images/attention-weights.png}
\caption{Way of computing attention\footnotemark[8]}
\end{figure}{}
\end{column}
\end{columns}

\end{frame}

\begin{frame}{Pet problem to work with}
\framesubtitle{Decoder}
    \begin{figure}
    \includegraphics[width = .3\textwidth]{images/attention-decoder-network.png}
    \caption{Our attention-decoder model\footnotemark[8]}
    % \label{fig:my_label}
\end{figure}{}
\end{frame}

\begin{frame}{Time to code}

Now we will do a little bit of live coding

\begin{block}{Where can You find the code?}

\url{https://github.com/tugot17/paying-attention-to-attention}

\end{block}

\end{frame}

\section{Attention applications}

\begin{frame}{Attention applications}
\framesubtitle{Automatic image captioning}

\begin{columns}
\begin{column}{0.4\textwidth}
    \begin{itemize}
    \item Nearly the same solution as previously

    \item Instead of encoder output, latent space from a pre-trained state-of-the-art network (e.g Inception V3)

    \item Attention method similar as before (e.g Bahdanau Attention)

    \end{itemize}
\end{column}
\begin{column}{0.7\textwidth}  %%<--- here
\begin{figure}
\includegraphics[width = .9\textwidth]{images/automatic-captions.png}
\caption{Show, Attend and Tell: Neural Image Caption Generation with Visual Attention\cite{image-caption}}
\end{figure}{}
\end{column}
\end{columns}

\end{frame}

\begin{frame}{Attention applications}
\framesubtitle{Attention UNet}

\begin{columns}
\begin{column}{0.4\textwidth}
    \begin{itemize}
    \item State-of-the-art image segmentation solution

    \item Improving model sensitivity and accuracy by attaching attention gates on top of the standard U-Net

    \end{itemize}
\end{column}
\begin{column}{0.6\textwidth}  %%<--- here
\begin{figure}
\includegraphics[width = .9\textwidth]{images/unet.png}
\caption{Segmentation for medical images with U-Net\cite{unet}}
\end{figure}{}
\end{column}
\end{columns}

\end{frame}

\begin{frame}{Attention applications}
\framesubtitle{Latex code generation}

\begin{columns}
\begin{column}{0.4\textwidth}
    \begin{itemize}
    \item Seq2seq very similar to the Image Captioning

    \item Uses attention based RNN to generate Latex Code

    \item Real world application: \url{https://mathpix.com/}

    \end{itemize}
\end{column}
\begin{column}{0.6\textwidth}  %%<--- here
\begin{figure}
\includegraphics[width = .5\textwidth]{images/im-2-latex.png}
\caption{What You Get Is What You See:A Visual Markup Decompiler\cite{markup-decompiler}}
\end{figure}{}
\end{column}
\end{columns}

\end{frame}

\begin{frame}{Attention applications}
\framesubtitle{Many others, mainly some kind of seq2seq models}

\begin{columns}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Bert\cite{bert}, RoBerta\cite{roberta}, AlBert\cite{albert}, *Bert*

    \item GPT 2\cite{gpt2}

    \item Transformer - Attention is all You need\cite{attention-is-all-you-need}

    \end{itemize}
\end{column}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Residual Attention Network for Image Classification\cite{res-attention}

    \item TreeGen: A Tree-Based Transformer Architecture for Code Generation\cite{treegen}

    \item and many others ...

    \end{itemize}

\end{column}
\end{columns}

\end{frame}

\section{Summary}
\begin{frame}{Summary}
\framesubtitle{Many others, mainly some kind of seq2seq models}

\begin{columns}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Attention = teach Your NN to on what part of data should it be focused on\pause

    \item Attention is all You need - we can solve a wide variety of problems using attention \pause

    \item We can use nearly same solution for many problems\pause

    \end{itemize}
\end{column}
\begin{column}{0.5\textwidth}
    \begin{itemize}
    \item Embed, Encode, Attend, Predict Framework\pause

    \item Hot research topic - It's hard to stay up to date but it is relatively easy to publish\pause

    \item There are several methods for calculating attention (e.g Bahdanau Attention, Luong attention, etc.)

    \end{itemize}

\end{column}
\end{columns}

\end{frame}

\begin{frame}[allowframebreaks]
        \frametitle{References}
        \bibliographystyle{plain}
        \bibliography{references.bib}
\end{frame}

\begin{frame}
\huge{\centerline{Thank You for Your attention}}
\end{frame}

\end{document}


%!TEX root = ../gronskiy_phd_thesis.tex
\chapter*{Nomenclature}
\addcontentsline{toc}{chapter}{Nomenclature}

\hfill
\begin{minipage}[t]{.75\textwidth}
  \textit{PHILOSOPHERS: \\
    --- Why? \\
    --- Why not! \\
    MATHEMATICIANS:\\
    --- $y$? \\
    --- $y_0$!}\\
  \hrule
  \vspace{.2cm}
  \hfill
  \textsc{--- A folklore joke}
\end{minipage}

\section*{Coding Theory}
\begin{longtable}{ >{\raggedleft\arraybackslash}p{.3\textwidth} | p{.8\textwidth} }
$C$ & Channel capacity \\
$R$ & Rate of communication
\end{longtable}


\section{Validation of the model}
For the last set of tests, we returned over the initial optical bench. In this case, we wanted to compare the output of the model with the resolution of the camera, in order to understand if the precision of the measures we can make, is inferiorly limited by camera resolution, or if it is influenced also by some other factors. \\
To reach our goal, we decided to modify the bench a bit: we used a less distorted lens (the one used in the Section \ref{sec:exp1} introduced too much tangential distortion), and we changed the laser to a similar one to that used in Section \ref{sec:exp2}, less noisy and thick than the initial one (configuration in Table \ref{tab:conf3}). Thus, we calibrate the system again, using the company's software. At this point, we performed two different test:
 \begin{itemize}
   \item First, we repeated the experiments in Section \ref{sec:exp1}, using the same known target object.
   \item Second, we empirically evaluated the resolution of the camera, and compared the results with both the nominal value of the camera, and the results obtained in the first point.
 \end{itemize}
 \input{./src/chapters/ch5-Analysis/tab-conf3.tex}

% --------- %
\subsection{Model validation}
In the first phase, the experiments were performed as described in Section \ref{sec:exp1}, but now we have limited the propagation of the errors to the single points, rather than proceeding with the measurements as before. Thus, we put the target in three know position: at the beginning, in the middle and at the end of the \acs{FOV}. This allowed us to cover the entire \acs{FOV} of the camera. In Table \ref{tab:exp3-res} we reported the obtained results. As before, the values are the means of the propagate error for each point over the same sub-filter. We repeated our analysis twice because, as we can see, in the first test, the target was taken in wrong positions. However, we reported all the results better to find some correlations with the results discussed in the next subsection. \\
  \input{src/chapters/ch5-Analysis/tab-exp3-res.tex}
For the moment, the only consideration that we can make is the comparison with the results in Section \ref{sec:exp1}: the use of a less distorted lens, and the use of a better laser reduced significantly the theoretical error that we can commit.

% --------- %
\subsection{Camera resolution}
In the second phase, we decided to integrate our library with a method that allowed us to evaluate the resolution of the camera. To do that, we put the step motor, used to calibrate the system, parallel with the optical axis; in this way the reference object (used to create the grid of points required by Tsai) is perpendicular with the camera. In this case we are not interested in determining the edges of the object, but only to acquire a horizontal laser line. So, we saved six frames: two at the beginning of the \acs{FOV}, two in the middle, and two at the end; in each pair of the acquisitions, the laser lines are $5 \, mm$ apart. This value was chosen arbitrarily to appreciate the distance to the naked eye, and to be able to control the displacement between one position and the other with a tape measure. In Figure \ref{fig:line-pos} are shown the three positions of the target, with respect of the \acs{FOV} of the camera.
  \begin{figure}[!b]
    \centering
    \begin{minipage}[c]{.32\textwidth}
      \centering
      \includegraphics[width=\textwidth]{./images/analysis/exp3/250.jpg}
    \end{minipage}
    \hfill
    \begin{minipage}[c]{.32\textwidth}
      \centering
      \includegraphics[width=\textwidth]{./images/analysis/exp3/380.jpg}
    \end{minipage}
    \hfill
    \begin{minipage}[c]{.32\textwidth}
      \centering
      \includegraphics[width=\textwidth]{./images/analysis/exp3/510.jpg}
    \end{minipage}

    \caption{Positions of the target along the \acs{FOV} of the camera}
    \label{fig:line-pos}
  \end{figure}
In Table \ref{tab:exp3:res2} we reported our results: the first two columns are, respectively, the distances and the errors between the pairs of laser lines, while the third column contains the resolution of the camera, evaluated in the three position cited above. As we can see, the filters did not change resolutions substantially, so we can conclude that the resolution of our camera was in the range $\left( 0.27, 0.55 \right) \, mm/pix$. \\
  \input{src/chapters/ch5-Analysis/tab-exp3-res2.tex}

If we compare the values in Table \ref{tab:exp3-res} with the ones in Table \ref{tab:exp3:res2}, it is clear that the target wasn't acquired in the same positions, but despite that, we can say that the values are the same. From our point of view, this was an interesting result: if on the one hand, it has further validated the model, determining a physical lower bound that we have reached, on the other it confirmed what we said in Section \ref{sec:exp2} about the ``sensitive points'' of the model. In our case, the performances of the hardware were good enough to make the software's effects on the final results negligible. As we have already said, the smaller the pixel size is, the smaller the gain done by the filters.


\documentclass{sig-alternate-05-2015}
\usepackage{subfigure}
\usepackage{balance}
\usepackage{multirow}
\usepackage{color}
\usepackage{chngpage}
\usepackage{url}
\usepackage{amsmath}
\usepackage{caption}
\usepackage{algorithm}
\usepackage{algpseudocode}

\newtheorem{theorem}{Theorem}[section]

\newcommand{\para}[1]{{\vspace{2pt} \bf \noindent #1 \hspace{8pt}}}

\newenvironment{packed_itemize}{
\begin{itemize}

}{\end{itemize}}

\makeatletter
\def\@copyrightspace{\relax}
\makeatother

\begin{document}

\title{Markov Chain Monte Carlo}
\subtitle{Introduction, Comparison \& Analysis}
\author{
    \alignauthor Tzu-Heng Lin, 2014011054, W42\\
    \affaddr{Department of Electronic Engineering, Tsinghua
      University, Beijing, China\\}
    \email{lzhbrian@gmail.com}
}

\maketitle
\begin{abstract}
\footnote{Tzu-Heng Lin is currently an undergraduate student in Dept.of Electronic Engineering, Tsinghua University. His research interests include Big Data Mining, Machine Learning, etc. For more information about him, please see www.linkedin.com/in/lzhbrian. The code in this paper can be found in www.github.com/lzhbrian/MCMC.

Feel free to contact him at any time via lzhbrian@gmail.com or linzh14@mails.tsinghua.edu.cn}Markov Chain Monte Carlo (MCMC) is a technique to make an estimation of a statistic by simulation in a complex model. Restricted Bolztmann Machine(RBM) is a crucial model in the field of Machine Learning. However, training a large RBM model will include intractable computation of the partition functions, i.e.Z($\theta$). This problem has aroused interest in the work of estimation using a MCMC methods.
In this paper, we first conduct Metropolis-Hastings Algorithm, one of the most prevalent sampling methods, and analyze its correctness \& performance, along with the choice of the accepting rate. We then implement three algorithms: TAP, AIS, RTS, to estimate partition functions of an RBM model. Our work not only give an introduction about the available algorithms, but systematically compare the performance \& difference between them. We seek to provide an overall view in the field of MCMC.
\end{abstract}

%
%  Use this command to print the description
%
\printccsdesc

\input{Introduction}

\input{Relatedwork}

\input{Metropolis_hastings}

\input{RBM}

\section{Conclusion} \label{sec:conclusion}
In this paper, we discuss about the Markov Chain Monte Carlo method which are now undoubtedly one of the most important sampling methods.

We comprehensively introduce the concept of Metropolis-Hastings Algorithm and conduct an experiment to verify its correctness. We also make some analysis about how accepting rate would interfere the sampling result.

We systematically compare three methods of partition function estimation which are crucial works in training a Restricted Bolztmann Machine or a Deep Belief Network.

As future work, we would like to join more methods to the comparison and if could, propose some improvement to the algorithms available.

\renewcommand{\baselinestretch}{1.1}
\balance

\small

% Acknowledgement
\section{Acknowledgement} \label{sec:acknowledgement}
I would like to thank Yubo Chen, Liren Yu, Yuanxin Zhang, XueChao Wang, Changran Hu, for the discussion with me on the algorithms. Without them, I wouldn't have the possibility to accomplish this work in such a short time. This paper is a project of Stochastic Process Course in Tsinghua University, taught by Prof. Zhijian Ou.

% Reference
\bibliographystyle{abbrv}
\bibliography{ref}

\end{document}


\chapter{Optimizing the CNN}%\label{make}

\section{Processing the Images}
For the following steps, a simulated dataset containing roughly \num{800000} gamma events
and \num{400000} hadron events has been used.
Each recorded event holds the counts of every arrived photon for all \num{1440} pixels
for \num{100} consecutive time frames of \num{0.5}\,\si{\nano\second}.
To reduce these \num{144000} variables,
the counts of photons for each pixel have been summed along the time axis (time series summation).

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{Plots/3d_Camera_Image.pdf}
    \caption{An event contains \num{100} consecutive images of the Cherenkov flash. To minimize the number of variables, all values for a single pixel have been summed up for each event.}
    \label{fig:example_event}
\end{figure}

Naturally, many background photons are contained in the image.
Since the telescope triggers all events in approximately the same time frame,
the photons of the air shower can be cut out by removing preceding and succeeding time frames filled with noise.
This results in cleaner, denoised images for the pattern detection in the CNN.

\begin{figure}
    \centering
    \includegraphics[scale=1]{Plots/Arrivaltimes.pdf}
    \caption{The brightest time frames for \num{10000} events are shown in the histogram. The distribution peaks around the triggering time. By cutting out this cluster, noise can be reduced.}
    \label{fig:arrivaltimes}
\end{figure}

For \num{10000} events, the time frame with the most photons arriving was computed.
These bright time frames most likely contain the events to examine.
Creating a histogram of these time frames,
highlights that the telescope triggers the events between frames \num{35} and \num{60} nearly every time.
As a result, only this range of frames will be used for the denoised images,
dropping all other frames, which contain mostly noise.

In the following paragraph, all architectures will be evaluated on the images
containing all photons as well as the denoised ones.

\section{Comparing Architectures}
In this section each tested network architecture is composed of a convolution layer followed by a pooling layer (c)
and fully-connected layers (f).
The architecture notation follows this example:

\begin{itemize}
\item A \enquote{\texttt{3c\_2f}} architecture translates to a network
starting with three convolution-and-pooling layers and ending with two fully connected layers.
\end{itemize}

There are four important hyperparameters for networks of this kind:

\begin{itemize}
\item The number of images in one batch fed to the network (batch-size)
\item The size of the patch/kernel in the convolution layers (patch-size)
\item The number of feature maps the convolution layers compute (depth)
\item The number of neurons contained in the fully-connected layers (neurons)
\end{itemize}

For all parameters a random grid search was performed over the course of the following steps.
To reduce the impact of random fluctuations in the network's performance caused inter alia by the grid search,
\num{50} networks have been trained for each architecture.

The common procedure of using a separate training, validation and testing dataset has been adopted.
To measure the network's performance, the area under the Receiver-Operating Characteristic (ROC-AUC) \cite{roc_auc} is used.
The training of a network is terminated when the ROC-AUC-score of the validation dataset does not rise anymore (early stopping).

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.35]{Plots/Architectures_AUC_denoised.pdf}
  \caption{CNNs on denoised images}
  \label{fig:heatmap_denoised}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.35]{Plots/Architectures_AUC_noised.pdf}
  \caption{CNNs on noisy images}
  \label{fig:heatmap_noised}
\end{subfigure}
\caption{The ROC-AUC performances of \num{30} architectures are compared on both denoised and noisy images. Since denoised images perform better, the second plot shows the difference between the denoised and noisy images' ROC-AUC scores.}
\label{fig:heatmaps}
\end{figure}

By comparing the networks trained on noisy and denoised images, it becomes apparent
that denoised images produce a much more reliable and therefore better network.

Looking at the architectures, it appears unambiguous that deeper networks perform much better than shallow ones.
The number of feature-generating convolution layers seems to have a greater impact
than the number of fully connected layers on the ROC-AUC score.

By examining the best-performing denoised deep network architectures,
only sleight differences can be seen.
Since a sample of only \num{50} trained networks guarantees no statistical certainty,
no distinct best architecture can be proclaimed.

\begin{figure}
    \centering
    \includegraphics[scale=1]{Plots/Best_denoised_architectures.pdf}
    \caption{For the comparison of the best denoised architectures the \num{10} best performing networks from \num{50} are being compared in this plot.}
    \label{fig:top_cnn_architectures}
\end{figure}

It can be seen that deeper networks perform better on the given task
and the more convolution layers included in the architecture, the higher the ROC-AUC score.
Additionally, using only the important part of the recorded events
and dropping some noise, increases the network's performance as well.

For a meaningful conclusion concerning the architectures, the hyperparameters of the networks will be investigated.
Since the optimal values for the hyperparameters could not be determined beforehand,
all parameters for the above investigations were randomly chosen from a suitable value range.
Therefore a conclusion can be drawn afterwards to narrow this range down or investigate a more promising value range.

Since only the best networks are of interest, the hyperparameters of the high performing ones are investigated.
For this analysis, the performance of all computed networks are compared regardless of their architecture.
As all parameters (batch-size, patch-size, depth and neurons) show only a small impact on the ROC-AUC score,
only small gains in performance can be achieved through optimizing the hyperparameters.
For this reason the random grid search will be performed for further investigations.

\begin{figure}
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.55]{Plots/Best_Hyperparameter_Batch_Size.pdf}
  \caption{Parameter batch-size}
  \label{fig:batch_size}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.55]{Plots/Best_Hyperparameter_Patch_Size.pdf}
  \caption{Parameter patch-size}
  \label{fig:patch_size}
\end{subfigure}

\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.55]{Plots/Best_Hyperparameter_Depth_1.pdf}
  \caption{Parameter depth (first c-layer)}
  \label{fig:depth}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[scale=0.55]{Plots/Best_Hyperparameter_Hidden_Nodes.pdf}
  \caption{Parameter hidden-nodes}
  \label{fig:hidden_nodes}
\end{subfigure}
\caption{Only a small impact on the ROC-AUC can be detected by comparing the hyperparameters. Since optimizing the parameters specifically appears complicated, the random grid search will be utilized henceforth.}
\label{fig:parameter}
\end{figure}

\section{Regularizing the Network}
The best-performing \enquote{\texttt{6c\_4f}} architecture is fixed for further investigations
so that the feature space to explore is reduced.
To stabilize behavior of the network, regularizing dropout layers will be integrated into the network's architecture.

Since the layer can be combined with every existing layer in the network
and can likewise adopt different values for the amount of data to drop out in every layer,
a large feature space has to be evaluated.
As adding dropout to a network extends the training time many times over,
only one dropout layer at once can be tested at every position.
In the first round of testing, a dropout rate of \num{50}\,\% will be used.
Investigating every possible permutation of multiple dropout layers is not possible
because of the size of the feature space that must be explored.

\begin{figure}
    \centering
    \includegraphics[scale=1]{Plots/Dropout_Positions.pdf}
    \caption{Using dropout layers increases the performance every time, but inserting the layer between the convolution and the fully connected layers, unlocks its potential the most.}
    \label{fig:random_dropout}
\end{figure}

Positioning the dropout layer between the convolution layers and the following fully-connected layers
appears to have the most positive impact on the network's performance.
Regularizing the fully connected layers seems to strengthen the network as well.
On the contrary, dropping too much information in the convolution layers weakens the network overall.
Therefore, the dropout rate is most important, when it is combined with the layer's position.

It has been decided that dropout layers shall be implemented after every layer, but using different dropout rates.
The rates in table \ref{tab:dropout_rates} were taken from corresponding literature \cite{deeplearning}.

\begin{table}
    \centering
    \caption{The chosen dropout rates for the different dropout positions}
    \label{tab:dropout_rates}
    \begin{tabular}{cc}
        \toprule
        {Dropout position}  & {Dropout rate} \\
        \midrule
        c- d -c & 0.90 \\
        c- d -f & 0.75 \\
        f- d -f & 0.50 \\
        \bottomrule
    \end{tabular}
\end{table}

Since dropout prolongs the network's training cycle and increases the difficulty of training deep networks in general,
pretraining is used to enable efficient training of many layers.

A short network containing dropout layers is trained for a few epochs
and the layers adjust their inner values using gradient decent.
Afterwards new layers are appended to the network and the short training epoch starts again.
In this case training epochs consisting of \num{0}, \num{1000}, \num{2000}, \num{5000} and \num{8000} batches are being compared.
To complete the training after the network has grown to its full depth, a normal training epoch with early stopping is used.

\begin{figure}
    \centering
    \includegraphics[scale=1]{Plots/Pretraining_Amounts.pdf}
    \caption{Implementing pretraining in the training process increases the network's performance in general. Since too much pretraining appears to be counterproductive, \num{5000}\,batches will be used henceforth.}
    \label{fig:pretraining_amounts}
\end{figure}

Using different amounts of pretraining batches, the results illustrate an increase in the performance
when compared to the networks without pretrained layers.
By pretraining the network too much, there seems to be an decrease in the performance through overfitting.
Therefore, it is advisable to choose the pretraining amount carefully.
In this case \num{5000} batches of images have been chosen.


% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{hhline}
\usepackage{multirow}
\usepackage[capitalize,nameinlink]{cleveref}
\hypersetup{colorlinks = true, citecolor = blue, linkcolor = blue}
\definecolor{blue}{HTML}{00a4ff}
\usepackage{subcaption}
\usepackage{placeins}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\renewcommand{\emph}[1]{\textcolor{blue}{#1}}

\title{Homework 2}
\author{Pedro Matias \\
\textsf{pmatias@uci.edu}}

\begin{document}

\noindent{\Large \textbf{Homework 2}}

\noindent\textbf{Pedro Matias}

\noindent\textsf{pmatias@uci.edu}

% \maketitle
% \noindent
% \begin{minipage}[t]{.5\textwidth}
% {\Large \textbf{Homework 2}\\[1em]Homework 1}
% \end{minipage}% This must go next to `\end{minipage}`
% \begin{minipage}[t]{.5\textwidth}
% \begin{flushright}
%     Pedro Matias\\
%     23201116\\
%     {\small \texttt{pmatias@uci.edu}}
% \end{flushright}
% \end{minipage}

\setcounter{section}{2}

\section*{2.1$\;\;$ Implement an $n$-gram Language Model}

We implemented two distinct $n$-gram models: one does not use any sort of UNK substitution, and the other uses the classic UNK substitution for rare words (details below). To account for rare $n$-grams we used Laplace smoothing in both models with varying values of $\lambda$ (see next section). To account for OOV words in the \textbf{first} model, we leverage the usage of Laplace smoothing, by defining (for some $\lambda$) the distribution $p(w\mid \textsf{context})$ as follows
\[
\frac{C(\textsf{context}\cdot w) + \lambda}{C(\textsf{context}) + \lambda(V + \textsf{NOOV})}
\]
Notice the addition of \texttt{NOOV} in the denominator. The intuition behind this is the following. We resolve OOV words by ``pretending'' we have seen them $\lambda$ times, one for each occurrence in a fictitious $n$-gram, which is itself resolved by ``pretending'' we have seen it $\lambda$ times, using Laplace smoothing. But having pretended that we have seen all OOV words, we need to increase the size of our vocabulary accordingly and, thus, the addition of \texttt{NOOV} in the denominator. This model is much simpler than the second model and it managed to improve upon the unigram model for $n=2$, even though it didn't perform as good as the second model.

For the \textbf{second} model, we define non-UNK words according to the parameter $\textsf{voc\_ratio}$, which defines the fraction of the training vocabulary which occurs most frequently in the training corpus. We use Adjusted Perplexity in our hyper-parameter search, so as to minimize bias caused by high probability UNK tokens, by adjusting $p(\mathrm{UNK}\mid \mathrm{context})$ with a multiplicative factor of $1/\mathrm{NOOV}$, where $\mathrm{NOOV}$ is the number of words OOV.

\paragraph{Implementation details: }

\begin{itemize}
  \item To simplify the implementation, we keep counts for all $n$-grams, as well as all $(n-1)$-grams. The latter are are used to count the number of occurrences of the given \textsf{context}. Notice that this is equivalent to counting all $n$-grams that start with \textsf{context}, since we append a \texttt{END\_OF\_SEQUENCE} token to each sequence.
  \item To avoid numerical underflow, we perform all computations (except for counting $n$-grams) in $\lg$ space.
  \item We add $n-1$ tokens \texttt{START\_OF\_SEQUENCE} in the beginning of each sequence (in both training and prediction), so we can simplify the handling of word probability distributions conditioned on fractional contexts (i.e. of size less than $n-1$).
\end{itemize}

\section*{2.2$\;\;$ Hyper-parameter Search}\label{sec:search}

For the first ``no-UNKs'' model we simply varied $n$ and $\lambda$ -- see \cref{fig:nounks} for ranges and a comparison against the unigram model. We did not conduct any finer hyper-parameter search since this model performed worse than the one using UNKs.

For the second model, we varied $n$, $\lambda$ and \textsf{voc\_ratio}. In anticipating that large $n$-grams would yield too much sparsity (given the small size of the dataset) and, hence, bad performance, we fine-tuned the remaining parameters on basis of $n\in\{2,3,4\}$ as follows. For each value of $n$, we first conducted a ``zoomed-out'' search of comprehensive ranges of $\lambda$ and \textsf{voc\_ratio} (left column in \cref{fig:search}). We then conducted a finer ``zoomed-in'' search on the values of $\lambda$ and \textsf{voc\_ratio} delimited, respectively, by the previous best values (right column in \cref{fig:search}).

The table below shows the best configurations for each $n\in \{1,2,3,4\}$.

\begin{center}
\begin{tabular}{r|cccc}
  $n$ & 1 & 2 & 3 & 4 \\\hline
  $\lambda$            & --   & 0.01 & 0.01   & 0.01 \\
  \textsf{voc\_ratio}  & --   & 0.09 & 0.0172 & 0.0059 \\\hline
  \textbf{Avg dev PPL} & 1460 & 597  & 1027   & 1631
\end{tabular}
\end{center}

\section*{2.3$\;\;$ Empirical Evaluation of Language Models}

See \cref{fig:cross_ppl} for a comparison between the Unigram model and our best bigram model. Our bigram model performed better in all fronts. The difference in perplexity is more noticeable in Cross-Domain cases, specially when the crossed-domains included the Reuters dataset, either for training or for evaluation (see Section~\nameref{sec:discussion}). Our model also presented worse PPL when crossing against Reuters, but still much better than unigram -- for example, when training using Gutenberg and evaluating against Reuters, our model improved PPL from 44906 (Unigram case) to 9378. With regards to In-Domain cases, our model's improvement was less significant in absolute terms (additive factor), but similar in relative terms (multiplicative factor). The biggest improvement over unigram in relative terms happened when using Reuters: reducing PPL from 1467 to 251.

\section*{2.4$\;\;$ Language Generation and Scoring}

See \cref{tbl:generated} for a list of sentences generated by both the unigram and our best bigram model, under common prefixes. See \cref{tbl:uniVsBi,tbl:compDomains} for examples of sentences we came up with that highlight the differences between, respectively, (i) the unigram and our best bigram model, and (ii) domains.

\section*{2.5$\;\;$ Discussion}\label{sec:discussion}

Regarding the hyper-parameter search, we note that smaller values of $\lambda$ performed better, since big values had a tendency of producing uniform distributions, but not as small so it avoids smoothing. As predicted, values of $n$ of 3 and 4 already have bad performance, due to the extremely low-frequency of $n$-grams in our small datasets. Probably for the same reason of high sparsity, we obtained better PPLs noticed for smaller values of \textsf{voc\_ratio}, but not so small as to being overpenalized by the $1/\textsf{NOOV}$ factor, when adjusting PPL. As $n$ increases, so does the sparsity and, thus, so decrease the best values of \textsf{voc\_ratio}.

Regarding In/Cross Domain analysis, we note that the big difference in PPL in Cross-Domain cases that include Reuters (either in training or in evaluation), can probably be explained by the fact that Reuters' vocabulary, which is very much ``financial'' related, is a lot more different than the vocabulary of the remaining domains. This effect is exacerbated in the unigram model, since it mostly takes into account individual high-frequency vocabulary words.

Regarding the generation of sentences, we note that the \textit{order} taken into account in our bigram model really makes a difference in generating more grammatically correct sentences. For example, the sentence ``The company with Saudi Arabia'' is highly unlikely to be generated by a unigram model. Since the unigram model only cares about word frequencies, it often occurs that two adjacent words are incompatible, such as ``The this But'', or ``How the because''.

Regarding sentence scoring, in order to come up with sentences that work better under the unigram model, we had to rely on frequent words that do not often appear together, so we would be able to worsen the PPL for our bigram model. Thus, each of our examples is vocabulary specific. We highlight ``Israel 15 and 16 from him'' (under Gutenberg domain), which yields PPLs for unigram (resp. bigram) of 412 (resp. 10781), or ``It has shares of him of 000'' (under Reuters domain), which yields PPLs for unigram (resp. bigram) of 429 (resp. 1421). Regarding cross-domain sentence scoring, the difficulty mostly relied on differentiating between Brown and Gutenberg, given the high similarity in vocabulary of these domains.

\section*{3$\;\;$ Statement of collaboration}

Briefly discussed with Ramtin and Evrim the range of PPL values obtained, which helped me detect a bug in the program. We also discussed how smoothing and UNKs work, when dealing with OOV words or rare $n$-grams.

\begin{figure}[h]
\centering
  \includegraphics[width=.4\textwidth]{figures/nounks.png}
  \caption{Average In-Domain dev Perplexity for the ``no-UNKs'' model with values of $n\in\{2,3,4\}$, for varying values of $\lambda$. Performance was better than the unigram model ($n=1$)  for $n=2$ and $\lambda \in \{0.001,0.01\}$.}
  \label{fig:nounks}
\end{figure}

\begin{figure*}[ht]
\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=2.png}
\end{subfigure}
\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=2_zoom.png}
\end{subfigure}

\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=3.png}
\end{subfigure}
\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=3_zoom.png}
\end{subfigure}

\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=4.png}
\end{subfigure}
\begin{subfigure}{0.5\textwidth}
  \includegraphics[width=\textwidth]{figures/n=4_zoom.png}
\end{subfigure}
\caption{Hyper-parameter search for $n$-gram models. The left column portraits initial search ranges for \textsf{voc\_ratio} and $\lambda$, and the right column portraits more fine-tuned ranges specific for each $n$, which changes across rows.}
\label{fig:search}
\end{figure*}

\FloatBarrier

\begin{figure*}
\centering
\begin{subfigure}{0.45\textwidth}
  \includegraphics[width=\textwidth]{figures/unigram_cross_ppl_train.png}
\end{subfigure}
\hspace{5pt}
\begin{subfigure}{0.45\textwidth}
  \includegraphics[width=\textwidth]{figures/unigram_cross_ppl_test.png}
\end{subfigure}

\vspace{1em}

\begin{subfigure}{0.45\textwidth}
  \includegraphics[width=\textwidth]{figures/bigram_cross_ppl_train.png}
\end{subfigure}
\hspace{5pt}
\begin{subfigure}{0.45\textwidth}
  \includegraphics[width=\textwidth]{figures/bigram_cross_ppl_test.png}
\end{subfigure}
\caption{In/Cross-Domain Perplexity using train dataset (left) and test dataset (right), for the unigram model (top) and our best model $[n=2,\lambda=0.01,\textsf{voc\_ratio}=0.09]$ (bottom). Rows (resp. columns) for each case represent the dataset in which the model was trained (resp. evaluated).}
\label{fig:cross_ppl}
\end{figure*}

\begin{table*}
\centering
\input{uni_vs_bi.tex}
\caption{Comparison of sentences generated across domains to highlight differences/similarities between the unigram and our best bigram model.}\label{tbl:uniVsBi}
\end{table*}

\begin{table*}
\centering
\input{comp_domains.tex}
\caption{Comparison of sentences generated by our best bigram model to highlight differences/similarities between domains.}
\label{tbl:compDomains}
\end{table*}

\FloatBarrier

\begin{table*}
\vspace{-50pt}
\input{sentences.tex}
\caption{Sentences generated by both the unigram and our best bigram model. Prefixes given to each simulation are highlighted, except for the first case where the prefix was empty.}\label{tbl:generated}
\end{table*}

\end{document}


\openepigraph{Memory... is the diary that we all carry about with us.}{---Oscar Wilde}

\section{Excel Tips}

This document gives a brief introduction to using excel for data analysis. The guide covers basic excel operation, formula use, and pivot tables. Many first time users will find aspects of excel confusing, this guide is intended to give you enough basic knowledge to get you started. There are extensive help guides published on the internet, and if you find yourself confronted with a problem that is not discussed here you will often find valuable help by searching Google for the answer. If you have suggestions for excel tips that should be included in this document please let your lab instructor know.

\subsection{What is Excel?}

Excel is a spreadsheet tool. It allows you to organize and analyze data that are stored in column and row format. Excel is extremely powerful. With enough know-how almost all of the data analysis that statistics programs like SPSS are built for, can be accomplished directly in Excel.

\subsection{What is Excel used for in this course?}

You can use excel (or another spreadsheet program) to:

\begin{itemize}

\item Store your data,

\item to perform basic analysis of your data (e.g., getting averages and standard deviations, computing correlations etc.), and

\item to create figures, graphs, and tables to present your data.
\end{itemize}

\section{Using Excel}

\subsection{The excel window}

Excel is a spreadsheet that contains different rows (the numbers), and different columns (the letters).
The individual cells can be used to hold data in the form of numbers or letters, to analyze data, and to report data using tables and figures.

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/Excel1.pdf}
      \caption{The excel window}
      \label{fig:excelwindow}
\end{figure}

\subsection{Inputting data}

Type or paste individual numbers or letters into individual cells, and then press return.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel2.pdf}
      \caption{Inputting data}
      \label{fig:excel2}
\end{figure}

\subsection{Addressing cells}

Excel uses a Letter-Number system to address or point to specific cells. As you can see, all of the numbers in this example have been added to rows 2-9 in Column B. Using the Letter- Number system, the B2=1, B3=5, B4=4, B6=7, and so on.

\subsection{Setting one cell to equal another}

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel3.pdf}
      \caption{Making one cell equal another}
      \label{fig:excel3}
\end{figure}

If you click on an empty cell, you can make this cell have the same contents as another cell by typing the (=) sign, then clicking on the cell you want to duplicate.
E.g., click on A2, type (=), then click on B2. B2 initially had a 1 in it, now when you press enter, cell A2 will also have a 1 in it. Now if you change the contents of B2 to another number (say 5), the contents of A2 will also change to 5.
Note: this trick becomes handy later on to help you quickly manipulate data in excel

\subsection{Adding cells together, and copying commands across other cells}

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel4.pdf}
      \caption{Adding}
      \label{fig:excel4}
\end{figure}

Let's say you had two columns of numbers. For each row you want to compute the sum of the first and second number (e.g., the first two numbers are 1 and 1, you want to add them together to get 2.
Click on a new column (C2) and type the equal sign =
Now click on A2 (which contains 1) once with the mouse, and then click on B2 (which also contains 1). Now the formula in C2 will automatically contain =A2+B2
When you press enter the two numbers will be summed, and the answer in C2 with be 2.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel5.pdf}
      \caption{Applying across cells}
      \label{fig:excel5}
\end{figure}

If you want to do the same operation for all of the rows, all you have to do is click on C2. Notice there is a little blue square at the bottom right hand corner of the cell.

\subsection{Using a formula to add two numbers together}
You can do the same addition operation from above by using the sum formula. In this example you would click on C2, then type =sum(
Then you need to tell excel the range of columns and rows that you want to sum over. In this case you can just click on A2, and drag across to B2. This will make a temporary blue rectangle, which represents the cells that will be summed together
Complete the formula by using the )
Then press enter, and you will get the answer.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel6.pdf}
      \caption{Adding using the sum formula}
      \label{fig:excel6}
\end{figure}

You can also copy this formula down across the other cells in the same way as above
If you click on the little square, then drag down, the formula will be applied to each of the rows.
And, then you have all the answers without having to enter the formula separately for each row.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel7.pdf}
      \caption{Completing the formula}
      \label{fig:excel7}
\end{figure}

\subsection{Getting an average}

Using the same method as the sum formula, you can also compute the mean of a set of numbers by using the average function instead of the sum function. The process is the same. Select a cell, type =average( then select the cells you want (in the example B2:B9) and press enter.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel8.pdf}
      \caption{The average formal}
      \label{fig:excel8}
\end{figure}

\subsection{Other formulas}

\begin{itemize}
\item Max – finds the biggest number
\item Min- finds the smallest number
\item Stdev- computes the standard deviation
\item Countif – counts the number of times a specific value occurs
\end{itemize}

Excel has a dictionary of other functions that may be useful, you can look them using help, or using insert: function, from the menu.

\subsection{Selecting a range of cells}
Formulas usually require you to enter a range of cells to compute. The format is put the upperleftmost cell first (e.g., A2), and then the bottomrightmost cell last (e.g., C9). So, A2:C9 would signify the
following rectangle.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel9.pdf}
      \caption{selecting a range}
      \label{fig:excel9}
\end{figure}

\subsection{Copying a formula to another cell: relative coordinates}

If you were to now select cell A13 it would have the formula =A2:C9 inside. If you copied this cell, and then pasted it into the cell beside it A14, Excel would automatically move the rectangle over one. This is because without further specification, excel always treats cells in relative coordinates.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel10.pdf}
      \caption{Relative coordinates}
      \label{fig:excel10}
\end{figure}

\subsection{Absolute coordinates}
You can control whether or not excel uses relative coordinates. When you set the range you can insert the \$ sign to make sure that excel holds the rectangle in place.
For example:
A13 =A2:C9
-this formula has no \$s, as in the above example, if you copy this formula to another cell say
B13, then B13=B2:D9, and not the original A2:C9
A13=\$A\$2:\$C\$9
-This formula has \$s infront of both letter and number for each cell in the rectangle. Now, when
the formula is copied to another cell, the original rectangle will be used. E.g., B13=A2:C9 You can set the only columns or the row or both to absolute coordinates using the \$ sign.

\subsection{Sorting data}

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel11.pdf}
      \caption{Some numbers to sort}
      \label{fig:excel11}
\end{figure}

If you had a bunch of numbers in random order, you could easily sort them by selecting the column or row of numbers, then click on Data from the menu, and choose sort:

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/Excel12.pdf}
      \caption{Sorting}
      \label{fig:excel12}
\end{figure}

You will see a menu something like this. You can choose to sort the current column ascending (smallest to largest) or descending (largest to smallest). Click OK and the data will be rearranged in order.

\subsection{Making a histogram}

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/Excel13.pdf}
      \caption{Making a histogram}
      \label{fig:excel13}
\end{figure}

If you want to know how many responses occurred for a particular range of values you can create a histogram. The following example is a very simple way to count individual categories of values in a data set.
Column A has the example data. In column C, I created cells with values ranging from 1 to 7 (Cells C2 : C8). In cell D2, I typed in the countif(range,value) formula. The range refers to the selected data (\$A\$2:\$A\$30).

Note, there are \$ signs used before the letter and numbers so that the selected data will be the same when the formula is copied. In this case, it is the value 1, which is in Cell C2. The value refers to the entity that is being counted. When you press enter, excel will compute the number of times that 1 appears in the data. You can now drag cell (D2) with the formula down, and it will be used to count the rest of the numbers.

\subsection{Making a table}

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/Excel14.pdf}
      \caption{Making a table}
      \label{fig:excel14}
\end{figure}

Now that you have the frequency of each of the values from 1-7.
You can make a graph by selecting the column with the frequencies in it, and then click on INSERT, from the menu, and choose chart. Select a column chart, and then you will see:
You can then edit this chart. You should insert a title for the figure, an x-axis label (e.g., the numbers on the bottom represent different categories from the data), and a y-axis label (the numbers on the left side represent the frequency count).

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/Excel15.pdf}
      \caption{A figure for the histogram}
      \label{fig:excel15}
\end{figure}

\subsection{Paired Samples T-test}
Suppose you ran an experiment with one independent variable that has 2 levels. You used a within- subject design so each participant contributed data to each of the 2 levels. You want to find out if there was significant effect. That is, is the mean performance in Level 1 different from mean performance in Level 2. An appropriate test is the paired-samples t-test.
Here is some sample data in Excel.
 and the number 1 tells excel to use a paired samples t-test. The resulting p-value is <.05 so we now know that there was a significant effect. The means for level 1 were significantly smaller than the means for level 2.

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/Excel16.pdf}
      \caption{t test in excel}
      \label{fig:excel16}
\end{figure}

Tip: Even before you run a paired samples t-test you should have a good idea whether the test will be significant. You can get a ballpark estimate by computing the differences between means for each subject. This has been done in the example under the column labeled Difference. Here, the mean for level 1 has been subtracted from the mean for level 2 for each subject. This gives us a difference score. Look at all of the difference scores. You will see that almost all of them (except for 2) are positive. So, most of the subjects showed the effect. As a ballpark rule, when most of the subjects show the effect you should expect that a t-test will likely show a significant result.

\section{SPSS}

\subsection{Paired Samples T-test in SPSS}

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS1.pdf}
      \caption{Copy the data into SPSS}
      \label{fig:spss1}
\end{figure}

1. Copy the data for each level of the independent variable into separate columns in the data editor. In the example I've given new names (lev1, lev2) to each of the conditions.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS2.pdf}
      \caption{Choose t-test}
      \label{fig:SPSS2}
\end{figure}

2. Next,choose analyze from the menu, select Compare Means, then select Paired-Samples T Test

3. You will see the following menu. Select both of your variables (Lev1, Lev2) then press the arrow button to move them into the paired variables list

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS3.pdf}
      \caption{Select levels}
      \label{fig:SPSS3}
\end{figure}

4. It should look like this... now click the button for the analysis

5. The first box shows basic descriptive statistics, means, number per cell, standard deviation and standard error of the mean.

\begin{figure}
      \includegraphics[width=\linewidth]{LabmanualFigures/SPSS4.pdf}
      \caption{ttest output}
      \label{fig:SPSS4}
\end{figure}

6. The third box shows the t-value, the associated degrees of freedom (df), and the p- value (Sig. (2-tailed)

7. Here's how you write up your results in one sentence.

8. Means were significantly smaller for level 1 (174) than level 2 (190), t(15) = 2.51, p<.05.

\subsection{2 x 2 Repeated Measures ANOVA}
\subsection{From Data to Analysis}

Suppose you ran an experiment with 2 independent variables, each with 2 levels. If this is a within- subject design, each participant will contribute data to each of the 4 levels in the design. The appropriate test is a 2x2 repeated measures ANOVA. We begin by looking at the data in excel.

\begin{figure}
      \includegraphics[width=.7\linewidth]{LabmanualFigures/SPSS5.pdf}
      \caption{Sample data for a 2 x 2 factorial design}
      \label{fig:SPSS5}
\end{figure}

1. All of the four conditions are placed into 4 separate columns. The second IV is nested underneath the first IV.

2. The means and standard deviations can be computed directly in excel for each of the conditions

3. The means are then organized in a table, and a figure can be created so that we can see the pattern of results.

4. Main effects: It looks like there are two main effects. One for IV1: level 1 is smaller than level 2. One for IV2: level 1 is smaller than level 2

5. Interaction:It looks like there is an interaction. The difference between L1 and L2 of IV1 is larger for level 1 of IV2 (in red) than level 2 of IV2 (in blue).

6. We need to run a 2x2 repeated
measures ANOVA to determine whether the main effects are significant, and to determine if the interaction is significant.

\subsection{2x2 Repeated measures ANOVA in SPSS}

\begin{figure}
      \includegraphics[width=.6\linewidth]{LabmanualFigures/SPSS6.pdf}
      \caption{Copy data into SPSS}
      \label{fig:SPSS6}
\end{figure}

1. Copy the data into SPSS. Make sure each column represents a different condition in the design.

2. Give names to each of the variables. The first column represents IV1 Level1 \& IV2 Level 1. The second column represents IV1 Level 1 \& IV2 Level 2. The third column represents IV1 Level 2 \& IV2 Level 1. The fourth column represents IV1 Level 2 \& IV2 Level 2.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS7.pdf}
      \caption{Choose the model}
 	\label{fig:sp7}
\end{figure}

3. Choose Analyze, General Linear Model, Repeated Measures from the menu.

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS8.pdf}
      \caption{Name variables}
      \label{fig:SPSS8}
\end{figure}

\begin{figure}
      \includegraphics[width=.5\linewidth]{LabmanualFigures/SPSS9.pdf}
      \caption{ thing a}
      \label{fig:SPSS9}
\end{figure}

4. Name IV1, specify 2 levels

5. Name IV1, specify 2 levels

6. Name dependent variable, click define

7. Select all four conditions, press the first right arrow, if you want more options see below, otherwise press ok.

\begin{figure}
      \includegraphics[width=\linewidth]{LabmanualFigures/SPSS10.pdf}
      \caption{Select conditions}
      \label{fig:SPSS10}
\end{figure}

8. If you pressed options, then you can ask SPSS to report descriptive statistics for each condition. Choose the factors that you want, then press the arrow button to move them into the display means field. Make sure you click descriptive statistics. Then click continue.

\begin{figure}
      \includegraphics[width=\linewidth]{LabmanualFigures/SPSS11.pdf}
      \caption{Option to report descriptive statistics}
      \label{fig:SPSS11}
\end{figure}

\subsection{Finding numbers in the SPSS analysis output}

SPSS gives you lots of information, you need to know what you are looking for. When you report the results from a 2x2 ANOVA, you will have 2 main effects, and 1 interaction. This means you will be looking for 3 F-values, 3 MSEs (Mean squared error terms), and 3 associated p-values. You will also need to know the means for the main effects and the means for the interaction.

\begin{figure}
      \includegraphics[width=\linewidth]{LabmanualFigures/SPSS12.pdf}
      \caption{Descriptive Statistics}
      \label{fig:SPSS12}
\end{figure}

If you chose the option for descriptive stats, then you will see a small table with Means and standard deviations for each condition in the design. You will use these means to compute averages for the main effects. You will use these means to report the pattern for the interaction.

\subsection{The ANOVA table}
All of the information that you need to report main effects and interactions is available in  the "tests of within-subjects effects" table.

\begin{figure}
      \includegraphics[width=\linewidth]{LabmanualFigures/SPSS13.pdf}
      \caption{ANOVA table}
      \label{fig:SPSS13}
\end{figure}

\subsection{Main effect for IV 1}

Each main effect is listed under source. Each main effect has a corresponding error term listed below. The source for the main effect of independent variable 1 is labeled IV one. You will see it has a corresponding df of 1, an F-value of 112.371, and a p-value <.05. This main effect is significant. You would report this in the following way.

The main effect for independent variable 1 was significant, F(1,15) = 112.371, MSE = 656.47, p<.05.

The 15 comes from the df from the error term for IVone. As well, the MSE (656.47) comes from the Mean Square from the error term. When you report the main effect, you will also need to report the pattern. The above sentence simply tells the reader that there was a significant difference between the levels of IVone, however it does not explain whether level 1 was larger or smaller than level 2. To report the main effect you will need to compute the averages for each level of IVone. You can do this directly in excel, or you can have SPSS compute these numbers for you by checking the appropriate options before running the analysis. You will need all of the numbers from the above descriptive statistics table.

First, we find the average for level 1 of IVone (477.5 + 499)/2 = 488

Second, we find the average for level 2 of IVone (659.1875 + 746.5)/2 = 703

Now we can write a complete description of the main effect for IVone.

The main effect for independent variable 1 was significant, F(1,15) = 112.371, MSE = 656.47, p<.05. Mean performance was lower for level 1 (488) than level 2 (703).

\subsection{Main effect for IV 2}

Following the same steps as above we look for IV two in the source table and find the dfs, the F-value, the MSE for the error term, and the p-value.

The main effect for independent variable 2 was significant, F(1,15) = 124.92, MSE = 379.14, p<.05. Mean performance was lower for level 1 (568) than level 2 (623).

\subsection{The interaction effect}

Reporting the interaction uses the same information that you would use to report the main effect. The interaction is listed in the source table as IVone*IVtwo. It has corresponding dfs, F-value, MSE for the error term, and a p-value.

The interaction between independent variable one and two was significant, F(1,15) = 36.89, MSE = 469.64, p<.05. The difference between level one and level two for independent variable two was smaller for level one (499 - 478 = 21) than level two (747 - 659 = 88) of independent variable 1.

\subsection{Post Hoc Tests to interpret the interaction}

Post hoc tests are used to clarify the nature of the interaction. In the above example we found a significant interaction. The difference between level 1 and level 2 for IV two was 21 and 88 for level 1 and level 2 of the first independent variable. The significant interaction tells us that 21 and 88 are significantly different from each other. However, we still do not know whether each comparison is significant in an of itself. For example it may be the case that the difference between level 1 and level 2 for IV 2 was only significant for the 2nd level of IV1 (88), and not significant for the 1st level of IV1 (21). In this example you can run a paired t-test, or a one-way repeated measures ANOVA on the specific comparison that you are interested in. For example, you would compare the means from IV1L1/IV2L1 and IV1L1/IV2L2. This comparison would test whether the difference of 21 that was found was actually significant from 0. You will report the statistics for the post-hoc tests in the same manner as you would report t-tests, or F-statistics from the above examples.


% ----------------------------------------------------------------------
\lecture{Modeling}{modeling}
% ----------------------------------------------------------------------
\part{Modeling}
% ----------------------------------------------------------------------
\begin{frame}{Extended syntax}
\vfill
\begin{center}{%
\begin{picture}(300,120)(-150,-60)
\put(-80,+40){\makebox(0,0){\framebox(80,20){Problem}}}
\put(-80,-40){\makebox(0,0){\framebox(80,20){\alert{\textbf{Logic Program}}}}}
\put(+80,+40){\makebox(0,0){\framebox(80,20){Solution}}}
\put(+80,-40){\makebox(0,0){\framebox(80,20){Stable Models}}}
\put(-80,+30){\vector(0,-1){60}}
\put(-40,-40){\vector(+1,0){80}}
\put(+80,-30){\vector(0,+1){60}}
\put(-110,  0){\makebox(0,0){{Modeling}}}
\put(+120,  0){\makebox(0,0){{Interpreting}}}
\put(   0,-55){\makebox(0,0){{Solving}}}
\end{picture}}
\end{center}
\end{frame}
% ----------------------------------------------------------------------
\input{introduction/language-constructs}
% ----------------------------------------------------------------------
\begin{frame}{Modeling and Interpreting}
\vfill
\begin{center}{%
\begin{picture}(300,120)(-150,-60)
\put(-80,+40){\makebox(0,0){\framebox(80,20){Problem}}}
\put(-80,-40){\makebox(0,0){\framebox(80,20){Logic Program}}}
\put(+80,+40){\makebox(0,0){\framebox(80,20){Solution}}}
\put(+80,-40){\makebox(0,0){\framebox(80,20){Stable Models}}}
\put(-80,+30){\vector(0,-1){60}}
\put(-40,-40){\vector(+1,0){80}}
\put(+80,-30){\vector(0,+1){60}}
\put(-110,  0){\makebox(0,0){\alert{Modeling}}}
\put(+120,  0){\makebox(0,0){\alert{Interpreting}}}
\put(   0,-55){\makebox(0,0){{Solving}}}
\end{picture}}
\end{center}
\end{frame}
% ----------------------------------------------------------------------
\section{Elaboration tolerance}
%
\input{modeling/elaboration-uniformity}
% ----------------------------------------------------------------------
\section{ASP solving process}
%
\input{modeling/workflow-elaboration}
\input{modeling/workflow-example-coloring}
% ----------------------------------------------------------------------
\section{Methodology}
%
\input{modeling/methodology}
\input{modeling/methodology-coloring}
% ----------------------------------------------------------------------
\section{Case studies}
% ----------------------------------------------------------------------
\subsection{Satisfiability}
\input{modeling/sat}
% ----------------------------------------------------------------------
\subsection{Queens}
\input{modeling/queens}
% ----------------------------------------------------------------------
\subsection{Traveling salesperson}
\input{modeling/salesperson}
% ----------------------------------------------------------------------
\subsection{Reviewer Assignment}
\input{modeling/reviewer}
% ----------------------------------------------------------------------
\subsection{Planning}
\input{modeling/planning}
% ----------------------------------------------------------------------
%
%%% Local Variables:
%%% mode: latex
%%% TeX-PDF-mode: t
%%% TeX-master: "main"
%%% End: