                                                                                \documentclass[a4paper]{article}
% \usepackage[margin=1.25in]{geometry}
\usepackage[inner=2.0cm,outer=2.0cm,top=2.5cm,bottom=2.5cm]{geometry}
% \usepackage{ctex}
\usepackage{color}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{bm}
\usepackage{hyperref}
\usepackage{multirow}
\usepackage{enumerate}

\newcommand{\homework}[5]{
    \pagestyle{myheadings}
    \thispagestyle{plain}
    \newpage
    \setcounter{page}{1}
    \noindent
    \begin{center}
    \framebox{
        \vbox{\vspace{2mm}
        \hbox to 6.28in { {\bf Optimization Methods \hfill #2} }
        \vspace{6mm}
        \hbox to 6.28in { {\Large \hfill #1 \hfill} }
        \vspace{6mm}
        \hbox to 6.28in { {\it Instructor: {\rm #3} \hfill Name: {\rm #4}, StudentId: {\rm #5}}}
        \vspace{2mm}}
    }
    \end{center}
    % \markboth{#4 -- #1}{#4 -- #1}
    \vspace*{4mm}
}

\newenvironment{solution}
{\color{blue} \paragraph{Solution.\\}}
{\newline \qed}

\begin{document}
%==========================Put your name and id here==========================
\homework{Homework 2}{Fall 2019}{Lijun Zhang}{Weikang Li}{181220031}

\paragraph{Notice}
\begin{itemize}
    \item The submission email is: \textbf{njuoptfall2019@163.com}.
    \item Please use the provided \LaTeX{} file as a template. If you are not familiar with \LaTeX{}, you can also use Word to generate a \textbf{PDF} file.
\end{itemize}

\paragraph{Problem 1: First-order Convexity Condition}
~\\
If $f$ is a continuous function on some interval $\mathbf{I}$,
\begin{enumerate}[a)]
    \item Prove that $f$ is a convex function if and only if $\forall x_1,x_2\in\mathbf{I}$,
    \begin{equation}\label{equa:1}
    f\left(\frac{x_1+x_2}{2}\right)\leq \frac{1}{2}[f(x_1)+f(x_2)].
    \end{equation}
    \item Prove that $f(x)=e^x$ is a convex function.
    \item If $m,n>0,p>1$ and $1/p+1/q=1$, prove that $mn\leq\frac{m^p}{p}+\frac{n^q}{q}.$
\end{enumerate}

\begin{solution}
	a)\;For this part, I referred to the Mathematical Analysis, Mei Jiaqiang.
	\begin{proof}The necessity is easy to prove by simply setting $\theta = \frac{1}{2}$.  \\ Next, we are going to prove the sufficiency:\\
   Let $a<b \in \mathbf{I}$, define $l(x)$ as a linear function that satisfies $l(a)=f(a),\,l(b)=l(b)$.\\It's easy to see that $g(x) = f(x) - l(x)$ still fulfills the inequality above. According to the definition of convex function, we have to prove that $g(x)\leq 0, \forall\,x \in [a,b]$. Because of the continuity of $g(x),\,g$ max to $M$ at $x_0 \in [a,b],$ so we just have to show that $M=0$. \\If $x_0=a$ or $x_0=b$, then $M=0.$
\\If $x_0 \in (a,b),$ consider such set: $A = \{\,y \in [a,b]~|~g(x) = M, \forall\,x \in [x_0,y]\,\}$, and $y_0=sup(A)$. Because of the continuity of $g(x),\,\,$ we have $y_0 \in A$ and $g(x)=M,\,x \in [x_0,y_0]$. \\We say that $y_0=b$, in fact, if $y_0<b, \exists\, \delta > 0,\,s.t.\,(y_0-\delta,y_0+\delta) \subset (a,b).$\\ So, if $0 \leq h < \delta:$
\begin{equation}\nonumber
M = g(y_0) = g\left(\frac{(y_0-h)+(y_0+h)}{2}\right) \leq \frac{1}{2}\left[g(y_0-h)+g(y_0+h)\right] \leq M
\end{equation}
So we have $g(x)=M, \forall\,x\in(y_0-\delta,y_0+\delta),$ Specially, $[y_0,y_0+\delta)\subset A,$which is contradicts with the definition of $y_0$, means that $y_0=b$, and, $M=g(b)=0.$\\
	\end{proof}
	b)\;According to the First-order Convexity Condition, consider $x,y\in\textbf{dom f}$, we have:
	\begin{equation}\nonumber
	f(y)=e^y \ ,\ f(x)+f'(x)(y-x)=e^x(1+y-x)
	\end{equation}	 It is easy to see that $e^{y-x} \geq 1+y-x$, therefore we have:
	\begin{equation}\nonumber
	f(y) \geq f(x)+f'(x)(y-x)
	\end{equation}	which means $f(x)$ is convex.\\
	c)\;Consider $f(x)=log(x)$, it is easy to see that $f$ is concave on $\mathbf{R_{++}}$, which means:
	\begin{equation}\nonumber
	log(\theta x+(1-\theta)y) \geq \theta log(x)+(1-\theta)log(y),\;x,y\in\textbf{dom f}
	\end{equation}	Let $\theta=1/p,\,x=m^p,\,y=n^q$, we have:
	\begin{equation}\nonumber
	log(\frac{1}{p} m^p+ \frac{1}{q} n^q) \geq \frac{1}{p} log(m^p)+\frac{1}{q}log(n^q),
	\end{equation} Therefore:
	\begin{equation}\nonumber
	mn \leq \frac{m^p}{p}+\frac{n^q}{q}
	\end{equation}
\end{solution}

\paragraph{Problem 2: Second-order Convexity Condition}
~\\
Let $\mathcal{D}\subseteq\mathbf{R}^n$ be convex. For a function $f:\mathcal{D}\to\mathbf{R}$ and an $\alpha>0$, we say that $f$ is $\alpha$-exponentially concave, if $\exp(-\alpha f(x))$ is concave on $\mathcal{D}$. Suppose $f:\mathcal{D}\to\mathbf{R}$ is twice differentiable, give the necessary and sufficient condition of that $f$ is $\alpha$-exponentially concave and the detailed proof.
\begin{solution}
	First, we give the conclusion:
	$f$ is $\alpha$-exponentially concave if and only if $\alpha \nabla f(x)\nabla f(x)^T-\nabla^2f(x)\preccurlyeq 0.$\\
$Proof.$ Let $g(x)=\exp(-\alpha x)$ and $h(x)=g(f(x))$. \\According to the Second-order Convexity Condition, we have:
	\begin{center}
	$f$ is $\alpha$-exponentially concave $\iff h(x)$ is concave on $\mathcal{D} \iff \nabla^2h(x)\preccurlyeq 0$
	\end{center}
	From the chain rule of derivatives, we have:
	\begin{equation}\nonumber
	\begin{aligned}
	\nabla^2h(x) &= g'(f(x))\nabla^2f(x)+g''(f(x))\nabla f(x)\nabla f(x)^T\\&= \alpha e^{-\alpha f(x)}\left(\alpha \nabla f(x)\nabla f(x)^T-\nabla^2f(x)\right)\preccurlyeq 0
	\end{aligned}
	\end{equation}
	$\because \alpha>0$, $\alpha e^{-\alpha f(x)}>0$. Therefore, we have $\alpha \nabla f(x)\nabla f(x)^T-\nabla^2f(x)\preccurlyeq 0$
\end{solution}
\paragraph{Problem 3: Operations That Preserve Convexity}
~\\
Show that the following functions $f:\mathbf{R}^n\rightarrow\mathbf{R}$ are convex.
\begin{enumerate}[a)]
    \item $f(x)=\|{Ax-b}\|$, where $A\in \mathbf{R}^{m\times n}, b\in \mathbf{R}^{m}$ and $\|\cdot\|$ is a norm on $\mathbf{R}^{m}$.
    \item $f(x)=-(\textnormal{det}(A_0+x_1A_1+\cdots+x_nA_n))^{1/m}$, on $\{x|A_0+x_1A_1+\cdots+x_nA_n\succ 0\}$ where $A_i\in\mathbf{S}^m$.
    \item $f(x)=\mathbf{tr}((A_0+x_1A_1+\cdots+x_nA_n)^{-1})$, on $\{x|A_0+x_1A_1+\cdots+x_nA_n\succ 0\}$ where $A_i\in\mathbf{S}^m$.
\end{enumerate}
\begin{solution}
   a)\;It is easy to see that $f(x)=\|{Ax-b}\|$ is composed with norm and an affine function. We know that norm is convex, so we have $f$ is convex.\\
   b)\;It is easy to see that $f(x)$ is composed with $g(X)=-(\det(X))^{1/m}$ and an affine transformation. Next, we will show that $g$ is convex on its domain $\mathbf{S}^m_{++}$.\\  To show that, one of approaches is to convert the function to any line that intersects with its domain and verify its convexity by the single variable function that we just got.\\Here, we define $h(t)=g(Z+tV),$ where $Z\succ 0,$ and $V\in \mathbf{S}^n$ .
   \begin{equation}\nonumber
   \begin{aligned}
	h(t)\;&=\;-(\det(Z+tV))^{1/m}\\&=\;-(\det (Z^{1/2}(I+tZ^{-1/2}VZ^{1/2})Z^{1/2}))^{1/m}\\&=\;-(\det Z^{1/2}\det(I+tZ^{-1/2}VZ^{1/2})\det Z^{1/2})^{1/m}\\&=\;-(\det Z)^{1/m} \left(\prod_{i=1}^{n}(1+t\lambda_i)\right)^{1/m}
   \end{aligned}
   \end{equation}
   where $\lambda_1,...,\lambda_i$ are the eigenvalues of $Z^{-1/2}VZ^{1/2}.$ Because $\det Z>0$ and the geometric mean $\left(\prod_{i=1}^{n}(1+t\lambda_i)\right)^{1/m}$ is concave on $\mathbf{R}_{++},$ which can be proved directly by proving its Hessian matrix is negative semi-definite.\\Therefore, we have $h$ is convex and $f$ is convex.\\
   c)\;It is easy to see that $f(x)$ is composed with $g(x)=\mathbf{tr}(X^{-1})$ and an affine transformation. Next, we will show that $g$ is convex on its domain $\mathbf{S}^m_{++}$.\\ Here, we define $h(t)=g(Z+tV),$ where $Z\succ 0,$ and $V\in \mathbf{S}^n$ .
   \begin{equation}\nonumber
   \begin{aligned}
	h(t)\;&=\;\mathbf{tr}((Z+tV)^{-1})\\&=\;\mathbf{tr}(Z^{-1}(I+tZ^{-1/2}VZ^{1/2})^{-1})\\&=\;\mathbf{tr}(Z^{-1}Q(I+t\boldsymbol\Lambda)^{-1}Q^T)\quad\quad \text(Eigenvalue\;Decompopsition)\\&=\;\mathbf{tr}(Q^TZ^{-1}Q(I+t\boldsymbol\Lambda)^{-1})\quad\quad (\mathbf{tr}(AB)=\mathbf{tr}(BA))\\&=\; \sum_{i=1}^{n}(Q^TZ^{-1}Q)_{ii}(1+t\lambda_i)^{-1}
   \end{aligned}
   \end{equation}
   It is easy to see that $(1+t\lambda_i)^{-1}$ is convex. Therefore, $h$ is convex and $f$ is convex.
\end{solution}

\paragraph{Problem 4: Conjugate Function}
~\\
Derive the conjugates of the following functions.
\begin{enumerate}[a)]
    \item $f(x)=\max\{0,1-x\}.$
    \item $f(x)=\ln(1+e^{-x}).$
\end{enumerate}
\begin{solution}
a)\;$f(x)=\max\{0,1-x\}=
\begin{cases}
1-x &,{x<1}\\
0 &,{x\geq 1}
\end{cases}$, \quad
$f^*(y)=\sup_{x\in\mathbf{R}}(yx-f(x))=
\begin{cases}
yx-1+x &,{x<1}\\
yx &,{x\geq 1}
\end{cases}$.\\It is easy to see that: \\If $y<-1$ and $y\geq 0,\;yx-f(x)$ has no upper bound;\\If $-1\leq y<0$, $yx-f(x)$ reaches maximum value at $x=1$.\\ Therefore, $\textbf{dom }f^*=\{y\,|-1\leq y<0\},\,f^*(y)=y$.
\\b)\;$f(x)=\ln(1+e^{-x}),\,f^*(y)=\sup_{x\in\mathbf{R}}(yx-f(x))=f^*(y)=\sup_{x\in\mathbf{R}}(yx-ln(1+e^{-x}))$.\\Observed that $f(x)$ is monotonically decreasing on $\textbf{dom } f$, it is easy to see that $yx-f(x)$ has no upper bound for all $y>0$. \\Thus when $y\leq 0$, $yx-f(x)$ reaches maximum value if $f'(x)=y$, which is $x=ln(1+y)-ln(-y),\,-1<y<0.$\\Therefore $f^*(y)=(1+y)ln(1+y)-yln(-y),\,\textbf{dom }f^*=\{y\,|\,-1<y<0\}$.
\end{solution}

\paragraph{Problem 5: Optimality Condition}
~\\
Prove that $x^\star=(1,1,-1)$ is optimal for the optimization problem
\begin{gather*}
\begin{matrix}
\text{minimize~~} & (1/2)x^TPx+q^Tx+r\quad~~\\
\text{subject to} & -1\leq x_i\leq1,\quad i=1,2,3
\end{matrix}
\end{gather*}
where
\begin{equation*}
P=\begin{bmatrix}
13&12&-2\\
12&17&~6\\
-2&~6&12\\
\end{bmatrix},\quad\quad q=\begin{bmatrix}
-28.0\\
-23.0\\
~13.0\\
\end{bmatrix},\quad\quad r=1.
\end{equation*}
\begin{solution}
	According to the optimality condition of differentiable function:
	\begin{center}
	$x$ is optimal $\iff x \in \boldsymbol{X}$ and $\nabla f_0(x)^T(y-x)\geq 0,\,\forall\,y\in\boldsymbol{X}$
	\end{center}
Let $f_0(x)=(1/2)x^TPx+q^Tx+r$, we have:
	\begin{equation}\nonumber
	\nabla f_0(x^*) = x^*P+q^T = (-1,0,5).
	\end{equation}
Let $y=(y_1,y_2,y_3)$, where $-1\leq y_i\leq 1,\,i=1,2,3.$ Thus, we have:
	\begin{equation}\nonumber
	\nabla f_0(x^*)^T(y-x^*)=-y_1+5y_3+6\geq 0.
	\end{equation}
So, $x^*$ is optimal for this problem.
\end{solution}
\paragraph{Problem 6: Equivalent Problems}
~\\
Consider a problem of the form
\begin{gather}
\label{quasi}
\begin{matrix}
\text{minimize~~} & f_0(x)/\left(c^Tx+d\right)\quad\quad\quad~\\
\text{subject to} & f_i(x)\leq0,\quad i=1,\dots,m\\
&Ax=b\quad\quad\quad\quad\quad\quad\quad~~
\end{matrix}
\end{gather}
where $f_0,f_1,\dots,f_m$ are convex, and the domain of the objective function is defined as \[\{x\in\textbf{dom } f_0 ~|~c^Tx+d>0\}.\]
\begin{enumerate}[a)]
    \item Show that the problem (\ref{quasi}) is a quasiconvex optimization problem.
    \item Show that the problem (\ref{quasi}) is equivalent to
    \begin{gather}
    \label{convex}
\begin{matrix}
\text{minimize~~} & g_0(y,t)\quad\quad\quad\quad\quad\quad\quad\quad~\\
\text{subject to} & g_i(y,t)\leq0,\quad i=1,\dots,m\\
&Ay=bt\quad\quad\quad\quad\quad\quad\quad\quad\\
&c^Ty+dt=1\quad\quad\quad\quad\quad~~
\end{matrix}
\end{gather}
where $g_i(y,t)=tf_i(y/t)$ and $\textbf{dom }g_i=\{(y,t)~|~y/t\in\textbf{dom }f_i,t>0\}$, for $i=0,1,\dots,m$. The variables are $y\in\mathbf{R}^n$ and $t\in\mathbf{R}$.
  \item Show that the problem (\ref{convex}) is convex.
\begin{solution}
	a)\;To strat with, we need to verify the problem is quasiconvex. We have known that $f_0$ is convex, so $\textbf{dom }f_0$ is convex. Also, $f_0(x)/c^Tx+d\leq\alpha$ is equivalent to $f_0(x)-\alpha(c^Tx+d)\leq 0,$ which is a convex constraint, so the sublevel sets are convex.  \\
	b)\;Assuming that $x$ is feasible in problem (2)\\Here, we define $t=\frac{1}{c^Tx+d}$, $y=\frac{x}{c^Tx+d}.$ Then we have $t>0$, and it's easy to see that $t,y$ is also feasible for problem (3) when $g_0(y,t)=f_0(x)/(c^Tx+d)$.\\
	On the contrary, assuming that $y,t$ are feasible for the problem (3), which must leads to $t>0$. Define that $x=y/t,$ we have that $x\in \mathbf{dom }f_i$.\\
	Then, $f_i(x)=g_i(y,t)/t \leq 0$, and $A(y/t)=b$, which means $c^Tx+d=(c^Ty+dt)/t=1/t$. \\Therefore, we have $t=1/(c^Tx+d),$ and $f_0(x)/(c^Tx+d)=tf_0(x)=g_0(y,t)$. So x is feasible in problem (2). Thus, problem (2) is equivalent to problem (3).
\end{solution}
\end{enumerate}

\end{document}


\documentclass[a4paper,12pt]{report}
\usepackage{color}
\usepackage{graphicx}
\usepackage{subfig}
\usepackage{listings}
\usepackage{media9}
\usepackage{hyperref}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsmath}
\usepackage{extarrows}

% \def\reals{{\rm I\!R}}
\def\reals{\mathbb{R}}
\def\integers{\mathbb{Z}}
\def\fft{\xlongleftrightarrow{\mathcal{F}}}
\def\fs{\xlongleftrightarrow{\mathcal{FS}}}

\newtheorem{theorem}{Theorem}

\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}

\begin{document}

\title{ECE355: Signal Analysis and Communications}
\author{Aman Bhargava}
\date{September-December 2020}
\maketitle

\tableofcontents

\section{Introduction and Course Information}

This document offers an overview of the ECE355 course. They comprise my condensed course notes for the course. No promises are made relating to the correctness or completeness of the course notes. These notes are meant to highlight difficult concepts and explain them simply, not to comprehensively review the entire course.

Primary course topics include:

\begin{enumerate}
\item Signals and Systems (Chapter 2).
\item Frequency Domain Analysis (Chapters 3-5).
\item Sampling (Chapter 9).
\item Introduction to Communication Systems (Chapter 8).
\end{enumerate}

\paragraph{Course Information}
\begin{itemize}
\item Professor: Ben Liang
\item Course: Engineering Science, Machine Intelligence Option
\item Term: 2020 Fall
\end{itemize}

\section{Useful Connections}

Lots of the stuff we learn here is deeply connected to linear algebra, differential equations, and more. Here are a few connections I find useful when thinking about signal processing problems.

\begin{itemize}
\item Continuous time signals are an \textbf{infinite-dimensional vector space}, complete with inner products, projection, norms, etc.
\item Most signal transforms (including Fourier transform) is the \textbf{projection} (in the linear algebraic sense) of a signal onto a set of basis vectors. For equality to the original signal, you need infinite basis vectors (hence why the Fourier transform is an integral operation and why the Fourier series is an infinite sum).
\begin{itemize}
\item There are multiple options for basis vectors/spanning sets of vectors to project onto -- see Wavelet transform, Chirplet transform, etc.
\item Yes, this is also a shameless plug for my \href{https://archive.org/details/bhargava-a-4-margins-adaptive-chirplet-transform-based-machine-learning-for-p-30}{paper (link)}.
\end{itemize}
\item Fourier transform is Laplace transform's little brother.
\item Discrete time signals are countably infinite-dimensional vector spaces.
\begin{itemize}
\item The reason you need any only $n\in \langle N \rangle$ discrete-time Fourier series components to \textbf{perfectly reconstruct} a signal with period $N$ is because those $N$ signals are linearly independent (and orthonormal). Since you know the period is $N$, you can think of them all as just $N$-long vectors repeated infinte times.
\item Since the $n\in \langle N \rangle$ discrete-time vectors are orthonormal, you can project an arbitrary signal onto them just by taking inner products (which is exactly the operation we do) and reconstruct it perfectly.
\end{itemize}
\end{itemize}

Signal processing is a pretty fascinating (and inescapable) field, so making these connections can be very helpful for thinking deeply about problems.

\section{Useful Identities}

\begin{eqnarray}
sin^2 (x) = 1/2 (1-cos(2x)) \\
cos^2 (x) = 1/2 (1+cos(2x)) \\
sinxcosx = 1/2 sin2x
\end{eqnarray}

\subsection{Useful Sinusoid Shortcuts}
$$\cos\theta = \frac{e^{j\theta} + e^{-j\theta}}{2}$$
$$\sin\theta = \frac{e^{j\theta} - e^{-j\theta}}{j2}$$

\chapter{Signal Basics}

\section{Definitions}

\paragraph{Two types of signals: } Continuous ($f(x)$ defined $\forall x \in \reals$) and Discrete $f(n)$ defined $\forall n \in \integers$.

\paragraph{Power and Energy of a signal: }
\begin{itemize}
	\item Power of $x(t)$ is $|x(t)|^2$.
	\item Energy of $x(t)$ is defined on interval $[t_1, t_2]$ as
	$$E_{[t_1, t_2]} = \int_{t_1}^{t_2} |x(t)|^2 dt$$
	$$E_{n_1 \leq n \leq n_2} = \sum_{n=n_1}^{n_2} |x[n]|^2$$

	\item Average power of in $[t_1, t_2]$:
	$$P_{[t_1, t_2]} = \frac{E_{[t_1, t_2]}}{t_2 - t_1}$$
	$$P_{[n_1, n_2]} = \frac{E_{n_1 \leq n \leq n_2}}{n_2 - n_1 + 1}$$
	\item Total Energy:
	$$E_{\infty} = \lim_{T\to\infty} \int_{-T}^T |x(t)|^2 dt$$
	$$E_{\infty} = \lim_{N\to\infty} \sum_{n = -N}^{N} |x[n]|^2$$
\end{itemize}

\section{Signal Transformations}

\paragraph{Time Shifting: } \textit{Shifts $t_0$ units RIGHT}
$$y(t) = x(t-t_0)$$
$$y[n] = x[n-n_0]$$

\paragraph{Time Scaling: } \textit{Speeds original signal up by factor $a$)} (or slowed down by factor $\frac{1}{a}$). Time reversal occurs when $a<0$.
$$y(t) = x(at)$$

\paragraph{Continuous Scaling AND Shifting: } It is important to remember the following steps for $y(t) = x(at+b)$

\begin{enumerate}
\item \textbf{SHIFT}: $v(t) = x(t+b)$.
\item \textbf{SCALE}: $y(t) = v(at)$.
\end{enumerate}

\paragraph{Discrete Time Scaling AND Shifting: } Remember to IGNORE fractional indexes. Interpolation for `slowing down' a signal is a poorly defined process that will be covered later.

\section{Periodic Signals}

\paragraph{Definition: } A signal is periodic iff $\exists T > 0$ s.t. $x(t+T) = x(t)$ $\forall t\in\reals$.

\begin{itemize}
\item $T$ is the period of the signal.
\item \textbf{Fundamental} period is the smallest possible $T$.
\item If $x(t)$ is constant, then the fundamental period is undefined.
\end{itemize}

\section{Even and Odd Signals}

\paragraph{Even: } $x(t) = x(-t)$
\paragraph{Odd: } $x(-t) = -x(t)$

\textbf{ANY SIGNAL} can be decomposed into an even and odd component.

$$x_{even}(t) = \frac{1}{2} (x(t) + x(-t))$$
$$x_{odd}(t) = \frac{1}{2} (x(t) - x(-t))$$

$$x(t) = x_{even}(t) + x_{odd}(t)$$

\section{Complex Exponential}
\paragraph{Function Family: } $x(t) = c e^{at}$, $c, a\in \mathbb{C}$

\subsection{Complex Number Review}
\begin{itemize}
\item $z = a+jb$, $z\in\mathbb{C}$, $a, b \in \mathbb{R}$, $j = \sqrt{-1}$.
\item Magnitude = $r=|z| = \sqrt{a^2 + b^2}$.
\item Angle (phase) = $\theta = \arctan(\frac{b}{a})$.
\item $z = re^{j\theta} = r(\cos\theta + j\sin\theta)$.
\end{itemize}

\subsection{Useful Sinusoid Shortcuts}
$$\cos\theta = \frac{e^{j\theta} + e^{-j\theta}}{2}$$
$$\sin\theta = \frac{e^{j\theta} - e^{-j\theta}}{j2}$$

\subsection{Periodic Case}
Letting $c = 1$:
$$x(t) = e^{j\omega_0 t} = cos(\omega_0 t) + j\sin(\omega_0 t)$$
\begin{itemize}
\item Combination of two \textbf{real} signals.
\item $|x(t)| = 1$ $\forall t\in\reals$.
\item \textbf{PERIOD: } $T=\frac{2\pi}{|\omega_0|}$.
\end{itemize}

For more general $c\in\mathbb{C}$: \textit{We let $c = |c|e^{j\phi}$} where $\phi$ is the phase. Then $x(t) = ce^{j\omega_0 t}$, then:

$$x(t) = |c|e^{j(\omega_0 t + \phi)}$$

For fully general $c, a \in \mathbb{C}$: $x(t) = ce^{at} = ce^{(r+j\omega_0)t}$ where $a = (r+j\omega_0)$

$$x(t) = |c|e^{rt} e^{j(\omega_0 t + \phi)}$$
$$Re\{x(t)\} = |c|e^{rt} \cos(\omega_0 t + \phi)$$

Which leads to two cases (`forced harmonic' when $r>0$, `damped harmonic' when $r < 0$).

\subsection{Discrete Time Complex Exponential}

$$x[n] = e^{j\omega_0 n} = \cos(\omega_0 n) + j\sin(\omega_0 n)$$

\begin{itemize}
\item Signal `hops' around the unit circle in \textbf{increments of} $\omega_0$.
\item \textbf{NOT ALWAYS PERIODIC!} $\omega_0 \in a2\pi, a\in \mathbb{Q}$ for periodicity to hold.
\item $c\in\mathbb{C}$ just changes magnitude and phase.
\end{itemize}

\section{Unit Step and Impulse}

\subsection{Discrete Time}

\begin{equation}
	u[n] =
	\begin{cases}
		0 & n < 0 \\
		1 & n \geq 0 \\
	\end{cases}
\end{equation}

\begin{equation}
	\delta[n] =
	\begin{cases}
		0 & n \neq 0 \\
		1 & n = 0 \\
	\end{cases}
\end{equation}

\paragraph{Important Properties: }
\begin{itemize}
\item $\delta[n] = u[n] - u[n-1]$
\item $u[n] = \sum_{k=0}^\infty \delta[n-k]$
\item $u[n] = \sum_{m=n}^{-\infty} \delta[m]$, if we let $m = n-k$
\item $u[n] = \sum_{m=-\infty}^n \delta[m]$, if we let $m = n-k$
\end{itemize}

\textbf{Sampling property: } $x[n] \delta[n-n_0] = x[n_0] \delta[n-n_0]$

\subsection{Continuous Time Case}

\begin{equation}
u(t) =
\begin{cases}
	0 & t < 0 \\
	1 & t > 0 \\
\end{cases}
\end{equation}

\begin{equation}
\delta(t) = \frac{d}{dt} u(t)
\end{equation}

There are some more formal definitions, but this will do for now. Consider it a finite amount of energy in an infinitely small period.

\paragraph{Important Properties: }

\begin{itemize}
\item $\int_{\infty} \delta(t) dt = 1 = u(\infty) - u(-\infty)$
\item $u(t) = \int_{\infty}^{t} \delta(\tau) d\tau$
\item $u(t) = \int_{0}^{\infty} \delta(\sigma - t) d\sigma$
\item Sampling still holds: $x(t_0) = \int_{\infty} x(t) \sigma(t-t_0) dt$
\end{itemize}

\section{Basic System Properties}

\begin{enumerate}
\item \textbf{Memoryless} if $y(t_0)$ depends ONLY on $x(t_0)$ $\forall t_0$.
\item \textbf{Invertable} when you can recover input SIGNAL given output SIGNAL (not value).
\item \textbf{Causal}:
\begin{itemize}
\item \textit{Discrete Time: } If $y[n_0]$ does not depend on \textit{future information}.
\item \textit{Continuous Time: } If $y(t_0)$ does not depend on $x(t)$ for $t \geq t_0$.
\item \textbf{Equivalently: } If two inputs are identical for $t < t_0$, the outputs are identical for $t < t_0$.
\end{itemize}
\item \textbf{Stability: } Small input does not lead to infinite output. \textit{Definition: } System is \textbf{bounded-input-bounded-output} (BIBO) if bounded input leads to bounded output.
\item \textbf{Time Invariance: } Shifted input $\to$ shifted output with same time shift. If $y(t) = sin(x(t))$, then for input $x(t-t_0)$, the output is:
\begin{equation}
\begin{split}
y(t) &= sin(x(t-t_0)) \\
&= y(t-t_0)
\end{split}
\end{equation}
\item \textbf{Linearity: } Must satisfy \textit{additivity} and \textit{homogineity} (in other words: \textbf{superposition}).
$$ax_1(t) + bx_2(t) \to_S ay_1(t) + by_t(t)$$
Where $x_1 \to y_1$, $x_2 \to y_2$.
\begin{itemize}
\item Linear systems commute with scaling and addition.
\item Scaling then pushing through system is the same as pushing through system and scaling.
\item Adding signals together then pushing through the system is the same as pushing each individual signal through the system and then adding the outputs together.
\end{itemize}
\end{enumerate}

\paragraph{Initial Rest Condition: } if $x(t) = 0$ $\forall t < t_0$, then the corresponding output $y(t) = 0$ $\forall t< t_0$.

\chapter{Linear Time-Invariant Systems}

\section{Discrete Time LTI Properties}

The system's response to an impulse function $\delta[n]$ is called $h[n]$. \textbf{IF WE KNOW $h[n]$}, we can map any input to its respective output due to \textit{time invariance and linearity}!

\begin{equation}
y[n] = \sum_{k=-\infty}^{\infty} x[k] h[n-k]
\end{equation}

\subsection{Convolution in Discrete Time}

\paragraph{Convolution of $x[n], h[n] \to y[n]$ } is defined as:
\begin{equation}
y[n] = \sum_{k=-\infty}^{\infty} x[k] h[n-k] \equiv x[n] * h[n]
\end{equation}

Interpretations:
\begin{enumerate}
\item Weighted superposition of time shifted impulse responses (pretty clear).
\item Sum over dimension $k$ of function $x[k]h[n-k]$.
\begin{itemize}
\item $h[n-k] = h[-k+n]$ is \textbf{flipped} and shifted \textbf{right} by $n$.
\item Multiply $h[n-k]$ by $x[k]$ and \textbf{sum} the result to get $y[n]$.
\end{itemize}
\end{enumerate}

\section{Continuous Time LTI Systems}

\paragraph{Unit Impulse Response: } $h(t)$ results from input $\delta(t)$.

\begin{theorem}
\begin{equation}
\begin{split}
y(t) = \int_{-\infty}^{\infty} x(\tau)h(t-\tau) d\tau \\
y(t) \equiv x(t) * h(t)
\end{split}
\end{equation}
\end{theorem}

\section{Properties of LTI Systems}

\paragraph{Properties of Convolution: }
\begin{enumerate}
\item \textbf{Convolution is Commutative: } $x(t)*h(t) = h(t)*x(t)$
\item \textbf{Convolution is Associative: } $x(t)*[h_1(t)*h_2(t)] = [x(t)*h_1(t)]*h_2(t)$
\item \textbf{Convolution is Distributive: } $x(t)*[h_1(t) + h_2(t)] = x(t)*h_1(t) + x(t)*h_2(t)$
\end{enumerate}

\paragraph{Identity System: } If the unit impulse response is $\delta(t)$, then the system is the \textbf{identity system} -- it will produce the same output as the input.

\paragraph{Time shift note: } Shifting the input AND the step response yields double that shift. One must only shift one to shift the output correspondingly (time invariance property).

\paragraph{Properties of LTI: }
\begin{enumerate}
\item \textbf{Memory: } LTI is \textit{memoryless} iff $h(t) = K\delta(t)$ ($K\in\reals$), leading to $y(t) = Kx(t)$ being the \textbf{only memoryless LTI} family.
\item \textbf{Invertibility: } If an LTI is invertible, \textit{its inverse is also an LTI}.
\item \textbf{Causality: } LTI is causal iff $h(t) = 0\,\,\,\forall t < 0$. \textit{Note that `causality' is interchangable for `initial rest condition'}.
\item \textbf{Stability: } Conditions for LTI stability are as follows (tl;dr: bounded unit impulse response is the necessary and sufficient condition).
\begin{itemize}
\item \textit{Absolutely Integrable: } $\int_{-\infty}^\infty |h(\tau)|d\tau < \infty$
\item \textit{Absolutely Summable: } $\sum_{n=-\infty}^\infty |h[n]| < \infty$
\end{itemize}
\end{enumerate}

\section{Linear Constant-Coefficient Differential (Difference) Equations: LCCDE's}

\paragraph{General Form for Continuous Time: }

\begin{equation}
\sum_{k=0}^N a_k \frac{d^k}{dt^k} y(t) = \sum_{k=0}^{M} b_k \frac{d^k}{dt^k} x(t)
\end{equation}

\begin{itemize}
\item Always assume \textit{initial rest} condition.
\item LCCDE's are a subset of \textit{causal LTI systems}.
\item LCCDE's provide a \textit{close approximation} of most LTI systems. \textit{This is because the ``transfer function'' is a rational function that can easily approximate most functions}.
\end{itemize}

\paragraph{Standard Solution: } We would normally use \textbf{method of undetermined coefficients} -- this is a relatively unsophisticated method, but it's important to keep in mind.

\begin{enumerate}
\item We are given some $x(t)$ and are asked to solve for $y(t)$ given an LCCDE relationship (e.g. $A\frac{d}{dt}y(t) + B y(t) = x(t)$).
\item Assume a solution $y(t) = y_h(t) + y_p(t)$ where
\begin{itemize}
\item $y_h$ is the solution for the case of $x(t) = 0\,\,\forall t$. This is the \textit{natural response} or the \textit{unforced response}.
\item $y_p$ is the solution for the given $x(t)$.
\end{itemize}

\item We guess $y_h(t)$ is of the form $Ae^{st}$. We can substitute into the homogenous equation to solve a relationship between $A, s$ and initial conditions.
\item For $y_p(t)$ we guess again (usually the same form as $x(t)$). We substitute in and solve for coefficients.
\item Finally, we solve for remaining unknown coefficients given some initial conditions.
\end{enumerate}

\paragraph{Better tools for solving LTI/LCCDES's: } Solve them in the frequency domain using the Fourier transform and Laplace transform!

\paragraph{How to solve LTI via Transfer Function Laplace Transform}
\textbf{Example: } $\frac{d}{dt} y(t) + 4y(t) = x(t)$.

\begin{enumerate}
\item Assume there exists some $H(s)$ (eigenvalue of the eigenfunction family $e^{st}$).
\item Therefore, $H(s) [\frac{d}{dt} e^{st} = 4e^{st}] = e^{st}$
\item We can now solve for $H(s) = \frac{1}{s+4}$
\item If we can write the input $x(t)$ as the sum of complex exponentials, we can solve for $y(t)$!
\item For the case $x(t) = \cos(\pi t) = 0.5 e^{j\pi t} + 0.5 e^{-j\pi t}$, the corresponding output would be:
\begin{equation}
y(t) = \frac{1}{2} \frac{1}{(j\pi)+4} e^{(j\pi)t} + \frac{1}{2} \frac{1}{-j\pi+4} e^{-j\pi t}
\end{equation}
\end{enumerate}

\section{Discrete Case for LCCDE's}

\textit{Stands for Linear Constant Coefficient Difference Equations}:

\paragraph{General form for discrete LCCDE: }

\begin{equation}
\hat{a}_0 y[n] + \sum_{k=1}^N \hat{a}_n (y[n] - y[n-k]) = \hat{b}_0 x[n] + \sum_{k=1}^M \hat{b}_n(x[n] - x[n-k])
\sum_{k=0}^N a_k y[n-k] = \sum_{k=0}^M b_k x[n-k]
\end{equation}

\paragraph{Solution Options: }
\begin{enumerate}
\item We could solve in a manner similar to the CT case:
\begin{equation}
y[n] = y_h[n] + y_p[n]
\end{equation}

\item That said, there is a \textbf{more efficient} method that takes advantage of the discrete nature of this problem. By rearranging, we get:
\begin{equation}
y[n] = \frac{1}{a_0} [\sum_{k=0}^M b_k x[n-k] - \sum_{k=1}^N a_k y[n-k]]
\end{equation}
\end{enumerate}

\chapter{Fourier Representations of Periodic Signals}

\section{LTI Response to Complex Exponentials}

It turns out that the guess of $y_p(t) = Ae^{st}$ is \textbf{always} a good guess for LTI systems. In fact, it is \textbf{SCALED} every time! Given a system with an impulse reponse $h(t)$ that is fed an input of $e^{st}$:

\begin{equation}
\begin{split}
y(t) &= \int_{-\infty}^{\infty} h(\tau) e^{s(t-\tau)} d\tau \\
&= e^{st} \int_{-\infty}^\infty h(\tau) e^{-s\tau} d\tau \\
&= e^{st} H(s)
\end{split}
\end{equation}

Where $H(s) \equiv \int_{-\infty}^\infty h(\tau) e^{-s\tau} d\tau$ is the \textbf{Laplace Transform} of $h$ (a.k.a. the \textbf{transfer function}).

\begin{theorem}
If $x(t) = e^{st}$ and $H(s)$ exists:
\begin{equation}
y(t) = H(s)e^{st}
\end{equation}
In essence, the response of the LTI is a scaled version of the same complex exponential by factor $H(s)$ defined above.
\end{theorem}

\subsection{Discrete Time Case}

Let $e^{sn} \equiv z^n$. $z^n \to h[n] \to y[n]$. We define

\begin{equation}
H(z) = \sum_{k=-\infty}^{\infty} h[k] z^{-k}
\end{equation}
As the \textbf{$z$-transform}.

\begin{theorem}
$y[n] = H[z]z^n$.
\begin{itemize}
\item $z^n$ is therefore an \textbf{eigenfunction} of any LTI.
\item $H[z]$ is the corresponding \textbf{eigenvalue} of that eigenfunction.
\end{itemize}
\end{theorem}

\paragraph{Cautionary Notes: }
\begin{itemize}
\item $(e^x)^z$ does not hold in general for $x, y, \in \mathbb{C}$.
\item $(e^x)^n$ DOES hold for $n\in \mathbb{Z}$.
\end{itemize}

\section{Continuous Time Fourier Series}

\paragraph{Guiding Fact: } Almost all periodic signals are approximated by a sum of weighted \textbf{harmonically related} complex exponentials.

\begin{equation}
x(t) = \sum_{k=-\infty}^{\infty} a_k e^{jk\omega_0 t}
\end{equation}

\paragraph{Note on utilizing frequency response with Fourier series signal representation: } If $x(t) = \sum_{k}^{} a_k e^{jk\omega_0 t}$:

\begin{equation}
y(t) = \sum_{k=-\infty}^{\infty} a_k H(jk\omega_0) e^{jk\omega_0 t}
\end{equation}

\paragraph{Finding Laplace Transform for LTI: } If given the system in implicit form: \textbf{input the eigenfunction} $x(t) = e^{st}$

\paragraph{Result of Passing Signal through LTI: }
\begin{enumerate}
\item Frequency-dependent \textbf{amplification}
\item Frequency-dependent \textbf{phase shift}
\end{enumerate}

\subsection{Calculating CT Fourier Series}

\begin{equation}
a_k = \frac{1}{T} \int_{T} x(t) e^{-jk\omega_0 t} dt
\end{equation}

Where $\int_{T}$ is integration over any period of length $T$.

\subsection{Convergence of Fourier Series}

\paragraph{Which periodic signals have Fourier series representations? }

\begin{theorem}
We define the finite fourier series
\begin{equation}
x_n(t) = \sum_{k=-N}^{N} a_k e^{j\omega_0 kt}
\end{equation}
where the error is $e_N(t) = x(t) - x_N(t)$, $E_N \int_{T}^{} |e_n(t)|^2 dt$.

\textbf{If $x(t)$} has finite energy in one period, then
\begin{equation}
\lim_{N\to \infty} E_N = 0
\end{equation}

\end{theorem}

If $x(t)$ satisfies \textbf{Dirchlet conditions} (nearly all signals), then $x(t) = FS(x(t))$ except at \textbf{isolated points}.

\paragraph{Gibbs Phenomena: } Small oscillations about discontinuities in a signal (e.g. approximations of a square wave).

\section{Properties of Continuous Time Fourier Series}

\paragraph{Setup: } Let $x(t)$ be periodic with fundamental period $T\to \omega_0 = \frac{2\pi}{T}$ that has Fourier series coefficients $a_k$.

\paragraph{Properties: }

\begin{enumerate}
\item \textbf{Linearity: }
\begin{equation}
Ax(t) + By(t) \underrightarrow{\,\,\mathcal{F}\,\,} Aa_k + Bb_k
\end{equation}

\item \textbf{Time Shift: }
\begin{equation}
x(t-t_0) \underrightarrow{\,\,\mathcal{F}\,\,} e^{-jk\omega_0 t_0} a_k
\end{equation}

\item \textbf{Time Scaling: } For $\alpha > 0$:
\begin{equation}
x(\alpha t) \underrightarrow{\,\,\mathcal{F}\,\,} a_k
\end{equation}

Where $a_k$ now has fundamental period $\alpha \omega_0$

\item \textbf{Time Reversal: }
\begin{equation}
x(t) \underrightarrow{\,\,\mathcal{F}\,\,} a_{-k}
\end{equation}
Therefore even functions have $a_k = a_{-k}$ and odd functions have $-a_k = a_{-k}$.

\item \textbf{Conjugation: }
\begin{equation}
x^* (t) \underrightarrow{\,\,\mathcal{F}\,\,} a_{-k}^*
\end{equation}
\begin{itemize}
\item Special case: $x(t)$ is a real signal so $x^*(t) = x(t)$. The we have $a_k = a_{-k}^*$
\item Known as ``conjugate symmetric'' or ``Hermitian''.
\item If you know $x(t)$ is real, then: $$x(t) = a_0 \sum_{k=1}^{\infty} 2 A_k \cos (k\omega_0 t + \theta)$$
\textit{Where} $a_k = A_k e^{j\theta_k}$.
\end{itemize}

\item \textbf{multiplication: } Multiplication in one domain corresponds to convolution in the other!
\begin{equation}
x(t) y(t) \underrightarrow{\,\,\mathcal{F}\,\,} c_k = \sum_{l = -\infty}^{\infty} a_l b_{k-l}
\end{equation}

\item \textbf{Parsevel's Transform: } The average power of $x(t)$ is given by:

\begin{equation}
\frac{1}{T} \int_{T}^{} |x(t)|^2 dt = \sum_{k=-\infty}^{\infty} |a_k|^2
\end{equation}

\end{enumerate}
\textit{Other properties can be found in table 3.1}

\section{Continuous Time Fourier Transform}

\textit{Recap: } Fourier series can approximate nearly all periodic signals. we now introduce the Fourier Transform, a system to approximate aperiodic signals!

\begin{theorem}

We define the Fourier Transform as follows:

\begin{equation}
X(j\omega) = \int_{-\infty}^{\infty} x(t) e^{-j\omega t} dt
\end{equation}

And its inverse operation as:

\begin{equation}
x(t) = \frac{1}{2\pi} \int_{-\infty}^{\infty} X(j\omega) e^{j\omega t} d\omega
\end{equation}
\end{theorem}

\paragraph{Important Notes on the Fourier Transform Properties: }

\begin{itemize}
\item $\text{sinc}(x) \equiv \frac{\sin(\pi x)}{\pi x}$
\item Wider signal in time domain leads to narrower signal in frequency domain.
\item $x(t) = \delta(t) \to X(j\omega) = 1$
\item $X(j\omega) = 2\pi \delta(\omega) \to x(t) = 1$
\item $x(t) = u(t) \to X(j\omega) = \delta(\omega) \pi + \frac{1}{j\omega}$
\end{itemize}

\subsection{Periodic Signal Fourier Transform}

\paragraph{Key struggle: } Periodic signals have infinite energy and therefore do not converge in the Fourier transform integral.

\begin{theorem}
For an arbitrary periodic $x(t) = \sum_{k = -\infty}^{\infty} a_k e^{jk\omega_0 t}$:
\begin{equation}
X(j\omega) = \sum_{k = -\infty}^{\infty} 2\pi a_k \delta(\omega - k\omega_0)
\end{equation}

In other words, a periodic signal is simply a collection of delta functions in the frequency domain.
\end{theorem}

Steps to find the Fourier Transform of periodic signal:
\begin{enumerate}
\item Find Fourier Series version of $x(t)$.
\item Use $a_k$ in the above formula $X(j\omega) = \sum_{k=-\infty}^{\infty} 2\pi a_k \delta(\omega - k\omega_0)$.
\end{enumerate}

\subsection{Properties of Continuous Time Fourier Transform}

\begin{enumerate}
\item \textbf{Linearity: } \begin{equation}
ax(t) + by(t) \fft aX(j\omega) + bY(j\omega)
\end{equation}

\item \textbf{Time Shift: } \begin{equation}
x(t-t_0) \fft e^{-j\omega t_0} X(j\omega)
\end{equation}

\item \textbf{Time + Frequency Scaling: } \begin{equation}
x(at) \fft \frac{1}{|a|} X(\frac{j\omega}{a})
\end{equation}

\item \textbf{Conjugation: } \begin{equation}
x^*(t) \fft X^*(-j\omega)
\end{equation}

\item \textbf{Differentiation and Integration: }
\begin{align}
x'(t) \fft j\omega X(j\omega) \\
\int_{-\infty}^{t} x(\tau) d\tau \fft \frac{1}{j\omega} X(j\omega)
\end{align}

\item \textbf{Duality: } If $x(t) \fft X(j\omega) = g(\omega)$ then $g(t) \fft 2\pi x(-\omega)$.

\item \textbf{Frequency Shifting: } \begin{equation}
e^{j\omega_0 t} x(t) \fft X(j(\omega-\omega_0))
\end{equation}

\item \textbf{Differentiation in Frequency Domain: } \begin{equation}
-jtx(t) \fft \frac{d}{d\omega} X(j\omega)
\end{equation}

\item \textbf{Integration in Frequency Domain: } \begin{equation}
\frac{-1}{j\omega} x(t) + \pi x(0) \delta(t) \fft \int_{-\infty}^{\omega} X(j\eta) d\eta
\end{equation}

\item \textbf{Parseval's Relation: } \begin{equation}
\int_{-\infty}^{\infty} |x(t)|^2 dt = \frac{1}{2} \int_{-\infty}^{\infty} |X(j\omega)|^2 dw
\end{equation}

\end{enumerate}

\subsection{Special Properties: Multiplication and Convolution}

Convolution and multiplication are the two most important properties to deeply understand about Fourier Transform. They let you use $\mathcal{F}(h(t)) = H(s)$ as you might use the laplace transform of $H$, but this time, you get more information about the \textbf{steady-state} harmonic response.

\paragraph{Convolution: } \begin{equation}
x(t)*h(t) \fft X(jw)H(jw)
\end{equation}
\begin{itemize}
\item \textbf{Eigen function:} $e^{st} \to \text{any LTI with }h(t) \to H(s)e^{st}$
\begin{itemize}
\item Since we can break down any signal into the weighted sums of $e^{st}$ terms with $s=j\omega$, we can determine frequency response for any function by multiplying to $H(s)$ in the frequency domain.
\end{itemize}
\item \textbf{Key point:} System with impulse reponse $h(t)$ and input $x(t)$ has output $\mathcal{F}^{-1}(H(jw)X(jw)) = \mathcal{F}(Y(jw)) = y(t)$.
\item An idealized low-pass filter is therefore a ``box'' in the frequency domain and a $sinc$ function in the time domain.
\end{itemize}

\paragraph{Multiplication: } \begin{equation}
x_1(t)x_2(t) \fft \frac{1}{2\pi} (X_1*X_2)(jw)
\end{equation}

%&= \frac{1}{2\pi} \int_{-\infty}^{\infty} X_1(j\theta) X_2(jw-j\theta)d\theta

\section{Solving LCCDE's with Fourier Transform}

\textit{This was high-key on the midterm before it was taught, so I put it in section 2.4 as well.}

\paragraph{Recall: } LCCDE's relate some output $y(t)$ to an input $x(t)$ in the following form:
\begin{equation}
\sum_{k=0}^{N} a_k \frac{d^k}{dt^k} y(t) = \sum_{k=0}^{M} b_k \frac{d^k}{dt^k} x(t)
\end{equation}

\paragraph{Frequency Response for an LCCDE: }
\begin{equation}
H(s) = \frac{\sum_{k=0}^{M} b_k s^k}{\sum_{k=0}^{N} a_k s^k}
\end{equation}

Since it's a rational function, you can approximate any $H(s)$ well. To get the \textbf{impulse response} $h(t)$, just compute $\mathcal{F}^{-1}(H(s))$.

You should probably use partial fractions to do that, it makes life a lot easier.

\section{Discrete Time Fourier Series}

\begin{itemize}
\item $x[n]$ has period $N$ $\to$ $w_0 = \frac{2\pi}{N}$
\item $e^{jk\omega_0 n}$ is \textbf{harmonically related} for $k\in \mathbb{N}$.
\end{itemize}

\begin{theorem}{Discrete Time Fourier Series: }
\paragraph{Synthesis Equation: }
\begin{equation}
x[n] = \sum_{k\in \langle N \rangle}^{} a_k e^{jk\omega_0 n}
\end{equation}

\paragraph{Analysis Equation: }
\begin{equation}
a_k = \frac{1}{N} \sum_{n\in \langle N \rangle}^{} x[n] e^{-jk\omega_0 n}
\end{equation}

\begin{itemize}
\item $n\in \langle N \rangle$ just means $n$ in any \textbf{continguous set} of $N$ integers.
\item \textbf{DTFS always converges!} It's basically projecting the vector of size $N$ (that repeats, but whatever) onto $N$ linearly independent basis vectors $e^{jk\omega_0 n}$. The linear algebra checks out (this is an extention of course content).
\item $a_k$ is \textbf{periodic} with period $N$ (it therefore makes little difference to represent $k$ for all integers).
\end{itemize}
\end{theorem}

\section{Discrete Time Fourier Transform}

\textit{Intuition: Like for the continuous time case, we take the limit as $N\to \infty$}.

\begin{theorem}{Discrete Time Fourier Series}
\paragraph{Synthesis Equation: }
\begin{equation}
x[n] = \frac{1}{2\pi} \int_{2\pi}^{} X(e^{j\omega}) e^{j\omega n} d\omega
\end{equation}

\paragraph{Transform/Analysis Equation: }
\begin{equation}
X(e^{j\omega} = \sum_{n=-\infty}^{\infty} x[n] e^{-j\omega n}
\end{equation}

\end{theorem}

\paragraph{Notes on DTFT: }
\begin{itemize}
\item Note that $e^{j\omega}$ is only allowable as the argument for $X$ because $e^{j\omega n} = (e^{j\omega})^n$.
\item $X(e^{j\omega})$ is periodic with $T=2\pi$.
\item Convergence does have some conditions, but they can be assumed for all problems in this class.
\item Values at $0, 2\pi$ correspond to \textbf{lower frequencies} while values at $\pi$ correspond to the highest possible frequency in discrete time.
\end{itemize}

\begin{theorem}{DTFT For Periodic Signals}
For any periodic signal $x[n]$ \begin{equation}
x[n] = \sum_{k\in \langle N \rangle}^{} a_k e^{jk\omega_0 n}
\end{equation}
Where $\omega_0 = \frac{2\pi}{N}$, we have DTFT
\begin{equation}
X(e^{j\omega} = \sum_{k = -\infty}^{\infty} 2\pi a_k \delta(\omega-k\omega_0)
\end{equation}
\end{theorem}

\subsection{Properties of Discrete Time Fourier Transform}

\begin{enumerate}
\item Periodic on $2\pi$: \begin{equation}
X(e^{j(\omega+2\pi)} = X(e^{j\omega})
\end{equation}

\item Linearity:
\begin{equation}
a_1 x_1[n] + a_2 x_2[n] \fft a_1 X_1(e^{j\omega}) + a_2 X_2(e^{j\omega})
\end{equation}

\item Time and Frequency Shift:
\begin{align}
x[n-n_0] \fft e^{-j\omega n_0} X(e^{j\omega}) \\
e^{j\omega_0 n} x[n] \fft X(e^{j(\omega-\omega_0)})
\end{align}

\item Conjugation:
\begin{equation}
x^*[n] \fft X^*(e^{-j\omega})
\end{equation}
\begin{itemize}
\item $x[n] \in \mathbb{R} \to X(e^{j\omega}) = X^*(e^{j\omega})$ (``Conjugate Symmetry'').
\item $\text{Even}(x[n]) \fft \Re\{X(e^{j\omega})\}$
\item $\text{Odd}(x[n]) \fft j\Im\{X(e^{j\omega})\}$
\end{itemize}

\item Differencing and Accumulation
\begin{itemize}
\item Difference:
\begin{equation}
x[n] - x[n-1] \fft (1-e^{-j\omega})X(e^{j\omega})
\end{equation}
\begin{equation}
x[n] - x[n-k] \fft (1-e^{-j\omega k}X(e^{j\omega}))
\end{equation}

\item Accumulation:
\begin{equation}
\sum_{m=-\infty}^{n} x[m] = x[n] * u[n] \fft X(e^{j\omega})U(e^{j\omega})
\end{equation}

\begin{equation}
= \frac{1}{1-e^{-j\omega}} X(e^{j\omega}) + \pi X(e^{(j\theta)}) \sum_{k=-\infty}^{\infty} \delta(\omega - 2\pi k)
\end{equation}
\end{itemize}

\item Time Reversal:
\begin{equation}
x[-n] \fft X(e^{-j\omega} )
\end{equation}

\item Time Scaling: $x[an]$ is not defined for all $a\notin \mathbb{Z}$.
\begin{itemize}
\item Time Contraction: $|a| > 1, a\in \mathbb{Z}$ has \textbf{no general formula}.
\item Time Expansion:
\begin{equation}
x_{(k)}[n] \equiv \begin{cases}
x[n/k], & \text{if }n\text{ is a multiple of } k \\
0, & \text{else}
\end{cases}
\end{equation}
\begin{equation}
X_{(k)}(e^{j\omega}) = X(e^{jk\omega})
\end{equation}
\end{itemize}

\item Differentiating in Frequency:
\begin{equation}
-jn x[n] \fft \frac{d}{d\omega} X(e^{j\omega})
\end{equation}

\item Parseval's Relation:
\begin{equation}
\sum_{n=-\infty}^{\infty} |x[n]|^2 = \frac{1}{2\pi} |X(e^{j\omega})|^2 d\omega
\end{equation}

\item Convolution in Time:
\begin{equation}
x[n] * h[n] \fft X(e^{j\omega})H(e^{j\omega})
\end{equation}

\item Multiplication in Time:
\begin{align}
x_1[n]x_2[n] \fft (X_1 *_p X_2)(e^{j\omega}) \\
= \frac{1}{2\pi} \frac{2\pi}{} X_1(e^{j\theta})X_2(e^{j(\omega - \theta)}) d\theta
\end{align}

\item Duality -- From CTFT to DTFT
\begin{equation}
x[n] \fft X(e^{j\omega}) \fs x[-n]
\end{equation}
\end{enumerate}

\section{LCCDE in Discrete Time}

\paragraph{Form of LCCDE: }
\begin{equation}
\sum_{k = 0}^{N} a_k y[n-k] = \sum_{k=0}^{M} b_k x[n-k]
\end{equation}

\begin{equation}
H(e^{j\omega}) = \frac{Y(e^{j\omega})}{X(e^{j\omega})} = \frac{\sum_{k=0}^{M} e^{-j\omega k}}{\sum_{k=0}^{N} a_k e^{-j\omega k}}
\end{equation}

Then simply use the resulting rational function $\to$ partial fraction $\to$ inverse DTFT.

\chapter{Reconstruction from Samples}

\section{Continuous Time Sampling}

\paragraph{Recall: Impulse Train Sampling -- } We multiply $x(t)$ with $p(t)$ (the ``picket fence'' function $p(t) = \sum_{n=-\infty}^{\infty} \delta(t-nT)$) to get $x_p(t)$ -- the sampled version of $x(t)$. It looks like a bunch of delta functions that match the ``value'' of $x(t)$ at each instance $t = nT$ where one exits.

\paragraph{Guiding Question: } How do we \textbf{interpolate} between discrete points recorded for a continuous waveform?

\paragraph{Answer: } Use an \textbf{ideal low-pass filter}. Make sure you visually understand the following things first.
\begin{itemize}
\item $x_p$ is $X(j\omega)*P(j\omega)$, which corresponds to \textbf{a bunch of copies of $X(e^{j\omega})$} spaced $w_s = \frac{2\pi}{T}$ apart.
\item To prevent \textbf{overlap in the frequency domain (aliasing)}, the \textbf{band limit} of the signal must be $w_c \leq w_s/2$ (draw it out to visualize it). This is the Nyquist rate.
\item We can recover $X(jw)$ by low-pass filtering $x_p(t)$ under $w_c \leq w_s/2 = \pi/T$.
\end{itemize}

And honestly, that's the central idea of the communications theory we learn in this course! For the theoretically perfect case, just convolve the sampled $x_p(t)$ with an idealized low-pass filter $sinc$ function:

\begin{align}
x_r(t) &= x_p(t) * h(t) \\
&= \sum_{n=-\infty}^{\infty} x(nT)\delta(t-nT)*h(t) \\
x_r(t) = x(t) &= \sum_{n=-\infty}^{\infty} x(nT)h(t-nT)
\end{align}

Which is pretty wild. Literally it just boils down to multiplying a bunch of $sinc$ functions \textbf{by the value of $x_p(t)$} at each sampled point. The only problem is that it is not \textbf{causal} and is therefore not practical in real time to perform idealized low-pass filtering.

\paragraph{Signal Reconstruction -- Key Facts and Equations: } In summary,
\begin{itemize}
\item Input signal is band limited by $\omega_m$.
\item Sampling occurs with period $T$ and frequency $\omega_s$ using function $p(t)$, a bunch of delta functions spaced $T$ distance apart in time.
\item \textbf{Nyquist Rate: } $w_s \geq 2\omega_m$.
\item To reconstruct $x(t)$, we \textbf{low-pass filter} $x_p(t)$ with an ideal low-pass filter with bounds $\omega_c$ which must be between $\omega_m$ and $\omega_s/2$. By default, we use $\omega_c = \frac{\omega_s}{2} = \frac{\pi}{T}$.
\end{itemize}

\subsection{Practical Interpolation Strategies}

Since an ideal low pass filter is, well, \textit{ideal}, we have a few methods to approximate the original signal in real time.

\paragraph{Zero-Order Hold: } Literally just keep the reconstruction constant as new samples are received.
\begin{equation}
x_r(t) = x(kT) \text{ for }kT \leq t \leq (k+1)T
\end{equation}
\begin{itemize}
\item Frequency response: $x_r(t) = \sum_{n=-\infty}^{\infty} x(nT) h_0(t-nT)$, where $h_0(t)$ is
\begin{align}
h_0(t) &= \begin{cases}
1 & \text{ if } t\in [0,T]  \\
0 & \text{ otherwise}
\end{cases} \\
H_0(jw) &= e^{-j\omega\frac{T}{2}} (\frac{2\sin(\omega\frac{T}{2})}{\omega}) \\
\end{align}

\item \textbf{Limits on Error: } You get a bit of high-frequency distortion from the left and right lobes of the $X_p$ signal.
\end{itemize}

\paragraph{First-Order Hold: } Simply applying linear interpolation ($x_r(t)$ is weighted average of two closest samples).
\begin{equation}
x_r(t) = \frac{(k+1)T-t}{T} x(kT) + \frac{t-kT}{T} x((k+1)T)\,\,\forall kT\leq t\leq (k+1)T
\end{equation}
\begin{itemize}
\item Frequency response: $x_r(t) = \sum_{n=-\infty}^{\infty} x(nT)h_1(t-nT)$ where
\begin{align}
h_1(t) &= \begin{cases}
\frac{t}{T} +1 & \text{ for } t\in [-T,0] \\
\frac{-t}{T} +1 & \text{ for } t\in [0, T] \\
0 & \text{elsewhere.}
\end{cases} \\
H_1(jw) &= \frac{1}{T} (\frac{2\sin(\omega\frac{T}{2})}{\omega})^2
\end{align}
\item Limits on Error: Reduces the error seen in the 0-order hold approximation, as one would expect from increasing... the order... of the approximation.
\end{itemize}

\subsection{Under Sampling and Aliasing}

\paragraph{Under Sampling: } When the sampling rate results in overlapping `lobes' for $X_p(j\omega)$. That is, when $\omega_s < 2\omega_m$.
\begin{itemize}
\item High frequency energy is added from the tails of the left and right `lobes'.
\item It is visible in the time domain that information is being lost.
\item Due to overlap, you can get \textbf{much lower frequencies} in the reconstructed signals, due to overlap of lobes in time domain.
\item Same phenomena as the wagon wheel effect.
\end{itemize}

\section{Discrete Time Sampling}

\paragraph{Core Idea: }
\begin{itemize}
\item Given some discrete time signal $x[n]$, we \textbf{sample} it by taking the $k$th samples.
\item To reconstruct the original signal, we use similar principles to before: Low-pass filter the signal in the frequency domain.
\item Like in Continuous Time, we have a Nyquist rate and a similar ``picture'' in the frequency domain.
\item \textbf{Difference to Deeply Understand: } How do frequency domain representations differ for DT?
\end{itemize}

\paragraph{Sampling: } Our sampling function is the same as before, just with \textbf{unit impulse functions}.
\begin{align}
p[n] &= \sum_{k=-\infty}^{\infty} \delta[n-kN] \\
P(e^{j\omega}) &= \frac{2\pi}{N} \sum_{k=-\infty}^{\infty} \delta(\omega-k\omega_w) \\
\end{align}

Where $\omega_s = \frac{2\pi}{N}$. $N$ takes the place of sampling period $T$ from the continuous time case. \textbf{Recall:} $P(e^{j\omega})$ is periodic in $\omega$ with period $2\pi$.

We can determine the \textbf{frequency domain representation} of our sampled $x_p[n] = x[n]p[n]$ by convolving their DTFT conterparts:

\begin{align}
X_p(e^{j\omega}) &= \frac{1}{2\pi} \int_{2\pi}^{} X(e^{j(\omega-\theta)})P(e^{j\omega})d\theta \\
&= \frac{1}{N} \sum_{k=0}^{N-1} X(e^{j(\omega-k\omega_s)})
\end{align}

\paragraph{End Result: } Try to visualize the above equation. It's just a bunch of translated copies of $X(e^{j\omega})$, spaces $\omega_s$ apart. But remember that DTFT is \textbf{periodic with period $2\pi$!} So you have `genuine' copies of $X(e^{j\omega} )$ at intervals of $2\pi$, too.

\paragraph{Information Loss Conditions: } Like before, we only lose information if there is overlap between the aforementioned copies of $X(e^{j\omega})$. That is, \textbf{we lose information if $\omega_s = \frac{2\pi}{N} < \omega_m$} (where $\omega_m$ is the band limit of the original signal $x[n]$).

\paragraph{Signal Reconstruction via Ideal Low-Pass Filter: } Our low pass filter is defined similarly to before.
\begin{align}
H(e^{j\omega}) &= \begin{cases}
N & \text{, for } |\omega| < \omega_c \\
0 & \text{, else. Periodic on } 2\pi.
\end{cases}
h[n] = N\frac{\omega_c}{\pi} \frac{\sin(\omega_c n)}{\omega_c n} \\
\end{align}

We reconstruct $x[n]$ to form $x_r[n]$ by convolving the sampled version with the low-pass filter $h[n]$ in the time domain (or multiplying in the frequency domain):

\begin{align}
x_r[n] &= x_p[n] * h[n] \\
x_r[n] &= \sum_{k=-\infty}^{\infty} x_p[kN] \cdot N \frac{\omega_c}{\pi} \frac{\sin(\omega_c(n-kN))}{\omega_c(n-kN)} \\
\end{align}

And that's pretty much it for discrete time sampling and reconstruction (at its core).

\section{Downsampling (a.k.a. ``Decimation'')}

\begin{theorem}{Decimation Definition: }
We define decimation (or ``downsampling'') as follows: The downsampled version of $x[n]$ with downsampling period $N$ is $x_b[n]$, given by
\begin{align}
x_b[n] &= x[nN] \\
\end{align}
\end{theorem}

\paragraph{Frequency Domain Representation of Decimated Signal: } We begin by observing that $x_b[n] = x[nN] = x_p[nN]$ if $x_p$ is sampled with period $n$. Therefore,
\begin{align}
X_b(e^{j\omega}) &= \sum_{k=-\infty}^{\infty} x_b[n] e^{(-j\omega k)} \\
&= \sum_{n=-\infty}^{\infty} x_p[n] e^{(-j\omega n/N)}  \\
X_b(e^{j\omega}) &= X_p(e^{(j\frac{\omega}{N})})
\end{align}

Therefore the frequency domain representation for $x_b[n]$ is the \textbf{exact same} as that of $x_p[n]$ but \textbf{stretched by factor} $N$.

\subsection{Upsampling and Resampling}

\paragraph{Upsampling} is the inverse of downsampling. In our context, it means transforming $x_b[n] \to x[n]$.
\begin{enumerate}
\item Add $N-1$ zeros between samples of $x_b[n]$ to create $x_p[n]$.
\item Perform same low-pass filtering from conventional discrete-time sampling to create the interpolated version and recover $x[n]$.
\begin{itemize}
\item Note that the same limitations on reconstruction hold as for the case of $x_p[n] \to x[n]$.
\end{itemize}
\end{enumerate}

\paragraph{Resampling} is how we ``speed up'' or ``slow down'' a discrete-time signal by a \textbf{fractional value}. For rational re-sampling rate $r = \frac{a}{b} | a,b\in \mathbb{R}$:
\begin{enumerate}
\item Up-sample by factor $a$.
\item Down-sample by factor $b$.
\end{enumerate}

\chapter{Communications Systems}

\section{Basic Components and Facts}

\paragraph{Problem with Regular Sampling: } To transmit human voices directly as electromagnetic waves, you would need a giant antenna.

\paragraph{General Solution: } We ``shift up'' the frequency representation of the original voice signal, send it, then shift it backdown, and play it.
\begin{enumerate}
\item $x(t)$: \textbf{``Base band''}, our original signal to transmit (generally low frequency relative to other parts). Also known as the ``modulat\textbf{ed}  signal''.
\item $\omega_c$: \textbf{Carrier frequency,} the frequency about which the transmitted signal is centered.
\item $y(t)$: \textbf{``Pass-band''} or \textbf{``modulated''} signal. This is what we send through the air (or channel) to be received, demodulated, and played back as $x(t)$.
\end{enumerate}

\section{Amplitude Modulation}

\begin{equation}
y(t) = x(t)c(t)
\end{equation}
Where $c(t)$ is the carrier signal.

\subsection{Form 1: Complex Exponential Carrier}
\begin{equation}
c(t) = e^{j(\omega_c t + \theta_c)}
\end{equation}
For now, we assume $\theta_c = 0$.
\begin{align}
y(t) &= e^{j(\omega_c t)} x(t) \\
Y(j\omega) &= X(j(\omega-\omega_c))
\end{align}

So $Y(jw)$ is literally just $X(jw)$ shifted up to center on $\omega_c$.

\paragraph{Demodulation: } Just shift back with another complex exponential.
\begin{equation}
x(t) = y(t) e^{-j\omega_c t}
\end{equation}

\paragraph{PROBLEM: } It's a pain to send real AND imaginary components (remember imaginary stuff is a convenient fabrication, so we would need to literally have two signals broadcasted and received independently). To solve this problem, we use different systems.

\subsection{Form 2: Sinusoidal Carrier}

\begin{equation}
c(t) = cos(\omega_c t + \theta_c)
\end{equation}

Again, we assume that $\theta_c = 0$ for convenience. The fourier series representation of $c(t)$ is just two delta functions at $\pm \omega_c$, so convolving that with $X(j\omega)$ yields two copies of $X(j\omega)$ centered at $\pm \omega_c$.

\begin{equation}
C(j\omega) = \pi [\delta(\omega-\omega_c) + \delta(\omega+\omega+c)]
\end{equation}

\begin{equation}
Y(j\omega) = \frac{1}{2\pi} [X(j(\omega-\omega_c)) + X(j(\omega+\omega_c))]
\end{equation}

\paragraph{Reconstructing: } Conveniently, re-convolving $Y(j\omega)$ with the same two delta functions in $C(j\omega)$ yields one larger ``lobe'' (copy of $X(j\omega)$) centered at 0 and two smaller ones centered at $\pm 2\omega_c$. We can then \textbf{low-pass filter} to extract the central lobe and get $X(j\omega)$.

\paragraph{Time Domain Representation: } We can also see this process algebraically in the time domain.
\begin{align}
y(t) &= x(t)\cos(\omega_c t) \\
w(t) &= y(t)\cos(\omega_c t) \\
&= x(t) \cos^2 (\omega_c t) \\
&= \frac{1}{2} x(t) + \frac{1}{2} x(t) \cos(2\omega_c t)
\end{align}

And we filter out the second term in the final expression with the low-pass filter.

\section{Non-Ideal Demodulation}

\subsection{Problems with Synthronous Demodulation}

\textit{Previously: We assumed that $\omega_c, \theta_c$ were shared perfectly between transmitter and receiver. This isn't true in practice and is known as \textbf{synchronous demodulation}}.
\begin{itemize}
\item Non-zero $\theta_c$ is incorporated fairly easily into equations from before for modulation/demodulation.
\end{itemize}

\paragraph{Phase Mis-Synchronization: } If the receiver thinks the phase of $\phi_c \neq \theta_c$, you can do a bunch of algebra to figure out that
\begin{equation}
x_{out}(t) = x_{in}(t) \cos(\theta_c - \phi_c)
\end{equation}

While you could just amplify the reconstruction for when $\phi_c \neq \theta_c$, you might run into $\phi_c - \theta_c = \pi/2 + k\pi$, leading to $\cos(\theta_c + \phi_c)$ being zero. Pretty bad situation overall.

\paragraph{Carrier Frequency Mis-Synchronization: } If the modulator has carrier frequency $\omega_c$ but the demodulator has frequency $\omega_d$, then
\begin{equation}
x_{out} = x_{in}(t) \cos((\omega_c-\omega_d) t)
\end{equation}

Leading to an oscillating envelope on the gain of the signal. Pretty bad for the \textit{audiophile experience}.

\paragraph{Solution: } Don't use synchronous modulation lmao

\subsection{Asynchronous Modulation}

\paragraph{Envelope Detector: } A relatively simple circuit that performs ``half wave rectification'' on an incoming modulated signal with high frequency carrier and low frequency base band. The circuit follows the wave on the positive derivative portion and then has a slow exponential discharge on the downward portion.

\begin{itemize}
\item When $\omega_c >> \omega_m$, $x_{reconstructed}(t) \approx x(t)$.
\item Usually is sent through a low-pass filter to get rid of noise afterward.
\item \textbf{Non-linearity} is introduced by the diode $\to$ harder to analyze.
\begin{itemize}
\item $X_{reconstructed}(j\omega)$ is harder to find.
\end{itemize}
\item \textbf{Assumes} $x(t) \geq 0$. If this is not true by default, we add \textbf{offset} $x(t) := x(t) + a, a\in \mathbb{R}$.
\begin{itemize}
\item Results in a delta function of amplitude $\pi a$ at $\pm \omega_c$ in the frequency domain representation of
\begin{equation}
y(t) = (x(t) + a)\cos(\omega_c t)
\end{equation}
\item \textbf{Transmitted carrier mode: } $x(t)$ has offset and therefore has aforementioned delta function spikes. \textbf{This enables envelope detection}. Corresponds to signal being greater than 0.
\item \textbf{Suppressed carrier mode: } no offset on $x(t)$ $\to$ no delta function spikes in frequency domain.
\item Transmitting the carrier takes \textbf{more energy}.
\item Tradeoff: More energy/work on the transmitter side OR more energy/work on receiver side.
\end{itemize}
\end{itemize}

\section{Frequency Division Multiplexing}

\paragraph{Goal: } Simultaneously transmit multiple signals through one channel (``wideband'' channel).

\paragraph{Solution: } Given
\begin{equation}
\vec{x}(t) = \begin{bmatrix}
x_1(t) \\
x_2(t) \\
\vdots \\
x_n(t) \\
\end{bmatrix};\,\,\,\,
\vec{c}(t) = \begin{bmatrix}
cos(\omega_1 t) \\
cos(\omega_2 t) \\
\vdots \\
cos(\omega_n t) \\
\end{bmatrix}
\end{equation}

We let
\begin{equation}
w(t) = \vec{x}(t)^T \vec{c}(t)
\end{equation}

Supposing that all $x_i(t) \in \vec{x}(t)$ are band limited by $\pm \omega_m$ and $\omega_i - \omega_j > 2\omega_m$, we would have $2n$ separated ``lobes'' representing each signal in the frequency domain that can be band-pass filtered and demodulated to reconstruct the original! This is basically how AM radio works. How bout that.

\section{Single-Sideband Sinusoidal AM}

\textit{Recall:} In our AM-modulated signals, we have two ``lobes'' in $Y(j\omega)$ centered at $\pm \omega_c$. In other words, we are sending \textbf{one redundant copy of the signal!}

\paragraph{Leveraging Extra Information: } We use a \textbf{low-pass filter} with cutoff $\omega_c$ (i.e. the frequency at which the two lobes are centered) BEFORE TRANSMISSION.
\begin{itemize}
\item When the convolution is done on the receiving side, we still end up with a \textbf{complete central lobe}, it's just not two times the size of the left and right lobes.
\item No information is lost!
\item We can utilize less bandwidth per signal and encode \textbf{more information in frequency-division multiplexing}!
\end{itemize}

\section{AM Pulse-Train Carrier}

\paragraph{Key Idea: } We aren't stuck with using sinusoidal carriers. It's a pain to create sinusoidal signals -- much easier to create a \textbf{pulse train}.

\begin{equation}
c_T(t) = \begin{cases}
1 & \text{, for } |t-kT| \leq \Delta, k\in \mathbb{Z} \\
0 & \text{, elsewhere.}
\end{cases}
\end{equation}

\paragraph{Frequency Domain Representation for $c(t)$: } Since $c$ is periodic, we calculate its Fourier series where $\omega_c = 2\pi/T$:
\begin{equation}
C(j\omega) = 2\pi \sum_{k=-\infty}^{\infty} a_k \delta(\omega-k\omega_c)
\end{equation}

\begin{align}
a_k &= \frac{1}{T} \int_{T}^{} c(t) e^{(-jk\omega_c t)} dt \\
a_k &= \frac{\sin(k\omega_c \frac{\Delta}{2})}{k\pi} \\
\end{align}

Once convolved with $X(j\omega)$, we end up with a bunch of $sinc$-scaled copies of $X(j\omega)$ at multiples of $\omega_c$.

To demodulate the signal, we apply the same techniques as before to capture the $\pm \omega_c$ signals. The main benefit of the method is the \textbf{low complexity for encoding} a signal.

\begin{itemize}
\item $\Delta$ only relates to the amplitude of the final signal -- it doesn't affect things otherwise.
\item $\lim_{\Delta \to 0^+}$ is just sampling.
\item \textbf{Time division multiplexing: } With pulse-train carriers, there's some empty space between pulses you can fill with other signals if you're clever .
\end{itemize}

\section{Angle Modulation}

Tbh this part was kind of half-assed. Here are some of the key points.

\paragraph{Angle Modulation: } Instead of $y(t) = c(t) x(t)$, put $x(t)$ INSIDE $c(t)$
\begin{equation}
y(t) = A_c \cos(\theta(t))
\end{equation}
Where $\theta(t)$ is a function of $x(t)$. The angle argument of the cosine is modulated by $x(t)$.

\paragraph{Phase Modulation: }
\begin{equation}
\theta(t) = \omega_c t + k_p x(t)
\end{equation}

\paragraph{Frequency Modulation: }
\begin{equation}
\theta(t) = \omega_c t + k_f \int_{0}^{t} x(\tau) d\tau
\end{equation}

\paragraph{Connection: }
\begin{itemize}
\item In frequency modulation, we use an integral because the instantaneous frequency is given by $\frac{d\theta}{dt}$. Solving for the derivative, we get that frequency is $\frac{d\theta}{d t} = w_c + k_f x(t)$, which is exactly what frequency modulation sounds like in the first place.
\item Instantaneous frequency in phase modulation is $\frac{d \theta}{dt} = \omega_c + k_p \frac{dx(t)}{dt}$.
\end{itemize}

\paragraph{Benefits and Detriments: }
\begin{itemize}
\item Transmission power is constant (i.e. not a function of the signal being communicated) -- makes it more reliable and higher quality (this is why all my homies listen to FM radio).
\item Non-linear modulation process makes it kind of hard to analyze, though.
\end{itemize}

\end{document}


\documentclass[a4paper,12pt]{article}
\usepackage[top=2cm,bottom=2cm,left=2cm,right=2cm]{geometry}
\usepackage{amsmath}
\usepackage{hyperref}
% \usepackage{listings}

% \lstset{
%   language=C++,                % choose the language of the code
%   numbers=left,                   % where to put the line-numbers
%   stepnumber=1,                   % the step between two line-numbers.
%   numbersep=5pt,                  % how far the line-numbers are from the code
%   showspaces=false,               % show spaces adding particular underscores
%   showstringspaces=false,         % underline spaces within strings
%   showtabs=false,                 % show tabs within strings adding particular underscores
%   tabsize=2,                      % sets default tabsize to 2 spaces
%   captionpos=b,                   % sets the caption-position to bottom
%   breaklines=true,                % sets automatic line breaking
%   breakatwhitespace=true,         % sets if automatic breaks should only happen at whitespace
%   title=\lstname,                 % show the filename of files included with \lstinputlisting;
% }

\title{Numerical Methods exercises}
\author{Margherita Billi, Filippo Raspanti, Eugenio Tampieri}
%\renewcommand{\arraystretch}{1.5}
\begin{document}
\maketitle
\section{Exercise 1}
\subsection{Part 1}
\par We are given the function $f(x)=2x^3-3x^2-12x+1$, whose derivative is $f'(x)=6x^2-6x-12$.
\par We have studied the first derivative in order to find and classificate the maximum and minimum values of the function using the Fermat’s theorem and the second consequence of the Lagrange’s theorem.
We’ve thus found that there is a maximum in the point M(-1;8) and a minimum in m(2;-19).
Provided that $\lim\limits_{x \to -\infty} 2x^3-3x^2-12x+1 = -\infty$, $\lim\limits_{x \to +\infty} 2x^3-3x^2-12x+1 = +\infty$ and that the function is increasing in the interval $]-\infty;-1[$ and in $]+2;+\infty[$ and decreasing in the interval $]-1;+2[$ we are able to state that the function intersects the $x$ axis in only 3 points.
\subsection{Part 2}
\par The code for this exercise can be found at \url{https://gitlab.com/eutampieri/calcolo-numerico} and is not included in this paper because it's over 300 lines long. In that repo there are, apart from this paper, three source files: \verb|numerical_methods.cpp|, \verb|funcgen.c| and \verb|functions.hpp|.
\par The first is the command line interface through which the user chooses a method and the bounds, the second is a tool to save into a file the coefficients for a polynomial function and the latter is the mathematical engine. It abstracts away the root finding methods for polynomial, logarithmic, exponential, composite and goniometric functions.
\subsection{Part 3}
\subsubsection{Section a}
By using the Bisection method to find the roots of the previously given function in the interval $[-2;4]$ the output is:
\begin{verbatim}
--- Numerical Methods ---
Leftmost point of the interval: -2
Rightmost point of the interval: 4
=========================
     Choose a method
=========================
	[b] for bisection method
	[s] for the secant method
	[n] for Newton's method
Your choice: b
A root has been found in 28 iterations: f(3.28183)=-3.52763e-07
\end{verbatim}
\par Using the bisection method we've found the rightmost solution in 28 iterations.
\par By applying the Secants method to find the roots of the previously given function in the interval $[-2;4]$ the output is:
\begin{verbatim}
--- Numerical Methods ---
Leftmost point of the interval: -2
Rightmost point of the interval: 4
=========================
     Choose a method
=========================
	[b] for bisection method
	[s] for the secant method
	[n] for Newton's method
Your choice: s
A root has been found in 9 iterations: f(-1.86358)=1.06581e-14
\end{verbatim}
\par The algorithm has successfully found the leftmost root in 9 iterations.
\subsubsection{Section b}
\par First we notice that in the given interval ($[-2;-1]$) the function has only one root.
\par First we notice that in the given interval ($[-2;-1]$) the function has only one root.
\par By running the program we get the following results:
\begin{verbatim}
--- Numerical Methods ---
Leftmost point of the interval: -2
Rightmost point of the interval: -1
=========================
     Choose a method
=========================
	[b] for bisection method
	[s] for the secant method
	[n] for Newton's method
Your choice: b
A root has been found in 27 iterations: f(-1.86358)=-6.4629e-08
\end{verbatim}
\begin{verbatim}
--- Numerical Methods ---
Leftmost point of the interval: -1
Rightmost point of the interval: -2
=========================
     Choose a method
=========================
    [b] for bisection method
    [s] for the secant method
    [n] for Newton's method
Your choice: s
A root has been found in 7 iterations: f(-1.86358)=0
\end{verbatim}
\subsubsection{Section c}
Results after running the algorithm:
\begin{enumerate}
    \item $x_0=-2$: \verb|A root has been found in 4 iterations: f(-1.86358)=-7.10543e-15|
    \item $x_0=-1,5$: \verb|A root has been found in 4 iterations: f(-1.86358)=0|
    \item $x_0=-1$: \verb|A root has been found in 1 iterations: f(-inf)=nan|
    \item $x_0=0$: \verb|A root has been found in 3 iterations: f(0.0817535)=-6.31051e-13|
\end{enumerate}
\subsection{Part 4}
     \begin{enumerate}
          \item For the bisection method, the interval must be wide at least $4,450\cdot 10^{-308}$ and at most $8,989\cdot 10^307$, because the \texttt{double} type used in this implementation (an IEEE-754 64 bit floating point) can hold a value $x \in [2,225\cdot 10^{-308};1,798\cdot10^{308}]$. Apart from that and provided that the solution is in the given interval, bisection is guaranteed to converge to the solution.
          \item From our data we can state that Newton's method is the fastest, followed by the secants one and by the bisection method. Moreover, looking at the outputs we notice that the secants algorithm is almost four times faster than the bisection one.
          \item The result can be explained taking into account the fact that $f'(-1)$=0, thus we perform a division by zero, that yelds $-\infty$.
     \end{enumerate}
\section{Exercise 2}
     \subsection{Part 1}
          \par We are given the function $f(x)=x^3-7x+9$, whose first derivative is $f'(x)=3x^2-7$. We have studied the first derivative in order to find and classify the maximum and minimum values of the function using the Fermat's theorem and the second consequence of the Lagrange's theorem. We have found that there is a point of maximum in $M(-1,53;16,13)$ and a point of minimum in $m(1,53;1,87)$.
          \par Provided that $\lim\limits_{x \to -\infty} x^3-7x+9 = -\infty$, that $\lim\limits_{x \to +\infty} x^3-7x+9 = +\infty$, that the function is increasing in the intervals $]-\infty;-1,53[$ and $]1,53;+\infty[$ and decreasing in the interval $]-1,53;+1,53[$ and that the minimum and the maximum are both positive ($f(x_{min})>0$, $f(x_{max})>0$); we can state that the function intersects only in one point the X axis.
     \subsection{Part 2}
          \par There's no need to modify the code, nevertheless you have to run again \verb|funcgen| to update the coefficients.
     \subsection{Part 3}
          \begin{enumerate}
               \item \begin{enumerate}
                    \item $x_0=1\cdot 10^3$: \verb|A root has been found in 28 iterations: f(-3.14092)=-1.87406e-11|
                    \item $x_0= - 1\cdot 10^3$: \verb|A root has been found in 19 iterations: f(-3.14092)=-3.55271e-15|
                    \item $x_0=7$: \verb|A root has been found in 45 iterations: f(-3.14092)=-3.55271e-15|
                    \item $x_0=42$: \verb|A root has been found in 29 iterations: f(-3.14092)=-4.14637e-10|
                    \item $x_0=14299$: \verb|A root has been found in 92 iterations: f(-3.14092)=-4.37765e-11|
                    \item $x_0=-10$: \verb|A root has been found in 7 iterations: f(-3.14092)=-8.41737e-10|
               \end{enumerate}
               \item Rightmost point: $-10$, leftmost point: $2$, output of the bisection method: \texttt{A root has been found in 29 iterations: f(-3.14092)=2.30148e-07}
               \item Rightmost point: $-10$, leftmost point: $2$, output of the secants method: \texttt{A root has been found in 37 iterations: f(-3.14092)=-3.19744e-14}
          \end{enumerate}
     \subsection{Part 4}
          \begin{enumerate}
               \item Newton method converges, assumed that the initial guess is not a stationary point or that a stationary isn't reached during the iterations.
               \item If we consider the initial guess as the rightmost point of the interval, then in the provided example ($x \in [-10;2]$) Newton's method is the fastest (7 iterations), followed by the bisection method (29 iterations) and by the secants method (37 iterations), thus not confirming the observations made in the previous exercise. Moreover, Newton's method doesn't seem to converge in a time related to the distance from the root: in facts, the time that it takes with an initial guess of 42 is less than the time needed to reach a root from an initial guess of 7.
          \end{enumerate}
\end{document}


\documentclass{article}

\begin{document}

    \tableofcontents

    \section{What are Algorithms?}

    The word algorithms comes from the Muslim scientist in the region of Iran, Khwarizmi. His date of birth is not known.

    He published a book called \textit{'Al Kitab Al Mukhtatasar fi hisab al-jab wa'l-muqabalah'}

    What is the algorithmic definitiion? It is as the folloing,
    \begin{itemize}
        \item An algorithm is any well-defined computational procedure that takes some values, or set of values, as input and produces some value, or set of values, as input.
        \item An algorithm is thus a sequence of computational steps that transform the input into output
    \end{itemize}

    What is an algorithm - a formal definition,
    \begin{itemize}
        \item An algorithm is a sequence of unambigious instructions for solving a problem, i.e, for obtaining a required output for an legitimate input in a finite amount of time
    \end{itemize}

    In the above, some keywords are,
    \begin{itemize}
        \item \textbf{Unnambiguity} - Must be very well defined
        \item \textbf{Finite amount of time} - should not done is some estimated amount of time
    \end{itemize}

    Another definition is,
    \begin{itemize}
        \item Recipe, process, method, technique, procedure, routine ...
    \end{itemize}

    \section{Why is the study of algorithms worthwhile?}

    To work on the time, and to demonstrate that our solution terminates, and with the correct answer,

    \begin{itemize}
        \item Suppose computers were infinitely fast and computer memory was free
        \item Would you have any reason to study algorithms?
        \item The answer is "YES"
        \item to demonstrate that your solution method terminates and does so with the correct answer
    \end{itemize}

    We study algorithms, because often we have limited resources, and we wish to complete the termination of the program with the correct answer, so that the fewest number of resources are used
    Putting in into summary,

    \begin{itemize}
        \item Processing time is a bounded resources
        \item Computers may be fsat but they are not infinitely fas. And memory maybe inexpensive, but it is not free.
        \item Use these resources wisely
        \item Insertion and Merge Sort
    \end{itemize}

    Let's take an example

    \begin{itemize}
        \item Suppose that computer A executes 10 billion instructions per second and computer B executes only 10 million instructions per seconds
        \item Insertion sort in machine language for computer A by world's crafftiest programmer resulting in 2n**2 time.
        \item Suppose an average programmer implements merge sort, using a high-level language with an inefficient compiler, taking 50 * n * log(n) \textit{Time}
        \item To sort 10million numbers, computer A takes,
    \end{itemize}
        \frac{2-(10^7)^2 instructions}{10^10 instructions/second} = 20,000 seconds (more than 5.5 hours).
    \begin{itemize}
        \item While B takes,
    \end{itemize}
        \frac{50 * 10^7 * log(10^7) instructions}{10^7 instructions/second} ~ 1163 seconds (less than 20 minutes)

    \section{Design Of Algorithms}

    \begin{enumerate}
        \item Understand the problem
        \item Decide on: computational means, exact vs approximate solving, algorithm design technique
        \item Design an algorithm
        \item Prove correctness. Go to step 2, 3
        \item Analyze the algorithm
        \item Code the algorithm
    \end{enumerate}

    \subsection{Algorithm Design Techniques / Strategies}
    \begin{enumerate}
        \item Brute Force
        \item Divide And Conquer
        \item Decrease And Conquer
        \item Transform And Conquer
        \item Space and Time
        \item Branch and bounded
        \item Backtracking
        \item Dynamic Programming
        \item Greedy Approach
    \end{enumerate}

    \subsection{Analysis Of Algorithms}
    \begin{

    \subsection{Important Problem Types}
    \begin{enumerate}
        \item Sorting
        \item Serarchoing
        \item Numerical
        \item Graphical
        \item Combinatorial problems
        \item String searching
    \end{enumerate}

    \subsection{}

\end{document}

Semester V/Algorithms/Sir Muhammad Waqas/Recorded Notes/Sep 1, 2020.tex


\chapter{Introduction to Core miniKanren}\label{mkintrochapter}

This chapter introduces the core miniKanren language, provides several
short example programs, and shows how to translate a simple Scheme
function into a miniKanren relation.

This chapter is organized as follows.  Section~\ref{coremksection}
introduces the core miniKanren language.  In
section~\ref{transappendosection} we show how to translate the
standard Scheme \scheme|append| function into a miniKanren relation.
Section~\ref{impureoperatorssection} describes several ``impure''
operators that, while not part of the pure miniKanren core language,
are useful when trying to model Prolog programs.

\section{Core miniKanren}\label{coremksection}

miniKanren extends Scheme with three operators: \scheme|==|,
\scheme|conde|, and \scheme|exist|. There is also \scheme|run|, which
serves as an interface between Scheme and miniKanren, and whose value
is a list.

\scheme|exist|, which syntactically looks like \scheme|lambda|,
introduces new variables into its scope; \scheme|==| unifies two
values.  Thus

\wspace

\noindent\scheme|(exist (x y z) (== x z) (== 3 y))|

\wspace

\noindent would associate \scheme|x| with \scheme|z| and \scheme|y|
with \scheme|3|.  This, however, is not a legal miniKanren
program---we must wrap a \scheme|run| around the entire expression.

\wspace

\noindent\scheme|(run1 (q) (exist (x y z) (== x z) (== 3 y)))| $\Rightarrow$ \begin{schemeresponsebox}(_.0)\end{schemeresponsebox}

\wspace

\noindent The value returned is a list containing the single value \schemeresult|_.0|;
we say that \schemeresult|_.0| is the \emph{reified value} of the fresh variable \scheme|q|.  \scheme|q| also remains fresh in

\wspace

\noindent\scheme|(run1 (q) (exist (x y) (== x q) (== 3 y)))| $\Rightarrow$ \begin{schemeresponsebox}(_.0)\end{schemeresponsebox}

\wspace

We can get back other values, of course.

\wspace

\begin{schemebox}
(run1 (y)
  (exist (x z)
    (== x z)
    (== 3 y)))
$\ $
\end{schemebox}
\begin{schemebox}
(run1 (q)
  (exist (x z)
    (== x z)
    (== 3 z)
    (== q x)))
\end{schemebox}
\begin{schemebox}
(run1 (y)
  (exist (x y)
    (== 4 x)
    (== x y))
  (== 3 y))
\end{schemebox}

\wspace

\noindent Each of these examples returns \scheme|`(3)|; in the rightmost example, the \scheme|y| introduced by \scheme|exist| is different from the \scheme|y| introduced by \scheme|run| because the variables are lexically scoped.  \scheme|run| can also return the empty list, indicating that there are no values.

\wspace

\noindent\scheme|(run1 (x) (== 4 3))| $\Rightarrow$ \schemeresult|`()|

\wspace

We use \scheme|conde| to get several values---syntactically,
\scheme|conde| looks like \scheme|cond| but without \scheme|=>|
or \scheme|else|.  For example,

\schemedisplayspace
\begin{schemedisplay}
(run2 (q)
  (exist (x y z)
    (conde
      ((== `(,x ,y ,z ,x) q))
      ((== `(,z ,y ,x ,z) q))))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
((_.0 _.1 _.2 _.0) (_.0 _.1 _.2 _.0))
\end{schemeresponse}

\noindent Although the two \scheme|conde|-clauses are different, the
values returned are identical.  This is because distinct reified fresh
variables are assigned distinct numbers, increasing from left to
right---the numbering starts over again from zero within each value,
which is why the reified value of \scheme|x|
is \schemeresult|_.0| in the first
value but \schemeresult|_.2| in the
second value.

Here is a simpler example using \scheme|conde|.

\schemedisplayspace
\begin{schemedisplay}
(run5 (q)
  (exist (x y z)
    (conde
      ((== 'a x) (== 1 y) (== 'd z))
      ((== 2 y) (== 'b x) (== 'e z))
      ((== 'f z) (== 'c x) (== 3 y)))
    (== `(,x ,y ,z) q))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
((a 1 d) (b 2 e) (c 3 f))
\end{schemeresponse}

\noindent The superscript \scheme|5| denotes the maximum length of the resultant
list.  If the superscript \scheme|*| is used, then there is no maximum
imposed.  This can easily lead to infinite loops:

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (let loop ()
    (conde
      ((== #f q))
      ((== #t q))
      ((loop)))))
\end{schemedisplay}

\noindent Had the \scheme|*| been replaced by a non-negative integer \scheme|n|, then
a list of \scheme|n| alternating \scheme|#f|'s and \scheme|#t|'s would be returned.
The \scheme|conde| succeeds while associating \scheme|q| with \scheme|#f|, which
accounts for the first value.  When getting the second value,
the second \scheme|conde|-clause is tried, and the association made between \scheme|q| and \scheme|#f| is forgotten---we
say that \scheme|q| has been \emph{refreshed}.  In the third
\scheme|conde|-clause,
\scheme|q| is refreshed once again.

We now look at several interesting examples that rely on \scheme|anyo|.

\schemedisplayspace
\begin{schemedisplay}
(define anyo
  (lambda (g)
    (conde
      (g)
      ((anyo g)))))
\end{schemedisplay}

\noindent \scheme|anyo| tries \scheme|g| an unbounded number of times.
Here is our first example using \scheme|anyo|.

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (conde
    ((anyo (== #f q)))
    ((== #t q))))
\end{schemedisplay}

\noindent This example does not terminate, because the call to
\scheme|anyo| succeeds an unbounded number of times.
If \scheme|*| is replaced by \scheme|5|, then instead we get
 \mbox{\schemeresult|`(#t #f #f #f #f)|}.
(The user should not be concerned with the order in which values are returned.)

Now consider

\schemedisplayspace
\begin{schemedisplay}
(run10 (q)
  (anyo
    (conde
      ((== 1 q))
      ((== 2 q))
      ((== 3 q))))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
(1 2 3 1 2 3 1 2 3 1)
\end{schemeresponse}

\noindent Here the values \scheme|1|, \scheme|2|, and \scheme|3| are interleaved;
our use of \scheme|anyo| ensures that this sequence will be repeated indefinitely.

Here is \scheme|alwayso|,

\wspace

\noindent\mbox{\scheme|(define alwayso (anyo (== #f #f)))|}

\wspace

\noindent along with two \scheme|run| expressions that use it.

\schemedisplayspace
\begin{schemebox}
(run1 (x)
  (== #t x)
  alwayso
  (== #f x))
$\ $
$\ $
\end{schemebox}
\hspace{2cm}
\begin{schemebox}
(run5 (x)
  (conde
    ((== #t x))
    ((== #f x)))
  alwayso
  (== #f x))
\end{schemebox}

\wspace

\noindent The left-hand expression diverges---this is because
\scheme|alwayso| succeeds an unbounded number of times,
and because \mbox{\scheme|(== #f x)|} fails each of those times.

The right-hand expression returns a list of five \scheme|#f|'s.  This
is because both \scheme|conde|-clauses are tried, and both succeed.
However, only the second \scheme|conde|-clause contributes to the
values returned. Nothing changes if we swap the two
\scheme|conde|-clauses.  If we change the last expression to
\mbox{\scheme|(== #t x)|}, we instead get a list of five
\scheme|#t|'s.

Even if some \scheme|conde|-clauses loop indefinitely, other
\scheme|conde|-clauses can contribute to the values returned by a
\scheme|run| expression.
% (We are not concerned with
% Scheme expressions looping indefinitely, however.)
For example,

\schemedisplayspace
\begin{schemedisplay}
(run3 (q)
  (let ((nevero (anyo (== #f #t))))
    (conde
      ((== 1 q))
      (nevero)
      ((conde
         ((== 2 q))
         (nevero)
         ((== 3 q)))))))
\end{schemedisplay}

\noindent returns \schemeresult|`(1 2 3)|; replacing \scheme|run3|
with \scheme|run4| causes divergence, however, since there are only
three values, and since \scheme|nevero| loops indefinitely.

\section{Translating Scheme Code to miniKanren}\label{transappendosection}

In this section we translate the standard Scheme function
\scheme|append| to the equivalent miniKanren relation,
\scheme|appendo|.  \scheme|append| takes two lists as arguments, and
returns the appended list.

\wspace

\noindent\mbox{\scheme|(append '(a b c) '(d e)) => |}\mbox{\schemeresult|`(a b c d e)|}

\wspace

Here is the definition of \mbox{\scheme|append|}.

\schemedisplayspace
\begin{schemedisplay}
(define append
  (lambda (l s)
    (cond
      ((null? l) s)
      (else (cons (car l) (append (cdr l) s))))))
\end{schemedisplay}

Rather than translate the Scheme definition directly to miniKanren, we
will massage the Scheme code to make it closer in spirit to a
miniKanren relation. Only after we have performed several
Scheme-to-Scheme transformations will we translate to
miniKanren\footnote{This approach differs from that of~\cite{trs},
  which translates Scheme functions directly to miniKanren.}.

First we replace the always-true \scheme|else| test with an explicit
\scheme|pair?| test, making the \scheme|cond| clauses
\emph{non-overlapping}\footnote{The concept of non-overlapping clauses
  is revisited in section~\ref{reconsiderrember}.}.

\newpage

%\schemedisplayspace
\begin{schemedisplay}
(define append
  (lambda (l s)
    (cond
      ((null? l) s)
      ((pair? l) (cons (car l) (append (cdr l) s))))))
\end{schemedisplay}

Next we replace \scheme|cond| with the \scheme|pmatch|
pattern-matching macro from Appendix~\ref{pmatch}.  The use of pattern
matching is close in spirit to unification, and lets us easily
translate the code to use \scheme|matche| or \scheme|lambdae| from
Appendix~\ref{matche}.

\schemedisplayspace
\begin{schemedisplay}
(define append
  (lambda (l s)
    (pmatch `(,l ,s)
      (`(() ,s) s)
      (`((,a . ,d) ,s)
       (cons a (append d s))))))
\end{schemedisplay}

We then perform an \emph{unnesting} step reminiscent of the
Continuation-Passing Style (CPS) transformation\footnote{More correctly, the unnested program is
  similar to one in A-Normal Form (ANF)~\cite{essencecompiling}.}
 (see, for example, \citet{eopl3}): we unnest
any nested calls, introducing \scheme|let|-bound variables
where necessary\footnote{Unlike in the CPS transformation we must
  unnest \emph{every} call, even those guaranteed to terminate.
  For example, unnesting \mbox{\scheme|(cons (cons 1 2) 3)|} results
  in \mbox{\scheme|(let ((tmp (cons 1 2))) (cons tmp 3))|}.}.

\schemedisplayspace
\begin{schemedisplay}
(define append
  (lambda (l s)
    (pmatch `(,l ,s)
      (`(() ,s) s)
      (`((,a . ,d) ,s)
       (let ((res (append d s)))
         (cons a res))))))
\end{schemedisplay}

After unnesting, we are ready to translate the Scheme function into a
miniKanren relation.  We add a superscript $o$ to the name, to
indicate the new function is a relation.  We add an ``output''
argument\footnote{When translating a Scheme predicate to a miniKanren
  relation we do not add an ``output'' argument. This is because success
  or failure of a call to the relation is equivalent to the Scheme
  predicate returning \scheme|#t| or \scheme|#f|, respectively.} and
change \scheme|pmatch| to \scheme|matche|.  We add the output argument
to the list of values being matched against by \scheme|matche|, and the
individual patterns.  Any value that would have previously been returned
must now be unified with the \scheme|out| argument, either explicitly
using \scheme|==| or implicitly using pattern matching.  We also
change the \scheme|let| to \scheme|exist| introducing a ``temporary''
logic variable.

\newpage

%\schemedisplayspace
\begin{schemedisplay}
(define appendo
  (lambda (l s out)
    (match-e `(,l ,s ,out)
      (`(() ,s ,s))
      (`((,a . ,d) ,s ,out)
       (exist (res)
         (appendo d s res)
         (== (cons a res) out))))))
\end{schemedisplay}

Since we are matching against all the arguments of \scheme|appendo|,
we can use \scheme|lambdae| rather than \scheme|matche|.
Also, we may wish to replace \mbox{\scheme|(cons a res)|} with
\mbox{\scheme|`(,a . ,res)|} to reflect our use of unification as
pattern matching.

\schemedisplayspace
\begin{schemedisplay}
(define appendo
  (lambda-e (l s out)
    (`(() ,s ,s))
    (`((,a . ,d) ,s ,out)
     (exist (res)
       (appendo d s res)
       (== `(,a . ,res) out)))))
\end{schemedisplay}

If we do not wish to use the \scheme|matche| or \scheme|lambdae|
pattern matching macros, we can rewrite \scheme|appendo| in core
miniKanren.

\schemedisplayspace
\begin{schemedisplay}
(define appendo
  (lambda (l s out)
    (conde
      ((== '() l) (== s out))
      ((exist (a d)
         (== `(,a . ,d) l)
         (exist (res)
           (appendo d s res)
           (== `(,a . ,res) out)))))))
\end{schemedisplay}

Of course we can use the \scheme|appendo| relation to append two
lists.

\noindent\mbox{\scheme|(run* (q) (appendo '(a b c) '(d e) q)) => |}\mbox{\schemeresult|`((a b c d e))|}

\noindent But we can also find all pairs of lists that, when
appended, produce \mbox{\scheme|'(a b c d e)|}.

\schemedisplayspace
\begin{schemedisplay}
(run6 (q)
  (exist (l s)
    (appendo l s '(a b c d e))
    (== `(,l ,s) q))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
((() (a b c d e))
 ((a) (b c d e))
 ((a b) (c d e))
 ((a b c) (d e))
 ((a b c d) (e))
 ((a b c d e) ()))
\end{schemeresponse}

\noindent Unfortunately, replacing \scheme|run6| with \scheme|run7|
results in divergence, for reasons explained in
Chapter~\ref{divergencechapter}.  We can avoid this problem if we swap
the last two lines of \scheme|appendo|.

\schemedisplayspace
\begin{schemedisplay}
(define appendo
  (lambda (l s out)
    (conde
      ((== '() l) (== s out))
      ((exist (a d)
         (== `(,a . ,d) l)
         (exist (res)
           (== `(,a . ,res) out)
           (appendo d s res)))))))
\end{schemedisplay}

\noindent This final version of \scheme|appendo| illustrates an
important principle: unifications should always come before recursive
calls, or calls to other ``serious'' relations.

%%% From diseq chapter

% Here is \mbox{\scheme|rember|}

% \schemedisplayspace
% \begin{schemedisplay}
% (define rember
%   (lambda (x ls)
%     (cond
%       ((null? ls) '())
%       ((eq? (car ls) x) (cdr ls))
%       (else (cons (car ls) (rember x (cdr ls)))))))
% \end{schemedisplay}

% To translate \mbox{\scheme|rember|} into the miniKanren relation
% \mbox{\scheme|rembero|} we add a third argument \mbox{\scheme|out|},
% change \mbox{\scheme|cond|} to \mbox{\scheme|conde|}, and replace uses
% of \mbox{\scheme|null?|}, \mbox{\scheme|eq?|}, \mbox{\scheme|cons|},
% \mbox{\scheme|car|}, and \mbox{\scheme|cdr|} with calls to
% \mbox{\scheme|==|}.  We also unnest the recursive call, using a
% temporary variable \mbox{\scheme|res|} to hold the ``output'' value of
% the recursive call.
% \newpage
% \schemedisplayspace
% \begin{schemedisplay}
% (define rembero
%   (lambda (x ls out)
%     (conde
%       ((== '() ls) (== '() out))
%       ((exist (a d)
%          (== `(,a . ,d) ls)
%          (== a x)
%          (== d out)))
%       ((exist (a d res)
%          (== `(,a . ,d) ls)
%          (== `(,a . ,res) out)
%          (rembero x d res))))))
% \end{schemedisplay}

\section{Impure Operators}\label{impureoperatorssection}

In this section we include several \emph{impure} operators that appear
in earlier work on miniKanren, notably~\citet{trs} and
\citet{DBLP:conf/iclp/NearBF08}: \scheme|project|, \scheme|conda|,
\scheme|condu|, \scheme|onceo|, and \scheme|copy-termo|.  These
operators are not considered part of core miniKanren, and are
inherently non-relational since they may not work correctly for every
goal ordering of a program; also, it is not legal to pass only fresh
variables to some of these operators, namely \scheme|onceo| and
\scheme|copy-termo|.  As a result we only use these operators to
demonstrate impure Prolog-like features, for example in
Chapter~\ref{alphatapchapter} during translation of the \leantapsp
theorem prover from Prolog to miniKanren.  Importantly, the final
version of the translated prover does not use any impure operators.

\scheme|project| can be used to access the values associated with
logic variables.  For example, the expression

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (exist (x)
    (== 5 x)
    (== (* x x) q)))
\end{schemedisplay}

\noindent has no value, since Scheme's multiplication function
operates only on numbers, not logic variables associated with numbers.
We can solve this problem by projecting \scheme|x|: within the body of
the \scheme|project| form, \scheme|x| is a lexical variable bound to
5.

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (exist (x)
    (== 5 x)
    (project (x)
      (== (* x x) q)))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
(25)
\end{schemeresponse}

\noindent Unfortunately, the expression

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (exist (x)
    (project (x)
      (== (* x x) q))
    (== 5 x)))
\end{schemedisplay}

\noindent has no value, since \scheme|x| is unassociated when
\mbox{\scheme|(* x x)|} is evaluated.  This example demonstrates that
\scheme|project| is not a relational operator\footnote{We explore a
  relational approach to arithmetic in Chapter~\ref{arithchapter}.}.

\scheme|conda| and \scheme|condu| are used to prune a program's search
tree, and can be used in place of Prolog's \emph{cut} ({\tt !})\footnote{More specifically,
\scheme|conda| corresponds to a \emph{soft-cut}~\cite{ClocksinClause}, while \scheme|condu|
  corresponds to Mercury's \emph{committed-choice}~\cite{Fergus-Henderson:1996uq,Naish:1995pr}.}.
The examples from chapter 10 of \emph{The Reasoned Schemer}~\cite{trs}
demonstrate uses of \scheme|conda| and \scheme|condu|, and
the pitfalls that await the unsuspecting programmer.

\scheme|conda| and \scheme|condu| differ from \scheme|conde| in that
at most one clause can succeed.  Furthermore, the clauses are tried in
order, from top to bottom.  Also, the first goal in each clause is
treated specially, as a ``test'' goal that determines whether to
commit to that clause; in this way, \scheme|conda| and \scheme|condu|
are reminiscent of \scheme|cond|.

For example,

\schemedisplayspace
\begin{schemedisplay}
(run* (x)
  (conda
    ((== 'olive x))
    ((== 'oil x))))
\end{schemedisplay}

\noindent returns \scheme|'(olive)| since \scheme|conda| commits to
the first clause when \mbox{\scheme|(== 'olive x)|} succeeds.
However,

\schemedisplayspace
\begin{schemedisplay}
(run* (x)
  (conda
    ((== 'virgin x) (== #t #f))
    ((== 'olive x))
    ((== 'oil x))))
\end{schemedisplay}

\noindent returns \scheme|'()| since \mbox{\scheme|(== #t #f)|} fails,
and since \scheme|conda| committed to the first clause once
\mbox{\scheme|(== 'virgin x)|} succeeded.
The expression

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (conda
    ((== #t #f))
    (alwayso))
  (== #t q))
\end{schemedisplay}

\noindent diverges.  The ``test'' goal for the first clause,
\mbox{\scheme|(#t #f)|}, fails.  The test goal for the second clause,
\scheme|alwayso|, succeeds; therefore \scheme|conda| commits to this
clause.  Since \scheme|alwayso| can succeed an unbounded number of
times, the \scheme|run*| expression diverges.

However, if we replace \scheme|conda| with \scheme|condu|, the
resulting expression

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (condu
    ((== #t #f))
    (alwayso))
  (== #t q))
\end{schemedisplay}

\noindent returns \scheme|'(#t)|.  This is because the test goal of a
\scheme|condu| clause can succeed at most once, which is the only
difference between \scheme|conda| and \scheme|condu|.

The next impure operator, \scheme|onceo|, can be trivially defined
using \scheme|condu|.  \scheme|onceo| takes a single argument, which
must be a goal; \scheme|onceo| ensures that when the goal is run it
produces at most a single answer.

\schemedisplayspace
\begin{schemedisplay}
(define onceo
  (lambda (g)
    (condu
      (g))))
\end{schemedisplay}

\noindent For example, \mbox{\scheme|(run* (q) (onceo alwayso))|} produces \scheme|'(_.0)|.

\scheme|copy-termo| creates a copy of its first argument, consistently
replacing unassociated logic variables with new variables; the
resulting copy is then associated with the second argument.

\schemedisplayspace
\begin{schemedisplay}
(run* (q)
  (exist (w x y z)
    (== `(a ,x 5 ,y ,x) w)
    (copy-termo w z)
    (== `(,w ,z) q))) $\Rightarrow$
\end{schemedisplay}
\nspace
\begin{schemeresponse}
(((a _.0 5 _.1 _.0) (a _.2 5 _.3 _.2)))
\end{schemeresponse}

\noindent A major theme of Chapter~\ref{alphatapchapter} is how
\scheme|copy-termo| can be replaced with a relational combination of
nominal unification and tagging, at least in certain cases.