\documentclass[10pt]{article}
\usepackage{amssymb,amsmath}
\usepackage{graphicx}
\usepackage{color}


\addtolength{\textheight}{1in}
\addtolength{\topmargin}{-0.5in}
\addtolength{\oddsidemargin}{-0.5in}
\addtolength{\evensidemargin}{-0.5in}
\addtolength{\textwidth}{1in}

\newcommand{\comment}[1]{}
\newcommand{\mmp}[1]{\textcolor{blue}{MMP: #1}}
\newcommand{\beq}{\begin{equation}}
\newcommand{\eeq}{\end{equation}}
\newcommand{\beqa}{\begin{eqnarray}}
\newcommand{\eeqa}{\end{eqnarray}}
\newcommand{\bit}{\begin{itemize}\setlength{\itemsep}{0cm}\setlength{\topsep}{0cm}}
\newcommand{\eit}{\end{itemize}}
\newcommand{\benum}{\begin{enumerate}\setlength{\itemsep}{0cm}\setlength{\parsep
}{0cm}}
\newcommand{\eenum}{\end{enumerate}}
\newcommand{\bprogram}{\begin{tabbing}mm\=mm\=mm\=mm\= \kill\\}
\newcommand{\eprogram}{\end{tabbing}}

% math mode commands

\newcommand{\bigOO}{{\cal O}}
\newcommand{\bbone}{{\mathbf 1}}
\newcommand {\argmax}[2]{\mbox{\raisebox{-1.7ex}{$\stackrel{\textstyle{\rm #1}}{\scriptstyle #2}$}}\,}
\newcommand{\dataset}{{\cal D}}
\newcommand{\rrr}{{\mathbb R}}
\newcommand{\fracpartial}[2]{\frac{\partial #1}{\partial #2}}

\newcommand{\setxx}[1]{\{x\,|\,{#1} \}}
\newcommand{\tilDD}{\tilde{\dataset}}
\newcommand{\tilN}{\tilde{N}}
\newcommand{\tilx}{\tilde{x}}
\newcommand{\hatl}{\hat{L}}
\newcommand{\llogit}{_{\rm logit}}
\newcommand{\supind}[1]{^{(#1)}}

\begin{document}
\setlength{\parindent}{0em}
\setlength{\parskip}{1.3em}

\begin{large}
\centerline{STAT 535 Homework 5}  % assignment number here

\centerline{Out Feb 25, 2026}       % out date here
\centerline{Due March  4, 2026}       % due date here
\end{large}

%\centerline{\bf } % title here

\centerline{\large \copyright Marina Meil\u{a} and Benjamin Schneider}
\centerline{\large mmp@uwaterloo.ca}

\vspace{2em}
\begin{small}
\end{small}

{\em All plots must be included in the .pdf writeup}

{\bf Problem 1 -- Decision regions for the neural network}
\\
In this problem, the inputs are of the form $[x_1\;x_2]^T\in[-10,10]^2$
and the dummy variable $x_0\equiv 1$.

Consider the following two-layer neural network
\beqa
f(x)&=&\beta_0+\sum_{k=1}^5\beta_k x\supind{1}_k\\
x\supind{1}_k&=&\phi(\sum_{j=0}^2w_{jk}x_j),\;{\rm for}\,k=1:5 \\
x&=& \left[\begin{array}{c}x_0\\x_1\\x_2\\ \end{array}\right]\\
W
&=& [w_{jk}]\;=\;
20\left[\begin{array}{ccccc}
	1&0&2&2&2\\
	1&1&0&-1&-0.5\\
	-1&1&-1&0&1\\
	\end{array}\right]
\\
\phi(u)&=&\frac{1}{1+e^{-u}}\;\;{\rm the\,sigmoid\,function}\\
\beta_{0}&=&-4.9,\;\beta_{1:5}\;=\;1
\eeqa
{\bf a.} Plot the decision boundary or alternatively the decision regions 
 $D_{\pm}=\{x\,|\,f(x)\lessgtr 0\}$ of this neural network on $[-10,\,10]^2$.

 % NOTE is x 20 a column concat ?
{\bf b.} Same as {\bf a} for $\beta_0=-3.9$.


{\bf Problem 2 -- Descent algorithms for training a neural network}
\\
This problem asks you to train a neural network to classify the data
sets given on the Assignments web page. The inputs are
2-dimensional, outputs are $\pm 1$, one data point/line. {\em Submit
  the code for this problem.}

Objective to minimize is
$\hatl\llogit(\beta,W)=-\frac{1}{n}$log-likelihood($\dataset|\beta,W$) and
$\beta\in\rrr^{m+1},W\in\rrr^{n\times m}$ are the neural net
parameters. However, you will also track the misclassification error $L_{01}$ during training and at testing. 

The algorithm is steepest descent with fixed step size. You need to implement the algorithm yourself.



{\bf a.} Plot the data set  $\dataset$ given {\tt hw5-nn-train-100.dat} $\in[-10,10]^2$ representing each class with a different color or symbol. 

{\bf b.} Based on the plot in {\bf a.}, is it possible to get $\hatl_{01}=0$ for  $m=2$? Explain in 1-2 sentences.

{\bf c.} Implement GD for a two layer neural network in a script {\tt hw5-main.py}. In the main function, {\tt m}	is the number of hidden units, {\tt eta} is the learning rate for training, and {\tt maxiter}	is the maximum number of training iterations. {\tt tol} is the threshold for the stopping criterion.

\textbf{Tip 1}: Use {\tt ./hw5-eval.sh} to check the accuracy of the evaluation and the output format.

\textbf{Tip 2}: We will evaluate your code on a different data set than the one provided, so try to make your solution general!

{\bf d.} Use the function you implemented in {\bf c.} to train a neural network on $\dataset$. Set the stopping criterion $1-\frac{\hatl^{t+1}}{\hatl^t}\leq tol$ with $tol=10^{-5}$. 
Choose a number $m\geq 5$ hidden units and an appropriate step size $\eta$. 
Obtain the best empirical $\hatl_{01}$ you can. {\em Note that larger $m$ values may be easier to train.} 
Explain how you
chose the initial points. {\em It's ok/recommended to plot the data and look at it or
even to make a sketch of the solution you want to find. Do not show these trials, but you can describe them briefly. Show only the results from the ``best'' training run you obtained.
The training algorithm will converge to a local optimum.  It's OK to
look at this local optimum and try other initial points if the found
optimum is bad.}

Report: $m$, $\eta$, number steps $T$ to stopping. 

Make the following plots
\benum
\item The training loss $\hatl\llogit^t$ and the test set loss $L\llogit^t$ versus $t=1:T$ on the same graph.
\item The training misclassification error  $\hatl_{01}^t$ and the test set misclassification error $L_{01}^t$ versus $t=1:T$ on the same graph.
 \item The relative change in loss $\frac{\hatl\llogit^{t-1}-\hatl\llogit^t}{\hatl\llogit^t}$ versus $t=1:T$.
\eenum

{\bf e.} If you arrived here, you succeeded to write code that trains a neural network, and track some of the variables during training. This is necessary, but not sufficient. You also need to use the plots you made to evaluate if the training is completed correctly. Based on your work in {\bf a,b,c,d} answer the following:

\benum
\item I believe that my training algorithm has converged to a (local) optimum. \hspace{2em}{\bf YES}\hspace{2em}{\bf NO}
\item Has your neural network overfitted the training data $\dataset$?  \hspace{2em}{\bf YES}\hspace{2em}{\bf NO}

\eenum


{\bf f.} Plot the final decision region superimposed on the data.
 
{\em Please make clear, well-scaled, well labeled graphs. }


{\bf g.} Repeat steps {\bf d, e} for the larger data set {\tt
  hw5-nn-train-10000.dat}. Use the same
parameter initialization as in {\bf d}. You can use {\tt hw5-nn-test.dat} as an eval set for your model.

{\bf h.} Plot the final decision region of the neural network from {\bf g.} superimposed on the plot from {\bf f.}. Okay to have a single plot for {\bf f,\,g}. 
{\em Do not plot the large data set to avoid cluttering the plot.}

\end{document}




