slides.tex

\documentclass{beamer}
\usepackage[utf8]{inputenc}

\usetheme{CambridgeUS}
\usecolortheme{default}
\usepackage{tikz}
\usepackage{caption}
\captionsetup{font=scriptsize,labelfont=normalsize}
\captionsetup[figure]{labelformat=empty}
\captionsetup[table]{labelformat=empty}
\captionsetup[subfigure]{labelformat=empty}
\setbeamertemplate{caption}{\raggedright\insertcaption\par}


\newcommand{\smallindent}{\hphantom{N}}

\usepackage{lmodern}
\usepackage{siunitx}
\usepackage{booktabs}
\usepackage{etoolbox}

\usepackage{amsmath,amssymb}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{xcolor,colortbl}

\definecolor{orange}{rgb}{0.8, 0.33, 0.0}
\definecolor{cblue}{rgb}{0.74, 0.83, 0.9}

\setbeamercolor{alerted text}{fg=orange}
\setbeamercolor*{palette primary}{fg = orange}
\setbeamercolor*{palette secondary}{fg = orange}
\setbeamercolor*{palette tertiary}{bg=orange, fg = white}
%\setbeamercolor*{palette quaternary}{bg=orange, fg = green}


\newcommand{\mycomment}[1]{}

%------------------------------------------------------------
%This block of code defines the information to appear in the
%Title page
\title[Master Thesis] %optional
{Advancing Packet-Level Traffic Predictions\\ with Transformers}
\date{\tiny September 1, 2022}
\author[ Siddhant Ray] % (optional)
{Siddhant Ray}

\institute[ETH Zürich] % (optional)
{
  D-ITET \\
  ETH Zürich
}



%\date[September 2022] % (optional)

\titlegraphic{
    \includegraphics[width=3cm]{figures/nsg_logo.pdf}
    \hspace{2cm}
     \includegraphics[width=3cm]{figures/eth_logo.pdf}
}

%End of title page configuration block
%------------------------------------------------------------



%------------------------------------------------------------
%The next block of commands puts the table of contents at the 
%beginning of each section and highlights the current section:

%\AtBeginSection[]
%{
%  \begin{frame}
 %   \frametitle{Table of Contents}
 %   \tableofcontents[currentsection]
 % \end{frame}
%}
%------------------------------------------------------------


\begin{document}

%The next statement creates the title page.
\frame{\titlepage}


%---------------------------------------------------------
%This block of code is for the table of contents after
%the title page
%\frametitle{Table of Contents}
%
%end{frame}
%---------------------------------------------------------


\section{Motivation}

\begin{frame}
\frametitle{Problems with machine learning methods in networks today}
\pause
\begin{itemize}
    \item<1-> \alert{No generalization: } Only work on specific tasks trained on 
    \pause
    \item<2-> \alert{Limited scope: } Models fail outside original training environment 
    \pause
    \item<3-> \alert{Resource intensive: } Always re-doing training from scratch
   
\end{itemize}
\end{frame}



%---------------------------------------------------------
%Changing visivility of the text
\begin{frame}
\frametitle{Why use Transformers?}
\pause
\begin{itemize}
    \item<1-> Efficient learning with \alert {\emph{attention}} mechanism
    \item<1-> Generalizing using \alert{\emph{large datasets}} available
    \pause
    \item<2-> State-of-art for \alert{\emph{sequence}} learning problems
     \item<2-> Network \alert{\emph{packet data}} is a sequence
\end{itemize}
\end{frame}


%---------------------------------------------------------
%Example of the \pause command
\begin{frame}
\frametitle{Transformer's unprecendented generalization in NLP \& CV}
\pause

BERT: Generalizing to many tasks in NLP

\begin{itemize}  
    \item<1-> Sentiment analysis
    \item<1-> Question answering
    \item<1-> Paraphrase detection
\end{itemize}
\pause

Vision Transformer: Generalizing to many tasks in CV

\begin{itemize}  
    \item<1-> Image classification
    \item<1-> Object detection
    \item<1-> Image segmentation
\end{itemize}



\end{frame}
%---------------------------------------------------------


\begin{frame}
\frametitle{Our Transformer prototype}

We present our Network Traffic Transformer (NTT):
\pause

\begin{figure}[!hbt]
  \begin{center}
    \includegraphics[scale=1.1]{figures/vision.pdf}
    \caption{Pre-train today, fine-tune and re-use tomorrow}
    \label{fig:vision}
  \end{center}
\end{figure}
    
    
\end{frame}

%---------------------------------------------------------

\section{Design}

%---------------------------------------------------------
%Highlighting text

\begin{frame}
\frametitle{NTT needs networking domain specific features}
\pause

\begin{figure}[!hbt]
  \begin{center}
    \includegraphics[scale=0.8]{figures/architecture_ntt.pdf}
    \caption{The Network Traffic Transformer (NTT) with
        an embedding layer, % for feature extraction,
        an aggregation layer,  a transformer encoder
        and a task-specific replaceable decoder.}
    \label{fig:ntt}
  \end{center}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{NTT needs networking domain specific features}

Feature selection for initial NTT's input data:
\pause 
\begin{itemize}
    \item<1-> \alert{Relative timestamp:} To learn sequence order
    \item<1-> \alert{End-to-end delay:} To learn network state information
    \item<1-> \alert{Packet size:} To learn packet state information
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Training objectives for the NTT architecture}

NTT's learning objectives: 
\pause 
\begin{itemize}
    \item<1-> \alert{Learn network dynamics:}  Reconstruct masked delay values
    \pause
    \item<1-> \alert{Scale to large sequences:}  Aggregate inputs ($ > 1000s$ of values)
    \end{itemize}
\end{frame}



\begin{frame}
\frametitle{Ensuring varied dynamics in our pre-training datasets}


\begin{figure}[h]
  \begin{center}
    \includegraphics[scale=0.8]{figures/simple_topo.pdf}
    \caption{Initial topology for data generation}
    \label{fig:topo}
  \end{center}
\end{figure}
 
 \pause

\begin{itemize}
    \item<1-> Varied start times across sender application flows
    \item<1-> Enough variance in pre-training data dynamics 
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Ensuring varied dynamics in our pre-training datasets}


\begin{figure}[h]
     
    \captionsetup[subfigure]{justification=centering}
    \begin{subfigure}[h]{0.5\textwidth}
    	\begin{center}
        \centering
        \includegraphics[scale=0.5]{figures/delay.pdf}
        \caption{Delay CDF, single simulation run}
        \end{center}
    \end{subfigure}%
    ~ 
    \begin{subfigure}[h]{0.5\textwidth}
    	\begin{center}
        \centering
        \includegraphics[scale=0.5]{figures/queue_profile_A.pdf}
        \caption{Bottleneck queue profile on the single-path topology}
        \end{center}    
    \end{subfigure}
    \caption{Distribution plots on pre-training data}
    \label{fig:datadist}
    \end{figure}

\end{frame}



%---------------------------------------------------------

\section{Evaluation}

%---------------------------------------------------------
%Two columns
\begin{frame}
\frametitle{Our NTT allows for generalization on network dynamics}


\begin{figure}[h]
  \begin{center}
    \includegraphics[scale=0.8]{figures/simple_topo_ft.pdf}
    \caption{Fine-tuning data generation, single path topology}
    \label{fig:topo_ft}
  \end{center}
\end{figure}

\pause 

\begin{itemize}
    \item<1-> Two bottleneck dynamics to learn 
    \item<1-> Packet-level fine-tuning task : Predict last delay
    \item<1-> Flow-level fine-tuning task : Predict Message Completion Time (MCT)
    \end{itemize}

\end{frame}


\begin{frame}
\frametitle{Our NTT allows for generalization on network dynamics}


\begin{table}[htbp]
\footnotesize
\centering
\sisetup{detect-weight,mode=text}
% for avoiding siunitx using bold extended
\renewrobustcmd{\bfseries}{\fontseries{b}\selectfont}
\renewrobustcmd{\boldmath}{}
\newrobustcmd{\B}{\bfseries}
\begin{tabular}{ l   c   c  c }
\toprule
\emph{all values $\times10^{-3}$} & Pre-training  & \multicolumn{2}{c}{Fine-tuning} \\
\cmidrule{3-4}
                                                       & {Delay}        & {Delay}                           & {log (MCT)} \\
\midrule
\em{NTT}                                                 &                &                                   &           \\
    \rowcolor{cblue}
    \smallindent Pre-trained                                 & \B 0.072          & \B 0.097                             & \B 65        \\
    \smallindent From scratch                                & {-}            & 0.313                             & 117       \\
    \noalign{\vskip 1mm}
    \em{Baselines}                                                                                                                 \\
    \smallindent ARMA                                            & 1.800        &  1.180                              &1412 \\
    \smallindent Last observed                               & 0.142          & 0.121                             & 2189      \\
    \smallindent EWMA                                        & 0.259          & 0.211                             & 1147      \\
    \noalign{\vskip 1mm}
    \em{NTT (Ablated)}                                                                                                        \\
    \smallindent No aggregation                              & 0.258          & 0.430                             & 61        \\
    \smallindent Fixed aggregation                           & 0.055          & 0.134                             & 115       \\[0.75mm]

    \smallindent Without packet size                         & 0.001          & 8.688                             & 94        \\
    \smallindent Without delay                               & 15.797         & 10.898                            & 802       \\
     \bottomrule

\end{tabular}
\caption{Mean Squared Error (MSE) for all NTT models and tasks for the single path topology (lower is better)}
\label{eval:table1}
\end{table}
\end{frame}

\mycomment{

\begin{frame}
\frametitle{Variable masked pre-training}

Variable masking strategies: For \emph{bi-directional} learning
\pause
\begin{itemize}
    \item<1-> Masking over last 16 delays, no mask over aggregates
    \item<1-> Masking over last 32 delays, no mask over aggregates 
    \pause
    \item<2-> Choose mask over encoded states,  also mask over aggregates
     \item<2-> Choose mask over aggregation levels,  also mask over aggregates
\end{itemize}


\end{frame}


\begin{frame}
\frametitle{Variable masked pre-training}


\begin{figure}[h]
    \centering
    \begin{subfigure}[h]{0.5\textwidth}
        \centering
        \includegraphics[scale=0.5]{figures/finetune_mct_loss_comparison.pdf}
        \caption{Pre-train with no mask over aggregated delays}
    \end{subfigure}%
    ~ 
    \begin{subfigure}[h]{0.5\textwidth}
        \centering
        \includegraphics[scale=0.5]{figures/finetune_mct_loss_comparison_agg.pdf}
        \caption{Pre-train with also mask over aggregated delays}
    \end{subfigure}
    \caption{Mean-square error on MCT predictions with different pre-training masks}
    \label{fig:mct_mask}
\end{figure}
\end{frame}

}

\begin{frame}
\frametitle{NTT works on multi-path topologies}


\begin{figure}[h]
  \begin{center}
    \includegraphics[scale=0.58]{figures/complex_topo.pdf}
    \caption{Fine-tuning data generation on multi-path topology}
    \label{fig:topo_ft_big}
  \end{center}
\end{figure}

\pause 
\vspace{-0.6cm}
\begin{itemize}
    \item<1-> Path delays vary as per number of links 
    \item<1-> Receiver ID as IP address proxy
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{NTT works on multi-path topologies}


\begin{table}[htbp]
\scriptsize
\centering
\sisetup{detect-weight,mode=text}
% for avoiding siunitx using bold extended
\renewrobustcmd{\bfseries}{\fontseries{b}\selectfont}
\renewrobustcmd{\boldmath}{}
\newrobustcmd{\B}{\bfseries}

\begin{tabular}{ l   c   c  }
\toprule
\emph{Model} &  MSE: Delay Prediction & \# of Epochs    \\
			&		\emph{all values$\times10^{-3}$}  & trained 	\\ 
			

\midrule
\em{NTT}                                                               &       \\      
    \rowcolor{cblue}                                
    \smallindent Pre-trained  +   Fine-tune (full)                                  & \B 0.004   &     \B 5    \\          
    \rowcolor{cblue}        
    \smallindent Pre-trained  +   Fine-tune ($10\%$)                                  & \B 0.035   & \B 12  \\                        
    \smallindent From scratch  + Fine-tune (full)                                       & 5.2    &   10 \\                      
     \smallindent From scratch  + Fine-tune ($10\%$)                                    & 8.2  & 15 \\                          
 \em{Baselines} &       \\  
    \smallindent ARMA  &       					4.2	& - \\  
    \smallindent Last observed &     				11.2	& -  \\  
    \smallindent  	EWMA &     					4.0	& -  \\  
\em{NTT (Ablated)}           &       \\       
     \smallindent Pre-trained  +   Fine-tune (full)  : No Receiver ID                              & 2.8   &     8    \\   
      \smallindent From scratch  + Fine-tune (full) : No Receiver ID                                  & 2.7    & 15  \\         
 
 \bottomrule

\end{tabular}
\caption{Fine-tuning NTT on the multi-path topology (lower is better)}
\label{eval:table5}
\end{table}

\end{frame}






%---------------------------------------------------------

\section{Future}

\begin{frame}
\frametitle{How do we improve the NTT further?}
\pause
\begin{itemize}
    \item<1-> NTT Scaling
    \begin{itemize}
    	\item<1-> Learn additional network features.
	\item<1-> Learn on larger topologies.
    \end{itemize}
    \vspace{0.2cm}
    \pause
    \item<2-> Federated Learning
    \begin{itemize}
    	\item<2-> Share models, not data.
	\item<2->Keep data private.
	\end{itemize}
    \vspace{0.2cm}
    \pause
    
    \item<3-> Continual learning
    \begin{itemize}
    	\item<3->  Re-train with time, prevent forgetting.
	\item<3->  Learn evolved dynamics.
	\end{itemize}
\end{itemize}
\end{frame}

\section{Concluding remarks}
\begin{frame}
\frametitle{Recap: Our NTT architecture demonstrates that}
%\pause
\mycomment{

\begin{figure}[h]
  \begin{center}
    \includegraphics[scale=0.55]{figures/questions.pdf}
    \label{fig:questions}
  \end{center}
\end{figure}
}

\begin{itemize}
   \pause
    \item<1-> Learning network dynamics is possible
      \begin{itemize}
    	\item<1-> NTT learns network dynamics from packet sequences
	\item<1-> Pre-trained NTT can be re-used easily
     \end{itemize}
    \vspace{0.2cm}
    \pause
    \item<2-> Generalizing power of the NTT
     \begin{itemize}
    	\item<2->  Can generalize to new environments: Packet level 
	\item<2-> Can generalize to new tasks: Flow level 
	\end{itemize}
\end{itemize}
\end{frame}



\end{document}