presentation.tex

\documentclass[utf8x,hyperref={pdfpagelabels=false}]{beamer}

\usepackage[utf8x]{inputenc}
\usepackage[OT1]{fontenc}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{tikz}
\usetikzlibrary{shapes.arrows}
%\logo{\includegraphics[width=.8in]{UdeM_NoirBleu_logo_Marie_crop}}


\usetheme{Malmoe}  % Now it's a beamer presentation with the lisa theme!
\usecolortheme{beaver}
\setbeamertemplate{footline}[page number]
\setbeamertemplate{navigation symbols}{}

\lstloadlanguages{Python}

\definecolor{darkgreen}{RGB}{0,93,21}
\definecolor{greenblue}{RGB}{40,110,126}
\definecolor{lightgray}{RGB}{246,246,246}
\definecolor{bordergray}{RGB}{193,193,193}
\definecolor{lightblue}{RGB}{0,114,168}
\definecolor{methblue}{RGB}{0,31,108}

\newcommand{\superscript}[1]{\ensuremath{^{\textrm{#1}}}}

\mode<presentation>

\title{Introduction to Theano}
\author{%
\footnotesize
Arnaud Bergeron \newline
(slides adapted by Frédéric Bastien from slides by Ian G.) \newline
(further adapted by Arnaud Bergeron)
}
\date{February 26, 2015}

\lstdefinestyle{theano}{
language=Python,
basicstyle=\fontfamily{pcr}\selectfont\footnotesize,
keywordstyle=\color{darkgreen}\bfseries,
commentstyle=\color{greenblue}\itshape,
%commentstyle=\color{blue}\itshape,
stringstyle=\color{violet},
showstringspaces=false,
tabsize=4,
backgroundcolor=\color{lightgray},
frame=single,
emph={[2]__init__,make_node,perform,infer_shape,c_code,make_thunk,grad,R_op},emphstyle={[2]\color{methblue}},
emph={[3]self},emphstyle={[3]\color{darkgreen}},
moredelim=**[is][{\color{red}}]{`}{`}
}

% We don't have code till the end of the file.
\lstdefinestyle{output}{
language={},
basicstyle=\ttfamily\footnotesize,
backgroundcolor=\color{white},
frame={},
breaklines=true,
emph={[2]},
emph={[3]},
}

\lstset{style=theano}

\newcommand{\code}[1]{\lstinline[emph={[2]}]|#1|}

\begin{document}

\begin{frame}[plain]
 \titlepage
% \vspace{-5em}
% \includegraphics[width=1in]{../hpcs2011_tutorial/pics/lisabook_logo_text_3.png}
% \hfill
% \includegraphics[width=.8in]{../hpcs2011_tutorial/pics/UdeM_NoirBleu_logo_Marie_crop}
\end{frame}

\section{Outline}
\begin{frame}{High level}\setcounter{page}{1}
  \begin{itemize}
  \item Overview of library (3 min)
  \item Building expressions (30 min)
  \item Compiling and running expressions (30 min)
  \item Modifying expressions (25 min)
  \item Debugging (30 min)
  \item Citing Theano (2 min)
  \end{itemize}
\end{frame}


\begin{frame}{Overview of Library}
  Theano is many things
  \begin{itemize}
  \item Language
  \item Compiler
  \item Python library
  \end{itemize}
\end{frame}

\begin{frame}{Overview}
  Theano language:
  \begin{itemize}
  \item Operations on scalar, vector, matrix, tensor, and sparse variables
  \item Linear algebra
  \item Element-wise nonlinearities
  \item Convolution
  \item Extensible
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Overview}
  Using Theano:
  \begin{itemize}
  \item define expression $f(x,y) = x + y$
\begin{lstlisting}
>>> z = x + y
\end{lstlisting}
  \item compile expression
\begin{lstlisting}
>>> f = theano.function([x, y], z)
\end{lstlisting}
  \item execute expression
\begin{lstlisting}
>>> f(1, 2)
3
\end{lstlisting}
  \end{itemize}
\end{frame}

\section{Building}

\begin{frame}{Building expressions}
  \begin{itemize}
  \item Scalars
  \item Vectors
  \item Matrices
  \item Tensors
  \item Broadcasting
  \item Reduction
  \item Dimshuffle
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Scalar math}
\begin{lstlisting}
from theano import tensor as T
x = T.scalar()
y = T.scalar()
z = x+y
w = z*x
a = T.sqrt(w)
b = T.exp(a)
c = a ** b
d = T.log(c)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Vector math}
\begin{lstlisting}
from theano import tensor as T
x = T.vector()
y = T.vector()
# Scalar math applied elementwise
a = x * y
# Vector dot product
b = T.dot(x, y)
# Broadcasting
c = a + b
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Matrix math}
\begin{lstlisting}
from theano import tensor as T
x = T.matrix()
y = T.matrix()
a = T.vector()
# Matrix-matrix product
b = T.dot(x, y)
# Matrix-vector product
c = T.dot(x, a)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Tensors}
   \begin{itemize}
    \item Dimensionality defined by length of ``broadcastable'' argument
    \item Can add (or do other elemwise op) on two
      tensors with same dimensionality
    \item Duplicate tensors along broadcastable axes to
      make size match
  \end{itemize}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
\end{lstlisting}
\end{frame}

\begin{frame}{Broadcasting}
\begin{tabular}{lcccccccl}
    &
    \begin{tabular}{cc}
        1 & 2 \\
        3 & 4 \\
        5 & 6 \\
    \end{tabular} &
    + &
    \begin{tabular}{cc}
        1 & 2 \\
    \end{tabular} &
    = &
    \begin{tabular}{cc}
        1 & 2 \\
        3 & 4 \\
        5 & 6 \\
    \end{tabular} &
    + &
    \begin{tabular}{cc}
        1 & 2 \\
        \color{blue} 1 & \color{blue} 2 \\
        \color{blue} 1 & \color{blue} 2 \\
    \end{tabular} &
    \hspace{-1.3em}
    \tikz[baseline={([yshift=-.5ex]current bounding box.center)}]{
        \draw [->, very thick] (0,0) -- (0,-1.2);
    } \\[1.5em]
    shape: & (3, 2) & & (2,) & & (3, 2) & & ({\color{blue}3}, 2) &
\end{tabular}
\vfill
\begin{itemize}
    \item Pad shape with 1s on the left : $(2,) \equiv (1,2)$
    \item Two dimensions are compatible when they have the same length or one of them is broadcastable
    \item broadcastable dimensions must have a length of 1
    \item Adding tensors of shape (8, 1, 6, 1) and (7, 1, 5) gives a tensor of shape (8, 7, 6, 5)
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Reductions}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
total = x.sum()
marginals = x.sum(axis=(0, 2))
mx = x.max(axis=1)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Dimshuffle}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
y = x.dimshuffle((2, 1, 0))
a = T.matrix()
b = a.T
# Same as b
c = a.dimshuffle((0, 1))
# Adding to larger tensor
d = a.dimshuffle((0, 1, 'x'))
e = a + d
\end{lstlisting}
\end{frame}

\begin{frame}{Exercices}
Work through the "Building Expressions" section of the ipython notebook.
\end{frame}

\section{Compiling/Running}
\begin{frame}{Compiling and running expression}
  \begin{itemize}
  \item \code{theano.function}
  \item shared variables and updates
  \item compilation modes
  \item compilation for GPU
  \item optimizations
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{\code{theano.function}}

\begin{lstlisting}
>>> from theano import tensor as T
>>> x = T.scalar()
>>> y = T.scalar()
>>> from theano import function
>>> # first arg is list of symbolic inputs
>>> # second arg is symbolic output
>>> f = function([x, y], x + y)
>>> # Call it with numerical values
>>> # Get a numerical output
>>> f(1., 2.)
array(3.0)
\end{lstlisting}
\end{frame}

\begin{frame}{Shared variables}
  \begin{itemize}
  \item It’s hard to do much with purely functional programming
  \item \emph{shared variables} add just a little bit of imperative programming
  \item A \emph{shared variable} is a buffer that stores a numerical value for a Theano variable
  \item Can write to as many shared variables as you want, once each, at the end of the function
  \item  Modify outside Theano function with \code{get_value()} and \code{set_value()} methods.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Shared variable example}
\begin{lstlisting}
>>> from theano import shared
>>> x = shared(0.)
# Can also use a dict for more complex code
>>> updates = [(x,  x + 1)]
>>> f = function([], updates=updates)
>>> f()
>>> x.get_value()
1.0
>>> x.set_value(100.)
>>> f()
>>> x.get_value()
101.0
\end{lstlisting}
\end{frame}

\begin{frame}{Which dict?}
  \begin{itemize}
  \item Use theano.compat.python2x.OrderedDict
  \item Not collections.OrderedDict
  \begin{itemize}
  \item This isn’t available in older versions of python, and will limit the portability of your code.
  \end{itemize}
  \item Not \code{\{\}} aka dict
  \begin{itemize}
  \item The iteration order of this built-in class is not deterministic so if Theano accepted this, the same script could compile different C programs each time you run it.
  \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Compilation modes}
  \begin{itemize}
  \item Can compile in different modes to get different kinds of programs
  \item Can specify these modes very precisely with arguments to \code{theano.function()}
  \item Can use a few quick presets with environment variable flags
  \end{itemize}
\end{frame}

\begin{frame}{Example preset compilation modes}
  \begin{description}[FAST\_RUN]
  \item[FAST\_RUN] Default. Spends a lot of time on
compilation to get an executable that runs
fast.
  \item[FAST\_COMPILE] Doesn’t spend much time compiling.
Executable usually uses python
instead of compiled C code. Runs slow.
  \item[DEBUG\_MODE] Adds lots of checks.
Raises error messages in situations other modes don't check for.
  \end{description}
\end{frame}

\begin{frame}{Compilation for GPU}
  \begin{itemize}
  \item Theano's current back-end only supports 32 bit on GPU
  \item CUDA supports 64 bit, but is slow in gamer card
  \item \code{T.fscalar}, \code{T.fvector}, \code{T.fmatrix} are all 32 bit
  \item \code{T.scalar}, \code{T.vector}, \code{T.matrix} resolve to 32 or 64 bit depending on theano’s floatX flag
  \item floatX is float64 by default, set it to float32
  \item Set the device flag to gpu (or a specific gpu, like gpu0)
  \item Optional: warn\_float64=\{'ignore', 'warn', 'raise', 'pdb'\}
  \end{itemize}
\end{frame}

\begin{frame}{Optimizations}
  \begin{itemize}
  \item Theano changes the symbolic expressions
    you write before converting them to C code
  \item It makes them faster
  \begin{itemize}
  \item $(x+y)+(x+y) \to 2\times(x + y)$
  \end{itemize}
  \item It makes them more stable
  \begin{itemize}
  \item $\exp(a)/\sum{\exp(a)} \to \operatorname{softmax}(a)$
  \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Optimizations (2)}
Sometimes optimizations discard error checking and produce incorrect output rather than an exception.
\begin{lstlisting}
>>> x = T.scalar()
>>> f = function([x], x/x)
>>> f(0.)
array(1.0)
\end{lstlisting}
\end{frame}

\begin{frame}{Exercises}
Work through the "Compiling and Running" section of the ipython notebook.
\end{frame}

\section{Modifying expressions}
\begin{frame}{Modifying expressions}
  \begin{itemize}
  \item The \code{grad()} method
  \item Variable nodes
  \item Types
  \item Ops
  \item Apply nodes
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{The \code{grad()} method}
\begin{lstlisting}
>>> x = T.scalar('x')
>>> y = 2. * x
>>> g = T.grad(y, x)
>>> from theano.printing import min_informative_str
# Print the unoptimized graph
>>> print min_informative_str(g)
A. Elemwise{mul}
 B. Elemwise{second,no_inplace}
  C. Elemwise{mul,no_inplace}
   D. TensorConstant{2.0}
   E. x
  F. TensorConstant{1.0}
 <D>
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{The \code{grad()} method}
\begin{lstlisting}
>>> x = T.scalar('x')
>>> y = 2. * x
>>> g = T.grad(y, x)
>>> from theano.printing import min_informative_str
# Print the optimized graph
>>> f = theano.function([x], g)
>>> theano.printing.debugprint(f)
DeepCopyOp [@A] ''   0
 |TensorConstant{2.0} [@B]
\end{lstlisting}
\end{frame}

\begin{frame}{Theano variables}
  \begin{itemize}
  \item A \emph{variable} is a theano expression.
  \item Can come from \code{T.scalar()}, \code{T.matrix()}, etc.
  \item Can come from doing operations on other variables.
  \item Every variable has a type field, identifying its \emph{type}, such as \code{TensorType((True, False), 'float32')}
  \item Variables can be thought of as nodes in a graph
  \end{itemize}
\end{frame}

\begin{frame}{Ops}
  \begin{itemize}
  \item  An Op is any class that describes a function operating on some variables
  \item Can call the op on some variables to get a
new variable or variables
  \item An Op class can supply other forms of
information about the function, such as its
derivative
  \end{itemize}
\end{frame}

\begin{frame}{Apply nodes}
  \begin{itemize}
  \item The Apply class is a specific instance of an application of an Op.
  \item Notable fields:
    \begin{description}[\texttt{outputs}]
    \item[\texttt{op}] The Op to be applied
    \item[\texttt{inputs}] The Variables to be used as input
    \item[\texttt{outputs}] The Variables produced
    \end{description}
  \item The \code{owner} field on variables identifies the Apply that created it.
  \item Variable and Apply instances are nodes and owner/
    inputs/outputs identify edges in a Theano graph.
  \end{itemize}
\end{frame}

\begin{frame}{Exercises}
Work through the "Modifying" section in the ipython notebook.
\end{frame}

\section{Debugging}
\begin{frame}{Debugging}
  \begin{itemize}
  \item DEBUG\_MODE
  \item Error message
  \item \code{theano.printing.debugprint()}
  \item \code{min_informative_str()}
  \item compute\_test\_value
  \item Accessing the FunctionGraph
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Error message: code}
\begin{lstlisting}
import numpy as np
import theano
import theano.tensor as T
x = T.vector()
y = T.vector()
z = x + x
z = z + y
f = theano.function([x, y], z)
f(np.ones((2,)), np.ones((3,)))
\end{lstlisting}
\end{frame}

\begin{frame}[fragile,allowframebreaks]{Error message}
\vspace{1em}
\begin{lstlisting}[style=output]
Traceback (most recent call last):
  File "test.py", line 9, in <module>
    f(np.ones((2,)), np.ones((3,)))
  File "/Users/anakha/Library/Python/2.7/site-packages/theano/compile/function_module.py", line 606, in __call__
    storage_map=self.fn.storage_map)
  File "/Users/anakha/Library/Python/2.7/site-packages/theano/compile/function_module.py", line 595, in __call__
    outputs = self.fn()
ValueError: Input dimension mis-match. (input[0].shape[0] = 3, input[1].shape[0] = 2)
Apply node that caused the error: Elemwise{add,no_inplace}(<TensorType(float64, vector)>, <TensorType(float64, vector)>, <TensorType(float64, vector)>)
Inputs types: [TensorType(float64, vector), TensorType(float64, vector), TensorType(float64, vector)]
Inputs shapes: [(3,), (2,), (2,)]
Inputs strides: [(8,), (8,), (8,)]
Inputs values: [array([ 1.,  1.,  1.]), array([ 1.,  1.]), array([ 1.,  1.])]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag  'optimizer=fast_compile'. If that does not work, Theano  optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high'  for a debugprint and storage map footprint of this apply node.
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Error message: exception\_verbosity=high}
\begin{lstlisting}[style=output]
Debugprint of the apply node: 
Elemwise{add,no_inplace} [@A] <TensorType(float64, vector)> ''   
 |<TensorType(float64, vector)> [@B] <TensorType(float64, vector)>
 |<TensorType(float64, vector)> [@C] <TensorType(float64, vector)>
 |<TensorType(float64, vector)> [@C] <TensorType(float64, vector)>

Storage map footprint:
 - <TensorType(float64, vector)>, Shape: (3,), ElemSize: 8 Byte(s), TotalSize: 24 Byte(s)
 - <TensorType(float64, vector)>, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Error message: optimizer=fast\_compile}
\begin{lstlisting}[style=output]
Backtrace when the node is created:
  File "test.py", line 7, in <module>
    z = z + y
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{debugprint}
\begin{lstlisting}
>>> from theano.printing import debugprint
>>> debugprint(a)
Elemwise{mul,no_inplace} [@A] ''
 |TensorConstant{2.0} [@B]
 |Elemwise{add,no_inplace} [@C] 'z'
   |<TensorType(float64, scalar)> [@D]
   |<TensorType(float64, scalar)> [@E]
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{min\_informative\_str}
\begin{lstlisting}
>>> x = T.scalar()
>>> y = T.scalar()
>>> z = x + y
>>> z.name = 'z'
>>> a = 2. * z
>>> from theano.printing import min_informative_str
>>> print min_informative_str(a)
A. Elemwise{mul,no_inplace}
 B. TensorConstant{2.0}
 C. z
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{compute\_test\_value}
\begin{lstlisting}
>>> from theano import config
>>> config.compute_test_value = 'raise'
>>> x = T.vector()
>>> import numpy as np
>>> x.tag.test_value = np.ones((2,))
>>> y = T.vector()
>>> y.tag.test_value = np.ones((3,))
>>> x + y
...
ValueError: Input dimension mis-match.
(input[0].shape[0] = 2, input[1].shape[0] = 3)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Accessing a function’s fgraph}
\begin{lstlisting}
>>> x = T.scalar()
>>> y = x / x
>>> f = function([x], y)
>>> debugprint(f.maker.fgraph.outputs[0])
DeepCopyOp [@A] ''
 |TensorConstant{1.0} [@B]
\end{lstlisting}
\end{frame}

\begin{frame}{Exercises}
Work through the "Debugging" section of the ipython notebook.
\end{frame}

\section*{}
\begin{frame}{Citing Theano}
Please cite both of the following papers in all work that uses Theano:
  \begin{itemize}
  \item Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and
     Bengio,Yoshua. Theano: new features and speed improvements. Deep Learning and Unsupervised Feature Learning NIPS 2012
    Workshop, 2012.
  \item Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-
     Farley, David, and Bengio,Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific
      Computing Conference (SciPy), June 2010. Oral Presentation.
  \end{itemize}
\end{frame}

\begin{frame}{Example acknowledgments}
We would like to thank the developers of Theano \textbackslash citep\{bergstra+al:2010-scipy,Bastien-Theano-2012\}.
We would also like to thank NSERC, Compute Canada, and Calcul Québec for providing computational resources.
\end{frame}


\begin{frame}
\begin{center}
\bibliography{strings,strings-short,ml,aigaion-shorter}
\Huge
Questions?
\end{center}
\end{frame}


\end{document}