-
Notifications
You must be signed in to change notification settings - Fork 0
/
MLE_ND_P5_SMC.html
392 lines (387 loc) · 21.7 KB
/
MLE_ND_P5_SMC.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
<!DOCTYPE HTML>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=0.9*device-width">
<title>MLE_ND_P5_SMC</title>
<script src="https://sagecell.sagemath.org/static/embedded_sagecell.js"></script>
<script>$(function () {
sagecell.makeSagecell({inputLocation:'div.linked',linked:true,evalButtonText:'Run Linked Cells'});
sagecell.makeSagecell({inputLocation:'div.sage',evalButtonText:'Run'});
});
</script>
</head>
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color:#b8e2fc;}; a, p {color:royalblue; font-family:'Roboto';}
h1 {color:#191970; font-family:'Orbitron'; text-shadow:4px 4px 4px #ccc;}
h2, h3 {color:slategray; font-family:'Orbitron'; text-shadow:4px 4px 4px #ccc;}
h4 {color:#191970; font-family:'Roboto';}
.sagecell .CodeMirror-scroll {min-height:3em; max-height:70em;}
.sagecell table.table_form tr.row-a {background-color:lightgray;}
.sagecell table.table_form tr.row-b {background-color:#b8e2fc;}
.sagecell table.table_form td {padding:5px 15px; color:royalblue; font-family:'Roboto';}
.sagecell_sessionOutput, .sagecell_sessionOutput pre {color:royalblue; font-family:'Roboto';}
</style>
<body>
<h1>Machine Learning Engineer Nanodegree</h1>
<h2>Deep Learning</h2>
<h1>📑 P5: Build a Digit Recognition Program</h1>
<h2>Step 0: Load and Create Datasets</h2>
<h3>Resources</h3>
<a href="https://scikit-learn.org/stable/index.html">🕸scikit-learn. Machine Learning in Python </a>
<a href="http://scipy-lectures.org/">🕸Scipy Lecture Notes </a><br/>
<h3>Code Library</h3>
<div class="linked"><script type="text/x-sage">
import numpy,pandas,skimage,pylab,urllib,random,h5py,sys; pylab.style.use('seaborn-whitegrid')
import warnings; from sklearn.exceptions import DataConversionWarning
for el in [FutureWarning,UserWarning,RuntimeWarning,DataConversionWarning]: warnings.filterwarnings("ignore",category=el)
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn import datasets,metrics,preprocessing
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
def ohe(x,categories='auto'): return preprocessing.OneHotEncoder(categories=categories).fit(x.reshape(-1,1))\
.transform(x.reshape(-1,1)).toarray().astype('int16')
def sas(X,y): return train_test_split(X,y,test_size=0.2,random_state=1)
def random_rotate(img): return skimage.transform.rotate(img,numpy.random.randint(-20,20))
def label5d(label):
zeros=numpy.full((5-len(label)),10)
if len(label) >= 5: return label
else: return numpy.array(numpy.concatenate((zeros,label),axis=0))
def concatenate(images,labels,image_size):
i=0
concat_images=numpy.array(numpy.zeros((image_size,image_size*5))).reshape(1,image_size,image_size*5)
concat_labels=numpy.array(numpy.zeros(5))
while i<images.shape[0]:
if images.shape[0]-i<=5:
image=images[i]; label=labels[i]
for j in range(images.shape[0]-i-1):
image=numpy.concatenate((image,images[i+j+1]),axis=1)
label=numpy.concatenate((label,labels[i+j+1]),axis=0)
label=label5d(label); image=skimage.transform.resize(image,(image_size,image_size*5))
concat_images=numpy.vstack((concat_images,[image]))
concat_labels=numpy.vstack((concat_labels,label))
i+=5
else:
random_n=numpy.random.randint(3,6); image=images[i]; label=labels[i]
for k in range(random_n-1):
image=numpy.concatenate((image,images[i+k+1]),axis=1)
label=numpy.concatenate((label,labels[i+k+1]),axis=0)
label=label5d(label); image=skimage.transform.resize(image,(image_size,image_size*5))
concat_images=numpy.vstack((concat_images,[image]))
concat_labels=numpy.vstack((concat_labels,label)).astype('int16')
i+=random_n
concat_images=concat_images[1:]; concat_labels=concat_labels[1:]
return concat_images,concat_labels
</script></div>
<p></p>
<h3>Experimental Datasets</h3>
<h4>Dataset #1. Scikit-learn. Digits.</h4>
<div class="linked"><script type="text/x-sage">
digits=datasets.load_digits(n_class=10)
X_train1,X_test1,y_train1,y_test1=sas(digits.data,digits.target)
cy_train1=ohe(y_train1); cy_test1=ohe(y_test1)
X_train1.shape,y_train1.shape,X_test1.shape,y_test1.shape
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
n=5; img=numpy.zeros((10*n,10*n))
for i in range(n):
for j in range(n): img[(10*i+1):(10*i+9),(10*j+1):(10*j+9)]=X_train1[i*n+j].reshape((8,8))
pylab.figure(figsize=(n,n)); pylab.imshow(img,cmap=pylab.cm.Blues)
pylab.title('Examples of 64-dimensional digits')
pylab.xticks([]); pylab.yticks([]); pylab.show()
print(y_train1[:n*n])
</script></div>
<p></p>
<h4>Dataset #2. MNIST</h4>
<div class="linked"><script type="text/x-sage">
path='https://s3.amazonaws.com/img-datasets/'; f='mnist.npz'
input_file=urllib.urlopen(path+f); output_file=open(f,'wb'); output_file.write(input_file.read())
input_file.close(); output_file.close(); mnist=numpy.load(f)
[X_train2,y_train2,X_test2,y_test2]=[mnist[el]for el in ['x_train','y_train','x_test','y_test']]
cy_train2=ohe(y_train2); cy_test2=ohe(y_test2)
X_train2.shape,y_train2.shape,X_test2.shape,y_test2.shape
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
n=5; fig,ax=pylab.subplots(figsize=(n,n+1),nrows=n,ncols=n,sharex=True,sharey=True)
ax=ax.flatten(); [ax[i].imshow(X_train2[i],cmap=pylab.cm.Blues) for i in range(n*n)]
ax[0].set_xticks([]); ax[0].set_yticks([]); pylab.tight_layout(); pylab.gcf()
ax[2].set_title('Examples of the 784-dimensional digits',fontsize=12); pylab.show()
print(y_train2[:n*n])
</script></div>
<p></p>
<h4>Dataset #3. Synthetic - two digits</h4>
<div class="linked"><script type="text/x-sage">
img1=numpy.concatenate((X_train2[20],X_train2[30]),axis=1); zero1=numpy.zeros((14,56))
img2=numpy.concatenate((zero1,img1,zero1),axis=0)
pylab.figure(figsize=(3,4)); pylab.imshow(img2,cmap=pylab.cm.Blues)
pylab.title('Example of two concatenated\n784-dimensional digits')
pylab.xticks([]); pylab.yticks([]); pylab.show()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
n1,n2=int(X_train2.shape[0]/10),int(X_test2.shape[0]/10); h1=random.randrange(0,28); h2=28-h1
X_train56=[numpy.concatenate((X_train2[:n1][i],X_train2[n1:][i]),axis=1) for i in range(n1)]
X_train56=numpy.array([numpy.concatenate((numpy.zeros((h1,56)),X_train56[i],numpy.zeros((h2,56))),axis=0) for i in range(n1)])
X_test56=[numpy.concatenate((X_test2[:n2][i],X_test2[n2:][i]),axis=1) for i in range(n2)]
X_test56=numpy.array([numpy.concatenate((numpy.zeros((h1,56)),X_test56[i],numpy.zeros((h2,56))),axis=0) for i in range(n2)])
y_train56=numpy.array([numpy.concatenate((y_train2.reshape(-1,1)[:n1][i],y_train2.reshape(-1,1)[n1:][i])) for i in range(n1)])
y_test56=numpy.array([numpy.concatenate((y_test2.reshape(-1,1)[:n2][i],y_test2.reshape(-1,1)[n2:][i])) for i in range(n2)])
k=20; pylab.imshow(X_test56[k],cmap=pylab.cm.Blues); pylab.title('Example of the third dataset')
pylab.show(); print(y_test56[k])
X_train56=X_train56.reshape(-1,1,56,56).astype('float32'); X_test56=X_test56.reshape(-1,1,56,56).astype('float32')
</script></div>
<p></p>
<h4>Dataset #4. Synthetic - five digits with rotation</h4>
<div class="linked"><script type="text/x-sage">
X_train2r=numpy.array([random_rotate(image) for image in X_train2])
X_test2r=numpy.array([random_rotate(image) for image in X_test2])
print(y_test2[k]); pylab.imshow(X_test2r[k],cmap=pylab.cm.Blues)
pylab.title('Example of rotation')
pylab.xticks([]); pylab.yticks([]); pylab.show()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
n=5000
X_train140,y_train140=concatenate(X_train2r[:n],y_train2.reshape(-1,1)[:n],28)
X_test140,y_test140=concatenate(X_test2r[:int(n/5)],y_test2.reshape(-1,1)[:int(n/5)],28)
del X_train2r,X_test2r
print(y_test140[k]); pylab.imshow(X_test140[k],cmap=pylab.cm.Blues)
pylab.title('Example of the final synthetic datasets')
pylab.xticks([]); pylab.yticks([]); pylab.show()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
with h5py.File('Synthetic5Digits.h5','w') as f:
f.create_dataset('X_train',data=X_train140); f.create_dataset('X_test',data=X_test140)
f.create_dataset('y_train',data=y_train140); f.create_dataset('y_test',data=y_test140)
f.close()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
f=h5py.File('Synthetic5Digits.h5','r'); keys=list(f.keys()); print(keys)
X_test140=numpy.array(f[keys[0]]); X_train140=numpy.array(f[keys[1]])
y_test140=numpy.array(f[keys[2]]); y_train140=numpy.array(f[keys[3]])
cy_train140=numpy.empty([len(y_train140),5,11]).astype('int16')
for i in range(5): cy_train140[:,i]=ohe(y_train140[:,i].reshape(-1,1),categories=[list(range(11))])
cy_test140=numpy.empty([len(y_test140),5,11]).astype('int16')
for i in range(5): cy_test140[:,i]=ohe(y_test140[:,i].reshape(-1,1),categories=[list(range(11))])
print(y_test140[k]); print(cy_test140[k]); pylab.imshow(X_test140[k],cmap=pylab.cm.Blues)
pylab.title('Example of the synthetic dataset (h5)')
pylab.xticks([]); pylab.yticks([]); pylab.show()
</script></div>
<p></p>
<h2>Step 1: Design and Test a Model Architecture</h2>
In this project, we will design and implement a deep learning model that learns to recognize sequences of digits.<br/>
Also, we will train the model using synthetic data generated by concatenating character images from <i>notMNIST</i> or <i>MNIST</i>.<br/>
To produce a synthetic sequence of digits for testing, we can, for example, limit ourself to sequences up to five digits, and use five classifiers on top of the deep network.<br/>
We would have to incorporate an additional <i>blank</i> character to account for shorter number sequences.<br/>
There are various aspects to consider when thinking about this problem:<br/>
- The model can be derived from a convolutional network or a multi-layer perceptron classifier.<br/>
- We could experiment sharing or not the weights between the softmax classifiers.<br/>
- We can also use a recurrent network in your deep neural net to replace the classification layers and directly emit the sequence of digits one-at-a-time.<br/>
We can use the module <i>keras</i> or <i>tensorflow</i> or <i>sklearn</i> to implement the model.
<h3>Examples of Models</h3>
Models N1 and N2 are classic examples, other models were designed for this project.
<h4>Model N1. Multi-Layer Perceptron</h4>
<div class="linked"><script type="text/x-sage">
class NeuralNetMLP(object):
def __init__(self,n_output,n_features,n_hidden=30,l1=0.0,l2=0.0,epochs=500,eta=0.001,
alpha=0.0,decrease_const=0.0,shuffle=True,minibatches=1,random_state=None):
numpy.random.seed(random_state); self.epochs=epochs; self.shuffle=shuffle; self.minibatches=minibatches
self.n_output=n_output; self.n_features=n_features; self.n_hidden=n_hidden
self.l1=l1; self.l2=l2; self.eta=eta; self.alpha=alpha; self.decrease_const=decrease_const
self.w1,self.w2=self._initialize_weights()
def _encode_labels(self,y,k):
onehot=numpy.zeros((k,y.shape[0]))
for idx,val in enumerate(y): onehot[val,idx]=1.0
return onehot
def _initialize_weights(self):
w1=numpy.random.uniform(-1.0,1.0,size=self.n_hidden*(self.n_features+1))
w2=numpy.random.uniform(-1.0,1.0,size=self.n_output*(self.n_hidden+1))
return w1.reshape(self.n_hidden,self.n_features+1),w2.reshape(self.n_output,self.n_hidden+1)
def _sigmoid(self,z): return 1.0/(1.0+numpy.exp(-z))
def _sigmoid_gradient(self,z): return self._sigmoid(z)*(1-self._sigmoid(z))
def _add_bias_unit(self,X,how='column'):
if how=='column': X_new=numpy.ones((X.shape[0],X.shape[1]+1)); X_new[:,1:]=X
elif how=='row': X_new=numpy.ones((X.shape[0]+1,X.shape[1])); X_new[1:,:]=X
else: raise AttributeError('`how` must be `column` or `row`')
return X_new
def _feedforward(self,X,w1,w2):
a1=self._add_bias_unit(X,how='column'); z2=w1.dot(a1.T); a2=self._sigmoid(z2)
a2=self._add_bias_unit(a2,how='row'); z3=w2.dot(a2); a3=self._sigmoid(z3)
return a1,z2,a2,z3,a3
def _L2_reg(self,lambda_,w1,w2): return (lambda_/2.0)*(numpy.sum(w1[:,1:]**2)+numpy.sum(w2[:,1:]**2))
def _L1_reg(self,lambda_,w1,w2): return (lambda_/2.0)*(numpy.abs(w1[:,1:]).sum()+numpy.abs(w2[:,1:]).sum())
def _get_cost(self,y_enc,output,w1,w2):
term1=-y_enc*(numpy.log(output)); term2=(1-y_enc)*numpy.log(1-output)
L1_term=self._L1_reg(self.l1,w1,w2); L2_term=self._L2_reg(self.l2,w1,w2)
return numpy.sum(term1-term2)+L1_term+L2_term
def _get_gradient(self,a1,a2,a3,z2,y_enc,w1,w2):
sigma3=a3-y_enc; z2=self._add_bias_unit(z2,how='row')
sigma2=w2.T.dot(sigma3)*self._sigmoid_gradient(z2)
sigma2=sigma2[1:,:]; grad1=sigma2.dot(a1); grad2=sigma3.dot(a2.T)
grad1[:,1:]+=(w1[:,1:]*(self.l1+self.l2)); grad2[:,1:]+=(w2[:,1:]*(self.l1+self.l2))
return grad1,grad2
def predict(self,X):
a1,z2,a2,z3,a3=self._feedforward(X, self.w1,self.w2)
return numpy.argmax(z3,axis=0)
def fit(self,X,y,print_progress=False):
self.cost_=[]; X_data,y_data=X.copy(),y.copy(); y_enc=self._encode_labels(y,self.n_output)
delta_w1_prev=numpy.zeros(self.w1.shape); delta_w2_prev=numpy.zeros(self.w2.shape)
for i in range(self.epochs):
self.eta/=(1+self.decrease_const*i)
if (print_progress and i%100==0): sys.stderr.write('\nEpoch: %d/%d'%(i+1,self.epochs)); sys.stderr.flush()
if self.shuffle:
idx=numpy.random.permutation(y_data.shape[0])
X_data,y_enc=X_data[idx],y_enc[:,idx]
mini=numpy.array_split(range(y_data.shape[0]),self.minibatches)
for idx in mini:
a1,z2,a2,z3,a3=self._feedforward(X_data[idx],self.w1,self.w2)
cost=self._get_cost(y_enc=y_enc[:,idx],output=a3,w1=self.w1,w2=self.w2)
self.cost_.append(cost)
grad1,grad2=self._get_gradient(a1=a1,a2=a2,a3=a3,z2=z2,y_enc=y_enc[:,idx],w1=self.w1,w2=self.w2)
delta_w1,delta_w2=self.eta*grad1,self.eta*grad2
self.w1-=(delta_w1+(self.alpha*delta_w1_prev)); self.w2-=(delta_w2+(self.alpha*delta_w2_prev))
delta_w1_prev,delta_w2_prev=delta_w1,delta_w2
return self
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
print('Dataset N1. Rows: %d, columns: %d'%(X_train1.shape[0],X_train1.shape[1]))
nn1=NeuralNetMLP(n_output=10,n_features=X_train1.shape[1],n_hidden=128,l2=0.1,l1=0.0,epochs=200,
eta=0.01,alpha=0.01,decrease_const=0.001,shuffle=True,minibatches=128,random_state=0)
nn1.fit(X_train1,y_train1,print_progress=True)
y_train1_pred=nn1.predict(X_train1); y_test1_pred=nn1.predict(X_test1)
nn1_train1_accuracy=1.*numpy.sum(y_train1==y_train1_pred)/X_train1.shape[0]
nn1_test1_accuracy=1.*numpy.sum(y_test1==y_test1_pred)/X_test1.shape[0]
print('Train accuracy: %.2f%%'%(nn1_train1_accuracy*100))
print('Test accuracy: %.2f%%'%(nn1_test1_accuracy*100))
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
# for the best results (97.5-99.5%)
# n=60000
# nn2=NeuralNetMLP(n_output=10,n_features=784,n_hidden=196,l2=0.01,l1=0.0,epochs=1000,
# eta=0.001,alpha=0.001,decrease_const=0.00001,shuffle=True,minibatches=50,random_state=1)
n=5000
print('Dataset N2. Rows: %d, columns: %d'%(n,X_train2.shape[1]*X_train2.shape[2]))
nn2=NeuralNetMLP(n_output=10,n_features=784,n_hidden=48,l2=0.01,l1=0.0,epochs=300,
eta=0.001,alpha=0.001,decrease_const=0.00001,shuffle=True,minibatches=8,random_state=1)
nn2.fit(X_train2[:n].reshape(-1,784),y_train2[:n],print_progress=True)
y_train2_pred=nn2.predict(X_train2.reshape(-1,784)); y_test2_pred=nn2.predict(X_test2.reshape(-1,784))
nn1_train2_accuracy=1.*numpy.sum(y_train2==y_train2_pred)/X_train2.shape[0]
nn1_test2_accuracy=1.*numpy.sum(y_test2==y_test2_pred)/X_test2.shape[0]
print('Train accuracy: %.2f%%'%(nn1_train2_accuracy*100))
print('Test accuracy: %.2f%%'%(nn1_test2_accuracy*100))
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
batches=numpy.array_split(range(len(nn2.cost_)),800)
cost_ary=numpy.array(nn2.cost_); cost_avgs=[numpy.mean(cost_ary[i]) for i in batches]
pylab.figure(figsize=(12,4)); pylab.plot(range(len(cost_avgs)),cost_avgs,lw=3,color='royalblue')
pylab.ylim([0,1500]); pylab.ylabel('Cost'); pylab.xlabel('Epochs'); pylab.title('Cost Function'); pylab.show()
print('Misclassified test MNIST images: {}'.format(len(y_test2[y_test2!=y_test2_pred])))
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
miscl_images=X_test2[y_test2!=y_test2_pred][:25]
correct_labels=y_test2[y_test2!=y_test2_pred][:25]; miscl_labels=y_test2_pred[y_test2!=y_test2_pred][:25]
fig,ax=pylab.subplots(nrows=5,ncols=5,sharex=True,sharey=True,figsize=(5,6)); ax=ax.flatten()
for i in range(25):
ax[i].imshow(miscl_images[i],cmap=pylab.cm.Blues)
ax[i].set_title('%d) t: %d p: %d'%(i+1,correct_labels[i],miscl_labels[i]))
ax[0].set_xticks([]); ax[0].set_yticks([]); pylab.tight_layout(); pylab.show()
</script></div>
<p></p>
<h4>Model N2. Gradient Descent Usage</h4>
<div class="linked"><script type="text/x-sage">
def NeuralNet2(X_train,y_train,X_test,D=64,K=10,h=128,step_size=.01,reg=.0001,epochs=800):
W1=.001*numpy.random.randn(D,h); b1=numpy.zeros((1,h))
W2=.001*numpy.random.randn(h,K); b2=numpy.zeros((1,K))
for i in range(epochs):
# evaluate class scores
hidden_layer=numpy.maximum(0,numpy.dot(X_train,W1)+b1) # ReLU activation
scores=numpy.dot(hidden_layer,W2)+b2
# compute the class probabilities
probs=numpy.exp(scores)/numpy.sum(numpy.exp(scores),axis=int(1),keepdims=True)
# compute the loss: average cross-entropy loss and regularization
correct_logprobs=-numpy.log(probs[range(X_train.shape[0]),y_train])
data_loss=1.*numpy.sum(correct_logprobs)/X_train.shape[0]
reg_loss=.5*reg*numpy.sum(W1*W1)+.5*reg*numpy.sum(W2*W2)
loss=data_loss+reg_loss
if i%100==0: print ("iteration %d: loss %f"%(i,loss))
# compute the gradient on scores
dscores=probs; dscores[range(X_train.shape[0]),y_train]-=1; dscores/=X_train.shape[0]
# backpropate the gradient to the parameters, at first into parameters W2 and b2
dW2=numpy.dot(hidden_layer.T,dscores); db2=numpy.sum(dscores,axis=int(0),keepdims=True)
dhidden=numpy.dot(dscores,W2.T) # next backprop into hidden layer
dhidden[hidden_layer<=0]=0 # backprop the ReLU non-linearity
# finally into W1, b1
dW1=numpy.dot(X_train.T,dhidden); db1=numpy.sum(dhidden,axis=int(0),keepdims=True)
# add regularization gradient contribution
dW2+=reg*W2; dW1+=reg*W1
# perform a parameter update
W1+=-step_size*dW1; b1+=-step_size*db1
W2+=-step_size*dW2; b2+=-step_size*db2
hidden_layer=numpy.maximum(0,numpy.dot(X_train,W1)+b1); scores=numpy.dot(hidden_layer,W2)+b2
y_train_pred=numpy.argmax(scores,axis=int(1))
hidden_layer=numpy.maximum(0,numpy.dot(X_test,W1)+b1); scores=numpy.dot(hidden_layer,W2)+b2
y_test_pred=numpy.argmax(scores,axis=int(1))
return y_train_pred,y_test_pred
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
y_train1_pred,y_test1_pred=NeuralNet2(X_train1,y_train1,X_test1,D=64,K=10,h=128,
step_size=.01,reg=.0001,epochs=800)
print('Dataset N1. Train accuracy: %.2f'%(numpy.mean(y_train1_pred==y_train1)*100))
print('Dataset N1. Test accuracy: %.2f'%(numpy.mean(y_test1_pred==y_test1)*100))
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
n=5000
y_train2_pred,y_test2_pred=NeuralNet2(X_train2[:n].reshape(-1,784),y_train2[:n],X_test2.reshape(-1,784),
D=784,K=16,h=32,step_size=1e-3,reg=1e-5,epochs=800)
print('Dataset N2. Train accuracy: %.2f'%(numpy.mean(y_train2_pred==y_train2[:n])*100))
print('Dataset N2. Test accuracy: %.2f'%(numpy.mean(y_test2_pred==y_test2)*100))
</script></div>
<p></p>
<h4>Model N3. Multi-Layer Perceptron Classifier (sklearn)</h4>
<div class="linked"><script type="text/x-sage">
clf1=MLPClassifier(hidden_layer_sizes=(256,),max_iter=30,solver='adam',verbose=0,random_state=1,learning_rate_init=.01)
clf1.fit(X_train1,y_train1); print(clf1.score(X_test1,y_test1))
y_test1_predictions=clf1.predict(X_test1)
pylab.figure(figsize=(12,5)); pylab.scatter(range(100),y_test1[:100],color='#b8e2fc',s=100)
pylab.scatter(range(100),y_test1_predictions[:100],color='#191970',s=25); pylab.show()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
clf2=MLPClassifier(hidden_layer_sizes=(272,),max_iter=5,solver='adam',verbose=0,random_state=1,learning_rate_init=.001)
clf2.fit(X_train2.reshape(-1,784),y_train2); print(clf2.score(X_test2.reshape(-1,784),y_test2))
y_test2_predictions=clf2.predict(X_test2.reshape(-1,784))
pylab.figure(figsize=(12,5)); pylab.scatter(range(100),y_test2[:100],color='#b8e2fc',s=100)
pylab.scatter(range(100),y_test2_predictions[:100],color='#191970',s=25); pylab.show()
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
</script></div>
<p></p>
<div class="linked"><script type="text/x-sage">
</script></div>
<p></p>
<h3>Additional Code Cell</h3>
<div class="linked"><script type="text/x-sage">
</script></div>
<p></p>
</body>
</html>