In [1]:
%pylab inline
from PIL import Image
import glob
import numpy as np
from randomhexcolor import randomhexcolor
np.set_printoptions(linewidth=200)

Populating the interactive namespace from numpy and matplotlib

In [39]:
def softmax(x):
expx = np.exp(x)
return expx/expx.sum(axis=0)

def loss(WT,X,y):
nimages = X.shape[1]

# forward pass to compute the loss
c = WT.shape[0]             # number of classes

#print(c,nimages)
S = np.dot(WT,X)
P = softmax(S)
Pyi = P[ y, np.arange(nimages) ]  # select the prob of the true class
li = -np.log(Pyi)           # cross-entropy
L = li.sum()                # this is the loss

# back-prop of the gradient of the loss

dLdli = np.ones_like(li)

dLdP = np.zeros_like(P)                     # most will be zero
dLdP[ y, np.arange(nimages) ] = dLdli * (-1/Pyi)  # modify the non-zero element in each row (see fancy indexing note below)

dLdS = np.zeros_like(S)
for m in range(c):             # apply the formula you derived for the derivatives of the softmax function last class
dLdS += dLdP[m]*(-P[m]*P)  # the "j!=m" terms
dLdS += dLdP*P                 # the "j=m" term

dLdWT = np.dot(dLdS,X.T)  # finally, this is the gradient of the loss

ypred = argmax(P,axis=0)
return L,dLdWT,ypred

# want to return the predicted ("most probable") classes too

'''
# finite-difference check of correctness of gradient computation

n = 5 # number of images
p = 4 # number of pixels per image
c = 3  # number of classes
X = np.random.rand(p,n)                  # fake images
y = np.random.choice( np.arange(c), n )  # fake true classes
WT = np.random.rand(c,p)                 # random weights

# evaluate loss and gradient at our randomly chosen base point

L0,dLdWT,ypred = loss(WT,X,y)
print(L0)
print(dLdWT)

# perturb one weight a little and find change in loss

delta = 1.0e-8
WT1 = np.array(WT)
WT1[2,3] += delta          # perturb the [2,3] component of WT a little
L1,foo,ypred = loss(WT1,X,y)     # get loss at perturbed point
approx = (L1-L0)/delta     # finite difference
print('supposedly exact  ',dLdWT[2,3])
print('finite-diff approx',approx)
print(ypred)

# TODO: write code to
# train the network on this data
# 	make a picture of the weights for each class
# 	find the % of the training images correctly classified
# 	make pictures of a sample of incorrectly classified images
# 	build confusion matrix

# incorporate
#	compare loss and success rate on training vs on a test set
#	biases
#	mini-batches
'''

allpngs = sorted( glob.glob('handwriting/pngs/*.png') )
selection = ['_0','_1','_2','_7','_8','_9','06','_m','_o','09']
c = len(selection)  # number of classes
pngs = [png for png in allpngs if png[-6:-4] in selection]

h,w = np.array(Image.open(pngs[0])).shape[:2]  # find first two dimensions of sample image
# build image array
X = np.empty((h*w,len(pngs)))
ytrue = np.array([selection.index(png[-6:-4]) for png in pngs])
for i,png in enumerate(pngs):
Xi = np.array(Image.open(png))[:,:,0]
X[:,i] = (255 - Xi.reshape(h*w))
X /= X.sum()
return X,ytrue

In [21]:

# just stick in all zeros for weights!
WT = np.zeros((c, X.shape[0]  ))

In [22]:
L,gradL,ypred = loss(WT,X,ytrue)
print('L',L)
print('ypred',ypred)

10 1081
L 2489.09448553
gradL [[ 0.  0.  0. ...,  0.  0.  0.]
[ 0.  0.  0. ...,  0.  0.  0.]
[ 0.  0.  0. ...,  0.  0.  0.]
...,
[ 0.  0.  0. ...,  0.  0.  0.]
[ 0.  0.  0. ...,  0.  0.  0.]
[ 0.  0.  0. ...,  0.  0.  0.]]
ypred [0 0 0 ..., 0 0 0]

Out[22]:
0.8302886538712364
In [23]:
stepsize = 10000
print('L',L)

10 1081
L 2488.91200925


# what fraction of images are correctly classified?¶

In [24]:
correct_fraction = (ypred==ytrue).sum()/len(ytrue)
correct_fraction

Out[24]:
0.67900092506938026

## Make pictures of the rows of WT (our class templates)¶

In [19]:
h = 125
w = 100
plt.figure(figsize=(12,4))
for j in range(c):
plt.subplot(1,c,j+1)
plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
plt.xticks([])
plt.yticks([])


## Make picture of some incorrectly classified images¶

In [35]:
incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
writer = pngs[sample].split('_')[2].split('.')[0]
plt.subplot(4,4,i+1)
plt.imshow( Image.open(pngs[sample])    )
plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
plt.xticks([])
plt.yticks([])

In [47]:
stepsize = 1e9
for i in range(50):
print('L',L)
h = 125
w = 100
plt.figure(figsize=(12,4))
for j in range(c):
plt.subplot(1,c,j+1)
plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
plt.xticks([])
plt.yticks([])

incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
writer = pngs[sample].split('_')[2].split('.')[0]
plt.subplot(4,4,i+1)
plt.imshow( Image.open(pngs[sample])    )
plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
plt.xticks([])
plt.yticks([])

correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
print(str(correct_percentage)+'%')

L 92.1371640112
L 89.0069195783
L 86.1664623717
L 83.616684593
L 83.4281869495
L 180.182844035
L 2846.72080891
L 18545.7528885
L 33889.5381532
L 32230.1890272
L 12457.1597118
L 10063.2586102
L 3597.75719291
L 2858.38868553
L 2595.1901149
L 2162.83759066
L 2273.50856161
L 3012.38503984
L 6465.76049401
L 1621.77815225
L 1351.07135896
L 514.776261644
L 368.406518038
L 313.873584464
L 276.954765163
L 252.200198147
L 237.830993566
L 234.871739842
L 227.559785786
L 236.47615447
L 204.81433502
L 216.916058526
L 180.509134731
L 195.751753774
L 163.550926861
L 183.904269504
L 154.642795577
L 178.347061691
L 149.145802353
L 172.784363654
L 141.967736577
L 163.239329678
L 132.304734348
L 150.02693271
L 121.696206595
L 135.323791274
L 110.792559099
L 119.210367878
L 99.0926178795
L 99.5242582899
98%

In [52]:
testpngs = glob.glob('handwriting/testpngs/*.png')

In [53]:
len(testpngs)

Out[53]:
180
In [54]:
Xtest,ytesttrue = load_images(testpngs)

In [56]:
L,gradL,ypred = loss(WT,Xtest,ytesttrue)
print('L',L)
h = 125
w = 100
plt.figure(figsize=(12,4))
for j in range(c):
plt.subplot(1,c,j+1)
plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
plt.xticks([])
plt.yticks([])

incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
writer = pngs[sample].split('_')[2].split('.')[0]
plt.subplot(4,4,i+1)
plt.imshow( Image.open(pngs[sample])    )
plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
plt.xticks([])
plt.yticks([])

correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
print(str(correct_percentage)+'%')

L 5624.9222966

/usr/lib/python3/dist-packages/ipykernel_launcher.py:12: DeprecationWarning: elementwise != comparison failed; this will raise an error in the future.
if sys.path[0] == '':
/usr/lib/python3/dist-packages/ipykernel_launcher.py:25: DeprecationWarning: elementwise == comparison failed; this will raise an error in the future.

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-56-9bfd9d117d53> in <module>()
23     plt.yticks([])
24
---> 25 correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
26 print(str(correct_percentage)+'%')

AttributeError: 'bool' object has no attribute 'sum'

# How to incorporate biases¶

Just add 1 pixel always on to each image

In [ ]:
def load_images(pngs):
h,w = np.array(Image.open(pngs[0])).shape[:2]  # find first two dimensions of sample image
# build image array
X = np.empty((h*w+1,len(pngs)))
ytrue = np.array([selection.index(png[-6:-4]) for png in pngs])
for i,png in enumerate(pngs):
Xi = np.array(Image.open(png))[:,:,0]
X[:-1,i] = (255 - Xi.reshape(h*w))
X[-1] = 255  # turn on all the fake pixels
X /= X.sum()
return X,ytrue


# To implement mini-batches¶

In [ ]:
Randomly divide images into batches