In [1]:
%pylab inline
from PIL import Image
import glob
import numpy as np
from randomhexcolor import randomhexcolor
np.set_printoptions(linewidth=200)
Populating the interactive namespace from numpy and matplotlib
In [39]:
def softmax(x):
    expx = np.exp(x)
    return expx/expx.sum(axis=0)

def loss(WT,X,y):
    nimages = X.shape[1]
    
    # forward pass to compute the loss
    c = WT.shape[0]             # number of classes

    #print(c,nimages)
    S = np.dot(WT,X)
    P = softmax(S)
    Pyi = P[ y, np.arange(nimages) ]  # select the prob of the true class
    li = -np.log(Pyi)           # cross-entropy
    L = li.sum()                # this is the loss

    # back-prop of the gradient of the loss

    dLdli = np.ones_like(li)

    dLdP = np.zeros_like(P)                     # most will be zero
    dLdP[ y, np.arange(nimages) ] = dLdli * (-1/Pyi)  # modify the non-zero element in each row (see fancy indexing note below)

    dLdS = np.zeros_like(S)
    for m in range(c):             # apply the formula you derived for the derivatives of the softmax function last class
        dLdS += dLdP[m]*(-P[m]*P)  # the "j!=m" terms
    dLdS += dLdP*P                 # the "j=m" term

    dLdWT = np.dot(dLdS,X.T)  # finally, this is the gradient of the loss

    ypred = argmax(P,axis=0)
    return L,dLdWT,ypred

    # want to return the predicted ("most probable") classes too

'''
# finite-difference check of correctness of gradient computation

n = 5 # number of images
p = 4 # number of pixels per image
c = 3  # number of classes
X = np.random.rand(p,n)                  # fake images
y = np.random.choice( np.arange(c), n )  # fake true classes
WT = np.random.rand(c,p)                 # random weights

# evaluate loss and gradient at our randomly chosen base point

L0,dLdWT,ypred = loss(WT,X,y)
print(L0)
print(dLdWT)

# perturb one weight a little and find change in loss

delta = 1.0e-8
WT1 = np.array(WT)
WT1[2,3] += delta          # perturb the [2,3] component of WT a little
L1,foo,ypred = loss(WT1,X,y)     # get loss at perturbed point
approx = (L1-L0)/delta     # finite difference
print('supposedly exact  ',dLdWT[2,3])
print('finite-diff approx',approx)
print(ypred)





# TODO: write code to
# train the network on this data
# 	make a picture of the weights for each class
# 	find the % of the training images correctly classified
# 	make pictures of a sample of incorrectly classified images
# 	build confusion matrix

# incorporate
#	compare loss and success rate on training vs on a test set
#	biases
#	mini-batches
'''



allpngs = sorted( glob.glob('handwriting/pngs/*.png') )
selection = ['_0','_1','_2','_7','_8','_9','06','_m','_o','09']
c = len(selection)  # number of classes
pngs = [png for png in allpngs if png[-6:-4] in selection]



def load_images(pngs):
    h,w = np.array(Image.open(pngs[0])).shape[:2]  # find first two dimensions of sample image
    # build image array
    X = np.empty((h*w,len(pngs)))
    ytrue = np.array([selection.index(png[-6:-4]) for png in pngs])
    for i,png in enumerate(pngs):
        Xi = np.array(Image.open(png))[:,:,0]
        X[:,i] = (255 - Xi.reshape(h*w))
    X /= X.sum()
    return X,ytrue
In [21]:
        
# just stick in all zeros for weights!
X,ytrue = load_images(pngs)
WT = np.zeros((c, X.shape[0]  ))
In [22]:
L,gradL,ypred = loss(WT,X,ytrue)
print('L',L)
print('gradL',gradL)
print('ypred',ypred)
abs(gradL).sum()
10 1081
L 2489.09448553
gradL [[ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 ..., 
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]]
ypred [0 0 0 ..., 0 0 0]
Out[22]:
0.8302886538712364
In [23]:
stepsize = 10000
WT += -stepsize*gradL  # step along the negative gradient
L,gradL,ypred = loss(WT,X,ytrue)
print('L',L)
10 1081
L 2488.91200925

what fraction of images are correctly classified?

In [24]:
correct_fraction = (ypred==ytrue).sum()/len(ytrue)
correct_fraction
Out[24]:
0.67900092506938026

Make pictures of the rows of WT (our class templates)

In [19]:
h = 125
w = 100 
plt.figure(figsize=(12,4))
for j in range(c):
    plt.subplot(1,c,j+1)
    plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
    plt.xticks([])
    plt.yticks([])

Make picture of some incorrectly classified images

In [35]:
incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
    writer = pngs[sample].split('_')[2].split('.')[0]
    plt.subplot(4,4,i+1)
    plt.imshow( Image.open(pngs[sample])    )
    plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
    plt.xticks([])
    plt.yticks([])   
In [47]:
stepsize = 1e9
for i in range(50):
    WT += -stepsize*gradL  # step along the negative gradient
    L,gradL,ypred = loss(WT,X,ytrue)
    print('L',L)
h = 125
w = 100 
plt.figure(figsize=(12,4))
for j in range(c):
    plt.subplot(1,c,j+1)
    plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
    plt.xticks([])
    plt.yticks([])

incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
    writer = pngs[sample].split('_')[2].split('.')[0]
    plt.subplot(4,4,i+1)
    plt.imshow( Image.open(pngs[sample])    )
    plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
    plt.xticks([])
    plt.yticks([])   
    
correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
print(str(correct_percentage)+'%')
L 92.1371640112
L 89.0069195783
L 86.1664623717
L 83.616684593
L 83.4281869495
L 180.182844035
L 2846.72080891
L 18545.7528885
L 33889.5381532
L 32230.1890272
L 12457.1597118
L 10063.2586102
L 3597.75719291
L 2858.38868553
L 2595.1901149
L 2162.83759066
L 2273.50856161
L 3012.38503984
L 6465.76049401
L 1621.77815225
L 1351.07135896
L 514.776261644
L 368.406518038
L 313.873584464
L 276.954765163
L 252.200198147
L 237.830993566
L 234.871739842
L 227.559785786
L 236.47615447
L 204.81433502
L 216.916058526
L 180.509134731
L 195.751753774
L 163.550926861
L 183.904269504
L 154.642795577
L 178.347061691
L 149.145802353
L 172.784363654
L 141.967736577
L 163.239329678
L 132.304734348
L 150.02693271
L 121.696206595
L 135.323791274
L 110.792559099
L 119.210367878
L 99.0926178795
L 99.5242582899
98%
In [52]:
testpngs = glob.glob('handwriting/testpngs/*.png')
In [53]:
len(testpngs)
Out[53]:
180
In [54]:
Xtest,ytesttrue = load_images(testpngs)
In [56]:
L,gradL,ypred = loss(WT,Xtest,ytesttrue)
print('L',L)
h = 125
w = 100 
plt.figure(figsize=(12,4))
for j in range(c):
    plt.subplot(1,c,j+1)
    plt.imshow( WT[j].reshape((h,w)), cmap='seismic')
    plt.xticks([])
    plt.yticks([])

incorrect_imagenumbers = np.where(ypred!=ytrue)[0]  # get indices of incorrect ones
nincorrect = len(incorrect_imagenumbers)
nsample = 16
samples = np.random.choice( incorrect_imagenumbers, min(nsample,nincorrect), replace=False)
plt.figure(figsize=(8,8))
for i,sample in enumerate(samples):
    writer = pngs[sample].split('_')[2].split('.')[0]
    plt.subplot(4,4,i+1)
    plt.imshow( Image.open(pngs[sample])    )
    plt.title( writer + ' ' + selection[ytrue[sample]]+' as '+selection[ypred[sample]] )
    plt.xticks([])
    plt.yticks([])   
    
correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
print(str(correct_percentage)+'%')
L 5624.9222966
/usr/lib/python3/dist-packages/ipykernel_launcher.py:12: DeprecationWarning: elementwise != comparison failed; this will raise an error in the future.
  if sys.path[0] == '':
/usr/lib/python3/dist-packages/ipykernel_launcher.py:25: DeprecationWarning: elementwise == comparison failed; this will raise an error in the future.
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-56-9bfd9d117d53> in <module>()
     23     plt.yticks([])
     24 
---> 25 correct_percentage = int(100*(ypred==ytrue).sum()/len(ytrue))
     26 print(str(correct_percentage)+'%')

AttributeError: 'bool' object has no attribute 'sum'

How to incorporate biases

Just add 1 pixel always on to each image

In [ ]:
def load_images(pngs):
    h,w = np.array(Image.open(pngs[0])).shape[:2]  # find first two dimensions of sample image
    # build image array
    X = np.empty((h*w+1,len(pngs)))
    ytrue = np.array([selection.index(png[-6:-4]) for png in pngs])
    for i,png in enumerate(pngs):
        Xi = np.array(Image.open(png))[:,:,0]
        X[:-1,i] = (255 - Xi.reshape(h*w))
    X[-1] = 255  # turn on all the fake pixels
    X /= X.sum()
    return X,ytrue

To implement mini-batches

In [ ]:
Randomly divide images into batches