Gradient descent

In [1]:
%config InlineBackend.figure_format = 'retina'
%pylab inline
import numpy as np
import matplotlib.pyplot as plt
from randomhexcolor import randomhexcolor
Populating the interactive namespace from numpy and matplotlib
In [2]:
def mycontourplot(f,xmin,xmax,ymin,ymax):
	x = np.linspace(xmin,xmax,300 )
	y = np.linspace(ymin,ymax,300 )
	X, Y = np.meshgrid(x, y)
	Z = f((X,Y))
	plt.imshow( -Z, interpolation='bilinear', origin='lower',
		        cmap='jet',extent=(xmin,xmax,ymin,ymax))
	plt.contour( X,Y,f((X,Y)),80 )

def f(X): 
	x,y = X
	return x**3 + y**3 + 3*x**2 + x*y - 3*y**2 + 12.

def df(X):
	x,y = X
	return np.array([3*x**2 + 6*x + y, 3*y**2 + x - 6*y])

plt.figure(figsize=(10,12))
plt.subplot(111,aspect=1)
mycontourplot(f,-2.,1.5,-1.,3.)

tol = 1.e-4
startingpoint = [1.,0.1]
stepsizes = [.02,.2,.3] #[.02,0.2,.3]
for stepsize in stepsizes:  # a.k.a. learning rate
	color = randomhexcolor(0.4)
	X = np.array(startingpoint)
	for i in range(100):
		gradf = df(X)
		newX = X - stepsize*gradf
		plt.plot( [X[0],newX[0]], [X[1],newX[1]], '.-', lw=2,color=color,markersize=8,alpha=0.8)
		X = newX
		
		if np.linalg.norm(gradf) <= tol: 
			print('gradf small')
			break
	print( X )
#plt.title('gradient descent with various learning rates');
[-0.43230969  2.06762005]
gradf small
[-0.44362993  2.07138813]
[-0.39709964  2.21047982]