3D plot

Author: j-w-yun

README.md

@@ -21,13 +21,17 @@ All methods start at the same location, specified by two variables. Both x and y
 For an overview of each gradient descent optimization algorithms, visit [this helpful resource](http://ruder.io/optimizing-gradient-descent/).
 
 #### Numbers in figure legend indicate learning rate, specific to each Optimizer.
+![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie11.gif)
 ![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie5.gif)
 
 #### Note the optimizers' behavior when gradient is steep.
+![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie9.gif)
+![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie10.gif)
 ![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie7.gif)
 ![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie6.gif)
 
 #### Note the optimizers' behavior when initial gradient is miniscule.
+![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie12.gif)
 ![](https://github.com/Jaewan-Yun/optimizer-visualization/blob/master/figures/movie8.gif)
 
 <!-- ## Additional Figures

figures/movie10.gif

@@ -1,8 +1,10 @@
 import imageio
 
 images = []
-filenames = ['figures/' + str(i) + '.png' for i in range(120)]
+filenames = ['figures/' + str(i) + '.png' for i in range(65)]
+
+imageio.plugins.freeimage.download()
 
 for filename in filenames:
     images.append(imageio.imread(filename))
-imageio.mimsave('figures/movie.gif', images)
+imageio.mimsave('figures/movie.gif', images, format='GIF-FI', duration=0.001)

figures/movie11.gif

@@ -1,3 +1,6 @@
+from matplotlib import cm
+from mpl_toolkits.mplot3d import Axes3D
+
 import matplotlib.pyplot as plt
 import numpy as np
 import tensorflow as tf
@@ -22,10 +25,10 @@ def cost_func(x=None, y=None):
         y = tf.placeholder(tf.float32, shape=[None, 1])
 
     # two local minima near (0, 0)
-    z = __f1(x, y)
+#     z = __f1(x, y)
 
     # 3rd local minimum at (-0.5, -0.8)
-    z -= __f2(x, y, x_mean=-0.5, y_mean=-0.8, x_sig=0.35, y_sig=0.35)
+    z = -1 * __f2(x, y, x_mean=-0.5, y_mean=-0.8, x_sig=0.35, y_sig=0.35)
 
     # one steep gaussian trench at (0, 0)
 #     z -= __f2(x, y, x_mean=0, y_mean=0, x_sig=0.2, y_sig=0.2)
@@ -53,7 +56,8 @@ def __f2(x, y, x_mean, y_mean, x_sig, y_sig):
 
 # pyplot settings
 plt.ion()
-plt.figure(figsize=(3, 2), dpi=300)
+fig = plt.figure(figsize=(3, 2), dpi=300)
+ax = fig.add_subplot(111, projection='3d')
 plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
 params = {'legend.fontsize': 3,
           'legend.handlelength': 3}
@@ -72,7 +76,9 @@ def __f2(x, y, x_mean, y_mean, x_sig, y_sig):
     z_val_mesh_flat = sess.run(z, feed_dict={x: x_val_mesh_flat, y: y_val_mesh_flat})
 z_val_mesh = z_val_mesh_flat.reshape(x_val_mesh.shape)
 levels = np.arange(-10, 1, 0.05)
-plt.contour(x_val_mesh, y_val_mesh, z_val_mesh, levels, alpha=.7, linewidths=0.4)
+# ax.contour(x_val_mesh, y_val_mesh, z_val_mesh, levels, alpha=.7, linewidths=0.4)
+# ax.plot_wireframe(x_val_mesh, y_val_mesh, z_val_mesh, alpha=.5, linewidths=0.4, antialiased=True)
+ax.plot_surface(x_val_mesh, y_val_mesh, z_val_mesh, alpha=.4, cmap=cm.coolwarm)
 plt.draw()
 
 # starting location for variables
@@ -91,58 +97,71 @@ def __f2(x, y, x_mean, y_mean, x_sig, y_sig):
     cost.append(cost_func(x_var[i], y_var[i])[2])
 
 # define method of gradient descent for each graph
-ops_param = [['Adadelta', 50],
-             ['Adagrad', 0.10],
-             ['Adam', 0.05],
-             ['Ftrl', 0.5],
-             ['GD', 0.05],
-             ['Momentum', 0.01],
-             ['RMSProp', 0.02]]
+# optimizer label name, learning rate, color
+ops_param = np.array([['Adadelta', 50.0, 'b'],
+                     ['Adagrad', 0.10, 'g'],
+                     ['Adam', 0.05, 'r'],
+                     ['Ftrl', 0.5, 'c'],
+                     ['GD', 0.05, 'm'],
+                     ['Momentum', 0.01, 'y'],
+                     ['RMSProp', 0.02, 'k']])
 
 ops = []
-ops.append(tf.train.AdadeltaOptimizer(ops_param[0][1]).minimize(cost[0]))
-ops.append(tf.train.AdagradOptimizer(ops_param[1][1]).minimize(cost[1]))
-ops.append(tf.train.AdamOptimizer(ops_param[2][1]).minimize(cost[2]))
-ops.append(tf.train.FtrlOptimizer(ops_param[3][1]).minimize(cost[3]))
-ops.append(tf.train.GradientDescentOptimizer(ops_param[4][1]).minimize(cost[4]))
-ops.append(tf.train.MomentumOptimizer(ops_param[5][1], momentum=0.95).minimize(cost[5]))
-ops.append(tf.train.RMSPropOptimizer(ops_param[6][1]).minimize(cost[6]))
+ops.append(tf.train.AdadeltaOptimizer(float(ops_param[0, 1])).minimize(cost[0]))
+ops.append(tf.train.AdagradOptimizer(float(ops_param[1, 1])).minimize(cost[1]))
+ops.append(tf.train.AdamOptimizer(float(ops_param[2, 1])).minimize(cost[2]))
+ops.append(tf.train.FtrlOptimizer(float(ops_param[3, 1])).minimize(cost[3]))
+ops.append(tf.train.GradientDescentOptimizer(float(ops_param[4, 1])).minimize(cost[4]))
+ops.append(tf.train.MomentumOptimizer(float(ops_param[5, 1]), momentum=0.95).minimize(cost[5]))
+ops.append(tf.train.RMSPropOptimizer(float(ops_param[6, 1])).minimize(cost[6]))
+
+# 3d plot camera zoom, angle
+xlm = ax.get_xlim3d()
+ylm = ax.get_ylim3d()
+zlm = ax.get_zlim3d()
+ax.set_xlim3d(xlm[0] * 0.5, xlm[1] * 0.5)
+ax.set_ylim3d(ylm[0] * 0.5, ylm[1] * 0.5)
+ax.set_zlim3d(zlm[0] * 0.5, zlm[1] * 0.5)
+azm = ax.azim
+ele = ax.elev + 40
+ax.view_init(elev=ele, azim=azm)
 
 with tf.Session() as sess:
     sess.run(tf.global_variables_initializer())
 
     # use last location to draw a line to the current location
-    last_x, last_y = [], []
-    plot_cache = []
-    for i in range(7):
-        last_x.append(x_i)
-        last_y.append(y_i)
-        plot_cache.append(None)
-
-    # available colors for each label
-    colors = ('b', 'g', 'r', 'c', 'm', 'y', 'k')
+    last_x, last_y, last_z = [], [], []
+    plot_cache = [None for _ in range(len(ops))]
 
     # loop each step of the optimization algorithm
     steps = 1000
     for iter in range(steps):
         for i, op in enumerate(ops):
             # run a step of optimization and collect new x and y variable values
-            _, x_val, y_val = sess.run([op, x_var[i], y_var[i]])
+            _, x_val, y_val, z_val = sess.run([op, x_var[i], y_var[i], cost[i]])
 
             # move dot to the current value
             if plot_cache[i]:
                 plot_cache[i].remove()
-            plot_cache[i] = plt.scatter(x_val, y_val, color=colors[i], s=3, label=ops_param[i][0])
+            plot_cache[i] = ax.scatter(x_val, y_val, z_val, s=3, depthshade=True, label=ops_param[i, 0], color=ops_param[i, 2])
 
             # draw a line from the previous value
-            if last_x[i] and last_y[i]:
-                plt.plot([last_x[i], x_val], [last_y[i], y_val], color=colors[i], linewidth=0.5)
+            if iter == 0:
+                last_z.append(z_val)
+                last_x.append(x_i)
+                last_y.append(y_i)
+            ax.plot([last_x[i], x_val], [last_y[i], y_val], [last_z[i], z_val], linewidth=0.5, color=ops_param[i, 2])
             last_x[i] = x_val
             last_y[i] = y_val
+            last_z[i] = z_val
+
+        if iter == 0:
+            legend = np.vstack((ops_param[:, 0], ops_param[:, 1])).transpose()
+            plt.legend(plot_cache, legend)
 
-        plt.legend(plot_cache, ops_param)
         plt.savefig('figures/' + str(iter) + '.png')
         print('iteration: {}'.format(iter))
-        plt.pause(0.001)
+
+        plt.pause(0.0001)
 
 print("done")