{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Gradient descent for other functions\n", "\n", "Gradient descent works well for other functions, even if they are non-convex; gradient descent, if it converges, converges to a local minimum. Lets look at the function $$f(a,b) = \\sin(a)\\cos(b).$$ We would like to find parameters $(a,b)$ that minimize the function $f$. \n", "\n", "The gradients are:\n", "$$\\frac{\\partial f}{\\partial a} = \\cos(a)\\cos(b),$$\n", "$$\\frac{\\partial f}{\\partial b} = -\\sin(a)\\sin(b).$$" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Code source: Sebastian Curi and Andreas Krause.\n", "\n", "# Python Notebook Commands\n", "%matplotlib inline\n", "%reload_ext autoreload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "# Numerical Libraries\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "from matplotlib import rcParams\n", "rcParams['figure.figsize'] = (10, 5)\n", "\n", "# IPython Libraries\n", "import IPython\n", "import ipywidgets\n", "from ipywidgets import interact, interactive, interact_manual\n", "\n", "\n", "# Custom Libraries\n", "from utilities.load_data import polynomial_data\n", "from utilities import plot_helpers\n", "from utilities.util import gradient_descent" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class sincos(object):\n", " structured = False\n", " def __init__(self):\n", " pass\n", " \n", " @property\n", " def number_samples(self):\n", " return 0\n", "\n", " def loss(self, w, *args):\n", " return np.sin(w[0]) * np.cos(w[1])\n", "\n", " def gradient(self, w, *args):\n", " return np.array([np.cos(w[0]) * np.cos(w[1]), -np.sin(w[0]) * np.sin(w[1])])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "rcParams['figure.figsize'] = (10, 5) # Change this if figures look ugly. \n", "rcParams['font.size'] = 16 # Change this if figures look ugly. \n", "\n", "eta_widget = ipywidgets.FloatSlider(value=4, min=1e-1, max=10, step=1e-1, readout_format='.1f', \n", " description='Learning rate:', style={'description_width': 'initial'}, \n", " continuous_update=False)\n", "n_iter_widget = ipywidgets.IntSlider(value=10, min=5, max=50, step=1, description='Number of iterations:',\n", " style={'description_width': 'initial'}, continuous_update=False)\n", "\n", "lr_widget = ipywidgets.RadioButtons(options=['Bold driver', 'AdaGrad', 'Annealing', 'None'], value='None',\n", " description='Learning rate heuristics:', style={'description_width': 'initial'})\n", "\n", "a0_widget = ipywidgets.FloatSlider(value=-0.9, min=-3, max=3, step=.1, readout_format='.1f', description='w_0:',\n", " style={'description_width': 'initial'}, continuous_update=False)\n", "\n", "b0_widget = ipywidgets.FloatSlider(value=1.1, min=-3, max=3, step=.1, readout_format='.1f', description='w_1:',\n", " style={'description_width': 'initial'}, continuous_update=False)\n", "def optimize_sincos(eta, n_iter, learning_rate_scheduling, a0, b0):\n", " loss_function = sincos()\n", " w0 = np.array([a0, b0])\n", " \n", " if learning_rate_scheduling == 'None':\n", " learning_rate_scheduling = None\n", "\n", " opts = {'eta0': eta,\n", " 'n_iter': n_iter,\n", " 'learning_rate_scheduling': learning_rate_scheduling\n", " }\n", " trajectory, _ = gradient_descent(w0, loss_function, opts=opts)\n", " contourplot = plt.subplot(111)\n", " dataplot = None\n", " contour_opts = {'x_label': '$w_0$', 'y_label': '$w_1$', 'title': 'Weight trajectory', 'legend': False}\n", " plot_opts = {'contour_opts': contour_opts}\n", "\n", " plot_helpers.linear_regression_progression(np.array([]), np.array([]), trajectory, np.array([]), \n", " loss_function.loss,\n", " contourplot, dataplot, options=plot_opts)\n", "\n", "\n", "interact_manual(optimize_sincos, eta=eta_widget, n_iter=n_iter_widget, learning_rate_scheduling=lr_widget,\n", " a0=a0_widget, b0=b0_widget);" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 2 }