Activation Functions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare some non-linear shaped data as a function of a single variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD8CAYAAACVZ8iyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG0FJREFUeJzt3X2MXfWd3/H3x4NN7Wx2Z1imxB7jGlZO1Jloy8KI0jaJ\n0o4XDIqAVG0EcsCE3fUyA9JGahVBkTZRoki7myarRl2bOo0VA9OQbLMEawsNtptutFKdZMzyZB7C\nQEB47IAXMKSFBWx/+8c5dzke34dz555zHz8v6cr3/u7v3vOdc6/nO7/Ho4jAzMwG27JOB2BmZp3n\nZGBmZk4GZmbmZGBmZjgZmJkZTgZmZoaTgZmZ4WRgZmY4GZiZGXBGpwPI6+yzz47169d3Ogwzs55x\n4MCBv42I0Tx1eyYZrF+/nrm5uU6HYWbWMyS9kLeuu4nMzMzJwMzMnAzMzAwnAzMzw8nAzMxwMjAz\n60qzs7B+PSxblvw7O1vu8XpmaqmZ2aCYnYXrr4eTJ5PHL7yQPAbYvLmcY7plYGbWZa677r1EUHHy\nJPzu75Z3TCcDM7MusnEj1Lo0/d/9XXnHdTIwM+si+/Z15rhOBmZmXWLjxs4d28nAzKxLNGoVTE2V\nd2wnAzOzLpBn6ujeveUd38nAzKwLNJopVGarAJwMzMy6QqOZQmW2CsDJwMys4zo5cFzhZGBm1mGN\nBo6np8uPIXcykLRT0suSHs+UfUXSU5IelXSvpOG0fL2ktyQ9nN7uyLzmIkmPSZqX9HVJKvZHMjPr\nHXlaBdu2lR9HMy2DbwGbFpXtAT4cEb8J/Ay4LfPcsxFxQXq7KVO+Hfg9YEN6W/yeZmYDo5PTSbNy\nJ4OI+BHw6qKyByPiePpwP7C23ntIWg38akTsj4gA7gSubi5kM7P+MDPTuE7ZA8cVRY4Z3Ag8kHl8\nnqS/kfRXkj6alo0BhzJ1DqVlZmYDZ/v2+s+3q1UABW1hLel24DhQWTZxBFgXEa9Iugj4vqSJJbzv\nVmArwLp164oI1cysK3RTqwAKaBlIugH4BLA57fohIt6OiFfS+weAZ4EPAguc2pW0Ni2rKiJ2RMRk\nREyOjo62GqqZWddo1CpYsaI9cVS0lAwkbQI+B1wZEW9mykclDaX3zycZKH4uIo4Ab0i6JJ1FdD1w\nXysxmJn1mjxbT+zcWX4cWbm7iSR9G/g4cLakQ8DnSWYPnQnsSWeI7k9nDn0M+KKkd4GTwE0RURl8\nniGZmbSSZIwhO85gZtb3brih/vNDQ+Vd0ayW3MkgIq6tUvzNGnW/B3yvxnNzwIfzHtfMrJ/MzsLx\n4/Xr7NrVnliyvALZzKyNGrUKoP2tAnAyMDNrmzytgnZsPVGNk4GZWZvkuaB9O7aeqMbJwMysTRpt\nU93ORWaLORmYmbXBWI69Ftq5yGwxJwMzs5LNzsLhw/XrdGqsoMLJwMysZHlmEHVqrKDCycDMrER5\nZhB1cqygwsnAzKxEeVoFnRwrqHAyMDMrSTevK1jMycDMrCRbtjSu0+mxggonAzOzEszOwokT9et0\nw1hBhZOBmVkJ8rQKumGsoMLJwMysYHlaBd0yVlDhZGBmVrDrrmtcp1vGCiqcDMzMCjQzA8kFgGvr\ntlYBOBmYmRWq0bWNoftaBeBkYGZWmJmZxnW6sVUATgZmZoXp1VYBOBmYmRViYqJxnbvvLj+OpXIy\nMDMrwBNP1H9e6sy1jfNqKhlI2inpZUmPZ8rOkrRH0jPpvyNpuSR9XdK8pEclXZh5zZa0/jOScizN\nMDPrXiMjjevcdVf5cbSi2ZbBt4BNi8puBfZFxAZgX/oY4HJgQ3rbCmyHJHkAnwf+KXAx8PlKAjEz\n6zUzM3DsWP06a9Z0d6sAmkwGEfEj4NVFxVcBu9L7u4CrM+V3RmI/MCxpNXAZsCciXo2I14A9nJ5g\nzMx6Qp5B44WF8uNoVRFjBudExJH0/i+Ac9L7Y8CLmXqH0rJa5aeRtFXSnKS5o0ePFhCqmVlx8kwl\n7abN6OopdAA5IgJosPauqffbERGTETE5Ojpa1NuamRUiT6ugmzajq6eIZPBS2v1D+u/LafkCcG6m\n3tq0rFa5mVnPyDOVtFsXmFVTRDLYDVRmBG0B7suUX5/OKroEeD3tTvoBcKmkkXTg+NK0zMysJ8zO\nNp5KCt27wKyaM5qpLOnbwMeBsyUdIpkV9EfAdyX9DvAC8Km0+v3AFcA88CbwGYCIeFXSl4CfpvW+\nGBGLB6XNzLpWnl1Ju3mBWTWKRtvrdYnJycmYm5vrdBhmNuBmZhqPFUhw8mR74qkfhw5ExGSeul6B\nbGbWhDyDxt2+wKwaJwMzs5zGqk6CP9X4ePcvMKvGycDMLIfZWTh8uHG9gwfLj6UMTgZmZjl8+tON\n6/TSVNLFnAzMzBrI0z00PNxbU0kXczIwM6sjb/fQa6+VH0uZnAzMzOrIs6agV/YfqsfJwMyshokJ\naLQUa/ny3tl/qB4nAzOzKvJuOfHOO+XH0g5OBmZmVeSZPdQP3UMVTgZmZovkmT0E/dE9VOFkYGaW\nMTOTb/ZQL68pqMbJwMwsI8/eQ2vW9PaagmqcDMzMUqtW5avXC9c0bpaTgZkZyTTSt95qXK/XrlOQ\nl5OBmQ28vNNIe3VH0jycDMxs4OWZRgq9uyNpHk4GZjbQ8o4T9Gv3UIWTgZkNrLzjBFNT/ds9VOFk\nYGYDKe84wfBwfy0uq6XlZCDpQ5IeztzekPRZSV+QtJApvyLzmtskzUt6WtJlrcZgZtasvOMEvb41\ndV5ntPoGEfE0cAGApCFgAbgX+AzwpxHxH7P1JY0D1wATwBpgr6QPRsSJVmMxM8tjxYp89fp9nCCr\n6G6iKeDZiHihTp2rgHsi4u2I+DkwD1xccBxmZlWNjMC77zauNwjjBFlFJ4NrgG9nHt8i6VFJOyWN\npGVjwIuZOofSMjOzUk1MwLFjjesNyjhBVmHJQNIK4Ergz9Oi7cBvkHQhHQG+uoT33CppTtLc0aNH\niwrVzAZQ3gHjZcsGZ5wgq8iWweXAQxHxEkBEvBQRJyLiJPAN3usKWgDOzbxubVp2mojYERGTETE5\nOjpaYKhmNmjyDhifGNDRyyKTwbVkuogkrc4890ng8fT+buAaSWdKOg/YAPykwDjMzE4h5avXb9tS\nN6Pl2UQAkt4H/Dbw+5niP5F0ARDA85XnIuKgpO8CTwDHgZs9k8jMyjI0lK9eP25L3YxCkkFE/D/g\n1xeVXVen/peBLxdxbDOzWlasgJMnG9dbubI/t6Vuhlcgm1lfWrUq3xRSgDffLDeWXuBkYGZ9Z9Wq\nfHsOAUSUG0uvcDIws77iRLA0TgZm1jfGxvIngkGeOVSNk4GZ9YWxMTh8OF/d8fHBnjlUTSGziczM\nOmlkJN82E5BMIe3nK5YtlZOBmfW0ZsYIhoc9hbQWJwMz61krVuSfPjo8PJh7DuXlZGBmPWloKN+C\nMkgWlTkR1OdkYGY9J+9eQwDLl3tRWR6eTWRmPaXZRPDOO+XF0k+cDMysJ8zONpcIVq50ImiGu4nM\nrOtNTOS7ME2FB4ub52RgZl2tmamjkKwj8PTR5rmbyMy6ltRcIhgfdyJYKicDM+s6MzPNjQ8ATE15\nZXEr3E1kZl2l2W4hgLvvhs2by4lnULhlYGZdodIaaDYRRDgRFMEtAzPruKW0BpYtgxO+enph3DIw\ns45ZamtgeNiJoGhuGZhZRzSzyVzW9LSvRVCGwpKBpOeBXwIngOMRMSnpLOA7wHrgeeBTEfGaJAH/\nCbgCeBO4ISIeKioWM+tezS4gy/JlKstTdDfRv4yICyJiMn18K7AvIjYA+9LHAJcDG9LbVmB7wXGY\nWZfZuDHpElpKIhgediIoW9ljBlcBu9L7u4CrM+V3RmI/MCxpdcmxmFkHVJLAvn1Le/30tLeWaIci\nxwwCeFBSAP8lInYA50TEkfT5XwDnpPfHgBczrz2Ulh3JlCFpK0nLgXXr1hUYqpmVrZXuIEg2mvPW\n0+1TZMvgIxFxIUkX0M2SPpZ9MiKCJGHkFhE7ImIyIiZHR0cLDNXMyjI2tvTuoIrpaSeCdiusZRAR\nC+m/L0u6F7gYeEnS6og4knYDvZxWXwDOzbx8bVpmZj1odhY+/enW38ebzHVOIS0DSe+T9P7KfeBS\n4HFgN7AlrbYFuC+9vxu4XolLgNcz3UnWxyYmkr8ay76NjHT6Jx0MlVZAq4lg2bJkgNiJoHOK6iY6\nB/hrSY8APwH+R0T8T+CPgN+W9AywMX0McD/wHDAPfAOYKSgO67BGv+xb6TpoxrFj+RPHxER7YuoX\n2c/48OHW32962gvIukEh3UQR8RzwT6qUvwJMVSkP4OYijm2dMzZWzC+DTnviifo7ZHogM2lpHTtW\n7HuOj3uX0W7i7Sgst8V/9fdDIsjjrbfqtyyWLUv6zPvJyMipP2ORiWB8POkSciLoLk4GVteqVe3v\n4uk1EUmfeb2EMdPFHaGVfv+yfvlXTE05CXQzJwM7TTYBNLuBmFW3fXu+8YuxseKPXdkMrtat7Bbe\n9HSSBPbuLfc41hpvVGdA6wuErBiHDzd/ha9u5AvS9x63DAbY7Gznu4AqfzW2erv77s7Eb6eqfJ5O\nBL3HyWAAVboNilgk1Eiln7jWraitiDdvzp84pqeLOaYlsp+xt5buXU4GA6SSBLaXtEdstb/yu7Gf\neNu2xgljeLjTUXavlSu7/zO25jkZDICyksDiX/799Ffha685WVRISTdc5Wcf9DUX/coDyH2sqP1i\nKjwo+J5G56FXB+QluOsuX2B+EDkZ9KmlXGC8GieApck7l77ohJ2HN4OzatxN1Gcqq4RbTQSeFdIe\nzQx8F3VzIrBq3DLoE0X8helWgNngcjLoA61uIuYNw8zM3UQ9rLJobKmJwBuGmVmFWwY9qpXto90S\nMLPFnAx60FL3rvGYgJnV4m6iHlLpFmpW5ZKCTgRmVouTQY8YG1vabCFfUtDM8nA3UQ8YGoKTJ5t7\njS/VaGbNcMugy0nNJ4LpaScCM2tOy8lA0rmSfijpCUkHJf1BWv4FSQuSHk5vV2Rec5ukeUlPS7qs\n1Rj6VbPjA8uX99+GcWbWHkW0DI4D/y4ixoFLgJsljafP/WlEXJDe7gdIn7sGmAA2AdskDRUQR9+o\n7DLajKkpeOedcuIxs/7X8phBRBwBjqT3fynpSaDelVyvAu6JiLeBn0uaBy4G/k+rsfSDpex2GVFO\nLGY2OAodM5C0Hvgt4Mdp0S2SHpW0U9JIWjYGvJh52SHqJ4+B0WwiqHQLmZm1qrBkIOlXgO8Bn42I\nN4DtwG8AF5C0HL66hPfcKmlO0tzRo0eLCrUrjY01lwjWrHG3kJkVp5BkIGk5SSKYjYi/AIiIlyLi\nREScBL5B0hUEsACcm3n52rTsNBGxIyImI2JydHS0iFC70shIc1tLTE97G2IzK1bLYwaSBHwTeDIi\nvpYpX52OJwB8Eng8vb8b+G+SvgasATYAP2k1jl7V7EVo3C1kZmUoYtHZvwCuAx6T9HBa9h+AayVd\nAATwPPD7ABFxUNJ3gSdIZiLdHBEDuUbWicDMukURs4n+Gqg2EfL+Oq/5MvDlVo/dy0ZGnAjMrHt4\nBXIHNHMxmsomc2ZmZXIyaLOxsfyJYPlybzJnZu3hZNBGExP5Zw2tXOmpo2bWPk4GbbJxY/51BMPD\n3mjOzNrLyaANZmdh3758dX01MjPrBCeDNsh7UZqVK50IzKwznAxKlnf30eXL3TVkZp3jZFCioZwb\ncy9f7sFiM+ssJ4OSrFqV7wply5Y5EZhZ5zkZlGBiIv/qYq8jMLNu4GRQsNnZ/FNIvbLYzLqFk0HB\n8s4cuvvucuMwM2uGk0GB8g4YT0/D5s3lxmJm1gwng4LkHTAeH4dt28qPx8ysGU4GBcg7YLxyJRw8\nWH48ZmbNcjJoUd4B42XLvKjMzLqXk0GL8g4YewqpmXUzJ4MWjIzkqzc9XW4cZmatcjJYopmZfBep\n8YCxmfUCJ4Ml2r69cZ3hYQ8Ym1lvcDJYglWr8tXzdtRm1is6lgwkbZL0tKR5Sbd2Ko5mbdyYbxqp\nVxibWS/pSDKQNAT8GXA5MA5cK2m8E7E0K88Vy8bHvcLYzHpLp1oGFwPzEfFcRLwD3ANc1aFYclux\nonGd5cs9TmBmvadTyWAMeDHz+FBadgpJWyXNSZo7evRo24KrZmIC3n23cT1fm8DMelFXDyBHxI6I\nmIyIydHR0Y7FkXeVsdcTmFmv6lQyWADOzTxem5Z1peuua1xneNjrCcysd3UqGfwU2CDpPEkrgGuA\n3R2Kpa6JiXwXofE0UjPrZWd04qARcVzSLcAPgCFgZ0R03bBr3u4hTyM1s16n6JFrL05OTsbc3Fxb\nj7lsWeNWwZo1sNC1HVxmNsgkHYiIyTx1u3oAuZNmZvJ1DzkRmFk/cDKoIc/eQ+4eMrN+4WRQxdhp\nKx5Ot2aNVxmbWf9wMlhkdhYOH25cz91DZtZPnAwWybOmwIvLzKzfOBlkbNzYeNDYi8vMrB85GWTk\n2ZHUi8vMrB85GaTyXM94aqr8OMzMOsHJgPzXM967t/xYzMw6wckArykwMxv4ZDAz07iO1xSYWb8b\n+GSQp1XgNQVm1u8GOhls3Ni4jtcUmNkgGOhkkGcqqdcUmNkgGNhkkGf/IbcKzGxQDGQyyLv/kFsF\nZjYoBjIZbNnSuI6nkprZIBm4ZDAzAydO1K8zPu6ppGY2WAYuGeSZSnqw667GbGZWroFKBnkWmHn/\nITMbRC0lA0lfkfSUpEcl3StpOC1fL+ktSQ+ntzsyr7lI0mOS5iV9XZJa/SHyytMq8P5DZjaIWm0Z\n7AE+HBG/CfwMuC3z3LMRcUF6uylTvh34PWBDetvUYgy5eIGZmVltLSWDiHgwIo6nD/cDa+vVl7Qa\n+NWI2B8RAdwJXN1KDHl5gZmZWW1FjhncCDyQeXyepL+R9FeSPpqWjQGHMnUOpWWlyjNW4FaBmQ2y\nMxpVkLQX+ECVp26PiPvSOrcDx4HZ9LkjwLqIeEXSRcD3JU00G5ykrcBWgHXr1jX78r/XaKxAcqvA\nzAZbw2QQEXV72yXdAHwCmEq7foiIt4G30/sHJD0LfBBY4NSupLVpWa1j7wB2AExOTja4OnF1eVoF\nd921lHc2M+sfrc4m2gR8DrgyIt7MlI9KGkrvn08yUPxcRBwB3pB0STqL6HrgvlZiaKRRq2BoyAvM\nzMwatgwa+M/AmcCedIbo/nTm0MeAL0p6FzgJ3BQRr6avmQG+BawkGWN4YPGbFiXPDKJdu8o6uplZ\n71Das9P1JicnY25urqnXNFrBIMHJky0EZWbWxSQdiIjJPHX7dgWyxwrMzPLr22Rwxx31n/dYgZnZ\ne/o2GTTq/fJYgZnZe/o2GdTjVoGZ2akGMhm4VWBmdqq+TQa1tpeYmnKrwMxssb5NBtu2JQlhaCh5\nPDSUPPYW1WZmp2t10VlX27bNew6ZmeXRty0DMzPLz8nAzMycDMzMzMnAzMxwMjAzM3po11JJR4EX\nlvjys4G/LTCcojiu5jiu5jiu5vRjXP8oIkbzVOyZZNAKSXN5t3FtJ8fVHMfVHMfVnEGPy91EZmbm\nZGBmZoOTDHZ0OoAaHFdzHFdzHFdzBjqugRgzMDOz+galZWBmZnX0TTKQ9G8lHZR0UtLkoudukzQv\n6WlJl9V4/XmSfpzW+46kFSXE+B1JD6e35yU9XKPe85IeS+vNFR1HleN9QdJCJrYratTblJ7DeUm3\ntiGur0h6StKjku6VNFyjXlvOV6OfX9KZ6Wc8n36X1pcVS+aY50r6oaQn0u//H1Sp83FJr2c+3z8s\nO670uHU/FyW+np6vRyVd2IaYPpQ5Dw9LekPSZxfVacv5krRT0suSHs+UnSVpj6Rn0n9Harx2S1rn\nGUlbCgkoIvriBvxj4EPA/wYmM+XjwCPAmcB5wLPAUJXXfxe4Jr1/BzBdcrxfBf6wxnPPA2e38dx9\nAfj3DeoMpefufGBFek7HS47rUuCM9P4fA3/cqfOV5+cHZoA70vvXAN9pw2e3Grgwvf9+4GdV4vo4\n8Jft+j7l/VyAK4AHAAGXAD9uc3xDwC9I5uK3/XwBHwMuBB7PlP0JcGt6/9Zq33ngLOC59N+R9P5I\nq/H0TcsgIp6MiKerPHUVcE9EvB0RPwfmgYuzFSQJ+FfAf0+LdgFXlxVrerxPAd8u6xgluBiYj4jn\nIuId4B6Sc1uaiHgwIo6nD/cDa8s8XgN5fv6rSL47kHyXptLPujQRcSQiHkrv/xJ4Ehgr85gFugq4\nMxL7gWFJq9t4/Cng2YhY6mLWlkTEj4BXFxVnv0O1fg9dBuyJiFcj4jVgD7Cp1Xj6JhnUMQa8mHl8\niNP/s/w6cCzzi6danSJ9FHgpIp6p8XwAD0o6IGlriXFk3ZI21XfWaJrmOY9lupHkr8hq2nG+8vz8\nf18n/S69TvLdaou0W+q3gB9XefqfSXpE0gOSJtoUUqPPpdPfqWuo/QdZJ84XwDkRcSS9/wvgnCp1\nSjlvPXVxG0l7gQ9Ueer2iLiv3fFUkzPGa6nfKvhIRCxI+ofAHklPpX9FlBIXsB34Esl/3i+RdGHd\n2Mrxioircr4k3Q4cB2ZrvE3h56vXSPoV4HvAZyPijUVPP0TSFfJ/0/Gg7wMb2hBW134u6ZjglcBt\nVZ7u1Pk6RUSEpLZN9+ypZBARG5fwsgXg3MzjtWlZ1iskTdQz0r/oqtUpJEZJZwD/GrioznsspP++\nLOleki6Klv4T5T13kr4B/GWVp/Kcx8LjknQD8AlgKtIO0yrvUfj5qiLPz1+pcyj9nH+N5LtVKknL\nSRLBbET8xeLns8khIu6XtE3S2RFR6j48OT6XUr5TOV0OPBQRLy1+olPnK/WSpNURcSTtMnu5Sp0F\nknGNirUkY6UtGYRuot3ANelMj/NIMvxPshXSXzI/BP5NWrQFKKulsRF4KiIOVXtS0vskvb9yn2QQ\n9fFqdYuyqJ/2kzWO91Ngg5JZVytImti7S45rE/A54MqIeLNGnXadrzw//26S7w4k36X/VSuBFSUd\nk/gm8GREfK1GnQ9Uxi4kXUzy/77UJJXzc9kNXJ/OKroEeD3TRVK2mq3zTpyvjOx3qNbvoR8Al0oa\nSbt0L03LWlP2iHm7biS/xA4BbwMvAT/IPHc7yUyQp4HLM+X3A2vS++eTJIl54M+BM0uK81vATYvK\n1gD3Z+J4JL0dJOkuKfvc3QU8BjyafhlXL44rfXwFyWyVZ9sU1zxJ3+jD6e2OxXG183xV+/mBL5Ik\nK4B/kH535tPv0vltOEcfIeneezRznq4Abqp8z4Bb0nPzCMlA/D9vQ1xVP5dFcQn4s/R8PkZmFmDJ\nsb2P5Jf7r2XK2n6+SJLREeDd9HfX75CMMe0DngH2AmeldSeB/5p57Y3p92we+EwR8XgFspmZDUQ3\nkZmZNeBkYGZmTgZmZuZkYGZmOBmYmRlOBmZmhpOBmZnhZGBmZsD/B8si8Q00W5DBAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd3cec8f550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xarr = [ x/100 - 10 for x in range(2000) ] # from -10 to 10, 0.1 steps\n",
    "yarr = [ (((x-2)**3 + 7*(x-2)**2 + 400)) for x in xarr ]\n",
    "\n",
    "plt.scatter(xarr, yarr, color='blue')\n",
    "\n",
    "# reshape for sklearn if your data only has a single example\n",
    "#X_train = np.asarray(xarr).reshape(1, -1)\n",
    "# or reshape if data has single feature:\n",
    "X_train = np.asarray(xarr).reshape(-1, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "\n",
    "X_scaled = scaler.transform(X_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "hidden_layers = (4, )\n",
    "#activation = 'relu'\n",
    "activation = 'tanh'\n",
    "#activation = 'identity'\n",
    "regularization = 0.0001\n",
    "max_iter = 10 * 1000\n",
    "warm_start = False\n",
    "learning_rate='constant'\n",
    "#learning_rate = 'adaptive'\n",
    "\n",
    "#nn = MLPRegressor(hidden_layer_sizes=hidden_layers, activation=activation, alpha=regularization, max_iter=max_iter, warm_start=warm_start, solver='sgd', learning_rate=learning_rate, tol=1e-12, random_state=1234)\n",
    "nn = MLPRegressor(hidden_layer_sizes=hidden_layers, activation=activation, max_iter=max_iter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pbalm/miniconda3/envs/neuralnets/lib/python3.6/site-packages/sklearn/neural_network/multilayer_perceptron.py:563: ConvergenceWarning: Stochastic Optimizer: Maximum iterations reached and the optimization hasn't converged yet.\n",
      "  % (), ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fd3cec844a8>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD8CAYAAACVZ8iyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHWBJREFUeJzt3X2QXXWd5/H3p5sEAz6kkYhJBzYwi+50Wy6jXQyzq5Y7\n3YtAWaJbuxbIQxQ10o1bY9XuKixVo6U1W844OlW4GjeOjEjaB2YchJ3CkaTXHWuqFrXjYCABJCAM\n6UQIkIBWkJD0d/8458JJ5/Y9p3PPuY+fV9Wtvn3O797769O376d/D+d3FBGYmVl/G2h3BczMrP0c\nBmZm5jAwMzOHgZmZ4TAwMzMcBmZmhsPAzMxwGJiZGQ4DMzMDTmh3BYo69dRTY926de2uhplZ19i2\nbduTEbGqSNmuCYN169YxOzvb7mqYmXUNSY8WLetuIjMzcxiYmZnDwMzMcBiYmRkOAzMzw2FgZtYx\npqdh3ToYGEi+Tk+37rW7ZmqpmVkvm56GK6+E+fnk+0cfTb4HuOyy6l/fLQMzsw6wfv1LQVAzPw8f\n+lBrXt9hYGbWZlNTcORI/X2//W1r6uAwMDNrs40b210Dh4GZWVu1cpC4EYeBmVkbXX114/3Ll7em\nHg4DM7M2+s1vGu+/8cbW1MNhYGbWJlNT+WVaMa0UHAZmZm2TN3A8OdmaeoDDwMysLYq0Cr785err\nUeMwMDNrg7xWQasGjmsKh4GkGyU9IenezLbPSbpf0nZJt0pamW5fJ+k5SXent69kHvNmSfdI2iXp\nBkkq90cyM+tsRaaTtmrguGYpLYOvAxcs2LYFeENEvBH4BXBdZt9DEXFOestOntoIfBg4O70tfE4z\ns56Wt8TE4GDrBo5rCodBRPwIeHrBtjsj4nD67V3A2kbPIWk18MqIuCsiAvgG8O6lVdnMrLvlLTFx\n002tqUdWmWMGVwHfz3x/pqR/kvQPkt6abhsGdmfK7E63mZn1hYmJ/DKtbhVASUtYS7oeOAzUesL2\nAmdExFOS3gx8T9LocTzvBmADwBlnnFFGVc3M2mpmpvH+Vk4nzWq6ZSDp/cA7gcvSrh8i4vmIeCq9\nvw14CHgdMMfRXUlr0211RcSmiBiLiLFVq1Y1W1Uzs7Yq0ipo5XTSrKbCQNIFwMeBd0XEwcz2VZIG\n0/tnkQwUPxwRe4FnJZ2XziK6EritmTqYmXWLvFbB+Hhr6lFP4W4iSd8C3g6cKmk38EmS2UMnAlvS\nGaJ3pTOH3gZ8WtILwDxwdUTUBp+nSGYmrSAZY8iOM5iZ9aQiJ5lt3Vp9PRajtGen442NjcXs7Gy7\nq2FmdlzyzqgaHy8/DCRti4ixImV9BrKZWcU6vVUADgMzs8p12tIT9TgMzMwq1IlLT9TjMDAzq9D6\n9Y33S+05yWwhh4GZWUWmpuDIkcZlbr65NXXJ4zAwM6tI3lgBdEarABwGZmaVKDJW0K6lJ+pxGJiZ\nVSBvrADat/REPQ4DM7OSTU/njxV0UqsAHAZmZqXrtlYBOAzMzEpVZAZRp7UKwGFgZlaqIjOIOq1V\nAA4DM7PSFFmDqBNbBeAwMDMrTbe2CsBhYGZWim5uFYDDwMysFN3cKgCHgZlZ04pc27iTWwXgMDAz\na1retY2hs1sF4DAwM2tKL7QKwGFgZtaUvFaB1PmtAnAYmJkdtyKtgk65XkGeJYWBpBslPSHp3sy2\nUyRtkfRg+nUo3S5JN0jaJWm7pDdlHrM+Lf+gpAKreJiZdZ68VsHgYOdcryDPUlsGXwcuWLDtWmAm\nIs4GZtLvAS4Ezk5vG4CNkIQH8Eng94FzgU/WAsTMrFsMFfjUuumm6utRliWFQUT8CHh6weaLgdqP\nfBPw7sz2b0TiLmClpNXAO4AtEfF0ROwHtnBswJiZdaypKThwoHGZNWu6p1UA5YwZnBYRe9P7vwJO\nS+8PA49lyu1Oty22/RiSNkialTS7b9++EqpqZta8IieYzc1VX48ylTqAHBEBRInPtykixiJibNWq\nVWU9rZnZcSuy7MTISPX1KFsZYfB42v1D+vWJdPsccHqm3Np022Lbzcw6XpFWwY4d1dejbGWEwe1A\nbUbQeuC2zPYr01lF5wHPpN1JPwDOlzSUDhyfn24zM+too6P5ZbrhBLN6TlhKYUnfAt4OnCppN8ms\noM8Ct0j6IPAo8N60+B3ARcAu4CDwAYCIeFrSZ4CfpuU+HRELB6XNzDrK9DTs3JlfrhtOMKtHSTd/\n5xsbG4vZ2dl2V8PM+tTAAOR9XE5OdlYYSNoWEWNFyvoMZDOzHFNT+UEAnRUES+UwMDPLUWTQePPm\n6utRJYeBmVkDw3XPgjrayEh3nWBWj8PAzGwRU1OwZ09+uW6cSrqQw8DMbBFFuoe6dSrpQg4DM7M6\niixPDd09aJzlMDAzq6PIpSy7fdA4y2FgZrbA8uX5ZXph0DjLYWBmljE6Ci+8kF+uFwaNsxwGZmap\noktO9FL3UI3DwMwsdcUV+WW67aI1RTkMzMxIZg8VWXKi2y5aU5TDwMyM/ps9tJDDwMz63uBgfple\n7R6qcRiYWV8bGoL5+fxyvdo9VOMwMLO+NTUFBw7kl+uVJScacRiYWd8qsvbQypW9s+REIw4DM+tL\nRcYJAPbvr7YencJhYGZ9p+g4QS/PHlrIYWBmfaXoOEGvrT2Up+kwkPR6SXdnbs9K+pikT0may2y/\nKPOY6yTtkvSApHc0Wwczs6KKjBMsW9Z7aw/lOaHZJ4iIB4BzACQNAnPArcAHgL+IiD/Plpc0AlwC\njAJrgK2SXhcRR5qti5lZI0XHCQ4dqrYenajsbqJx4KGIeLRBmYuBb0fE8xHxS2AXcG7J9TAzO8pJ\nJ3mcoJGyw+AS4FuZ7z8qabukGyUNpduGgccyZXan28zMKjE6Cs89l1+u38YJskoLA0nLgXcBf51u\n2gj8DkkX0l7g88fxnBskzUqa3bdvX1lVNbM+UnRZ6n4cJ8gqs2VwIfCziHgcICIej4gjETEPfJWX\nuoLmgNMzj1ubbjtGRGyKiLGIGFu1alWJVTWzfnH55cXK9eM4QVaZYXApmS4iSasz+94D3Jvevx24\nRNKJks4EzgZ+UmI9zMwAkIqV69dxgqymZxMBSDoZ+PfARzKb/0zSOUAAj9T2RcQOSbcAO4HDwDWe\nSWRmZSs6c6ifxwmyFEWu5tABxsbGYnZ2tt3VMLMucNJJxQaMV6yAgwerr0+7SNoWEWNFyvoMZDPr\nKUNDxYIAejsIlsphYGY9Y2io2FITUOwSl/3EYWBmPWF42EHQDIeBmXW90VHYs6dY2X64UM3xcBiY\nWVcbHS12Uhkk1zHuhwvVHA+HgZl1raUEwcqVvX8d42Y4DMysKy0lCFas6J8rlh2vUk46MzNrpeHh\n4mMEy5Z5CmkRDgMz6ypLmT66bJnXHCrKYWBmXaPomcUAAwMOgqVwGJhZVxgcLHZxmpojXvFsSRwG\nZtbxiq4+WuOTypbOs4nMrGNNTy8tCAYGHATHyy0DM+tIExMwM1O8vAeLm+MwMLOOs5QZQ+AgKIPD\nwMw6ylLHB1au9AllZfCYgZl1hKmppQfByIiDoCxuGZhZ2y3l/IGa8XHYurWa+vQjtwzMrG1qrYGl\nBsHmzQ6CsrllYGZtcTytAfDU0aq4ZWBmLTUxcXytgWXLHARVKi0MJD0i6R5Jd0uaTbedImmLpAfT\nr0Ppdkm6QdIuSdslvamsephZZ6qdQLaUcwdqxsc9dbRqZXcT/buIeDLz/bXATER8VtK16fefAC4E\nzk5vvw9sTL9aLzje9r/1rADel96WSgAztTt9Zs2all2Rp+puoouBm9L7NwHvzmz/RiTuAlZKWl1x\nXawsw8PJv3iL3RwEtoCauPW1PXuSv7cWKDMMArhT0jZJG9Jtp0XE3vT+r4DT0vvDwGOZx+5Otx1F\n0gZJs5Jm9+3bV2JVbclqbXyp+FVFzKx5Lfp7K7Ob6C0RMSfpNcAWSfdnd0ZESFrS8E9EbAI2AYyN\njXnoqF2WujaAmXWd0sIgIubSr09IuhU4F3hc0uqI2Jt2Az2RFp8DTs88fG26zTrNUk8JNbOuVEo3\nkaSTJb2idh84H7gXuB1YnxZbD9yW3r8duDKdVXQe8EymO8k6hYPArP3WrGnJy5TVMjgNuFXJh8cJ\nwDcj4u8l/RS4RdIHgUeB96bl7wAuAnYBB4EPlFQPK0tJQdAtfXuOvWLK/n36uOdo4WyiUsIgIh4G\n/nWd7U8B43W2B3BNGa9tFVi+fEnFG31APMlKXkN3ryQmwc03w2WXtbsm7VHFTOHNm/v3eHYqn4Fs\nRxsdhRdeyC0W6W0ecRmbGSDq3ro9CCA56/XyyxefSTswkEy26hVDQ9XNFN68OTmeDoLO4zCwl0xP\nw86ducUC+DUrGCAYZJ5v0d9/2d0cFvVOGSl74pjkEOgGDgN7yRVX5BYJ4E7GeRUHq69Pj8gLi+xt\nYqL81z/ppMVfr8op7CtWJD/7/LxDoBs4DCwxMUHkrAIWwJeY5AK8dnBVZmaKhcZSbq0+IXxkJAmB\ng/5/oas4DIzRUYiZmYYzO2otgv/Ml0t//cnJ5MOj2dvISOlVs4KyXUE7drS7NnY8fD2DPlY7sfgZ\nTmpYLkhmBR1vi2BkpDUfEEt5Da+lV47JSfhy+f8fWBs4DPrQ8PBLfcVfZIpX8Fxuq6DorKBuuRRh\nXhdG9hjZS3zx+d7lbqI+Ups5kv2Qu4aNuUFwOZsX3V/rH67duiEIipiba9wltXJlu2vYGtnunwgH\nQS9zGPSB0dH6M0ceOXah2KME8M+sOWbqaLaPv1/7h/fvbxwWLVpBoHQLx288E6h/OAx6WO1i4/VO\nHfgiU5zBntxWwbp0/cDsh4T7iPPltSxqt/Fjzs+vVm2652I3/277l8cMetTy5Y1PJC7SPXTTiknC\n0wMr1Svdatb93DLoMbWLjTcKghcKLA8m4AMH/W+iWb9wy6CHDA4mfbyNHEYMUGC1yM2LDxqbWe9x\ny6AH1FoDjYLg75lgvmgQjIx41NCsz7hl0OXyxgaeYIhTSVYeK7R2/LJl/TtFyKyPuWXQpWrXp68F\nQe0//4W3UzmAWMJFRA4dqqjGZtbJ3DLoNhMTxMwM7wPet2BX01eN8jiBWd9yGHSLdDGdoKJLBU5O\nepzArI+5m6jT1U4fTldVqyQINm/22UZmfc4tg05WZK5os3KuYWBm/aHploGk0yX9UNJOSTsk/VG6\n/VOS5iTdnd4uyjzmOkm7JD0g6R3N1qEnVR0E4+MOAjN7URktg8PAf4mIn0l6BbBN0pZ0319ExJ9n\nC0saAS4BRoE1wFZJr4uIIyXUpTcsX15NEEhw880eGzCzYzQdBhGxF9ib3v+1pPug4XKYFwPfjojn\ngV9K2gWcC/y/ZuvSE4aGGp84sED2f/ujxhNadUUZM+sJpQ4gS1oH/B7w43TTRyVtl3SjpKF02zDw\nWOZhu2kcHv1jaiq59FiOyNzuZJyTVwRauPykg8DMlqC0MJD0cuC7wMci4llgI/A7wDkkLYfPH8dz\nbpA0K2l23759ZVW1c23cmFskgEMMMEAwQPC58a2+8LiZNa2UMJC0jCQIpiPibwEi4vGIOBIR88BX\nSbqCAOaA0zMPX5tuO0ZEbIqIsYgYW7VqVRlV7Vyjo+QN59YuNvMykuGVzZu9BLKZlaPpMQNJAr4G\n3BcRX8hsX52OJwC8B7g3vX878E1JXyAZQD4b+Emz9ehq09PEzp251xfYzgjnkHT/eCKQmZWpjNlE\n/xa4ArhH0t3ptv8OXCrpHJLPsUeAjwBExA5JtwA7SWYiXdPvM4mOXH4Fgw3211oEDgIzq0oZs4n+\nkfonxt7R4DF/AvxJs6/dC/7qpCnen9NBNE9y+cmBATjS17FpZlXxGchtNDoK9z6Xf/nJK9jMsmVe\nUNTMquO1idpkdBS+sHOiYZnaOMHfLLvMQWBmlXIYtMHEBOzcCeczk9sq+IMVOxwEZlY5h0GLTU/D\nzExyBbJGAvjaskmfQ2BmLeEwaLHLL4cvMvXiFcgWcwT48CEvK21mreEwaCGln/7XkD9ofIKvOmZm\nLeQwaJHB9ESC3zY8oyAJAq1Z45VFzaylHAYtMDSUrEj9WwZZznzDVoEA5uquzmFmVhmfZ1Cx0dFk\nIdJDiBMocNnKyckW1MrM7GgOgwpNTSVTSF9ADFIgCFas8LWIzawt3E1UoZGNUxwpGgSA55GaWbu4\nZVCRf9Yw17CnWAiAu4fMrK0cBhV4SkOcnnMewVFGRtw9ZGZt5W6ikj0zOMQpSw0CX6LSzNrMYVCi\n3wwN88p5B4GZdR+HQVmmpjj5QP4YwYtXLpicdBCYWcfwmEFJYmPjJSYgCYIA5EuVmVmHccugBEfU\neIkJSELgBQYYcBCYWQdyGDRreJiBnCUmAvg1K1je35d6NrMO5jBoxvQ0safxOEEtCF4ZPqHMzDqX\nw6AJ85dfkRsEhxjgf292EJhZZ2tbGEi6QNIDknZJurZd9ThuU1OIxv3/AZy15ohXozazjteWMJA0\nCHwJuBAYAS6VNNKOuhyvvNlDAVzBZq9GbWZdoV0tg3OBXRHxcEQcAr4NXNymuizZT4cmGu4PYDsj\nTIebBGbWHdoVBsPAY5nvd6fbjiJpg6RZSbP79u1rWeXyjB2YyW0V/K9Jn1BmZt2joweQI2JTRIxF\nxNiqVavaXR0A7tFow/0BbBqY9LpzZtZV2hUGc8Dpme/Xpts62tQUvIGdDVsFhxFXH3ESmFl3aVcY\n/BQ4W9KZkpYDlwC3t6kuhX1i4zE9WUcJ4Obxm1tTGTOzErUlDCLiMPBR4AfAfcAtEdHRnewfOmma\nMxpcrCaA3azhqq0eNDaz7tO2heoi4g7gjna9/lJMTcGm5y7PXYju9Oj4ni4zs7o6egC5U3xi43Du\n7KH71oy3qjpmZqVzGOT4H6ONu4dqRua2tqQ+ZmZVcBjkuHZn/vpDe0bcKjCz7uYwaGCHRgutPzS8\nw60CM+tuDoNF3DgxzUjOOQUBDGze3KoqmZlVxmGwiPUz+d1D+9eM4CVJzawXOAzq+N7wFAMFuodO\nmevoUyPMzApzGNRx8Z785akHJidbVR0zs8o5DBbYOZi/EN3TK9bglejMrJc4DDL+cWqa353PHzR+\n9UGfaWxmvcVhkHHuxvfnBsEvxt09ZGa9x2GQunFimmUcblhmHvhXW909ZGa9x2GQunTmQ7mtgkGf\nU2BmPcphQLIq6cv47aL7A5hb6XMKzKx3OQyAj2xsPIMIYO1+n1NgZr2r78NgYgLe2GDZCS9PbWb9\noO/D4GszjS9lCV6e2sx6X1+HwX8bzr+UpZenNrN+0Ndh8Nk9jRejAy9PbWb9oW/D4JtDjRejc6vA\nzPpJU2Eg6XOS7pe0XdKtklam29dJek7S3entK5nHvFnSPZJ2SbpBUt4/55W49ED+YnRuFZhZv2i2\nZbAFeENEvBH4BXBdZt9DEXFOers6s30j8GHg7PR2QZN1WLJf66SG+70qqZn1m6bCICLujIjaGg53\nAWsblZe0GnhlRNwVEQF8A3h3M3VYqt8MDfNynsttFXhVUjPrJ2WOGVwFfD/z/ZmS/knSP0h6a7pt\nGNidKbM73dYaw8OcfGDx2UPgVoGZ9acT8gpI2gq8ts6u6yPitrTM9cBhYDrdtxc4IyKekvRm4HuS\n8k/zPfa1NwAbAM4444ylPvxog4PE/Hzu7KF5YNCtAjPrM7lhEBETjfZLej/wTmA87fohIp4Hnk/v\nb5P0EPA6YI6ju5LWptsWe+1NwCaAsbGxxtehbGRoCAoEgRejM7N+1exsoguAjwPvioiDme2rJA2m\n988iGSh+OCL2As9KOi+dRXQlcFszdSjkwIHcIi8uO+HF6MysD+W2DHL8T+BEYEs6Q/SudObQ24BP\nS3qBpOfl6oh4On3MFPB1YAXJGMP3Fz5pqwXwJCu97ISZ9a2mwiAi/uUi278LfHeRfbPAG5p53aUK\naLjkxCEGuHR8P44CM+tXzbYMusJTrOTVHDgmEAJ4jmWczCHCSWBmfawvlqNYxX6eZCUBR93uZJyT\nOcS4V50wsz7XFy0DgNewf9F9W90qMLM+1xctg0Z8fpmZmcPAq06YmdEnYbDYf/9uFZiZJfpizKD2\n3/+mTXDkCAwOwoYNbhWYmdX0RRhA8sHvD38zs/r6opvIzMwacxiYmZnDwMzMHAZmZobDwMzMAKXX\no+l4kvYBj5bwVKcCT5bwPGVynYrrxHq5TsV1Yr16uU7/IiJWFSnYNWFQFkmzETHW7npkuU7FdWK9\nXKfiOrFerlPC3URmZuYwMDOz/gyDTe2uQB2uU3GdWC/XqbhOrJfrRB+OGZiZ2bH6sWVgZmYL9GQY\nSPpPknZImpc0tmDfdZJ2SXpA0jsWefyZkn6clvuOpOUl1+87ku5Ob49IunuRco9IuictN1tmHeq8\n1qckzWXqddEi5S5Ij90uSddWWaf09T4n6X5J2yXdKmnlIuUqP1Z5P7ukE9Pf7a70/bOuinpkXu90\nST+UtDN9v/9RnTJvl/RM5vf6x1XWKfO6DX8fStyQHqvtkt5UcX1enzkGd0t6VtLHFpSp/FhJulHS\nE5LuzWw7RdIWSQ+mX4cWeez6tMyDktaXXTciouduwO8Crwf+LzCW2T4C/Bw4ETgTeAgYrPP4W4BL\n0vtfASYrrOvngT9eZN8jwKktOmafAv5rTpnB9JidBSxPj+VIxfU6Hzghvf+nwJ+241gV+dmBKeAr\n6f1LgO9UfGxWA29K778C+EWdOr0d+LtWvIeW8vsALgK+Dwg4D/hxC+s2CPyKZA5+S48V8DbgTcC9\nmW1/Blyb3r+23nscOAV4OP06lN4fKrNuPdkyiIj7IuKBOrsuBr4dEc9HxC+BXcC52QKSBPwh8Dfp\nppuAd1dRz/S13gt8q4rnr8C5wK6IeDgiDgHfJjmmlYmIOyPicPrtXcDaKl+vgSI/+8Uk7xdI3j/j\n6e+4EhGxNyJ+lt7/NXAfMFzV65XsYuAbkbgLWClpdYteexx4KCLKOIl1SSLiR8DTCzZn3zeLfd68\nA9gSEU9HxH5gC3BBmXXryTBoYBh4LPP9bo7943k1cCDzAVSvTFneCjweEQ8usj+AOyVtk7Shojpk\nfTRtst+4SFO1yPGr0lUk/03WU/WxKvKzv1gmff88Q/J+qlzaJfV7wI/r7P4DST+X9H1Jo62oD/m/\nj3a+ly5h8X/A2nGsTouIven9XwGn1SlT+fHq2ovbSNoKvLbOrusj4rZW12ehgvW7lMatgrdExJyk\n1wBbJN2f/mdRep2AjcBnSP6IP0PSfXXV8b5WWfWqHStJ1wOHgelFnqbUY9VNJL0c+C7wsYh4dsHu\nn5F0h/wmHQf6HnB2C6rVkb+PdPzvXcB1dXa361i9KCJCUlumeHZtGETExHE8bA44PfP92nRb1lMk\nTdYT0v/u6pVpun6STgD+A/DmBs8xl359QtKtJF0Vx/0HVfSYSfoq8Hd1dhU5fqXXS9L7gXcC45F2\noNZ5jlKPVR1FfvZamd3p7/dVJO+nykhaRhIE0xHxtwv3Z8MhIu6Q9GVJp0ZEpWvxFPh9VPJeKuBC\n4GcR8fjCHe06VsDjklZHxN60q+yJOmXmSMY0ataSjImWpt+6iW4HLklnfZxJkvo/yRZIP2x+CPzH\ndNN6oIqWxgRwf0TsrrdT0smSXlG7TzKQem+9smVY0F/7nkVe66fA2UpmWy0naW7fXlWd0npdAHwc\neFdEHFykTCuOVZGf/XaS9wsk75//s1h4lSEdj/gacF9EfGGRMq+tjVtIOpfkb77qgCry+7gduDKd\nVXQe8Eymq6RKi7bG23GsUtn3zWKfNz8Azpc0lHbhnp9uK0+VI+ftupF8mO0GngceB36Q2Xc9yayQ\nB4ALM9vvANak988iCYldwF8DJ1ZQx68DVy/Ytga4I1OHn6e3HSRdJlUes5uBe4DtJG/O1QvrlH5/\nEcmslYeqrlP6ertI+krvTm9fWVivVh2rej878GmSoAJ4Wfp+2ZW+f86q+Ni8haRbb3vm+FwEXF17\nbwEfTY/Jz0kG4P9NC35ndX8fC+ol4EvpsbyHzKy/Cut1MsmH+6sy21p6rEiCaC/wQvoZ9UGScaUZ\n4EFgK3BKWnYM+MvMY69K31u7gA+UXTefgWxmZn3XTWRmZnU4DMzMzGFgZmYOAzMzw2FgZmY4DMzM\nDIeBmZnhMDAzM+D/A17cW9hpcKLfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd3cec84588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn.fit(X_train, yarr)\n",
    "\n",
    "y_pred = nn.predict(X_train)\n",
    "\n",
    "plt.scatter(xarr, yarr, color='blue')\n",
    "plt.scatter(xarr, y_pred, color='red')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2275"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.n_iter_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(1,), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=10000, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=None,\n",
       "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([ 23.4775502]), array([ 22.48208766])]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.intercepts_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 2.3698002]]), array([[ 18.3433664]])]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.coefs_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135021.61485702879"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.loss_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1547.4110678542072"
      ]
     },
     "execution_count": 437,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#squared_loss = sklearn.metrics.mean_squared_error(yarr, y_pred)\n",
    "squared_loss = ((yarr - y_pred) ** 2).mean() / 2\n",
    "squared_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I think my loss function does not give the same result as the loss\\_ parameter because it doesn't evaluate the same points?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fd3cf007f28>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGhhJREFUeJzt3W2MXOd53vH/xTeZ6yTalcUq2iVR0gjhdjdAa2VBK3Ua\nGOaCopTAVALXoCFHrC2UCNdu7aJFKtVAFFj5EPclbgRENFRLDeUsLCmKUxGuXIWiFeSTZC1lmRIp\nq1y/KOKSkjYmKbulYZHS3Q/zrDncedmzM3Nmzsy5fsBgZ855Zubes8O5eM4z9xlFBGZmZtVW9boA\nMzMrHoeDmZnVcDiYmVkNh4OZmdVwOJiZWQ2Hg5mZ1XA4mJlZDYeDmZnVcDiYmVmNNb0uoFVXX311\nbN68uddlmJn1jSNHjvx9RGzIMrZvw2Hz5s3Mzs72ugwzs74h6eWsY31YyczMajgczMyshsPBzMxq\nOBzMzKyGw8HMzGosGw6S7pf0uqQXqpZdJemQpBPp50haLkl3S5qTdFTSdVX32ZPGn5C0p2r5r0h6\nPt3nbknq9C9pZmYrk2XP4c+AnUuW3Q4cjoitwOF0G+BGYGu67AX2QyVMgDuB9wHbgDsXAyWN+VdV\n91v6XGZm5TQ2BtKly9hY15562XCIiL8FzixZvAs4kK4fAG6uWv5AVDwFDEu6FrgBOBQRZyLiLHAI\n2JnW/UJEPBWV7yt9oOqxzMzKS4JTpy5fdupU1wKi1TmHayLidLr+KnBNuj4GvFI17mRa1mz5yTrL\nzczKa926xuuWBkZO2p6QTv/jjw7UsixJeyXNSppdWFjoxlOamXXXxARcuNDrKloOh9fSISHSz9fT\n8nlgU9W4jWlZs+Ub6yyvKyLujYjJiJjcsCHT6UHMzPrHzAwcP97rKoDWw+EgsPiJoz3Ao1XLb02f\nWroeeCMdfnoc2CFpJE1E7wAeT+t+JOn69CmlW6sey8ysXD72seXHjI7mXwcZTrwn6SvAB4CrJZ2k\n8qmjPwIelnQb8DLwkTT8MeAmYA44D3wcICLOSLoLeCaN+1xELE5yT1P5RNR64OvpYmZWLlk/xT/f\n8OBKRy0bDhHx0QarttcZG8AnGzzO/cD9dZbPAr+8XB1mZgNrZGT5MQD79uVbRxV3SJuZ9dLUFJw7\nt/y44WG4557860kcDmZmvTIzA4cPLz9u1So4ezb/eqqfsqvPZmZml2SZgAZ4661866jD4WBm1gvN\nGt2q/fmf51tHAw4HM7Nuy9roNj4Ot9ySfz11OBzMzLopa6Pb+vVw7Fj+9TTgcDAz66as8wznz+db\nxzIcDmZm3ZK10S26crq6phwOZmbdUMBGt2YcDmZmeStoo1szDgczszwVuNGtGYeDmVmeCtzo1ozD\nwcwsLwVvdGvG4WBmloexscI3ujXjcDAz67SpqWzf9dzjRrdmHA5mZp2UdQIaet7o1ozDwcysk7JO\nQBeg0a0Zh4OZWaf0WaNbMw4HM7NOyNroNjpamEa3ZhwOZmbtyjrPsHYtzM/nX08HOBzMzNqVdZ7h\nzTfzraODHA5mZu3o40a3ZhwOZmat6vNGt2YcDmZmrRiARrdmHA5mZis1II1uzTgczMxWakAa3Zpx\nOJiZrcTQULZxfdDo1ozDwcwsq7Ex+MlPlh/XJ41uzTgczMyymJ7ONgHdR41uzbQVDpL+raRjkl6Q\n9BVJ75C0RdLTkuYkPSRpXRp7Rbo9l9ZvrnqcO9LylyTd0N6vZGaWg/37s43ro0a3ZloOB0ljwL8B\nJiPil4HVwG7g88AXIuKXgLPAbekutwFn0/IvpHFIGk/3mwB2AvdIWt1qXWZmHbc641tSnzW6NdPu\nYaU1wHpJa4Ah4DTwQeCRtP4AcHO6vivdJq3fLklp+YMR8dOI+D4wB2xrsy4zs84YGYG3315+3Pbt\nfdfo1kzL4RAR88B/Af6OSii8ARwBzkXExTTsJDCWro8Br6T7Xkzj31W9vM59zMx6J+uZVoeH4Ykn\n8q+ni9o5rDRC5X/9W4BR4J1UDgvlRtJeSbOSZhcWFvJ8KjMru6yNbqtWwdmz+dfTZe0cVpoCvh8R\nCxFxAfgq8H5gOB1mAtgILE7bzwObANL6K4EfVi+vc5/LRMS9ETEZEZMbNmxoo3Qzs2VkbXR76618\n6+iRdsLh74DrJQ2luYPtwHHgSeDDacwe4NF0/WC6TVr/jYiItHx3+jTTFmAr8M026jIza0/WRrcB\nmoBeas3yQ+qLiKclPQI8C1wEvgXcC/wv4EFJf5iW3Zfuch/wZUlzwBkqn1AiIo5JephKsFwEPhkR\ngxnFZlZ8WRvd+vBMqyuh6NNzf0xOTsbs7GyvyzCzQTI9na2fYe3avuxnkHQkIiazjHWHtJnZopI1\nujXjcDAzg1I2ujXjcDAzK2mjWzMOBzMrtxI3ujXjcDCz8ip5o1szDgczK6+SN7o143Aws3Jaty7b\nuJJMQC/lcDCz8hkbgwsXlh834I1uzTgczKxcVvKNbseO5V9PQTkczKxc3OiWicPBzMoja6Nbn55W\nqJMcDmZWDkND2Rrd9u3Lv5Y+4HAws8E3MZHtTKvDw3DPPfnX0wccDmY22GZm4Pjx5ceVsNGtGYeD\nmQ02N7q1xOFgZoPLjW4tcziY2WByo1tbHA5mNnimprI1uq1fX+pGt2YcDmY2WFZyptXz5/Ovp085\nHMxssHgCuiMcDmY2OIaGso1zo9uyHA5mNhjc6NZRDgcz639ZG93WrnWjW0YOBzPrf1nnGUp+ptWV\ncDiYWX/LeqZVN7qtiMPBzPrXyEi2M6260W3FHA5m1p+mpuDcueXHudGtJQ4HM+s/bnTLncPBzPqP\nG91y11Y4SBqW9Iik70h6UdKvSrpK0iFJJ9LPkTRWku6WNCfpqKTrqh5nTxp/QtKedn8pMxtgbnTr\ninb3HP4E+N8R8Y+AfwK8CNwOHI6IrcDhdBvgRmBruuwF9gNIugq4E3gfsA24czFQzMwuMzaWrdFt\ndNSNbm1qORwkXQn8OnAfQES8GRHngF3AgTTsAHBzur4LeCAqngKGJV0L3AAciogzEXEWOATsbLUu\nMxtQ09PZz7Q6P59/PQOunT2HLcAC8D8kfUvSlyS9E7gmIk6nMa8C16TrY8ArVfc/mZY1Wm5mdsn+\n/dnGeQK6I9oJhzXAdcD+iHgv8P+4dAgJgIgIINp4jstI2itpVtLswsJCpx7WzIrOjW5d1044nARO\nRsTT6fYjVMLitXS4iPTz9bR+HthUdf+NaVmj5TUi4t6ImIyIyQ0bNrRRupn1DTe69UTL4RARrwKv\nSHpPWrQdOA4cBBY/cbQHeDRdPwjcmj61dD3wRjr89DiwQ9JImojekZaZWdllbXQbHnajW4etafP+\n/xqYkbQO+B7wcSqB87Ck24CXgY+ksY8BNwFzwPk0log4I+ku4Jk07nMRcabNusys32VtdPOZVnOh\nyrRA/5mcnIzZ2dlel2FmeZGyjevT97BekHQkIiazjHWHtJkVjxvdes7hYGbF4ka3QnA4mFlxuNGt\nMBwOZlYcbnQrDIeDmRWDG90KxeFgZr03NORGt4JxOJhZb01MZJuAdqNbVzkczKx3Zmbg+PHlx7nR\nrescDmbWO1m/0e3NN/Otw2o4HMysN7JOQLvRrSccDmbWfSs506ob3XrC4WBm3TU97TOt9gGHg5l1\nV9ZGN09A95TDwcy6x41ufcPhYGbdkbXRbd8+N7oVgMPBzPKXtdHNE9CF4XAws3ytpNHNE9CF4XAw\ns3y50a0vORzMLD9udOtbDgczy4cb3fqaw8HMOs+Nbn3P4WBmnedGt77ncDCzzpKyjXOjW6E5HMys\nc9atyzbOjW6F53Aws86YmIALF5Yf5wnovuBwMLP2udFt4DgczKx9bnQbOA4HM2tP1ka3iHzrsI5y\nOJhZ67I2urkDuu84HMysNStpdPMEdN9pOxwkrZb0LUlfS7e3SHpa0pykhyStS8uvSLfn0vrNVY9x\nR1r+kqQb2q3JzLrAjW4DrRN7Dp8GXqy6/XngCxHxS8BZ4La0/DbgbFr+hTQOSePAbmAC2AncIynj\nQUwz6wk3ug28tsJB0kbgN4AvpdsCPgg8koYcAG5O13el26T129P4XcCDEfHTiPg+MAdsa6cuM8uR\nG91Kod09h/8G/B6wOCP1LuBcRFxMt08CY+n6GPAKQFr/Rhr/s+V17nMZSXslzUqaXVhYaLN0M1sx\nN7qVRsvhIOk3gdcj4kgH62kqIu6NiMmImNywYUO3ntbMwI1uJbOmjfu+H/iQpJuAdwC/APwJMCxp\nTdo72AjMp/HzwCbgpKQ1wJXAD6uWL6q+j5kVhRvdSqXlPYeIuCMiNkbEZioTyt+IiFuAJ4EPp2F7\ngEfT9YPpNmn9NyIi0vLd6dNMW4CtwDdbrcvMcpB1AtqNbgOjnT2HRv4D8KCkPwS+BdyXlt8HfFnS\nHHCGSqAQEcckPQwcBy4Cn4yIt3Koy8xaMTKSbZwb3QaKok+TfnJyMmZnZ3tdhtlgm57O1s8wPOx+\nhj4g6UhETGYZ6w5pM2ssSzCsWuVgGEAOBzOrL+s8w1s+CjyIHA5mVitro5s7oAeWw8HMLreSRjd3\nQA8sh4OZXTI97UY3AxwOZrZoZib7mVbd6DbwHA5mVpG1A7pPP/5uK+NwMDM3ulkNh4NZ2U1N+Rvd\nrIbDwazMZmbg8OHlx7nRrXQcDmZllnWewY1upeNwMCsrN7pZEw4HszIaG3OjmzXlcDArm6kpOHVq\n+XHr17vRrcQcDmZlknUCGuD8+XxrsUJzOJiViRvdLCOHg1lZuNHNVsDhYFYGExPZGt1GR93oZoDD\nwWzwreRMq/Pz+ddjfcHhYDbofKZVa4HDwWyQudHNWuRwMBtUbnSzNjgczAaRG92sTQ4Hs0GzkjOt\nutHNGnA4mA0an2nVOsDhYDZIhoayjXOjmy3D4WA2KMbG4Cc/WX6cG90sA4eD2SCYns42Ae1GN8vI\n4WA2CNzoZh3WcjhI2iTpSUnHJR2T9Om0/CpJhySdSD9H0nJJulvSnKSjkq6reqw9afwJSXva/7XM\nSsSNbpaDdvYcLgL/LiLGgeuBT0oaB24HDkfEVuBwug1wI7A1XfYC+6ESJsCdwPuAbcCdi4FiZsvI\n2ui2fbsb3WxFWg6HiDgdEc+m6z8GXgTGgF3AgTTsAHBzur4LeCAqngKGJV0L3AAciogzEXEWOATs\nbLUus9LI2ug2PAxPPJF/PTZQOjLnIGkz8F7gaeCaiDidVr0KXJOujwGvVN3tZFrWaLmZNbKSRrez\nZ/OvxwZO2+Eg6eeAvwQ+ExE/ql4XEQF07CulJO2VNCtpdmFhoVMPa9Z/3OhmOWsrHCStpRIMMxHx\n1bT4tXS4iPTz9bR8HthUdfeNaVmj5TUi4t6ImIyIyQ0bNrRTuln/cqObdUE7n1YScB/wYkT8cdWq\ng8DiJ472AI9WLb81fWrpeuCNdPjpcWCHpJE0Eb0jLTOzpdzoZl2ypo37vh/4HeB5Sc+lZf8R+CPg\nYUm3AS8DH0nrHgNuAuaA88DHASLijKS7gGfSuM9FxJk26jIbTG50sy5SZVqg/0xOTsbs7GyvyzDr\nHinbuD79N235k3QkIiazjHWHtFk/WL062zg3ulmHOBzMim5oCN5+e/lxbnSzDnI4mBXZxES2CWg3\nulmHORzMimpmBo4fX36cG90sBw4Hs6Jyo1upDQ1VPoNQfRnr4rkjHA5mRZS10c0T0ANlbOxSENQ7\nmnjqVPcCwuFgVjRZG93Gxz0BPQAmJi4FQpY2lixjOsHhYFYkWRvd1q+HY8fyr8dyMT19KRCyTCv1\nQjsd0mbWaVm/0e38+XzrsI6bns7+5y0Ch4NZUbjRbeBMTHR+z2B0tLOP14jDwawI3Og2MEZG4Ny5\nfB57dLR7p83ynINZr7nRra9NTV3+cdM8gmH9+sops7p5PkWHg1kvudGt71RPJkvZvpCvVfv2VUKh\nF1NMPqxk1ktudCu8bk8kb99ejB1Eh4NZr3gCupDymERezvh48T6Z7MNKZr0wMpJtAtqNbrlaeoio\nm30H4+OVQ0YRxQsGcDiYdd/0dLZZSze6ddTSiWOp+30Hi3MIRQ2Eag4Hs27L8o60apUb3VpUb28g\n74njRqTKUcHFQOinr/X2nINZN2WdZ/AE9LLy7CdoR1EmlNvlcDDrlqyNbvv25V9LH+jFxHArutmY\n1k3lDYei/rfDym10tL+OPbSgX970GxnUMFiqnHMODgYrorVr++Zdp/o00yu99FswbN9+ac6g213K\nvVTOPQcHgxXM28DqC2+Cel1JuZVlryCLcu45mBVIAB/DjW7dVt1nULa9gizKuedgVhABHGWcr+BG\nt7x4b6A15dxzGB7udQVmBPBj1vNPKXg3VB9Y2k/gvYH2lXPP4ezZQk5KR68LsK46yriDYQUGpX+g\nX5QzHKDnpz8eGsp2Cn+zQec3/WIq52GlLhsbq/04X9mCod7kX5EvPhHqyiz9uOdKLg6GYipMOEja\nKeklSXOSbu91Pa2qd16XU6d6XVVrRkc792Zb9JOMLXXLLb0PqH66+A1+8BQiHCStBv4UuBEYBz4q\naby3VWWzNAy6fZbHlVrJG74n8szKqxDhAGwD5iLiexHxJvAgsKvHNdU1M1PsMKg+JbDf8M2sVUWZ\nkB4DXqm6fRJ4X49qqTE2VqxDQ/v2Dfzpd8ysx4qy55CJpL2SZiXNLiws5PY8S/cOehUMjfYCHAxm\nlrei7DnMA5uqbm9Myy4TEfcC9wJMTk5GJwuYmurNl4FApSevx5+sNTO7TFH2HJ4BtkraImkdsBs4\nmPeTVn9tYLeCoV4np4PBzIqmEHsOEXFR0qeAx4HVwP0RkeuHH1evzva9K+3yeV3MrB8VIhwAIuIx\n4LFuPNfISH7B4DAws0FQmHDopk6eUslhYGaDqChzDn1j6ZyBg8HMBlEp9xxWynsHZlY2pdxzyPJ1\nDtUnEnMwmFnZlDIczp6tHxDVgeATiZlZmZX2sJJ7C8zMGivlnoOZmTXncDAzsxoOBzMzq+FwMDOz\nGg4HMzOroYiOnvm6ayQtAC934KGuBv6+A4/TSa4puyLW5ZqyK2Jdg1zTP4yIDVkG9m04dIqk2YiY\n7HUd1VxTdkWsyzVlV8S6XFOFDyuZmVkNh4OZmdVwOKSvHS0Y15RdEetyTdkVsS7XhOcczMysDu85\nmJlZjVKEg6R/IemYpLclTS5Zd4ekOUkvSbqhwf23SHo6jXtI0roO1/eQpOfS5QeSnmsw7geSnk/j\nZjtZQ53n+gNJ81V13dRg3M607eYk3Z5nTen5/rOk70g6KumvJNU9AXs3ttVyv7ukK9Lfdi69fjbn\nUUfV822S9KSk4+n1/uk6Yz4g6Y2qv+vv51lT1fM2/Xuo4u60rY5Kui7net5TtQ2ek/QjSZ9ZMib3\nbSXpfkmvS3qhatlVkg5JOpF+jjS475405oSkPZ2ujYgY+Avwj4H3AH8DTFYtHwe+DVwBbAG+C6yu\nc/+Hgd3p+heBfTnW+l+B32+w7gfA1V3aZn8A/PtlxqxO2+zdwLq0LcdzrmsHsCZd/zzw+V5sqyy/\nOzANfDFd3w08lPO2uRa4Ll3/eeD/1KnpA8DXuvEaWsnfA7gJ+Dog4Hrg6S7Wthp4lUoPQFe3FfDr\nwHXAC1XL/hNwe7p+e73XOHAV8L30cyRdH+lkbaXYc4iIFyPipTqrdgEPRsRPI+L7wBywrXqAJAEf\nBB5Jiw4AN+dRZ3qujwBfyePxc7ANmIuI70XEm8CDVLZpbiLiryPiYrr5FLAxz+drIsvvvovK6wUq\nr5/t6W+ci4g4HRHPpus/Bl4ExvJ6vg7bBTwQFU8Bw5Ku7dJzbwe+GxGdaKpdkYj4W+DMksXVr5tG\n7zc3AIci4kxEnAUOATs7WVspwqGJMeCVqtsnqf3H9C7gXNUbUr0xnfLPgdci4kSD9QH8taQjkvbm\nVEO1T6Vd/Psb7Npm2X55+gSV/23Wk/e2yvK7/2xMev28QeX1lLt0COu9wNN1Vv+qpG9L+rqkiW7U\nw/J/j16+lnbT+D9kvdhW10TE6XT9VeCaOmNy314D82U/kp4AfrHOqs9GxKPdrmepjPV9lOZ7Db8W\nEfOS/gFwSNJ30v88Ol4TsB+4i8o/6ruoHO76RKvP1am6FreVpM8CF4GZBg/T0W3VTyT9HPCXwGci\n4kdLVj9L5fDJ/03zSP8T2NqFsgr590jzhx8C7qizulfb6mciIiT15COlAxMOETHVwt3mgU1Vtzem\nZdV+SGUXd03631+9MW3XJ2kN8NvArzR5jPn083VJf0Xl0EbL/8CybjNJ/x34Wp1VWbZfx+uS9C+B\n3wS2RzoAW+cxOrqt6sjyuy+OOZn+vldSeT3lRtJaKsEwExFfXbq+Oiwi4jFJ90i6OiJyPZdQhr9H\nLq+lDG4Eno2I15au6NW2Al6TdG1EnE6H1l6vM2aeypzIoo1U5lQ7puyHlQ4Cu9OnSrZQ+V/BN6sH\npDefJ4EPp0V7gDz2RKaA70TEyXorJb1T0s8vXqcyMftCvbGdsOR47281eK5ngK2qfJprHZXd84N5\n1ZTq2gn8HvChiDjfYEw3tlWW3/0gldcLVF4/32gUZp2Q5jPuA16MiD9uMOYXF+c9JG2j8h6Qd2Bl\n+XscBG5Nn1q6Hnij6tBKnhrurfdiWyXVr5tG7zePAzskjaRDvjvSss7Jcya+KBcqb24ngZ8CrwGP\nV637LJVPnbwE3Fi1/DFgNF1/N5XQmAP+Argihxr/DPjdJctGgceqavh2uhyjcoglz232ZeB54CiV\nF+u1S2tKt2+i8qmY7+ZdU3q+OSrHWp9Lly8uratb26re7w58jkpwAbwjvV7m0uvn3Tlvm1+jchjw\naNX2uQn43cXXFvCptE2+TWVC/5914W9W9++xpC4Bf5q25fNUfaowx7reSeXN/sqqZV3dVlSC6TRw\nIb1H3UZlXuowcAJ4ArgqjZ0EvlR130+k19Yc8PFO1+YOaTMzq1H2w0pmZlaHw8HMzGo4HMzMrIbD\nwczMajgczMyshsPBzMxqOBzMzKyGw8HMzGr8f91pxlxQ9ee0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd3cf36c7b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# tune something by hand\n",
    "nn.intercepts_ = [np.array([-5]), np.array([500.])]\n",
    "nn.coefs_ = [np.array([[10.]]), np.array([[100.]])]\n",
    "\n",
    "y_pred = nn.predict(X_train)\n",
    "\n",
    "plt.scatter(xarr, yarr, color='blue')\n",
    "plt.scatter(xarr, y_pred, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3283.9473643924321"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squared_loss = np.linalg.norm(yarr - y_pred)\n",
    "squared_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(10,), learning_rate='adaptive',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=1234,\n",
       "       shuffle=True, solver='sgd', tol=1e-12, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "losses = []\n",
    "for i in range(10):\n",
    "    nn.fit(X_scaled, yarr)\n",
    "    losses.append(nn.loss_)\n",
    "    \n",
    "losses.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[9632.6542831268725,\n",
       " 9637.4471533925516,\n",
       " 9648.3762939959743,\n",
       " 9656.0743096938877,\n",
       " 9664.0157891645649,\n",
       " 9669.9523768377767,\n",
       " 9686.5937609535831,\n",
       " 9693.5423830722048,\n",
       " 9750.8958593802327,\n",
       " 9756.0456168197743]"
      ]
     },
     "execution_count": 456,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses.sort()\n",
    "losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses= [9632.6542831268725,\n",
    " 9637.4471533925516,\n",
    " 9648.3762939959743,\n",
    " 9656.0743096938877,\n",
    " 9664.0157891645649,\n",
    " 9669.9523768377767,\n",
    " 9686.5937609535831,\n",
    " 9693.5423830722048,\n",
    " 9750.8958593802327,\n",
    " 9756.0456168197743]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  2.,  1.,  1.,  2.,  0.,  0.,  0.,  0.,  2.]),\n",
       " array([ 9632.65428313,  9644.9934165 ,  9657.33254987,  9669.67168323,\n",
       "         9682.0108166 ,  9694.34994997,  9706.68908334,  9719.02821671,\n",
       "         9731.36735008,  9743.70648345,  9756.04561682]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE7VJREFUeJzt3XuQnXd93/H3p/KFcGkQaEmJpbXMxJliB2zDRtCBFJME\nITstoiRNpFxwMjCaMDgNuduhY1MxmQHShhmCE6MJqiFpbBIHt+pEYJSQYBpikOQI27IxyMKNpTCx\nsBwIMcWV+faP82x5vN7VHu2e1R7p937NnNF5fpdnfz895+xnn9s5qSokSW36Z8s9AEnS8jEEJKlh\nhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ07Y7kHMJtVq1bV2rVrl3sYknTK2Lt375er\nauJE+41lCKxdu5Y9e/Ys9zAk6ZSR5H8vpJ+HgySpYYaAJDXMEJCkhhkCktQwQ0CSGjZvCCRZk+Qv\nktyTZH+Sn5+lTZK8J8mBJHcmeVGv7ookX+geV4x6ApKkhRvmEtFjwC9V1R1JngHsTbKrqu7ptbkM\nOL97vAT4XeAlSZ4FXAtMAdX13VFVj4x0FpKkBZl3T6CqvlRVd3TP/xG4FzhnRrONwAdr4HbgmUme\nC7wa2FVVR7tf/LuADSOdgSRpwU7onECStcAlwKdnVJ0DPNhbPtSVzVUuSRoDQ98xnOTpwJ8Ab6mq\nr456IEm2AFsAJicnF7yetVf96aiGdMp44B0/tNxDOOmWazu3+H/dopZeX0PtCSQ5k0EA/Leq+vAs\nTQ4Da3rLq7uyucqfpKq2VdVUVU1NTJzwx19IkhZgmKuDArwfuLeqfmuOZjuA13dXCb0U+EpVfQm4\nFVifZGWSlcD6rkySNAaGORz0MuCngLuS7OvKfh2YBKiq64GdwOXAAeBR4Ge6uqNJ3g7s7vptraqj\noxu+JGkx5g2BqvpfQOZpU8Cb56jbDmxf0OgkSUvKO4YlqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhS\nwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYfN+s1iS\n7cC/AR6qqu+Zpf5XgJ/ore/5wET31ZIPAP8IPA4cq6qpUQ1ckrR4w+wJ3ABsmKuyqn6zqi6uqouB\nq4FPzPge4Vd29QaAJI2ZeUOgqm4Dhv1y+M3AjYsakSTppBnZOYEkT2Wwx/AnveICPpZkb5Ito/pZ\nkqTRmPecwAn4t8BfzTgU9PKqOpzkOcCuJJ/r9iyepAuJLQCTk5MjHJYkaS6jvDpoEzMOBVXV4e7f\nh4BbgHVzda6qbVU1VVVTExMTIxyWJGkuIwmBJN8OvAL4H72ypyV5xvRzYD1w9yh+niRpNIa5RPRG\n4FJgVZJDwLXAmQBVdX3X7N8BH6uqf+p1/Q7gliTTP+cPq+qjoxu6JGmx5g2Bqto8RJsbGFxK2i87\nCFy00IFJkpaedwxLUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapgh\nIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSw+YNgSTbkzyUZNbvB05yaZKvJNnXPa7p1W1I\ncl+SA0muGuXAJUmLN8yewA3AhnnafLKqLu4eWwGSrACuAy4DLgA2J7lgMYOVJI3WvCFQVbcBRxew\n7nXAgao6WFWPATcBGxewHknSEhnVOYF/leSzST6S5MKu7BzgwV6bQ13ZrJJsSbInyZ4jR46MaFiS\npOMZRQjcAZxbVRcBvw3894WspKq2VdVUVU1NTEyMYFiSpPksOgSq6qtV9bXu+U7gzCSrgMPAml7T\n1V2ZJGlMLDoEkvyLJOmer+vW+TCwGzg/yXlJzgI2ATsW+/MkSaNzxnwNktwIXAqsSnIIuBY4E6Cq\nrgd+BHhTkmPA14FNVVXAsSRXArcCK4DtVbV/SWYhSVqQeUOgqjbPU/9e4L1z1O0Edi5saJKkpeYd\nw5LUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEg\nSQ0zBCSpYYaAJDXMEJCkhhkCktSweUMgyfYkDyW5e476n0hyZ5K7knwqyUW9uge68n1J9oxy4JKk\nxRtmT+AGYMNx6r8IvKKqXgC8Hdg2o/6VVXVxVU0tbIiSpKUyzHcM35Zk7XHqP9VbvB1YvfhhSZJO\nhlGfE3gD8JHecgEfS7I3yZbjdUyyJcmeJHuOHDky4mFJkmYz757AsJK8kkEIvLxX/PKqOpzkOcCu\nJJ+rqttm619V2+gOJU1NTdWoxiVJmttI9gSSvBD4PWBjVT08XV5Vh7t/HwJuAdaN4udJkkZj0SGQ\nZBL4MPBTVfX5XvnTkjxj+jmwHpj1CiNJ0vKY93BQkhuBS4FVSQ4B1wJnAlTV9cA1wLOB30kCcKy7\nEug7gFu6sjOAP6yqjy7BHCRJCzTM1UGb56l/I/DGWcoPAhc9uYckaVx4x7AkNcwQkKSGGQKS1DBD\nQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQk\nqWGGgCQ1bKgQSLI9yUNJZv2O4Ay8J8mBJHcmeVGv7ookX+geV4xq4JKkxRt2T+AGYMNx6i8Dzu8e\nW4DfBUjyLAbfSfwSYB1wbZKVCx2sJGm0hgqBqroNOHqcJhuBD9bA7cAzkzwXeDWwq6qOVtUjwC6O\nHyaSpJNo3i+aH9I5wIO95UNd2VzlT5JkC4O9CCYnJ0c0rDasvepPl3sIkk5RY3NiuKq2VdVUVU1N\nTEws93AkqQmjCoHDwJre8uqubK5ySdIYGFUI7ABe310l9FLgK1X1JeBWYH2Sld0J4fVdmSRpDAx1\nTiDJjcClwKokhxhc8XMmQFVdD+wELgcOAI8CP9PVHU3ydmB3t6qtVXW8E8ySpJNoqBCoqs3z1Bfw\n5jnqtgPbT3xokqSlNjYnhiVJJ58hIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqY\nISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsOGCoEkG5Lcl+RAkqtmqX93kn3d4/NJ/qFX\n93ivbscoBy9JWpx5v1ksyQrgOuBVwCFgd5IdVXXPdJuq+oVe+58DLumt4utVdfHohixJGpVh9gTW\nAQeq6mBVPQbcBGw8TvvNwI2jGJwkaWkNEwLnAA/2lg91ZU+S5FzgPODjveKnJNmT5PYkr13wSCVJ\nIzfUF82fgE3AzVX1eK/s3Ko6nOR5wMeT3FVV98/smGQLsAVgcnJyxMOSJM1mmD2Bw8Ca3vLqrmw2\nm5hxKKiqDnf/HgT+kieeL+i321ZVU1U1NTExMcSwJEmLNUwI7AbOT3JekrMY/KJ/0lU+Sf4lsBL4\n617ZyiRnd89XAS8D7pnZV5K0POY9HFRVx5JcCdwKrAC2V9X+JFuBPVU1HQibgJuqqnrdnw+8L8k3\nGQTOO/pXFUmSltdQ5wSqaiewc0bZNTOW3zZLv08BL1jE+CRJS8g7hiWpYYaAJDXMEJCkhhkCktQw\nQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTME\nJKlhQ4VAkg1J7ktyIMlVs9T/dJIjSfZ1jzf26q5I8oXuccUoBy9JWpx5v14yyQrgOuBVwCFgd5Id\ns3xX8Ieq6soZfZ8FXAtMAQXs7fo+MpLRS5IWZZg9gXXAgao6WFWPATcBG4dc/6uBXVV1tPvFvwvY\nsLChSpJGbZgQOAd4sLd8qCub6YeT3Jnk5iRrTrCvJGkZjOrE8P8E1lbVCxn8tf+BE11Bki1J9iTZ\nc+TIkRENS5J0PMOEwGFgTW95dVf2/1XVw1X1jW7x94AXD9u3t45tVTVVVVMTExPDjF2StEjDhMBu\n4Pwk5yU5C9gE7Og3SPLc3uJrgHu757cC65OsTLISWN+VSZLGwLxXB1XVsSRXMvjlvQLYXlX7k2wF\n9lTVDuA/JHkNcAw4Cvx01/dokrczCBKArVV1dAnmIUlagHlDAKCqdgI7Z5Rd03t+NXD1HH23A9sX\nMUZJ0hLxjmFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlh\nhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlq2FAhkGRDkvuSHEhy1Sz1v5jkniR3JvnzJOf26h5P\nsq977JjZV5K0fOb9eskkK4DrgFcBh4DdSXZU1T29Zn8DTFXVo0neBLwL+LGu7utVdfGIxy1JGoFh\n9gTWAQeq6mBVPQbcBGzsN6iqv6iqR7vF24HVox2mJGkpDBMC5wAP9pYPdWVzeQPwkd7yU5LsSXJ7\nktcuYIySpCUy7+GgE5HkJ4Ep4BW94nOr6nCS5wEfT3JXVd0/S98twBaAycnJUQ5LkjSHYfYEDgNr\nesuru7InSPKDwFuB11TVN6bLq+pw9+9B4C+BS2b7IVW1raqmqmpqYmJi6AlIkhZumBDYDZyf5Lwk\nZwGbgCdc5ZPkEuB9DALgoV75yiRnd89XAS8D+ieUJUnLaN7DQVV1LMmVwK3ACmB7Ve1PshXYU1U7\ngN8Eng78cRKAv62q1wDPB96X5JsMAucdM64qkiQto6HOCVTVTmDnjLJres9/cI5+nwJesJgBSpKW\njncMS1LDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlh\nhoAkNcwQkKSGGQKS1DBDQJIaZghIUsOGCoEkG5Lcl+RAkqtmqT87yYe6+k8nWduru7orvy/Jq0c3\ndEnSYs0bAklWANcBlwEXAJuTXDCj2RuAR6rqu4B3A+/s+l7A4IvpLwQ2AL/TrU+SNAaG2RNYBxyo\nqoNV9RhwE7BxRpuNwAe65zcDP5DBN85vBG6qqm9U1ReBA936JEljYJgQOAd4sLd8qCubtU1VHQO+\nAjx7yL6SpGVyxnIPYFqSLcCWbvFrSe5bzvEAq4AvL/MYFut0mAOM0TzyzgV3HZs5LMLpMAcY43mc\nwOtrtjmcu5CfOUwIHAbW9JZXd2WztTmU5Azg24GHh+wLQFVtA7YNN+yll2RPVU0t9zgW43SYA5we\n83AO4+N0mMco5zDM4aDdwPlJzktyFoMTvTtmtNkBXNE9/xHg41VVXfmm7uqh84Dzgc+MYuCSpMWb\nd0+gqo4luRK4FVgBbK+q/Um2AnuqagfwfuD3kxwAjjIICrp2fwTcAxwD3lxVjy/RXCRJJ2iocwJV\ntRPYOaPsmt7z/wP8+zn6/gbwG4sY43IZm0NTi3A6zAFOj3k4h/FxOsxjZHPI4KiNJKlFfmyEJDWs\nqRBI8vNJ7k6yP8lbeuU/l+RzXfm7ZvSZTPK1JL/cKzvux2iM0xySvDDJX3fldyV5Slf+4m75QJL3\ndDf3jeU8kpyZ5APdeO9NcnWv/Vhti+7jU/Z1jweS7Ou1n/UjVE6VOSR5VZK93XbYm+T7e+sZu9fT\n8bZFVz/27+15Xk+jeW9XVRMP4HuAu4GnMjgX8mfAdwGv7J6f3bV7zox+NwN/DPxyt7wCuB94HnAW\n8FnggnGcQ9fmTuCibvnZwIru+WeAlwIBPgJcNq7bAvhxBnee0/V5AFg7jttiRpv/AlzTPb+gG9/Z\nwHnduFecYnO4BPjOXt/DvXZj93qaax69srF/bx9nW4zsvd3SnsDzgU9X1aM1uKv5E8DrgDcB76iq\nbwBU1UPTHZK8FvgisL+3nmE+RmNc5rAeuLOqPtuVP1xVjyd5LvDPq+r2GrxqPgi89iTNYSHzKOBp\nGdyD8m3AY8BXGc9tAUD319ePAjd2RXN9hMopM4eq+puq+ruuej/wbRlc/j2ur6dZ59GVnSrv7enx\nzpzDyN7bLYXA3cD3JXl2kqcClzO4ke27u/JPJ/lEku8FSPJ04NeA/zRjPcv5URgnNIeuvJLcmuSO\nJL/am8OhZZoDnPg8bgb+CfgS8LfAf66qo4zntpj2fcDfV9UXuuW5xnoqzaHvh4E7usAe19fTtCfM\n4xR7b0+buS1G9t4em4+NWGpVdW+SdwIfY/ALZR/wOIP/g2cx2H36XuCPkjwPeBvw7qr62kk+vDmn\nBczhDODlXdmjwJ8n2cvgs52WzQLmsa6r/05gJfDJJH+2HGOfdpw5TNtM7y/PcbTQOSS5kMEnBa8/\nGeOczwLm8TZOnff2tJlzGNl7u6U9Aarq/VX14qr618AjwOcZJOWHa+AzwDcZfC7HS4B3JXkAeAvw\n6xncNDf0R2GMwRwOAbdV1Zer6lEG93q8qBvv6uWawwLm8ePAR6vq/3aHiP4KmGI8twXdYavXAR/q\nNZ9rrKfSHEiyGrgFeH1V3d8Vj+vraa55nErv7bnmMLr39rAnLk6HB9860TgJfA54JvCzwNau/LsZ\n7A5mRr+38a2TR2cABxmc3Js+eXThOM6BwV/Nd/DEk00/VLOfPLp8XLcFg133/9qVP43BHegvHMdt\n0S1vAD4xo+2FPPHE8EEGJyJPpTk8sxvf62ZZz9i9nuaax4x+Y/3ePs62GNl7+6RtpHF4AJ/sfoF8\nFviBruws4A8YHJO7A/j+471QuuXLGaT0/cBbx3kOwE8yOPl1N/CuXvlUV3Y/8F5mBN84zQN4OoOr\nOPZ3fX5lXLdFV34D8LOztH9rN8776F2xcarMAfiPfOtQxfRj+hfX2L2ejrctevVj/d6e5/U0kve2\ndwxLUsOaOicgSXoiQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIb9P4DDyFgKhfN5AAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd3ce762630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}