generative_net.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "\n",
    "import os\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Convolution2D, MaxPooling2D, BatchNormalization\n",
    "from keras.layers import Flatten, Dense, Dropout, ZeroPadding2D, Reshape, UpSampling2D\n",
    "from keras.layers.noise import GaussianNoise\n",
    "from keras.layers.local import LocallyConnected1D\n",
    "from keras.regularizers import l1\n",
    "from keras import backend as K\n",
    "from keras.utils.layer_utils import print_summary\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import cv2\n",
    "import h5py\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\" # uncomment this line to run the code on the CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filter_id = 668 # mosque class id\n",
    "N = 3200 # feature vector size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def max_loss(y_true, y_pred):\n",
    "    return (1.-K.sum(tf.mul(y_true,y_pred),axis=-1))\n",
    "\n",
    "def max_metric(y_true, y_pred):\n",
    "    return (1.-max_loss(y_true,y_pred))\n",
    "\n",
    "def get_model():\n",
    "    # generator\n",
    "    inputs = Input(shape=(N,), name='input')\n",
    "    batch_size = tf.shape(inputs)[0]\n",
    "    \n",
    "    g0 = Reshape((N,1))(inputs)\n",
    "    g0 = GaussianNoise(2.0)(g0)\n",
    "    g1 = LocallyConnected1D(nb_filter=1, filter_length=1,\n",
    "                            activation='linear', bias=False,\n",
    "                            border_mode='valid')(g0)\n",
    "    g2 = Reshape((128,5,5))(g1)\n",
    "    g3 = UpSampling2D(size=(2, 2))(g2) # 10x10\n",
    "    g3 = Convolution2D(512,2,2,activation='relu',border_mode='valid')(g3) # 9x9\n",
    "    g3 = BatchNormalization(mode=0, axis=1)(g3)\n",
    "    g3 = Convolution2D(512,2,2,activation='relu',border_mode='same')(g3) # 9x9\n",
    "    g3 = BatchNormalization(mode=0, axis=1)(g3)\n",
    "    g4 = UpSampling2D(size=(2, 2))(g3) # 18x18\n",
    "    g4 = Convolution2D(256,3,3,activation='relu',border_mode='valid')(g4) # 16x16\n",
    "    g4 = BatchNormalization(mode=0, axis=1)(g4)\n",
    "    g4 = Convolution2D(256,3,3,activation='relu',border_mode='same')(g4) # 16x16\n",
    "    g4 = BatchNormalization(mode=0, axis=1)(g4)\n",
    "    g5 = UpSampling2D(size=(2, 2))(g4) # 32x32\n",
    "    g5 = Convolution2D(256,3,3,activation='relu',border_mode='valid')(g5) # 30x30\n",
    "    g5 = BatchNormalization(mode=0, axis=1)(g5)\n",
    "    g5 = Convolution2D(256,3,3,activation='relu',border_mode='same')(g5) # 30x30\n",
    "    g5 = BatchNormalization(mode=0, axis=1)(g5)\n",
    "    g6 = UpSampling2D(size=(2, 2))(g5) # 60x60\n",
    "    g6 = Convolution2D(128,3,3,activation='relu',border_mode='valid')(g6) # 58x58\n",
    "    g6 = BatchNormalization(mode=0, axis=1)(g6)\n",
    "    g6 = Convolution2D(128,3,3,activation='relu',border_mode='same')(g6) # 58x58\n",
    "    g6 = BatchNormalization(mode=0, axis=1)(g6)\n",
    "    g7 = UpSampling2D(size=(2, 2))(g6) # 116x116\n",
    "    g7 = Convolution2D(128,4,4,activation='relu',border_mode='valid')(g7) # 113x113\n",
    "    g7 = BatchNormalization(mode=0, axis=1)(g7)\n",
    "    g7 = Convolution2D(128,4,4,activation='relu',border_mode='same')(g7) # 113x113\n",
    "    g7 = BatchNormalization(mode=0, axis=1)(g7)\n",
    "    g8 = UpSampling2D(size=(2, 2))(g7) # 226x226\n",
    "    g8 = Convolution2D(3,3,3,activation='relu',border_mode='valid')(g8) # 224x224\n",
    "    g8 = BatchNormalization(mode=0, axis=1, name='image', gamma_regularizer=l1(0.01))(g8)\n",
    "    \n",
    "    \n",
    "    # discriminator  \n",
    "    vgg1 = ZeroPadding2D((1,1),input_shape=(3,224,224))(g8)\n",
    "    vgg2 = Convolution2D(64, 3, 3, activation='relu')(vgg1)\n",
    "    vgg3 = ZeroPadding2D((1,1))(vgg2)\n",
    "    vgg4 = Convolution2D(64, 3, 3, activation='relu')(vgg3)\n",
    "    vgg5 = MaxPooling2D((2,2), strides=(2,2))(vgg4)\n",
    "\n",
    "    vgg6 = ZeroPadding2D((1,1))(vgg5)\n",
    "    vgg7 = Convolution2D(128, 3, 3, activation='relu')(vgg6)\n",
    "    vgg8 = ZeroPadding2D((1,1))(vgg7)\n",
    "    vgg9 = Convolution2D(128, 3, 3, activation='relu')(vgg8)\n",
    "    vgg10 = MaxPooling2D((2,2), strides=(2,2))(vgg9)\n",
    "\n",
    "    vgg11 = ZeroPadding2D((1,1))(vgg10)\n",
    "    vgg12 = Convolution2D(256, 3, 3, activation='relu')(vgg11)\n",
    "    vgg13 = ZeroPadding2D((1,1))(vgg12)\n",
    "    vgg14 = Convolution2D(256, 3, 3, activation='relu')(vgg13)\n",
    "    vgg15 = ZeroPadding2D((1,1))(vgg14)\n",
    "    vgg16 = Convolution2D(256, 3, 3, activation='relu')(vgg15)\n",
    "    vgg17 = MaxPooling2D((2,2), strides=(2,2))(vgg16)\n",
    "\n",
    "    vgg18 = ZeroPadding2D((1,1))(vgg17)\n",
    "    vgg19 = Convolution2D(512, 3, 3, activation='relu')(vgg18)\n",
    "    vgg20 = ZeroPadding2D((1,1))(vgg19)\n",
    "    vgg21 = Convolution2D(512, 3, 3, activation='relu')(vgg20)\n",
    "    vgg22 = ZeroPadding2D((1,1))(vgg21)\n",
    "    vgg23 = Convolution2D(512, 3, 3, activation='relu')(vgg22)\n",
    "    vgg24 = MaxPooling2D((2,2), strides=(2,2))(vgg23)\n",
    "\n",
    "    vgg25 = ZeroPadding2D((1,1))(vgg24)\n",
    "    vgg26 = Convolution2D(512, 3, 3, activation='relu')(vgg25)\n",
    "    vgg27 = ZeroPadding2D((1,1))(vgg26)\n",
    "    vgg28 = Convolution2D(512, 3, 3, activation='relu')(vgg27)\n",
    "    vgg29 = ZeroPadding2D((1,1))(vgg28)\n",
    "    vgg30 = Convolution2D(512, 3, 3, activation='relu')(vgg29)\n",
    "    vgg31 = MaxPooling2D((2,2), strides=(2,2))(vgg30)\n",
    "\n",
    "    vgg32 = Flatten()(vgg31)\n",
    "    vgg33 = Dense(4096, activation='relu')(vgg32)\n",
    "    vgg34 = Dropout(0.5)(vgg33)\n",
    "    vgg35 = Dense(4096, activation='relu')(vgg34)\n",
    "    vgg36 = Dropout(0.5)(vgg35)\n",
    "    vgg37 = Dense(1000, activation='relu', name='vgg_class')(vgg36)\n",
    "    \n",
    "    # create model\n",
    "    model = Model(input=inputs, output=[vgg37,g8])\n",
    "    \n",
    "    # set weights\n",
    "    offset = 33\n",
    "    f = h5py.File('vgg16_weights.h5')\n",
    "    for k in range(f.attrs['nb_layers']):\n",
    "        g = f['layer_{}'.format(k)]\n",
    "        weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]\n",
    "        model.layers[k+offset].set_weights(weights)\n",
    "        model.layers[k+offset].trainable = False\n",
    "    f.close()\n",
    "    \n",
    "    # compile model\n",
    "    model.compile(optimizer='adamax', loss=[max_loss,'mse'], metrics=['mse'], loss_weights=[1.,0.])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# create neural network\n",
    "model = get_model()\n",
    "print_summary(model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def print_img(model,niter=0):\n",
    "    z = np.ones((1,N))\n",
    "    \n",
    "    out = model.predict(z, batch_size=1)\n",
    "    activ = np.squeeze(out[0])\n",
    "    img = np.squeeze(out[1])\n",
    "    \n",
    "    # change to RGB colors\n",
    "    # and rescale image\n",
    "    img -= np.min(img)\n",
    "    img = np.sqrt(img)\n",
    "    img /= np.max(img)\n",
    "    img *= 256.\n",
    "    img = cv2.cvtColor(img.astype('uint8').transpose(1,2,0), cv2.COLOR_BGR2RGB)\n",
    "    plt.figure(figsize=(10,10))\n",
    "    plt.imshow(np.flipud(img))\n",
    "    plt.title('filter activation: '+str(activ[filter_id]))\n",
    "    plt.show()\n",
    "\n",
    "print_img(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# training the model\n",
    "batch_size = 1\n",
    "n_samples = 50\n",
    "z = np.ones((n_samples,N))\n",
    "vgg_nclasses = 1000\n",
    "\n",
    "dummy_labels1 = np.ones(shape=(n_samples,vgg_nclasses))*(-10./vgg_nclasses) # put a penalty to the other classes\n",
    "dummy_labels1[:,filter_id] = 1                                              # give a positive unit weight for the target class\n",
    "dummy_labels2 = np.zeros(shape=(n_samples,3,224,224))\n",
    "\n",
    "for k in range(10):\n",
    "    out = model.fit(z, [dummy_labels1,dummy_labels2], batch_size=batch_size, nb_epoch=1, verbose=1)\n",
    "    print_img(model,k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}