transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "colab": {
      "name": "transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "dI_0yAksTQ2v",
        "FBgIwI2HTQ3E"
      ]
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S8luGcCybclt",
        "colab_type": "text"
      },
      "source": [
        "fork of fastai\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WLN3VEf6TQ0P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fastai.text import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SL7t7hJTQ0S",
        "colab_type": "code",
        "outputId": "9c8843f9-00e5-4c93-ffd0-a51bc55f3dc7",
        "colab": {}
      },
      "source": [
        "path = Config().data_path()/'giga-fren'\n",
        "path.ls()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[PosixPath('/home/jhoward/.fastai/data/giga-fren/cc.en.300.bin'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/data_save.pkl'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/models'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/giga-fren.release2.fixed.en'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/giga-fren.release2.fixed.fr'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/questions_easy.csv'),\n",
              " PosixPath('/home/jhoward/.fastai/data/giga-fren/cc.fr.300.bin')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2N4t4HtOTQ0W",
        "colab_type": "text"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wRUhPdYlTQ0X",
        "colab_type": "text"
      },
      "source": [
        "We reuse the same functions as in the translation notebook to load our data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "by7E5t0bTQ0Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def seq2seq_collate(samples, pad_idx=1, pad_first=True, backwards=False):\n",
        "    \"Function that collect samples and adds padding. Flips token order if needed\"\n",
        "    samples = to_data(samples)\n",
        "    max_len_x,max_len_y = max([len(s[0]) for s in samples]),max([len(s[1]) for s in samples])\n",
        "    res_x = torch.zeros(len(samples), max_len_x).long() + pad_idx\n",
        "    res_y = torch.zeros(len(samples), max_len_y).long() + pad_idx\n",
        "    if backwards: pad_first = not pad_first\n",
        "    for i,s in enumerate(samples):\n",
        "        if pad_first: \n",
        "            res_x[i,-len(s[0]):],res_y[i,-len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])\n",
        "        else:         \n",
        "            res_x[i, :len(s[0])],res_y[i, :len(s[1])] = LongTensor(s[0]),LongTensor(s[1])\n",
        "    if backwards: res_x,res_y = res_x.flip(1),res_y.flip(1)\n",
        "    return res_x, res_y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rT1rgLZETQ0b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Seq2SeqDataBunch(TextDataBunch):\n",
        "    \"Create a `TextDataBunch` suitable for training an RNN classifier.\"\n",
        "    @classmethod\n",
        "    def create(cls, train_ds, valid_ds, test_ds=None, path='.', bs=32, val_bs=None, pad_idx=1,\n",
        "               dl_tfms=None, pad_first=False, device=None, no_check=False, backwards=False, **dl_kwargs):\n",
        "        \"Function that transform the `datasets` in a `DataBunch` for classification. Passes `**dl_kwargs` on to `DataLoader()`\"\n",
        "        datasets = cls._init_ds(train_ds, valid_ds, test_ds)\n",
        "        val_bs = ifnone(val_bs, bs)\n",
        "        collate_fn = partial(seq2seq_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)\n",
        "        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs//2)\n",
        "        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)\n",
        "        dataloaders = [train_dl]\n",
        "        for ds in datasets[1:]:\n",
        "            lengths = [len(t) for t in ds.x.items]\n",
        "            sampler = SortSampler(ds.x, key=lengths.__getitem__)\n",
        "            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))\n",
        "        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHiEk64ITQ0d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Seq2SeqTextList(TextList):\n",
        "    _bunch = Seq2SeqDataBunch\n",
        "    _label_cls = TextList"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rtJxO854TQ0k",
        "colab_type": "text"
      },
      "source": [
        "Refer to notebook 7-seq2seq-translation for the code we used to create, process, and save this data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDptH1LWTQ0l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = load_data(path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dejdmXoTTQ0o",
        "colab_type": "code",
        "outputId": "ac0f3d5a-48eb-43cf-ced3-dd8f0b140c71",
        "colab": {}
      },
      "source": [
        "data.show_batch()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>xxbos quelles questions devraient être traitées respectivement au niveau international et au niveau national , ou quelle xxunk devrait être établie entre la réglementation internationale et la réglementation nationale ?</td>\n",
              "      <td>xxbos which issues should be dealt with internationally and which nationally , or what division should be made between international regulation and national regulation ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos comment la culture et les arts y vivent - ils , et comment la société civile les prend - elle en considération dans le développement de la ville ?</td>\n",
              "      <td>xxbos where do art and culture fit in , and how is civil society taking them into consideration in developing the city ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos qu’arrivera - t - il si les entreprises canadiennes et les gouvernements ne se xxunk pas sur la question du conflit entre le travail et la vie personnelle ?</td>\n",
              "      <td>xxbos what will likely happen if canadian organizations and governments do not deal with the issue of work – life conflict ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos qu'adviendra - t - il de l'examen de rendement et de la rémunération au rendement de xxunk et comment se xxunk - t - il à cet égard ?</td>\n",
              "      <td>xxbos what happens to xxunk 's evaluation review and performance pay and how will this make him feel ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos quels avantages prévoit - on en général pour la région du delta de beaufort par suite de la signature de xxunk / d'une entente future sur l'autonomie gouvernementale ?</td>\n",
              "      <td>xxbos what benefits to the beaufort delta region generally are expected as a result of this aip / future self - government ?</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0MPEl0ULTQ0q",
        "colab_type": "text"
      },
      "source": [
        "## Transformer model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6FxHDm4GTQ0r",
        "colab_type": "text"
      },
      "source": [
        "![Transformer model](https://github.com/fastai/course-nlp/blob/master/images/Transformer.png?raw=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vpbx5eTiTQ0r",
        "colab_type": "text"
      },
      "source": [
        "### Shifting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OR4lt6JxTQ0s",
        "colab_type": "text"
      },
      "source": [
        "We add a transform to the dataloader that shifts the targets right and adds a padding at the beginning."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kYpTq5JjTQ0t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "v = data.vocab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1wMcrgWNTQ0v",
        "colab_type": "code",
        "outputId": "6fcc9feb-b54c-45f1-d2b6-379ba6ef017d",
        "colab": {}
      },
      "source": [
        "v.stoi['xxpad']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUvUIVgmTQ0y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def shift_tfm(b):\n",
        "    x,y = b\n",
        "    y = F.pad(y, (1, 0), value=1)\n",
        "    return [x,y[:,:-1]], y[:,1:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pSNUPTRbTQ00",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.add_tfm(shift_tfm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cA0yPUxFTQ02",
        "colab_type": "text"
      },
      "source": [
        "### Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GLOgriNATQ03",
        "colab_type": "text"
      },
      "source": [
        "The input and output embeddings are traditional PyTorch embeddings (and we can use pretrained vectors if we want to). The transformer model isn't a recurrent one, so it has no idea of the relative positions of the words. To help it with that, they had to the input embeddings a positional encoding which is cosine of a certain frequency:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yMsy1cfsTQ05",
        "colab_type": "code",
        "outputId": "1961197f-0d91-47d2-f42d-924f9c7d483b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "d = 30\n",
        "torch.arange(0., d, 2.)/d"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.0000, 0.0667, 0.1333, 0.2000, 0.2667, 0.3333, 0.4000, 0.4667, 0.5333,\n",
              "        0.6000, 0.6667, 0.7333, 0.8000, 0.8667, 0.9333])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ORiU3XZTQ07",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \"Encode the position with a sinusoid.\"\n",
        "    def __init__(self, d):\n",
        "        super().__init__()\n",
        "        self.register_buffer('freq', 1 / (10000 ** (torch.arange(0., d, 2.)/d)))\n",
        "    \n",
        "    def forward(self, pos):\n",
        "        inp = torch.ger(pos, self.freq)\n",
        "        enc = torch.cat([inp.sin(), inp.cos()], dim=-1)\n",
        "        return enc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O75wxYqGWyFC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.ger??"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kC3MxUUTQ0-",
        "colab_type": "code",
        "outputId": "2101c41d-2255-40bb-e715-2241412d0747",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "tst_encoding = PositionalEncoding(20)\n",
        "res = tst_encoding(torch.arange(0,100).float())\n",
        "_, ax = plt.subplots(1,1)\n",
        "for i in range(1,5): ax.plot(res[:,i])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXWYXNXdgN8zvu6S3c1mJUpc8GBJ\nSZEEt6BtgeAF2n6F4FCgUCgtLRq0eJAWgiQhEIKHuBNdybrN6szu6P3+uHNXx2d2Z9vM+zz7bDLX\nzt255/7Oz4UkSUSJEiVKlCgKqkgPIEqUKFGiDC+igiFKlChRovQhKhiiRIkSJUofooIhSpQoUaL0\nISoYokSJEiVKH6KCIUqUKFGi9CEqGKJEiRIlSh+igiFKlChRovQhKhiiRIkSJUofNJEeQDCkp6dL\nBQUFkR5GlChRovxXsXHjxkZJkjJ87fdfKRgKCgrYsGFDpIcRJUqUKP9VCCHK/dkvakqKEiVKlCh9\niAqGKFGiRInSh6hgiBIlSpQofYgKhihRokSJ0oeoYIgSJUqUKH0Ii2AQQrwshKgXQuzwsF0IIf4h\nhNgvhNgmhJjRa9sVQoh9rp8rwjGeKFGiRIkSPOHSGF4FTvGy/VRgjOtnEfAsgBAiFbgXOBI4ArhX\nCJESpjFFiRIlSpQgCEsegyRJ3wghCrzscibwmiT3EV0rhEgWQowATgRWSZJkBBBCrEIWMG+HY1yh\n0mK28tWees6alosQIoQTHYR9qyBjPIw6BkI51yDilJxUd1Szr3kfxi4jJpsJs92MVqUlxZBCsj6Z\n3PhcipOL0aj+K1NgBo/OZmivA6sJ7F2QMx10sYN6SUmScJrMOFpacLa34Whrx9nRjtPcibPTjNTV\nhWSzIdnsSDZb34PVKoRGi9BoEHodqphYVDEGVHFxqOITUCfEo0pKQpOSwq4GM1a7k+n5/71rtlaz\njYNGM5PzkkI7ka0Tdv4H4jIh/0jQJ4RngMOMoZrduUBFr/9Xuj7z9PkAhBCLkLUN8vPzB2eU/Xj8\n8z28sfYg+alxzBwV4KSQJNjwEmx5C6o29nw+8kiY/TsY+8shERAby5u57YNtPHfpDEZnDnyIS1tL\nWVOxhm8qv+Fn48+YbCaf5zSoDYxPHc+MrBnMGzWPw9IOC01w/rezdSksuxEc1p7P0kbDBa9x309Q\nmB7HFccUBHRKSZJwNDVhrajAVl2Nraoae20Ntvp67PUN2BsacBiNSBZLeO/FDV26GFoMCZSOG4Uu\nKwtNVibanBy0I3LQ5uWiGzkSVUzMoI8jFH737ha+3d/Ihrt+QaJBG/gJnE7Y8T58cT+0VcqfCRWM\nmApz74HiOeEdsBtqW7tY8k0Ji08bj1Y9uO7h/5plnyRJS4AlALNmzZIG+3rNJivvb5QfgI+2VAUu\nGL5+FNb8GbKnwNx7YdxpUPYtfP8kvH0hTLkQzn5+UIVDeZOJq1/bgNFkZfn2Wm6aKwsGk83ER/s/\nYumepZS0lgAwLmUcZxSfwdiUsYxJGUNWbBYxmhhitbHYHDZaLa0YLUZKW0vZ2biTnU07eW3na7y8\n42Vy43M5rfA0Fo5fSEasz2x7v5AkiXc3VHDQaKa9y06XzcFVxxUxNmsYrdCcTljzMHzzGBQcB7N+\nDbp4sLTDyjuQXphLR+fl3Os4AbtT4srZhQNOIdntWMvLsezbj+XAfqwHSrCUlWIrP4jT1FdIq5KS\n0GZmosnMRF9UhDotFU1KCuqUFFSJiagTElHFx6OKjUUVG4PQ61HpdAitFrTabuEtSRI4HEh2u/xj\nseDs7ELqNOM0mXC0d8gaSGsr1WU1fPHNLpIt7SS0mEmt3Yq9tnaABqLJykI3ahS6okL0RcXoRxej\nHzcOTVra4P39/eSHA418ubsegDV7Gjhjak5gJzAb4c3zoWqDLAjOfEr+vPwH2PlveOsiuOQ9KDoh\nzCPvYcWOWm7/9zasdidnT88NXfPxwVAJhipgZK//57k+q0I2J/X+fM0Qjckrb607SJfNyZS8JD7e\nWs3d8w/zX0pvek0WCtMugTOf7nn5Z46Hmb+St337VxgxDY6+flDG32yy8utX1iNJEiNTY/jhQBML\nj0nhpe0v8Z/9/8FkMzElfQp3HHkHJ+adyIj4ER7PpVVpidXGMiJ+BBPTJjK/aD4ArZZWVh9czcqy\nlby4/UVe3fkqC4oXcMXEKyhKKgpp/PvrO7jtg+2oVYIEg4a2ThuxOg33nTExpPOGDYcd/rMIdnwA\n0y+F0/8GGl3P9sLjafzXZTxuf57pSSbu/ARicXBWUhddO3fQuWMHlp93Y9m/v2fVLwTa3Fx0RYXE\nzpyFLj8f7cg8dLm5aHNyUMXFhWXoQgjQaBAa1/SPj/e473PLf+al9vGMzUrgtU4ba/7vRDQCHE1N\n2KqrsVZUYqs4iLX8INayMto+W46zra37eHV6OoZx4zAcdhiGSZMwTJyINjdnyDRMp1Pi4c9+Jjc5\nBovdwec7awMTDJIEH14HtdvgzGdg6kJQud4DxSfBUdfBq6fD2wvhsn9D/lFhHX+XzcH9H+/i7XUH\nmZKXxJMXTacwPTzPgTeGSjAsA24UQryD7GhulSSpRgixEni4l8N5HrB4iMbkEYvdwas/lHH82Ayu\nOHoUV/5rA9/sbWDuhCzfB+/9HD6+BYrnwoInB2oEai3MuRsa9sDnd8GIKVAwO+zjX/T6BipbOnnr\nqiP5ZHs5S/e+yen//harw8q8gnlcOuFSJmdMDuk6Sfokzh5zNmePOZuDbQd5bddrfLj/Qz7c/yEX\njL2AG6ffSJI+uJXNtspWAFbcfBxjshI4/7kf2FbZEtJ4w8q6JbJQmHM3HPf7gd9zfCbPxS/muI33\nMrfpY/Lb9pC0rJYypwMAdVIS+sMmkLJwIfrx4zCMHYuusHBYmWScTolPttYwe0w6lx0lz4NPtlVz\n9vQ8NBkZaDIyiJk6tc8xkiThaGzEsn8/XXv2YNmzl67du2l65RWw2wFZWMRMnUrM1KnEzpyBYfJk\nVDqduyGEzEdbq9hR1cbfLpzKTyVGPtlWg8XuQK9R+3eCtc/C3hVwyqMw/ZKB22NT4bIP4dXTZK3i\nV5/IWkWYeHdDBW+vO8g1JxTx+5PHodMMTYZBWASDEOJt5JV/uhCiEjnSSAsgSdJzwGfAacB+wAz8\n2rXNKIT4E7DedaoHFEd0JPlkaw0N7Rb+en4hRxenkRKr5T+bq3wLhpYKeO9XkD0JLviXLATcIQSc\n9Sy8MEfe/5pvIDFA9dYLX+2uZ31ZM389fypmzQ6+7LgPTXoDYxJn8+Dxt1GQVBC2aynkJ+Zz11F3\ncf2063l2y7O8u/ddlpct56ZpN3H+uPNRicAe6O1VrcTq1BRlyKvZKXnJvLG2HLvDiWaQ7as+aauG\nrx6C0b/oIxTsDQ2Y1v6E6ae1dK7fwHnlcr2yFnUsI7Pq+HjiSbQUjOW2m88Z0lVzsGyuaKaqpZPf\nzxvLSeMyGZeVwLNrDnDm1FxUKvdjF0J0C424o4/u/txpsWDZu5fO7dvp2rqNzq1b6fjyS/kYnY6Y\nKVOIPfJI4o46EsPUqWERFF02B4+v3Muk3ETOnJpLUoyWd9ZX8OOBJk4cl+n7BFWbYNU9MO50OPIa\nz/slZMHlH8GLJ8NHN8Kir3u0ihDZVtlKRoKexadOCMv5/CVcUUkLfWyXgBs8bHsZeDkc4wgHkiTx\n4neljMtK4Lgx6QghWDA1h6XrK2jvspHgzXH19SPgtMGFb/iOVjAkyvu9OBeW3QSXfhC2e/i5ph2h\n7mKj6Tk+/vIjipJGU7f/HKZk/WJQhEJvUg2p3HnUnZw/7nweXfcoD/70IF8c/IIHj32QrDg/NC4X\n26tamZiTiNr1ApqSl4TF7mRvXQeH5SQO1vD9Y8Xt4LTjnPsQ5u9/wPTdd5i+/w7Lvv0Asr1/6jRe\njJ/MrAUncdH0TsR/fkNRViz3GSdwd57b+Iphx7It1eg1Kk4+LAuVSnDtiUXcunQrX+2p90977oVK\nrydm8mRiJk+Giy8GwN7cTOemTZg3bMS8fj2Nzz5L49NPIwwGYmfNIm72scTPno2uuDgoIfrG2nKq\nWjp57LwpqFSCY4rTidWpWbWrzrdgsLTD+7+B+CzZp+Dr+kl5cPID8O+rYPu7MPWigMfrjt21bYzP\nHnq/WjTzuR8/Hmji55o2rjyusPthPGt6Lha7kxU7aj0f2LBXjkA6/CpI9jNqKnM8HP8H2P8FVIav\njPjGuk0kFj/Jp2Ufc9Xkq3hvwVImp8/g+wNNYbuGL8amjOXFeS9yz9H3sLVhK+csO4fPyz7361i7\nw8mu6jYm5faYoSa7/r29KrLmJNva92letoqKrVPZe+pCKq66iua33kKTkUnmH35PwfvvM/bHH9hx\n/T18MOZEJp98LGLquTBhAb9seIX4jjKaTVbfF4owdoeTT7fXMGd8ZvdiaP6UHHKTY3jx29KwXEOT\nkkLC3Llk3fZHCt9/j7FrfyTvmadJPu88bFVV1D/yKCXzF3DgFydT+6cH6fj2O5xW//92Px5oYmxW\nPMeMTgfAoFVzwtgMVu2qw+n0Eb+y7gVoLoVzlsjmIn+YdK7sN/zyT3JYa4jYHfJCaMKIoV8IRQVD\nP9bsbUCnUfVxUE0fmcyotFg+3FLl+cCvHgRtrGxaCITDr4KYFPj6L0GOuAdJknh3z7tstT+CVq3h\n9VNf5+YZN6NT6zi2OI3tlS20ddl8nyhMCCE4f+z5vDv/XfIT8vn917/nrxv+isNlZ/fEgQYTnTYH\nU3pFXhSkxZFg0LDV5XsYSiwlJTQ+9zyl557L/l/dTe2GZCxGB8nnnsvIJc8z9qe15L/8EmlXXUXM\npIkItZq1JU0kGjQ9k/q0x5HUeu7QvMneuvYhv4dAWVtipLHDyoJe80CrlrWHrZUtcmRTmFEnJpIw\nZw7Zd91J8fLPGP3lF2Tffz/6ceNo+eADKq6+mn1HHU3lrbfS+umnODo6vJ6vpNHE6My+jvV5E7Oo\nb7ew1Zu/ytYJa5+RQ1ALjvX/BlQqmPegHM7603P+H+eBsiY5fyQSGsN/TbjqUFHaaKIgLRaDtsc5\nJYTgrGm5/GP1Purbu8hMMPQ9qHoz7PoITrgN4tIDu6A+AY6+AVY/KJ8nZ3pQ47Y6rDz808N8sO8D\nHKZxnFl4O1MypnRvP7o4nX+s3s9PJUZOPiwwM0CoFCQV8Nppr/GXdX/h1Z2vsq9lH385/i8k6tyv\nhLZXyS//yb00BpVKMDk3ie1DJBgsJSW0LV9O+4oV3SYiQ/EIMqa0kXDto+hOusSreePHkiaOLErr\nNoWRkI1l5iLm/vhXlpXsGNTQxnCwbGsVcTo1c8b3NbkUZcRhtjqob7eQlWjwcHR40ObmknLhBaRc\neAHOri7MP/1E+5eraV+9mvblKxA6HXGzZ5N4yi+JnzMHda/oKqvdyUGjmflT+kbbzRmXhVol+HxX\nneeEvc1vgKkh8EUeQOFxMPYU+PYJmH45xAUfrru7Vo7uGhc1JUWeskYTBWkDw8GOKkpDkmBvrZtV\nypcPyKv+o926UXxzxCIwJMHXjwV1eIe1g+u/uJ4P9n3AuUVXYK64gskj+k6IGaOS0WtU/HCgMbgx\nhohWpeXOo+7k7qPu5qfqn7jk00uo6nCvgW2vbCFWp6Ywve9qb0peMrtr27DYvWscwWKrqaHppZco\nOetsSk47ncannkaVlETWnXcy+qvVFC6wkn5SAXofQqG6pZPyJjNHFfV9KcQdezUOoSJr92uDMv5w\nsr6smdlj0vsskIDuuVHa6DsRMpyoDAbiTziBEQ/cz5iv1zDqrTdJWXgRXTt3Uv3H29h37Gwqb76F\n9i++wGm1ctBowuGUKMroO5eTYrUcVZTKql117i/ksMMP/4C8I2BUANpCb35xP1g74Lsngjvexe6a\ndtQqMUDrGQqigqEXTqdEudHsNk44P00ub1Bu7DchqjbCgdUw+1b55R4MhiQ46gbY8ynUbAvo0MbO\nRn6z8jdsrNvIw7MfZlbSxYCKMVl9Hya9Rs3hBan8sH/o/AzuuGDcBbww7wWMXUYuX345JS0lA/bZ\nXtXKpJykntW2iyl5SdgcErtrwmeKcZpMtHz4IeVX/Ir9c+ZS/9jjCL2OrDsWM3rNGgreeIPUyy5F\nay2D2u1wxNU+HZE/unw5R/cTDCIhm+8NJzK96RPoHEaht/1wOCUqm80DBDPQPTeGWjD0RqjVxM6Y\nQdbixYz+ajWj3nqT5PPOw7x+PZU33sS+447H+NCDjDeWU+RmkXfC2Az213dgdOfr2fGBXMLmuN8F\nn3yaOR4OO0vWPGxdwZ0DWWMozojzP7Q2jEQFQy+qWzux2p0UuBEM2YkGdGoVB43mvhs2vQ6aGJj5\n69AufuQ1oE8MaJVR1VHFFcuvoLS1lH/M+QcLihewt64DlYDijIGT+pjRaeypa6ehffDLKHhjVvYs\nXv7lyzicDq5YcQU7m3Z2b7M7nOyq6et4VlB8DtuqQjMnSZKEedNmqu+8k73HHU/N7YuxVVeTfsMN\nFK9cQeHSpaRefjnarF5mlHVLZAE++Xyf5/+xpImUWK1b2/D2vIsxSF1Im18P6R4Gk5rWTmwOifzU\ngbWecpJj0KlVlEVQMPRGqFTEzphB9t13MebrNYx8YQnxxx1H7OqV/O2bfxK76GKaXnoJe2OPpqxo\nPRX957LTKc+/zMNgzC9DG9jMK6CrBX5eFvQpfq5pZ3x2ZCLwooKhF2WN8oMyKm3ghFCrBHmpMRxs\n6vUwWc3yCuOwM+Xw01CISYbpl8HPn8gp+D6o6ajhypVX0mJp4YV5L3Bc3nEA7KtrJz81doAJAHpW\nsOvLIp4qwrjUcbx26mvEamK5cuWVbG3YCsD+ho7ujPP+5CbHkBqnY1tFcKttR0sLxn/9i5IFCyi/\n+GLalq8g8dRTGPXmGxR/vpKMG29AN2rUwAPbauQJPv0y0PnOOl1b0sSRhWluY/0TCmfyk3M8zrXP\ny2aLYYiy+HEnGNQqwai0WEqGiWDojdBqiT/uOHIff4y3/vgsLx51MdrUVOofe5x9J55E5U2/peO7\n7xmZLPtGKpr7CYYDX0LDbln7DzUPoeB4SCmQqyAEQVuXjaqWTsaPiEwJmKhg6EVpk/ywe0o5z0+N\npby3YNj9CVja5JII4WDqhXIexM5/e92tzlTHlZ9fSZuljSXzljAtc1r3tr117YzxUE9I+bzPPUSQ\n/MR8/nXqv0g1pHLdquv4uennbueyO41BCMGUvKRu57S/dG7dSvXti9l3wonU/fkR1HHxjHjwT4z5\n5htyHnqI2JkzvcfJb3wVnA6Y9Ruf1+qw2Kls7mTKSPdmxbFZCbxsPwV1WwXs+Syg+xgqlJW0uwUS\nQEF63LDRGDyxp0Oi8uhfUPDWmxR99impl1+OecMGKq66CvUV53Pevq+oPdgv/HzrOxCTChPPDn0A\nKhXMuFyuj9Z0IODD99bK5tIJUY0h8pQ1mjBoVWT1jzpyMSo1lgqjuSdUb/Pr8qogWCdVf7KnyGrs\n1qUed2nsbOSqz6/C2GXkuZOfY2JaT+0gi91BWZOZsVnunVXxeg1JMVqqWoaHYADIjsvmxXkvEqeL\n45pV1/D9wZ3E6dQUeRDOU3KT2FvXjtnqfbXttFpp+fBDSs+/gLILL6L9889JOvssCj/8DwVL3yH5\nvPNQx/tRc8ZuhY2vwJiTIa3Y5+5VzXL8el6K+5fq2OwEVjln0W7IgfUv+r5+BDhoNKNWCUYkuZ8H\nRelxlDeZcfjKBYggJQ0d3Vnz+qIisv74f4z+eg05jz+OfsQIrtz5KUfe/iuq77iTzp07wdIhC+qJ\nZ3uuWBAo0y4BoQ5Ka/jZJRiiGsMwQIlI8pTun58WR7vFTrPZBs1lUPqN/OWHKf0dIeSqq5Xr3K4y\nTDYT139xPXXmOp6Z+0yfcFR5/PJk9VaBNDc5hsrm0JNvwklOfA4vznsRlVCxpvVBxuTZPH4HU/KS\ncUqwq7rN7XZ7UxMNTz3N/pPmUHP7YpwmE1l338Xob75hxH33YRg/PrDB7VsJHXVw+NV+7a4I3dxk\n9zWP0uP1pMQZWJd4srya7KgPbDxDQHmTmdzkGI+lRwrS47A6nFS3DK/nSKHZZKXZbKO4X0SSSqcj\naf7pjHr9NZ5YeB/bJs6mbflyys49j/KLzqOt1Ik08bzwDSQhWw5d3fIWOALLH9pd00aiQUP2IIcE\neyIqGHpR2uQ+VFVBsbmWN5lgy9uAkKsthpPJ58vn3dZXa7A5bfx+ze/Z27yXJ058ghlZMwYcqiRO\njXHTd0EhLyWme1U7nBiVOIrnfvE8dslKfexTtFrcm4sU30P/RDdLSQnVd93F/pPm0PjUUxgmTWTk\nSy9S9OknpF5yiX/agTt2fiibF/yst1/VIkeh5KV4LoY3JiueZfYjQXKG5JwcLCqMZo9mJBgekUne\nKGmUQ8r7h6r2Rjd6DC8efgFjvl5D5m23Yauupuq7VA5cfS/GN97EaQ6TVj3zCjDVw57lAR22u7ad\n8SMSI1ZPKyoYXNgdTiqMZrcRSQrKZDnY1AFb3pTL7iaP9Lh/UCTlyslP25bKJX+Ro2ge+PEBvq/+\nnnuPvpfZue6rse6ra0clvE+I3JQYqlo6ByVzNVTU9hw6Ky/D7Kzn1jW3YnOzyspMNJASq+VAgzz5\nzZs2UXH9DZScdjptH39C0jlnU/TZp+Q//zzxxx4b2sSydcmVNSfMB7V/uaBVzZ1o1YKMeL3HfcZl\nJfBlUxpS+jhZ8AwzDhrNjHTjeFZQBENZ0/AUDAfq5XEVuQm3VRiZGktlsxniE0g7/3SKT6kk98pj\n0aSlUffgg+w/aQ71Tz6J3RhioEbxXEjIkd8XfuJ0SuypbWdCBBLbFKKCwUV1Sxc2h0RhuucJMdJl\nN3aUfAetFbIZaTCYcpFsqqr4CYDntz3Ph/s/5Lqp13H2GM+Osb11HRSkxbmNSFLITY7BbHXQYh66\n0hj+UtXSicNcxJXjb2d97Xru/eFetwJsRKKBmM3rKLv0UsovvoTOTZtIv+EGRn+1mhH33Ye+KLRe\nEN0cWC0nKh12ZkD3MCIpxqMpDOQggA6Lg/bi+VD2ndwSdJjQ1mWj2WxzG5GkkJmgJ1anpqRhmAqG\nxg60auFVaxuZGoPNIVHX1gU7/40QDhKv+D0FS99h1FtvEjNrFk3PPsf+OXOpfehhbNXVwQ1GrYGJ\nZ8GBr2Q/hh9UtXTSYbEzPgI1khSigsGFEpHkzZQUo1OTmaAnveoLUOth3KmDM5gJC+S6S1vfZvXB\n1Ty95WkWFC3guqnXeT1sb327zyxJZbJUDUP7cG2rbIY5d+wZ3DjtRj4u+ZgXt/c4aCWnk7ZVq/jj\nfx7mvHcfx1ZZRdYddzB69Zdk3HQjmlQ/i535y8/LwJAMhf6Xr6hu6fToX1BQShz8nDoXkIaVOUkJ\nx/YmGIQQFKTFDVuNoaRBNgl7K8+uLPIqjGbY9i5kTYZMubR17IwZjHz6KYo+/YTEU06h+e232T/v\nl1TfdRfWgwcDH9D408FhkcNh/eDnGtl/FokaSQpRweBCCb/z1R1pVGoM41q/g6IT/YppDwp9PIw/\nnf17P2Hxt4uZlDaJe4+516tZxGJ3UN5k9tn6MjdZnhDDzQENUN3ahUpARoKeRVMWcVrhafxz8z/5\nruIb2pYvp/TMs6i66bfE2C08f+RCRn++ktTLL0MV6/klFjR2K+z+TJ7UAUSpVDV3kutlpQow1uUD\n2tyVDRkT5Obyw4QKLzkMvSnMiBu+PoaGDq/mVKDbVGas3C237JwyMHFRX1xMziN/ZvTKFaRceCFt\nyz7mwKmnUX3b7VjLyvwf0MijZD/V7k/92n1fvaxZRLKNbVgEgxDiFCHEHiHEfiHE7W62/00IscX1\ns1cI0dJrm6PXtogtnUobTcTp1GQkeLYNAxwRV0+WoxbGnzao42ktPpGbk7TEqDT87aS/oVd7H1dJ\ng1wbpn8pjP7kDmuNoZOMBD1atQohBPcedQ9nVWbTecl1VN36OySHg5zH/sLG+5/jwxEzsYhBLBVQ\n+jVYWgMyI1ntTurau8jxoTEkxWrJStTLwQITz5Z7B7fVhDrisNCd3ObF+QxQmBZHZbNcKWA4YXfI\nxfOK3GT+9yYn2YAQEH/AlUsy6VyP+2pzc8m++y6Kv1hF6qWX0rZyJQdOn0/17Yv90yDUGtm6sHeF\nX9FJVS2dpMbpiNNHrsZpyIJBCKEGngZOBQ4DFgohDuu9jyRJt0qSNE2SpGnAP4HeGVydyjZJks4I\ndTzBUtZkYlRanE9n5bGOdQB0FZ48aGNxSk4W139NtUbD35Nmkh2X7fOY/a5VhreIJICUWC0xWvWw\njEyqae0iOykGSZJoX7OG2gsvY+HrFagdsHRhDtn/foekBQvITo3r3n/Q2PWhXKKk6ES/D6lt7UKS\nIM+HYADZMVrWaJLtz8PInFRuNJMcqyXRW0MqZM3a4ZQGZg9HmIpmuZyHpzwYBb1GTXaigZyGb+X8\noaQ8n+fWZmaStfh2Rq/6XBYQy5dz4NTTqLn7bmw1PgT7uNOgqxXKv/d5nfq2rkGvXOuLcGgMRwD7\nJUkqkSTJCrwDeFtmLQTeDsN1w0pZo8mvJtvjW79li7OYSvvgOYZe3fkq39as5Y9SKtPKN/p1TE2r\n/KIfmer9pSSEcEUmDa8JDfKLdVZzCeULL6by2utwmkzk/OVRDO88z38KG3lo/Z8ByHYlXin3HHYc\nNlntH3cqaLxrar1RtDBfpiSQ76GuzQIZ4+SkxmESnVRhNDPKhxkJ6I7eG24Z0CUNSqiq74qk45Kc\nFJi3w5h5AV1Dk5FB1uLbKV71OSkLF9L64UccmPdLah9+GHuThyKVxXPkmmp+mJNq27rITvT/uRsM\nwiEYcoGKXv+vdH02ACHEKKAQWN3rY4MQYoMQYq0Q4qwwjCdgbA4nFc2dFHiJSAKgvZbUlu2scswc\ntLISW+q38I9N/2DeqHlcNOZcqNsh95L2QV2bhVidmng/1M/c5JhhZ0rq+vlnfrPs75z3xsPYamrI\nvv9+ij/9hKQzzuCYkbO5dsq1fFzyMR/t/4gRSfKLt3awNIay76CzOSAzEvQIBl+mJIDMRD317V1y\n1NW40+QItK6hb0LUH1+hqgp5dwOGAAAgAElEQVRFwzSXQYmU6p/c5o6TdDtR4wxYMChoMzPlpkIr\nV5B45hk0v/EmB06eR8M/n8LR0e/voouVhcPuT7vD0D1R2zr4vS58MdTO54uA9yVJ6l1Qf5QkSbOA\ni4G/CyHc1h0QQixyCZANDQ0NYR1UZXMnDqfkNSIJkG2EwBfOGYMiGFq6WvjD139gRNwI7jvmPsS4\nU+QN+3y3xKxzqZ/+xO3nDqMkN2tlFVX/90dKzz6H4qZySs79DcUrV5By4QUIbY85Y9GURRyefTgP\n/fQQFmS1fdBMSftdUWdFJwV0mPI39VRKojdZCQZsDknOoh89FyQHlHwd1HDDhd3hpKq506fjGSAl\nTkdSjHb4CYbGDlLjdCTH6nzue7h1Hc1SPJbs4JpjKWhzcsh58EGKPvmEuNmzaXz6aQ7Mm4fxjTeR\nbL18CuNPh7YqqNni8Vw2h5Mm0/+GYKgCemd55bk+c8dF9DMjSZJU5fpdAqwB3H5LkiQtkSRpliRJ\nszIyMkIdcx/8jUhiz3Kk5HwqtQUDy2+HiCRJ3P393Ri7jDx+4uMk6BIgfaxci2nvSp/H17dZyPTh\nOFfITY6h2WzzWW9oMHG0tVH32GOUnHoq7Z9/juqSK/j1yXdgP/9iVIaBk0KtUvPIcY9gUBu484fb\nSI4bRFPSgdUw6mh5lRcAVS1mMhL0XvNIFBRzWF1bF+QdDroEv8MZB4ua1i7sTvfltt1RmD78IpMq\nmzv90nhwOilqWcs3zilUt4Unp0dfVEjeP56k4N2l6EePpu7BBylZcAZtq1bJmuHYU0CovJqTGtot\nSFLP8xEpwiEY1gNjhBCFQggd8st/gCdNCDEeSAF+7PVZihBC7/p3OnAssCsMYwoI5eH2lvWM1QQl\naxDjTic/LT7sguH9fe+zpnINt868tacwnhByXfjSr+US316oa/ffYdWdyxABrUGy2TC+/gYHTp6H\n8eVXSDz9dIpXrqD+oisx6WK6zUTuyIzN5MHZD7K3eS+GrOWDY0pqq4H6XX6XwOhNdUuXzxwGhSyX\nDbmurUsOhy06Afav9mlmGEz8jUhSKByGVVbr2yxk+bNAqtmCwdrEV45pYZ/LMVOmkP+vV8l77llQ\nq6m66beUX3YZnaW1cujqnhUej61tk5/pSNVIUghZMEiSZAduBFYCPwPvSpK0UwjxgBCid5TRRcA7\nUt9U1gnABiHEVuAr4BFJkoZcMBw0monTqUmL86J+lqwBexeMO4X81Bi5XlKYKG8r57H1j3HUiKO4\nZEK/bOqxv5SvW/atx+MlSXKZkvzXGAAqh9DPIEkSHV9/TcmZZ1H30EPoD5tA4b8/IOeRP6PNzu42\nC/kywxyfdzyXTrgUk/5rSjo2h3+gB1zuryAEQ5UfyW0KSt/w+jZX06TRc6H1IDTuC/i64cJbHwZ3\n5CQbqGu3DKsqq/XtXWT6Mw/2rUJC8LVzysCGPWFACEHCiSdS9NGHZN93L9aSUsrOO4/qbzXYynaC\nyX2L3TrXPPDrHgaRsPgYJEn6TJKksZIkFUuS9JDrs3skSVrWa5/7JEm6vd9xP0iSNFmSpKmu3y+F\nYzyBUt/eRVaSD/t8yddyVEH+MYxKi6OiuRNnGCaE3Wnnjm/vQKvS8uCxD6IS/b6Sgtmgjev2b7ij\nrctOl83pt8aQO8Qag6WkhIpF11BxzbXgdJL3zDPkv/wyhgkTuvepDWBC3DzjZuJVOTQYXvNYbC9o\nDqyGuEzInOh73144nZIsGPyISIKe+6xzrRApnuu6fuTMSQeNZrRq4VVr601mggGHU3LfIjMCWO1O\nms22bqHrlX0rIXcWJnXyoIbcCo2GlIsuonjlCtKuupK2DWUc+DSTxsfvx2kZ2Emx7n9FY/hfoKHd\n4rXoGSCbc0YdDRod+amx3clMofLCthfY1riNe46+h6y4rIE7aPRysb69n3s0M9S3KS9V/x6mzAQD\nGpUY9MgkR3s7dY88SskZZ9K5eTOZt99G0bKPSJhz0gAhXNPaRXq8zq/+tgaNgXkZtyCp23lw7cPh\nG7DTCSVfyX/vAEupN5msWO1OvzUGvUZNSqy223RAyihIGw37IygYmszkpcQO6LXtCSUZNNKtYhUa\nOuRx+PS1dTRA1SbE2HnkpsRQaRz8BZI6IYHMP/yBoo+XEZ/jpOHtLyg5fT7tX3zRpx5YbZsFnVpF\nqjfrxRAQFQzID7bXl2p7ndzyz1UzR3FuVYT4QO027mbJtiXML5rPLwu89JgtOhHaKsFY4nZzncsc\n4ZdtFbk944hkw6BpDJLTScuHH3Lg1NMw/utfJJ99lrxi+tWvEDr3D3xta2dADrfJGZOxNs5hRdln\nrCzz7Zz3i9ptYG4K2owEnvswuCMr0dD93QGy1lD2XUgN5EPB31BVBeUFXB+GBVI46Fkg+ZgHB74E\nJBhzMiNTY4c0SU9XUEjerw8nf74aVUwMlTfeRMXVi7CUlAKyxpCZqI9YuW2FqGAA6n1pDKXfyL8L\njwd6HIehrJRsTht3f383yYZkbj9iQBWRvriu68nPoKifgYS4DVYuQ9fPP1N+yaXU3L4YbW4OBe++\ny4g//QlNWprX42pau8hO9P+lmpNkwNp4EgXx43n4p4dp7moOdeg9/oWiEwM+VBGy/uQwKGQlGvq+\nVEf/AuydcPCHgK8fDmpaO8lN9v8ZGm4aQ71rHBnxPu6h9Bu5dlH2VEamxAyKj8ErRScSF19B4UtP\nkHXHHXRu2ULJmWdS/9e/0tTYGvFQVYgKBkwWO2arw3uNpNKvwZAEI6YCdAuRhhBWSi9vf5ndxt3c\nddRdJOnd9wfuJn0sxGfJq0k3KCatQBxWucmxYdUYHB0d1D78MKXnnoe1vJwRDz1EwdtvEzN5kl/H\n17R2+RX/ryBrF2pOz7mZNmsbj6x7JMiR9+LAasiaJHfeCpDuzm1++hhAXmB0+xgACo4FtS4i5iS7\nw0mTyUqGP/Z5F92CoWN4CQaf86DsW9l3p1IxMjWWZrON9q4hLENfJFseRMV3pF5+GcUrlpM0fz5N\nL7zINS8v5pjq7RHvl3LICwZltePVLln6NRQcByrZ/p0Sq0OtEkFPiH3N+3hu23OcWnAqc/Pn+j5A\nCPlBLv3WrZ+hvs1CgkFDrM7/olt5KTHUtXeFXARNkiTali+n5NTTaH79DZIvvIDi5Z+RfO45CD/t\n9GarndZOW0CmJMVB6ujKZtHkRXxW+hlfV4SQIGY1wcG1sn8hCKpbukhw9dT2l6xEAw29o3p0cZB/\ndI/mMoQYTVYkCTLi/bdtx+o0xOs1PZFVEaahrQsh8B5d2FwOLQfl+UxPFFwfAT3YpI+FhBHdCY2a\n9HRy/vwwo956k3aNgdPe+zsV116LtbJy6MbUj0NeMHSrn54EQ3OZ/CD1qsmvUgnS4nRBqdAOp4N7\nvr+HRF0ii49c7P+BBcdBRy007R+wqb69y+/kNoXclBgkKbSyEtaKCiquXkTVrb9Dk5FBwbtLGXHv\nvaiTfGhA/aj1M1S1NzE6NcmxWmpbu7hq8lWMSRnDA2sfoN3aHtC1uyn7Hpy2nuigAKn0o9x2fzIT\nDTglaOq9wCg6Qc6jMHmouTNIKIscX9WF+5OZoB9WGkNanN5rH4buInYFchfEHu1/CCOrhJDfJ6Vf\nywEPLhyHTeaGE26h5Pwr6Vy/gZL5C2hc8kLf7Okh4pAXDA2+1E+lTIFi53eRkaCnsSPwh+mdPe+w\no2kHtx9xOymGFP8PVK6v+Dt6UdcWeAp9XncuQ+D2Vclmo/H5JZTMX0Dn5s1k3XEHBe+9S8zkyQGf\nC3oEQ6DZntmJBmpau9CqtTxwzAM0djbyxMYnghoDZd/IZpz8o4I6vKqlMyD/AvQEC/RxQI86Vv59\n8Ec3RwweyjxI9xWd14/0BD0Nw0RjqG/3I/u/7DuITYOM8YA8foDGoRZuRSfKgQ71O7s/qmvrwqFS\nYz93IUWffkLc7GNpeOIJSs85F/PmQcjZ8UJUMLjs8x6dz6XfyPb9jHF9Ps5I0AesMdSaavnn5n9y\nbO6xnFJwSmADTS2Se8e6cUDXBVGmV1ndBtqwp3PrVkrPPY+Gv/2N+BNOoOizT0m9/DKEOvjeCD3J\nbYG9WEckGahtk8c/KX0Sl064lPf3vs+Wes+1aDxS/gPkzgRtYGNQ8KdzW3+U76yPGSNnOmgM8niG\nEGWR89+tMfiR3Fb2rSx8XWZOZd4PvWBwWSBK1nR/1B1dmGhAO2IEI596irxnnsbR0UH5xZdQ+8Cf\ncHT41x40VA55wVDfbkGjEqS4K7olSbJgKDxeVv96kR4fuGB4dN2jOJwO7jryrsDD0YSAwuPkFU8v\nP4MkSXKdpAAzJZWXkr/34DSZqH3oYcouWoijtZW8Z54m7x9Pos1yk3sRIMGWARiRHENNS89L9YZp\nN5AVm8UDax/A5gxA/bZ0QM1WGHVMQNdXMFlkH0nAGoPrfmt7CwaNXq6d5Efd/nASrMaQkaDvDhON\nND7rhfXzLwAkxWjRqMTQR1Yl5si+hl6CoVtz7jUPEubMoejjj0m59FKa336bktPn07Vnz6AP75AX\nDA3tFtLj9e6btzfsBlO9256/GQl6mkwWv7Ofvzr4FV8c/IJrp15LXoLvpiBuKTgOTA3yuFy0mG1Y\nHU6yAogmATBo1SToNX5NiI7vvqdkwRk0v/EGKQtlNTdhTuCx/p6oae0kOVZLjC4wrWNEooEmk5Uu\nm1ysN1Yby+IjF7OveR9v7HrD/xNVrgenHfKDEwx+BTC4IT1eh0ow8MU66lg5p2IIy3A3dshl2wPt\nGpaZYMBkdWCyRK4gI4DDKdHYYfGe9dzPvwAuf2G8bug1BpAXnAfXgkP+29V6CDtXx8eRfecdFCx9\nB8PEiejy8wd9aFHB0GHxrD6Xusw2hccN2JQRr8fmkGjt9L0yNdvMPLzuYUYnj+byiZcHP1hlHKU9\n5iQlVDWY2Oe0eB1NXsoZOFpbqV58BxVXXYXQ6xn15htk33M36njfTVACoba1K6gSAIpPondUzNz8\nuZw48kSe3fos1R3V/p3o4I9y1cuRRwQ8BugxQ6QHKBg0ahXp8fq+PgaQNRfJCRXrghpPMDS0e5kH\nXhguuQxNJgtOyUeoaj//gkJ6fHD+wpDJPxqsHd1+hrq2LhINGo8LpJgpUxj5zNOoYoIzdwbCIS8Y\n6tu8TIiDP0JiLiSPGrApPYAY7he2v0CtqZZ7jr4Hrcr/cMYBpBRAUr7sKHXRY5cMfFKnx+tp9DCh\n21evpmT+AlqXLSPtmmso/PA/xM6YEdSwfRFoDoOC4pOo7ld+e/ERcrTXn9f92b8Tlf8A2ZPBEFxX\nPuWl4jVM0gNZiYaBpVXyDgeVZkjNSYrmHCiZwySXQVkceNXa+vkXFGTBEIHxK4EOB9cCsmCIdLlt\nhUNeMDR0eLFLVvwEI48c4F+AXk4rHyulstYyXt35KmcUn8H0zNAaggAuP8P33WFuwWQ9K8gaQ9/x\n25ubqfrD/1F5/Q2o09IofO9dMm+9BZV+8Ko91rp6PQfKCFeWbv+Q25z4HK6Zcg1rKtbwbaXnqrQA\n2K2yKSlIMxL0aAzBrLjlJLd+z5AuFnJmDKkDurHDj3phblDuOdK5DA3dYece5oEb/4JCRoLnBdKg\nkpQHSSO7I9Bqg4guHCwOacHgcEo0eTIltVTI3ZY8hC/6k/UpSRKPrJOby9w689awjJlRx0CnsTuf\nQbFPB/NS6q9Ct3/5pdxYZMUK0m+8kcJ3l2I47LDwjNsDXTYHTSZrUBqDYn5y18ntssMuoyCxgEfX\nP4rV4cVMULNFLmsepOMZoMn1Nwym8FlmosG983bUMVC1yWcfjnDR0GEhPSGI8XebkiLrgFZKi3hc\n5LnxLygo8yAi2cb5R8kagyRR1xp4dOFgcUgLBsUu6falWvGT/HvkkW6P7UmM8SwYVles5vvq77lh\n2g2kx6SHPN4+43GNr67NQnKs1q+uYf1Jj9fTbLZiMTZTfdttVN5wI5r0dArff4+MG2/wWPAunCgr\nzWBU6Di9hkSDxm0nN51ax+1H3E55Wzmv7XrN80mUVXn+0QFfX6HJJH8HWm+JVR7ISjB0V2btw6hj\n5YS7qg1Bj8tfrHYnLWab7xpDbkiJ1aFRie5E0UihPEceF0jl30NMygD/AshBAFaHk7bOCDjQ84+C\n9hoczeU0dFgiXm5b4ZAWDF6jSQ6ulfsgZLmv9ZMYo0GnVnnUGLrsXfxl3V8YkzKGi8ZfFLYxkzZa\nfsAreuySgUYkKaTH65heu4eys86i9ZNPSb/+OllLGD9w8gwWyks92AmR6Sor4Y5jc49lzsg5LNm2\nhFpTrfsTlP/gqkUVfLvYxg5LUP4FgOwkD5pn/pGyQ3wIzEmKOTEYjUGlEkGFboeb+nYLSTFeFkgV\n6+RFlZsyLRGt+TRStkh07P0Oh1Mi63/JxyCEOEUIsUcIsV8IMaBUqBDiV0KIBiHEFtfPVb22XSGE\n2Of6uSIc4/GXBm/lMCrWQt4sULsP3xNCeE1ye3Xnq1Sbqll8xGI0qsBCAL0ihPyAuyJW6toDz2EA\nOS9h7FvP8NCPL+CIiaNg6VIyfvvbIdESehNsYpVCWpz3yKr/O/z/cEpOntjgJiPa6ZAXACFoCyDf\nQzCOW+jpoTGgNIkhSXaIeyicGE4aXeUggvExgCuXIeKCwUtZGLMRGvd6jDpLj1SSG0DmBNAnYS+T\nFwD/MxqDEEINPA2cChwGLBRCuDNML5UkaZrr50XXsanAvcCRwBHAvUKIAOpEhEZ3Ncb+K+6uNqjb\n6fOFkR7vvl5SnamOl3e8zMmjTubw7MPDNt5uRh4hP+hmI/VBZD13btlCyTnnkPDFJ3ww+gSanlhC\nzKTAOpaFC6NrtZoWQPG23qTH6/vWGupHXkIev570a5aXLR+YEV2/CyytIfkXQH6hBCsYsrpbfLqx\n0ecfA5UbwDG4tXIaOuRrBxpuq5AZRBWAcFPvLdy20mWOyxuGgkGlhpFHoK+WF3rBRBcOBuHQGI4A\n9kuSVCJJkhV4BzjTz2N/CaySJMkoSVIzsAoIsFZE8HjM9qxcL8eR57v3Lyh4qpf05KYnsTvt/G7m\n78I21j64/AzOg+uob7f4/TBJNhv1Tz5J2cWXgM2O7slneXHSAhoj2JlR+fu5zTz3g1QfGgPAryf+\nmsyYTB5d9yhOqZctv9xVjyhEwdDUYSU9SMGW1b/FZ29GHiH3Z6jbEcrwfPI/oTF4y3quXAdCDbnu\nw60jnouRfxTxbftIouN/R2MAcoGKXv+vdH3Wn3OFENuEEO8LIUYGeOyg0NBuIUHvJqGk4ifZvpvn\nfbXvzpS0rWEbH5d8zOWHXR58hrMvcmaASkNn6Y+yXdKPh8lSUkrZwotpevY5ks44g8JlH5FxnPxC\nbIpEco+LUBy3IGsaLWYbdofn8uGx2lhumXkLO5p28GnJpz0bKtfJ9aeSg88ktdqdtHbaSAvypZoS\nq0OrFtS5eykpz1/F+qDH5w/BVlZVyEzQYzT1Kh8+xEiSJIede5oHFT9B9iS5rLkbkmO0qFUiMhoD\ndFsmDlfvC/o5CjdD5Xz+GCiQJGkKslbwr0BPIIRYJITYIITY0NDQEJZBNbRbyHC32j64FrImgj7B\n6/Hp8X0nhCRJPLr+UdJj0rl6ytVhGaNbdLGQPaU7McZbGQBJkmh+5x1KzzkHW0UFuf94kpxH/ow6\nPp5Eg3cH+lBgNFmDdtwC3RPJaPYu3E4vOp2JaRP5+6a/Y7a5QkAr18t+pBAwurSVYE1hKpUgM8Hg\nXmNIypPr9lcOsmBwLZCCiWwDyHBXPnwIaeu0Y7U73WsMDjtUbvQYXQg9ZfQbh7L0dm9yZ2AXGmbr\n9/vdb3uwCYdgqAJG9vp/nuuzbiRJapIkSXlqXgRm+ntsr3MskSRpliRJszIygo8g6U2Du5aeDrts\nkxzpu/xyRoJenhAuO/mKshVsa9jGb6f/ljit+9VJ2Bh5JIb6LWiwezQl2ZuaqLzuemrvu5/YmTMp\nXLaMxHnzurcLIdeJiaTG0NhhJS0u+FWSIlR83YNKqLjtiNuoN9fz6s5X5YbwzWU+tUJfdJfDCGGl\nl5mod58gJoQsuCoHtzSG17IwfqDMoUiZk5QcBrf3UL8LbCavggEimP0MoI2hXDeGWarBL47nL+EQ\nDOuBMUKIQiGEDrgIWNZ7ByHEiF7/PQP42fXvlcA8IUSKy+k8z/XZkOB2QtTtkB8kP+ry92Q/W7E6\nrDy56UnGpYzjjOIzBmO4fRl5BGpHFxPEQbempI5vv6XkzLMw/fADWXcsZuQLS9BmZQ7YL6ITApfG\nEORqG3oEg9GHnwFgeuZ0Tik4hVd2vEJdqat9ZtgEQ/D3kJmg79v7uTd5R8gCrCM8WrI7gi2HoaBE\nxUVK8/QYRAI9+Ug+vuf0CJcP36GawFj7PjkTfxgQsmCQJMkO3Ij8Qv8ZeFeSpJ1CiAeEEMob8rdC\niJ1CiK3Ab4FfuY41An9CFi7rgQdcnw0J9W1dAx8mH4ltvekd//z27rep6qji97N+j1oVfG8Cv3GN\nb6Zqb58Xq9Nqpe7Pf6bi6kVoUlIoeO89Ui+/3GObzfQIawxNHZbQBEOAESU3z7gZu2Tn6d1vyvWI\nXH28g0X524XyYk2N03sWbMoLbRAT3RrDpDFEqmFPvbee55XrIT7bpx8pPV4XmbIYLjY6R6PDBnXb\nIzaG3oTFxyBJ0meSJI2VJKlYkqSHXJ/dI0nSMte/F0uSNFGSpKmSJJ0kSdLuXse+LEnSaNfPK+EY\njz+YLHZMVsfACVG5XrbrJo90f2AvlJdBeXMDz297ntm5szk6J7SYeL9JyqVFl8WRmv3oNbIgshw4\nQNkFF2L812ukXHIJBe+9i2HcWK+nSYugxmB3OGk220gdAlOSQl5CHgvHL+RDcxl7s8fL/poQUP52\noTgN0+J0NJtt7ku450yTBdggVlqVNYbghXNEE8TwUUCv4icYebjbeme9USIMI1IWA/ihq0D+R+XG\niFy/P4ds5rPHwmdVG+VOXn6gHPt59RuYbKbBC0/1wAH9RGao9iJJEi3vv0/peedjr60l75lnyL77\nLlQG39FKch5AZCZEs9nmGkPwL6UkV0RJ/2KA3rhm0lXEO538LYTrKjSZrBi0KuIC7CXRm7R4HQ6n\nhxLu2hg50W2QHNBdNgftXfaQNAaDVk2iQROxhj317RZitGri+/eS6KiXzXD+aP/xerksRtfQl8Ww\n2B3styRh0qYNSQkUfzhkBUO9u3IYZiMYS/yOVInTa4iNbWF722ecPfpsxqSMGYyhemSnahzpliaq\nb7qemrvuJmbqVAo/+oiEOSf5fY7uOjERmBDKyzyY4nMKKpUgNU7nl49BIamthkXNrXxna2Rtzdqg\nrw1ydd20OH3gHfl6ody/x3yMvCPkgnqO8H9H4XCeg6s0SQR9DJmJbr4DRcvyQzBEMsmt2WQDBMaU\nKT3JeBHmkBUMbsthVG2Sf/upMQAYMlcBaq6fdn0YR+cflbVxlK7MoG31N2Tceiv5L73o1sHsjUhO\nCGN3H4PQXkppcbrAGq1Urmdhezs5MRn8dcNf+ya9BUijyRp0xrCCcv8ewz3zDpcDIhp+dr89BLyW\nhQmAjHgPkVVDQIOnchgVP4Fa55cfSZkHkUhyU+ZeZ+Z0MB6QF6gR5pAXDH0eqKoNgJAbsvvBrqZd\n2GI2kub4BZmxgb2QQ0FyOml66SXOWPYOEoJRt5xE+jWLEOrAzRmK4zcSDuhGk+K4Dc2kI4fcBjCh\nK9ejN6Ry08zfsdu4m+Wly4O+dmO7hfQQNB7o0Rg8O6BdGuwg+BlCrVWlkJkYuaieJk8hz1WbZDOc\nxve9KQUEI7JAcn3vTmVBWr1pyMfQn0NaMKhVom8phqqNclleH4ltCk9uehINcYhW/003oWI3Gqm4\n9lrqH3uctTmTiD07hVh9edDni6TGoLzMQzElgbziDsSUROUGyJvFaUWnMS5lHE9tfgpbkPWImkyh\nhXpCL+Hs6R5SCiAuY1DMDB7LwgRIJDUGo8lKSv9nyOmA6s1+a//d8yACGoPy7OryZwJiWDigD1nB\nUN/eJTdjVzINJcn1wvDvQVpbs5Yfqn9gvOFsmtqH5s9oXr+e0rPOxrz2J5LuuJMHZ11GS9YUqNrc\n3dEtUJQJEYmsVaPJikpAcpB1khRS4wIIue1sgYbdkHc4KqHi5hk3U9lRyXt73wv4uk6nJK9WQ9R4\nlMWJR+EmhGxOGoREt56oqtDuIT1BT6fNgdk6tL4qp1Oi2ewme75xr2x+81MwpMTqXGUxhl5zVhYE\naSnp8sJ0GDigD1nBIJdi6LVKai6VO6P58SA5JSd/2/g3RsSNYFbqfFrMtoGNVsKI5HTS+NxzlF/x\nK1QxMRQsfQfraWeDEJgzpoK1HZr2BXXulFgtQkBDJExJHdbuCRkK6fE62i12LHaH750VNd1lnpmd\nO5tZWbN4ftvzPaUy/KSty4bdKYW82tZpVCQYNN61nrzD5a59YbY/N7j6GCghz8GSGhsZk2Rrpw2n\n5EbrrHKtunP861OudgUxRMaUZEGjEiTGaOSFaeUGeaEaQQ5ZwdBksvZ9mLodz74jklaVr2JX0y5u\nmHYDIxITXOcbnAfKbjRScfUiGv7+JImnnkrBBx9gmDChe5XhVPwhVcGpnxq1itTYAG30YcJoCi25\nTaG7XpI/5qRKlx/JtQAQQnDLzFswdhm9d3pzg7K6DMs9+KoSq1QGrdnieZ8gCDW5TUGZS80+alaF\nG+Vv5lYw6BPlxlZ+EqmGQ00dsilMCCG/fzqN8kI1ghyygqG5v2Co3ACaGMj03uPY7rTz1OanGJ08\nmvlF87sdp4PxQJk3bJBNR+vXk33//eQ8/hjqeLkGk/ISjMkeD7qEoAUDyC+2yPgYQquTpJAaSJJb\n1UZIHyM3wnExNWMqc/Pn8urOV2nuavb7uuEK9QRcIbdevoMR0+TfVeF1TIaa3KaQ6stPMkgogmig\nYNjkSg70/xWXHql50ImpMj8AACAASURBVLuQpBJoEGE/wyErGAZqDBvlB8lDxzaFjw98TFlbGTdO\nvxG1Sj0otdwlp5PGF16g/IpfIWIMFCx9h5QLL+gTp628RFITYuRxhyAYlCS3oabJZO1+oYSC8mLz\nOaklSXZIujEv3DT9Jjrtnby842W/rxuOchgKqXE+voOYZEgtlscfRmSNIfQeAIopqXmIBYPyN+sz\nl21dcs2zAMLOQXagR8LHYOz9LsqYANrYiPsZDknBYHM4ae+y93wZdivUbPX5IFkdVp7d+iyT0iYx\nZ+QcoCcGPaCoGC84WlqovP4GGv76BAknn0yhy3TUn26HVZxeHnftDnlCBEGkymI0dYQe6gkBfAdt\n1dBR57ZhS3FyMfOL5vP27repN9f7dd1wOW5BNiX5HH/ujGGvMYRrHviL0Z0pqW4HOO1++xcUMlyF\n9Ia6CoBcSNK1uFBrZO0wwoluh6RgaO7/MNXvBIfFp2B4b+971Jhq+O2M33av3sM5ITq3b6f0nHPp\n+P57su66i9y/PYE6Pt7tvsYOKzFatdxkKHcmOG1Bd/qKRCE9q13Otg5HY5JUf3MxlNW2hzyVa6de\ni8PpYMm2JX5dt6nDgkoE332uN6nxOprNPkqT5MyA9mporw35eiCXwzBZHSH1w1BI0GvQqkUEBIOb\nkGdFew5QY0iP12O1O2m3DG1kVVOHpe93kDcTardFtNLqISkYBjisunvCenY8m21mlmxbwhHZR3DU\niJ6S3HE6NTq1ymejGG9IkoTxrbcov/gSJCQK3nyD1Esv8VpmoY/6qUyAIM1J6fF62i12umx+RPWE\nCY+24SBI0MsNhxp9BQBUb5IL0mVPdrt5ZMJIzh17Lh/s/YCK9gq3+/SmoUP+DsLRXCUtTofNIXkv\nTaJoOmHSGnq+g9CFsxByTtDQCwYbcTp13yZDVRshPgsScwI6l5LkNpQOaGWB1Gce5M4Eh1XuJREh\nDknBMEBjqN4MsemQ5Lmi6lu738LYZeSm6Tf1eWEL4arVE+SK22kyUf1/f6TugT8Re8zRFP3738RM\nmeLzuKbefQwSc+TSwkG+MNIj4DgMRx8DBaXhkM/voHozZE6QC9N5YNGURahVap7b+pzP6zZ1hJ7c\npuAz+xlkgSZUYfMz9JhhtGE5nz/9t8ON0WQZ6Keq2iS/XAOsXxVus7A/uF0gKRptmP1JgXBICoYB\nGkP1Znk15uFBare288qOVzg+73imZU4bsD0lThdUmJ6lpITSCy+k7bPPyLjlZkY++yzq5GS/ju2j\nMQhX+GUIGgMMbdZnj9MwfC9Wry+lbsez93InmbGZLBy/kI8PfMyBlgNe920MsZdEb3oEg5fvQBcn\nOyfDVDJBLt4WHlMYyPcw5M5nk7Xb8Q3ICYxN+wL2L4CfwjnMNHXXC+t1D8mjICYlKhiGmj5S2mqS\nM2G9vDBe3/U6bdY2bph2g9vtqXHagB+mthUrKTvvfBxNRvJffIH0a6/12EzHHcb+UVW5M+QJ0dkS\n0Dgg8GY34SDUXsn9SYvXe8/FaC6Dzma/Xhi/mfQbYjQxPLPlGa/7NZmsYdMYegrp+XJAT5dXxGFw\nkCq5N+EUbkNtSmo295sHSp6HmwADX3TnYgzhPfTMg17PkXDVa/tvFwxCiFOEEHuEEPuFELe72f47\nIcQuIcQ2IcSXQohRvbY5hBBbXD/L+h87GCiTLzlGK0fzSE6PgqGlq4XXdr3GyaNO5rA09zkOXjtw\n9UOy2ah75FGqbrkF/ZgxFP7n38Qdc0zg92Dq57AKIQEqPQKF9LpNSWHSGNJ9aQzKKtuPAokphhQu\nmXAJn5d/zh6j5z684crDgACCGHJmyAlQLcHXx1JQXoDh1BhC8bUFg7HD2lfr7M549q8QZm+Uv8NQ\nmsM8lp7Pme7qVx2ZHhchCwYhhBp4GjgVOAxYKITo/wbdDMySJGkK8D7wl17bOiVJmub6GYJmyfIq\nIzlWi0bdy147YqCJCODVna9itpm5fqrnstqpsf5pDPaGBsp//WuMr75KyiWXMOr119BmZwc8frPV\nTpfN2XdCKOOvDkYwDH0HLqPJ2lMGIAz4rJdUvRnUep8JjApXTLyCBG0CT2952u32LpuDDou922EZ\nKt2d6PwJWYWwOKCNZhsiDLWqFFLjdLSYbdgdg1cepjeSJLnykXr5SKo2QUohxKYGfL4YnZoYrXpI\nNQa3piSQBYPTDnU7h2wsvQmHxnAEsF+SpBJJkqzAO8CZvXeQJOkrSZKUQjRrgbwwXDdo+tglqzfL\njtvEEQP2a+xs5K3db3Fq4amMTvGcWp8ap6ety47Ny4Qwb9xI6Tnn0rVzFzmPPUb23XchdMFNSLcP\nU2yq3Nc2CPXToFUTpxv6CZGqlAEIA2nxPoq4VW2G7Emg8e9vnqRP4vKJl/NVxVfsbBw4ObtzGMIQ\nVQXydxCrU/teYGROlHsMhMHPYDRZSHZ1wAsHyqq3xV0nukGg0+bAYu+3QKreEpQZSWGotR6jyYpa\nJUiK6RcA0O2AjkwJ7nAIhlygd2xfpeszT1wJ9C6AbxBCbBBCrBVCnOXpICHEItd+GxoaGkIacJ9y\nGF4ckq/seAWLw8J1U6/zej5lxeLOAS1JEsbXXpML4MXGUrD0HZIWzA9p/G6TekC+jyBr6aQMsX24\nyWQJS6iqgte+Ek6n/HcJ0CF56YRLSdIn8dSWpwZs67YNh8mUBIrW40Nr0+gga5Is6EKk2WQbWK46\nBIbaeTtggWRqhLZKj9q/Pwy1n6TJJBeSVPUXzom5cqn1ICwA4WBInc9CiEuBWcBjvT4eJUnSLOBi\n4O9CiGJ3x0qStESSpFmSJM3KyMgIaRzd9dstHXJ5XjeCocHcwNI9S5lfNJ+CpAKv50vpdlr1XSk5\nzf/P3nmHyVXVjf9zpu+U7T2bbHoPqfQiAlIUCCogKIp0C4K9/HxfRQQF8ZUXBUEQFMur0ouINBOQ\nmoQQ0pb0tr23md2Z3Znz++POnZ3dnXJn5t7ZjZnP8+TZ3dvmTO4993u+3aeEov7kp7g/9CGmP/Yo\njrlzMxq7On5gfJhe1bIRJ2uKZHulpKfjFpKYYjp2QaA/5ZWk2+bmikVX8HrD62xqHT1B1Xug54s1\naSE9lSkrFEEXyizvpHNsRE+GZLvC6rh7oL5Eq9MXDEVZjqzqHOsrVJlgB7QegqEBiE4AqAlvG4UQ\n4gzg+8D5UsrIskhK2RD+uRdYC6TuNUqRSNGq5s2AjPkgPbT1IYZDw3zhqC8kvd5Iz96R1V7gwAH2\nX3Ipvc89R9lXv0rNr36J2aOtAZCW8UMcuySktcoocmZ3QujRxyCakkR9JZJkPCfi0vmXUuwo5t73\n7x21Xc8EPRXNq9XqFYqg69id0eeNi2zLEHWhkq0Kq51j70GT6i9M3sozHsVOa9ZNSXHvQfVypZ1r\nILVy8Hqgh2BYD8wRQswQQtiAS4BR0UVCiOXAb1CEQmvU9iIhhD38eylwImBoup+Uki5VY4jjeG71\ntfLIjkc4f9b5TM2Pn/SmUjxGY+hbu5Z9F17EcEsLUx94gNIvXJdSKGoyYpYBgJEJkYY5aSJsq7qa\nkhJpDA0bweqC0tS1NafVyRWLruDNxjd5r3Vk9dYZvtf6CgaN0W3V6QcaRNM5NtQzQ4oT3QMD6Bxr\nSmrcpBQajKqcmyrFLvs4zd9IOvoTFJKsXq5ETDZvydp4VDJ+W0kph4HrgReAOuARKeU2IcTNQgg1\nyugOwA08OiYsdQGwQQjxPrAGuE1Kaahg6B0cZjgklYepcZNiy/NUjDrmt1t+S0iGuPaoazVdU1Wh\nO/sHabv7Huq/8EWsU2uY/vjjuE86Uffv0OENYDObcNvHRPQ4i5U2kGmon0pyUnYmRCSiR09TUiIf\nQ9MmqDoKTOk1o7l43sUUO4pHRSh1hZ2G+Q59oqog3Lvam6ReEkDpPLA4MurNoC6Q9BQMRVmusDrO\npNq4KSMzEij+wn6tTZ90YFTJ7bFEIg2zb07S5amWUv4D+MeYbT+I+v2MOOe9CcQuXGMQo8phxHA8\nN3ubeWznY6yevZoaj7bgqSKXDefQALW/+CHtW9dTsPp8Kn/0I0yOzMsZx6IzUURP1bK0BYM6ITLt\n5pWMuKawDHDaLORZzeNNSaGgsuJacXn617Y6uXLxlfx8w895t+VdVlasjDgN9YqqAuX/IzAcwhsI\njhf60ZgtigO66f20P0tdIOkpGKxmE/nJOtHpSIc3gNUs8NgtUY7n6zK6ZrS/sLLA2HkwFAzRMzAU\nP4Ahvwo8VRMiGI64zGf1pVRq8ytOyTFmpAe3PIiUkmuOukbzNUN79/Cr135JyfaNVPzXf1F1222G\nCQVIZpdcpiQ/pdgCcmS1Z7zW0Bmrhr4OqCvuUbTvhCFfxivJi+ddTImjJJIN3TU2fl4HIlE9Wpy3\n1cugaXPavb71Tm5TyWZUT1e0cNbB8QxR2n8WvkPET5XI1zZBDugjTjCoE2KKb6eyIUpjaPY28/iu\nx7lgzgVMcSeKuB2h94UX2fepS3AN+3n6c99PWhVVD0YV0BuL+n1SNDOoL7lsTIj2SCkG/UxJECeq\nR31hZBDCCJBnyeOqJVexrnkd65vX0+kL6P5SjZjDtLSJrVqm9PruTFzPKR4dY80wOpFNwTCq2VZj\n5o5nyG6L0k4tmnPVMmVx4+8zfDzRHHGCIXIzeuuUDVErjIi2sCS5tiCDQVp/cScNN96IY84c7r30\nv9leOtOQMY8locagTowUHZMRjSEbE8IgjSFmqGHTJqUjVumcjK9/0dyLKM0r5b7379PdPg8jBQWz\n4YCOmFQN0Biy5nyOzoVpytzxDNl1oMfsPjeW6mWAzLoD+sgTDOEXn7tzi1Jm21UKjNYWqt2J67gH\nu7s5dN0X6Lj/fgovvphpf/wDlvKKrK2UEgqGvCKlJEDKGsMEqNB6v1hj9QNo3KSUq07T8RyNw+Lg\nysVXsq55HR3BOv1NYam8lMrmKyU+0nRAjwv11IlsVljt8g1FaQyZO54h2seQBcGgVWOAjPxJ6XDk\nCQZvALvFhLll8yi1U6u2MLhjJ/suuhjvO+9Q+aMfUXXzjzDZbFlTof3DSkRPwoepOnUHdLZVaL0j\neiBG+XPV8ZyhGSmai+ZeRImjhEHXPw15qYJG4Wy2KiU+0tQY4mbPZ4gacpuN9piRzmc6ZDyrFOZZ\nESI7C6TOfg0mVU+FUrInyxnQR6RgmOocRnTsjqwwtGoLvc8/z/5LLkEODjL9j3+g6FMXR/apeQBG\nT4iRCZ3gYapeDt0HU3JAF4QnRDayVrt8+kf0gHIPfIHgSCe6jt0w5NVlJanisDi4dN7lmF178Ipd\nul0XwGkzY7eYtL+UqpYpK8k0HNBd3gA2iwmnTd/Im2KXlUBQiawykqGg0vmsSA07h7QSGMdiMZso\nyEu9jH46dHoDmES4ynMiqpdlFJqcDkekYFhhr1f+CK8wVG3h6iVXxzxHBoO0/s//0PC1r+OYP5/p\njz9G3rLRL5uicKihz+AJockumUb8szohsqUx6BmqqjJO69HJ8TyWUyrPIzTs5t3ev+l6XSGE4kDX\nKpyrww7orn0pf5ZaDkN/4Rz2kxi8wFDvccmoRNXknQ+1UOzMTrJn3DpJY1Ed0AGv4WNSOeIEQ4c3\nwFJzeCJVLaXF28Ljux5n9ezVMSORIv6EB35L4ac+Re3Dv8daXj7uuGzZ6DU1uFEnSPPmlK4d00Zv\nAErxNn1DPWHEgR75Dk2bwJKXVsZzInx+E4GOD7G3fxMbW/StflnstiXu4hZNBglQemeeq0Si2wx+\nsY7SnHVyPKtkq16S5ntQtTScAb3V8DGpHHGCocsbYG5oL3iqwV3OQ1sfiqstDO7cyb6LP6X4E27+\nEVU/uiluqexsxT9rsg3nFYUzoFOMTEqzRWmq6F2KQWWccG7cpNjhzfr6Mjq9Qwx1HUu+tUhTb+hU\nSKXpk+KAtqVlZjDuHqiRVcb29lA1kiKXVTGn6WguzJa/ULNgqM6+A/qIEwyd3gDTA7ugailtvjYe\n2/kY5806b1yWc++LL7L/kksJDfioffhhii6+OM4VFTR34MoQzVnDVUtTfpCUCZGFBDev/jkAMCYX\nIxRSNCadzUgQjliRNi6e+1neanprXOXVTNBcYRXCJbgXpeWYjNQL05lIZJXBpiRVIyk3eaHnkK73\nOWuas1bh7KlSSnBn0c9wRAkG/3CQoL+f0sEDUL2Mh7Y+RFAGR2U5y1CI1rvuouGGG7HPmc2Mxx7H\nuSK5Uyt7GoM/HNGTxBRTtVSxPafQA7o4CxVWgyFJt0Gr1VG1ejr3KBVIdVxJqqgvpc8uvJQiexH3\nbdZPa0g53LMqnAGdYtBDwho9GVA01s9jEOo8K+0P5yNlmNgWjao5ZyOQRJNwFmIk0CBLHFGCocs7\nxAJxEIGkvWQmj+58lHNnnstUj1JBNdjXR/2XvkzHvfdR8MlPUPvHP2KtGO9PiEX2JsSQdocVpORn\nUJv1GDkhegeGCEn9SzHASGRVp2/IMMczKBPaYTVR7HRz+aLLeaPhDba06ZOAVOyy4Y2OrEpG9TLw\n90DnXs2fMRQM0Tc4bMg9cNnM2CwmwxPEVI3E0xXurqeT4xkUrWcoKOn3x+kGqAOhkFTyMLTeg6ql\n0FoHQwOGjSmaI0owdHoDLDEpjuff99YxFBqKaAv+vfvYf/Gn6H/9daXe0S23YEqh9Wa+w4LFJAyf\nEEq2pwbHbRqJMdkINTQqsQqiIqu8AUXtNtuhbJ7unxPd4ObS+ZdSaC8c168hXVLOQE/jPmuq0ZMm\nQoisaJ6d3gAFeVbMze9DYa3iV9OJoiwEkvQNDhMMSe3mvOplIINZ6wF9xAmGxWIfzc5SHtn/dz42\n42PU5tfSt2YN+y++mGBPD9MeejCtekdCiKxEM3SFNYakuEqUzO4U7M/ZKJs8qrqtAUT6SjS9H3Y8\n6x/9FG2fd1qdXL7ocv7d8O+YvaFTJeWaVeULwGRNSTBEAhgM0BggO87bTl/YFKaz4xmyUzdsZIGk\n8fnMoNdKOhxZgsEXYLFpHw+XVeMPBrh68VW033sv9V/6MrZp05jx2KO4jjkm7euXZGlCaH6ppuiA\nzkbIrVEZtyrFThvd/YPK9zbAjATj78El8y4h35avi69BjerRXOXWYoeKhWkJBiNChiFLgqE/wNQ8\nv9LKVkf/AmSnbpimRNVoCqZCXnHWMqB1EQxCiLOFEDuEELuFEN+Nsd8uhPhbeP87QojpUfu+F96+\nQwhxlh7jiUdPTw+l5iYeN/dxbuUZ2H5wF213/ZL8c8+l9v/+jLU6cY2kZBRlIZohpeJtVUuV7F+N\nlRkjgsHACaFONiMiYtTr2voOgr9X9xeGyth74La5+dzCz7H20FrqOuoyura6gtRUYVWlaqmyktTo\nG1KFTtw+ABmSDcHQ5QuwxHJQ+UPn+1wSCbk1LkIvZa1NiLQiDdMlY8EghDAD9wDnAAuBS4UQC8cc\ndhXQJaWcDdwJ3B4+dyFKK9BFwNnAr8PXMwRz23b+UuiioDPIZ+/aRt8rr1D+7W9T/bPbdemfYHR7\nTMVhlYpgSK0yY0r9ANKkw2gzhtNGlW+H8odBgiFWuO2nF3waj9XDbzb/JqNrp2XOq1oKA11K2KYG\n1BwDIzUGw53P3gALZNjhrrNmWBQxJRmXi9GVjtZWvUxxQA8bmyMC+mgMxwC7pZR7pZQB4K/A6jHH\nrAYeDv/+GHC6UIz4q4G/Sin9Usp9wO7w9QzB1PUemzrc3PEHEO3dTH3gfkquvEK3sgBGr5R6Uo3o\nidglta0yshFZ1RWO6MnTuUaPSpHLxvTALqTJCuVj1yeZo9boGSucPTYPly28jFcOvsKOzh1pX7/Q\naRuJrNJKig5odSVsRFQSKPOgb3CYoWB6TYSSobYlnTG0G/JrIhWS9cJtt2A1C2M1hnSCMKqWQmhI\nEQ4Go4dgmAJEL1Xqw9tiHhPuEd0DlGg8VxeklPjfeZFvPCqxV05hxmOP4j5R337MRS4bPQNDDBs0\nIVJ+mNTKjBpfGB67ElllrI8hhRC9NCh2WVnAPkJlC5QEMJ1JZAr7zILP4La6uX/z/Wlf32wSFKqR\nVVqpWATCrNn+3OULkO+wYDUb42KM1Kwy6DlS25JWD+w0RCsUQlBkcGRVV7jKc541hQXS7I/AN3YY\nkpszlsPG+SyEuFYIsUEIsaGtrS2d83H2DrN7loV5jz6ObepU3cdY7LQipbKyN4K0Inqql2l+YUQi\nqwz2MRjlXwAoyrOy2LSfgZLFhlxftc/HEm4F9gI+veDTvHTgJXZ37U77M4pS1TyteUp5DI0LgI5U\n/FRpYLSvqssbwMUABb4Dhr0kjTYLq4UkU7JW2N3gqTRsTNHoIRgagOi3bE14W8xjhBAWoADo0Hgu\nAFLK+6WUq6SUq8rKytIa6IV/XsfHnn4Pk8uV1vnJKHan0IErDTrSEQxVS6F9BwR8mg43OrLKqOJt\nKlNM7RSJfroKFxly/WQRPZ9d8FnyLHkZ+RrSKsmQggPaqHIYKkb7qjq8ARaKAwikYX4ko83CmrOe\nJwg9BMN6YI4QYoYQwobiTH5mzDHPAJeHf78Q+JdU0mufAS4JRy3NAOYA63QYU0yExYJF54Jq0Rhd\nFmPEYZWiYJAhaNFWmVFRoY2zrabkPE+DSq9i329x6Z/YBtHlnmNH9BQ6Crl0/qW8sP8F9vZoz0aO\nJi2trXoZeNugrznpoUaVw1DJhsaw2DRSIdkIjM5JSqeIYUiGaPG2GDSi0WQsGMI+g+uBF4A64BEp\n5TYhxM1CiPPDhz0IlAghdgNfB74bPncb8AiwHfgn8GUppbENDQzE6C5oER9DKjb6FB2TSkSJcVEP\nRhXQUynprWNYmjhknWHI9bXkAHxu0edwWBxp+xrS0tpSSIDqMvgeGJ0Po+YjBV0VhplWSgw2JaVz\nD14+8DLnPHGOLomUydDFxyCl/IeUcq6UcpaU8tbwth9IKZ8J/z4opbxISjlbSnmMlHJv1Lm3hs+b\nJ6V8Xo/xTBRGNxLv8gbIs5pTi+jJrwZnqWY/Q5HLSlcqETEpEBhWavQYqTE4O7exS06hfdAY91lE\nMCSY1MWOYi6ZdwnP73ue/T37U/6MtIq4VSwGRNIFgJTSsJLbKmpHMsMEgzfAYrEfWalffaSxFDlt\ndPsMDCRJ0aQakiHu23wfNZ4a5hfPN2RM0Rw2zufDgUj8s4G21ZQntBDh1oAaNQanjW5fgGBI/0J6\n3QYntyEllpb32SZnGrba6/QG8GiI6Pncos9hM9l4YMsDKX9GsVMp4taXShE3u1tpSJTkPnsDQQLD\nIUPt2xaziUKnce0x+3t7mC0aME8xLjpHnWfdBgSSxAt5TsSag2vY1bWLa4+6FrPJsFSvCDnBoCN2\nixm33WKoxpBWUlLVUmirg6HBpIcWuWyEpFIFVW/SMoWlQl8TwtvGPutsw+zDWn0kpXmlXDTvIp7b\n+xyHerUlnqlE8knSMScl0Qw7tbSG1QEjexrYO7djFhKhQ4/neKR9DzTQHdbItQpnKSX3bb6P2vxa\nzp5+tu7jiUVOMOhMsYHhnp0+jQX0xlK1DELD0JrcNmmk49DoGj3qarkxb66hZgyt9+DKxVdiMVlS\n1hpK0rXRVy2Fvkbob417iOo/MtL5DMZG9RT3qD0YjNMYSgw0C6daDmPtobV80PkB1yy5BovJuOCZ\naHKCQWeMnBBd6UaTqLHeGvwMRiYnRXIAjHopNW4CBF2e+YYJ5y6f9ntQmlfKhXMv5Nk9z1LfV6/5\nM9Iu+6wh070rnYzbNEg5FyMFKn0f0GMqUPxnBjGuf7iOpLJAklJy7/v3UuOu4WMzP6b7WOKREww6\nU+KyGdbWMO3Y54KpSr16DREr6oQwZKVk9Eup6X0onYvLk2+gcB5K6R5cufhKTMLEb7f8VvM5aYc9\nq81qEiwA1GfTqAJ6Kkbmw0z376Ihb57iPzOIErdx8yAV4fxa/WvUddZx7VHXZk1bgJxg0B2jVkr+\n4SD9/uH07PMptAY0VmNIHtGTEU2boGqpoZFVHV5/SoKt3FnOJ+d+kqd3P01Df8zczXGoK8mUtR5H\nARTPSrgAiJgxDGjSE41h7TGHBqiVh2j3LND3umMwsjeJ1tLzqrYwxT2Fc2edq/s4EpETDDqjxj/r\nPSFUh1XaE7pqKbRsT1qZ0Wgfg5aInrToa4G+JqheZlhk1UAgyOBQKGXBduXiKxFCaNYa3HYLNrMp\nvSJuSUqgdHoD2MwmXAYVMVRR22OmFFmlgaHGLVgI0VdkTGa7is1iwuOwGLLI07pA+nfDv9nWsY1r\nllyD1WSQXy4OOcGgM8UuG4Fh/dtjqiaAtCN6qpdpqszosJpx2syGhNwaWg5DXSVXLYtEVuldsyrl\nrlthKl2VfGLOJ3hq91M09TclPV6pWWVNr+xz1TLorQdve8zd6j3Qq6JwPIxacQ8eeBcAf7lxOQwq\nJQaVD+/QsECSUnLf+/cxxT2F82edH/c4o8gJBp2JOA51frFm3OAmkgGtzc9ghMbQ5TMw4zbseKbq\nKMMybzMxhV295GoAzRFKStOnNDUGiKs1GF2rSqXYIBt9sHETndKNo6RW1+vGQgkk0b8KgJaQ59cb\nXmdL+xauXnI1VgPa0yYjJxh0ZiTMTd8HKuOWmEXTFRu0hsikErcxDnTDNYbSOWD3GNaasb0/HOqZ\nhjmv0lXJJ+d8kid3P6lJa0g77DkSmfRezN0d3kBa40+VYoM0BkvLZraGZlBksPMclLabRs2DRIsL\nVVuoclWxetbY1jbZIScYdMaoekkZhxmm0BrQqIgSQ2v0NL4X0YqM0hjU66Ub0aNqDVp8DWkXcXMU\nQPHMhBqDkXWSVAwpDzM0iLN7J1vljKxoPYbNgyQhz280vsHm9s0Tpi1ATjDojvrS0HuloV5PrUOT\nFlVLoWUbBBObKIpddmPit30GrVajHM9gXGRVphE9qtbwxO4naOxvTHhsRkXcEkSgZc2UZMQ9aN2O\nSQ6zJZQdwVDs9/XCTwAAIABJREFUNiayKlHIs5SSezfdS6Wrko/P/riun5sKOcGgMyP9YvXXGAry\nrFgyieipWgZBP7R9kPCwEreN9n6/rhMi3YgeTUQ5niEqOUl3U1IAq1ngsacfT65Va8ioiFv1MqX/\ns7dj1GY15NnorGcAp82M3WLSdx6E7/MWOYNCp/EraTWyqndQ38iqRCHPqrZwzZJrJkxbgJxg0J1I\nqKHOLyVdVnpqbZnG2PZnlWKXDf9wCJ+OkVWqzyXViB5NRDmeAfJsZvKsZgM0Bj8lLntGET3RvoZE\nWkNGRdwigQaj73Mk8zwLPgYhhP5VABo34TN76LNXG9aWNBojsp8TLZCklPx606+pdlVPqLYAOcGg\nO5EJYUBUUsaCoWgG2JM7oNOu1ZOALiMb0DdtgpLZYPdENhUbEGqolxlGS4RSRkXcVAf0mPucrTpJ\nKroLhqb3OWifE+mUaDSqANUzMilRyLMaiXTNUROrLUBOMBiCEfWSOvp1cBqaTFC9NKnGYEQ5AEPL\nYTRuGtf7t8hl1V1jaO/Xx0eiag1P7XoqbjZ0RsI5r1CJQhsTmqyll4Se6No3edgPLdvYZZpFURbM\nSBAVYajjIm+kb/to4aZqC1PcUyYsEimajASDEKJYCPGSEGJX+GdRjGOWCSHeEkJsE0JsFkJ8Kmrf\n74UQ+4QQm8L/jCuXmEWMWK0qGoMOE6J6udLmczj++IojDnT9VkpptSXVQl+LUlF0TKVNIxzoagN3\nPbh6ydUIIXhgc2ytIWMzRtUyaBztgI5EVWXBlAQ6L5BatkFoiPdDs8a9VI3CiOi2kbDz0XP53w3/\nZmvH1gn3LahkqjF8F3hFSjkHeCX891h8wOeklIuAs4H/FUIURu3/lpRyWfiftjZjkxy9S29LKVMu\n3haX6uUQDCQswW1EyWH1Wrr3YlBXxWNq85caZkrS56VU6arkorkX8dTup2L2a8i4NEn1Mug5CL7O\nyKbOOKtVoyjSsydD40YANgzVGuOnikEkwtAAwRCttUkpuWfTPUqW8+zsZznHIlPBsBp4OPz7w8AF\nYw+QUu6UUu4K/94ItAJlGX7upEZvH4M3ECQQDOmzWtXggC5x679S6uj3YzEJCjIJt43FGMezit5J\neoND4YgeHVfbVy25CovJwm82/2bcvkghvUw0Bhh1nzu9AUwC/e9BHEpcNvoGhwkM69Aes/E9ZF4x\n23wFWRNsahCDMRrDyHO05tAatnds5wtLv5D1mkjxyFQwVEgp1TTOZqAi0cFCiGMAG7AnavOtYRPT\nnUKI7Nxxgylx2ejzD+Mf1ieqRxUyutiGC2uVEtwJBIPTZsFh1TfUsKNfcdyaTDrX6InheAYocdsZ\nGAriC+gTajiS3KafYCh3lnPR3Iv4+96/c6D3wKh9ajfAtMpiQFRpjJH73OENUOi0Ydb7HsRB1XC7\n9dCeG94jWLWcoaBBkW1x0Ntf2OVThHO+Q/kOIRni15t+zTTPNM6dmd0KqolIKhiEEC8LIbbG+DfK\nQyKVoPe4ge9CiCrgj8AVUkp1CfE9YD5wNFAMfCfB+dcKITYIITa0tbUl/2YTyEhEiT5F3HR13Aqh\naA3JHNAue6QEhB50eP2UGBFNEsPxDPo7DjMuSRKHq5ZchdVk5b737xu3L+1CeqAI/+KZozWG/uwk\nt6noZpIM+KCtDm/JEiB7znNQNE+9NYYi58gC6eUDL7OjawdfWPqFrPZbSEZSwSClPENKuTjGv6eB\nlvALX33xx+wpKITIB54Dvi+lfDvq2k1SwQ/8DjgmwTjul1KuklKuKiub3JYovcM9dXfcVi9XqqwO\nDcQ9RO8J0d4foFRvp2dvk+J4rl4xbldpWAjpJdwyqZOUiNK8Ui6Zfwn/2PcP9nbvHbWv2GmjM5O+\nEtUroGFj5M9sZT2r6NY3uXkLyBDtBUqp7Ww5z8EYjUG9B8FQkF9v+jUzCmbw0Rkf1e0z9CBTU9Iz\nwOXh3y8Hnh57gBDCBjwJ/EFK+diYfapQESj+ia0ZjmdSoHc0g+5mjOrlSg/olvgOaL0nRIfXH3lZ\n60bYIcmU8YIhEnKrs8ZgROezKxZfgcPs4J5N94zanna9JJUpKxXB2dcMhEuSHI4aQ/g+N7nmA9nV\nGPSeB+39I10YX9j/Ant69vClpV/CbDK2P0aqZCoYbgM+IoTYBZwR/hshxCohhJrzfzFwCvD5GGGp\nfxZCbAG2AKXALRmOZ1IwUkBMn9Vqp+4aQ/hFmsCcVKxzi9KOfgNeSg0bQZihcnxtftVspfc9MCJr\nuNhRzGULL+PFAy/yQedIuZKMX0qqwAxrDROmMWTqY2h8DzxVNAWVaHij25JGo/Rk0M+k2t7vp8xt\nZzg0zL3v38vswtmcOf1M3a6vFxkJBillh5TydCnlnLDJqTO8fYOU8urw73+SUlqjQlIjYalSytOk\nlEvCpqnLpJT9mX+liUd3jcEXwGLKrEbPKPKrwVU+yswwFj0nhC8wjC8Q1N/H0LgRyheCzTlulyqE\n2nUSbnrUSUrE5Ysux2PzcM97I1pDcabhnpVHKYKzcSPBkNQnez4FCvOsCKGD1tawEaqXR/Ukyabz\n2c7gUEi3IIaOsEn12T3Psr93P9cvvx6TmHx5xpNvRP8BFDptCKFfZcnOsPqpW9ctDQ7oErd+EyLS\ngF7P1baUygtjyvKYux1WJapHP1NS5nWSEpFvy+fziz7P2vq1bG7bDIxEVnnTbY9pc0L5AmjYSLcv\ngJQGZZ7HwWI2UZBnzUxjGOyFjl1QvZxO7xBWs8BtkHCOhZ5BDIHhED0DQxS6TNz7/r0sLlnMaVNP\ny/i6RpATDAZgNgmKnPolWLX3G2Cfr14O7TvAH1tJK9ZxQqiOW12dz517YbBbsaPHocStn9aTDTPM\nZxZ8hiJ7Eb9671fAyP9XRg706uXQuJGuSBHD7AkG9fMymgdq+fDqFXSGq5Ia3ZY0miIdtX/1WTw0\nvIYmbxNfWf6VrH6XVMgJBoMoclp1MyUpgkHnCV29HGRIifiIgZ7ZzxGNQU/bsKrtxIhIUinR0U+i\nV52kRLisLq5achVvN73NuqZ1lHp0iKyasgIGuuhvVlKHsmmfB8UclpHmrAYYhDWGbDqeQV+zcEd/\nAESAtzsfYWXFSo6vPj7jaxpFTjAYRInLrqPGEKDMCI0BRibeGEYmROYr7khVTz1frA3vgiVPMZXE\nocStXy6GnnWSEvGpeZ+i3FnOXRvvojT8eW19GTxHYcEpw/6kidAYMnqpNr4HhdPAVaKY87IYqgr6\nLpDa+v3Yit6ib6iTG5bfMGm1BcgJBsMozjTUMIyUkrZ+P2UenQWDpwLya6B+Q8zdqulKH1OSARpD\nw0alDEaCgmOlbv3MeXrWSUqEw+Lgi0u/yOb2zezqfwfIUGOoWARmO/ZWpabUYScYwo5ngC7fBGgM\nOpberu/uwFa6lpVlx7OiIr6mOxnICQaDKNYpQazPr9Sa0d3HAFCzChpiCwY9e/Z29Adw2czk2XSK\n1Q4OK7bnBGYkUARRpzdAKJRZJzoj6iQl4oLZF1CbX8sfd9wHhDITDGYrVC6hoFNJEcpmRA+MFJRM\nqxugtx26D0Tuc3u/P6t5GAAeuwWrWegyD15pegRhHuCrK2/UYWTGkhMMBlHsVCZEpi+l9r6w49Zj\nwISoWQXdB6F/fMK6nq0ZO7z+iL1cF9rqYHggoeMZFNNVMCTpSacLWhRG1ElKhMVk4fpl17O7ezf5\nZVszN4dNWUFZfx0FdhN2S3YTqYrD7TH70omsanhX+VlzNINDQfoGh/XXnJOgV+OtNl8bm3qeJdS3\njGUVi3QanXHkBINBFLtshGSarRmjaFMFgxEaw5RVys8Y5iQhBKVuuy6mJN2T2xriZzxHo1eSm1F1\nkhJx5vQzmV88H1PRC7T2+jK7WPUKbKEBljljVqwxlIyi2+rXK3kY1csiwjHbggH06e3xm82/IcQw\nBf7JUygvETnBYBB6la5W7fOGCIaqpcrES2BO0sO22t6vcwG9xo3gKFCKxCVAF+ctxtVJSoRJmLhx\nxY0EzR3s8b+U2cXCAnSlZW+SA/Uno5pV9euhYiHYXJEF0kQIhpIMQ24P9R7i8Z2PUxw6mXJHjY4j\nM46cYDAIvcLcRnIADJgQNqfinIzjgNarE53uBfQaNip25yRRHXprDNkO9Tyx+kQKxULaLM/RH8ig\nKEDJHPqEi6Pkbv0Gp5HyfOX/TH2xayYUUu5zzdGjzi9zO3QdnxYydaD/atOvsJgsWHvPNGYeG0BO\nMBjESGvGzF5K7f1+TMJAM0bN0UpIYGh8MxU98gBCIRnJGtaFgA9atyc1I4F+hfSMrJOUCCEEK9yf\nQZr6+f2236d/IZOJrcxh7lCdbmPTihpmnbJgaN8J/t6IubOt30BfWxIyiTDc1rGN5/c9z2cXfpau\nvjzKJmD86ZATDAYReSnpoDEUuwxsrlKzSpmA7TvH7dKj9Hb3wBAhqaMZpvE9pTLs1GOTHloULk2S\nae9qo+skJWJu0UKGeo7i4W0P0+ZLrw+JlJINw7Oo9O8Df5/OI0xMUbgxUGvfYGonqubNsMbQ3jcx\nWpvymek13pJS8osNv6DIXsTlCz+v7wLJYHKCwSD0KinR1hcwVv1UHdAx/AzFrsy7oHVE7PM6fYdD\nSmy/+sJIhNkkKHbaaM9QuBldJykRpW4b/rYzGQoNxWzmo4V+/zDrg7MxERqJ9MkSJpOg1G1LXWOo\nX6/4kUpmA9DWP0iR04rNkv1Xlqopptp46/WG11nXvI7rll7H8LCdkNS5LIyB5ASDQdgtZgryrKmv\nlMZgSJ2kaEpmg71AmYhjd+kg3CLOc71MYfXrlTE7izUdrvR+ztzHkO3EMJVStx05VMqHq1fz+K7H\n2duTugO5vT/AppDygo11n42mzGNPQzBsUMKRTcorqq3PgCRPjYxkP2v/DsFQkF+8+wumeqZy8dyL\nR+bBBH2HVMkJBgMp99hp7c3cx2DoKsNkUuz19eNXkno40EfKYegwIaSEQ+ugJm6jv3GUuDIPuc1G\nnaR4qP9vp5RdisPi4M4Nd6Z8jdbeQXpx4cufBYeyLxjKPY6Ij0AT/n7FjxSlFU6kYFAz3lOZB8/s\neYbd3bu5ccWNWM3WEc05Z0rKUZHvoDXVlVIUUkrjNQZQJmDrNgh4R23WI+S2IxJuq8OLtWsf+Nph\nagqCQYeyGNmqkxQL9f/N73dy9ZKrWVu/lnVN61K6Rkv4GQxUr1I0hnSykDOgzJ2ixtD4nlLgMVow\nZGMexEFdIGkNufUN+bh7090sKV3CmbVKE562SB7GEWBKEkIUCyFeEkLsCv8sinNcMKp72zNR22cI\nId4RQuwWQvwt3Ab0P4bydFToKLyBIINDIePVz5pVykRs3DRqs7q6ySTztiMcVVWoR42bQ+EXYgqC\noVSHQnod/f6s1EmKRXQewGULLqPKVcXPN/yckBwfRRaP1l7FnGmrPRYGOqFjjyFjjUeZx057f4Cg\n1ioAqr8rnNkupaS9z4BCkhqpCIfcatX+H972MK2+Vr656psRv5Sh+UgGkKnG8F3gFSnlHOCV8N+x\nGIjq3nZ+1PbbgTullLOBLuCqDMczqSjLVwRDWnViiCqHYfTDFMmAHm1mKNZBY2gP2+d1iao6tA5s\nHiibr/mUEpeNvsHUI0pUBoeCeAPBCTMlOaxmPHYL7f0BHBYHN6y4gbrOOv6+9++ar9HW58dmMZE3\nM1zmOct+hjKPPdJBThP1G6B4VsSP5A0EGRgKTpgpyW234LSZNWn/Ld4Wfrftd5xZe+aoQnkd/X4s\nJkG+I7u1qtIlU8GwGng4/PvDwAVaTxSKKD0NeCyd8w8Hyj0OAsEQ3b70ymIY0uAmFq4SZSKqET/q\nZpsZW4b1kjr6dQzRq18HNSshhcbpqo0+3e+Q7TpJsSj12COmiI/O+CiLShZx18a7GBge0HR+a5/S\nZ1iUzQd7vvL/mEXUF7om7VlKRXCN8S9EXyfbCCEo99hp6U0eSPLL937JcGiYr6786qjtSva/DZNR\nYec6k6lgqJBSNoV/bwYq4hznEEJsEEK8LYRQX/4lQLeUUo2FrAemZDieSUV5+EFO189gaNbzWGqP\nh4Nvj0p0E0JQ6rJl1DdZN8etvx9atmnKX4gm0yQ39byJikoCZWGgao8mYeJbR3+LVl8rv9v6O03n\nt/QOKuYQk0kxz2TZAV2eimDoPgD9LYp5M8xECwaA8nxHUlPSto5tPLPnGS5beBlTPVNH7VOy/w8P\nMxJoEAxCiJeFEFtj/FsdfZxU7CXxbCa1UspVwKeB/xVCzEp1oEKIa8PCZUNbW3qJPtlmRDCkF7La\nFn4plWdjQkw7Pmx/3jVqc5nHnlHIbYdedZIa3g07JLX7FyDz9piGNBlKkbF+kpUVKzl7+tk8tPUh\nmvqbEpyp0Nrnp9wTLiUx9Rgl0CCLiW5lqSyQDryl/Jw20t3M0EKSGqnId9CSYB5IKfnZup9R7Cjm\n2iXXjtuv2zzIEkkFg5TyDCnl4hj/ngZahBBVAOGfMcs3Sikbwj/3AmuB5UAHUCiEUNNJa4CGBOO4\nX0q5Skq5qqysLIWvOHGU5yuTMd2Q1fY+P8LIchjRqBPx4FujNlfkOzSp0PHQrbKqav6oSVxqeyyq\nGStdjUH97pEX6wSgCIbR4//6yq8D8It3f5H0/NbewUjNImqOUQRsQ+zOfUZQmkpZjINvKolt5Qsj\nm9rCL+SJ1BgqwqHn8fyFL+x/gY2tG7l++fW4be5x+3WvF2YwmZqSngEuD/9+OfD02AOEEEVCCHv4\n91LgRGB7WMNYA1yY6PzDGXWln2ilkYj2fj9FThsWcxaiiotngqtsZMUWprLAQXNPeuMfHArS5x/W\nZ0IcWg+l8yAvZuBbXEZKk6QnnJt7lPMq8idWMPQMDBEYHjHzVbmruHLxlfxz/z95tyV+NvPgUJDe\nweERrVMVrIey52dw2S24bGaNguFtmHpcJLENlJeq2SSy3r0tmop8BwPh53ksviEfP9/wcxYUL+AT\nsz8xbn/Wws51JNM3zm3AR4QQu4Azwn8jhFglhPht+JgFwAYhxPsoguA2KeX28L7vAF8XQuxG8Tk8\nmOF4JhUuuwW33ZK+xmB0cls0QihaQwyNoXdwmIFA6lE9EcdtphNCdUimEKaq4rZbsFlMaWsMzb2D\nlLptE1KKQUUtHDdWuF2x+AoqnBXcvu52gqHY90d99lTtlbwiZTV+8E3jBhyDsigHely87UrNrtrj\nR21u61M6txlWL0wD5ZGQ1fGLpAe3PkiLr4XvHfs9zDECI/r9w/iHQ0eOxiCl7JBSni6lnBM2OXWG\nt2+QUl4d/v1NKeUSKeXS8M8Ho87fK6U8Rko5W0p5kZRSn87tk4hMchna+rK8yph2vOL8622MbKoM\nv1Ca0zAndegVu922Q/F/TDsu5VMzdaArjtuJ0xYgKpdhTF+JPEse31j1Deo663h81+Mxz1X9Q6P8\nVLUnwsF3IJhZE6lUUMpiJHmG1EXJtBNGbTak53mKqKbEljGLvEN9h/j91t/zsZkfY3n58pjnHm45\nDJDLfDacTJy3WY9kUF+8B9+ObKosCAuGNMxJ7Xo5bvf/W/lZe2Jap5e47Wmbkpp6BiPCcaJI1Ozm\n7Olnc3Tl0dy18S46BzvH7VcdvqN8JNNPhCGv0jc7S5R7NFQBOPAWWBxQPfoFO5HlMFTUJLex/rY7\n1t+B2WTmayu+Fvdc3QtJZoGcYDCY8gzKYmTdLll5FFhdo8xJ6mo5HeEW0RgyzWM48Abk10DR9LRO\nL3Hb0o5KaukdpKJgYgVDpKdBjO8ghOD7x34f35CPuzbeNW6/+iJTX2zAyIr8wBv6DzYOmgrpHXxT\nSba0jF5IZF1zjkEkkCTqO7xW/xprDq3h2qOupcIVL1I/i/lIOpITDAZTniSaIR6+wDC+QDC7jUnM\nFiV+PEowZKQxhCdERg1upIT9ryur3DTLXqdbzNA/HKTTG5h4jcGTOOR2VuEsPrvwszyx6wnebxut\nBbT2KRm3oxy3ngoomQP7sysY+gaHGRyK46vy90PT5nH+hVBI0uGdeI3BHfYXqoJ2YHiAn7zzE2YW\nzOTyhZcnPFcNO5+okh7pkBMMBlPuUXoa9MeIZkiEak/O+kqp9gQlkWywBxiZEOn4GJp7BvGEz0+b\n9l3gbYPpJ6V9iaqCPNr6/aOierSgCpOJFgxOm1KSIZED/bql11HuLOfWt28d5Yhu7VVequMybqef\nGE5oTK9USKok7eRWvw5kcFT+AkDPwBBDQTkpXqrRC4wHNj9AQ38D/3Xcf2E1Jy5zoZqSiiYwSTJV\ncoLBYCLRDCmakyLVGLM9IaYdp8S5R9XTqcjXVg5gLI3dA1QVZvhSzdC/AFBd6EDK8fbhZKjCcKJN\nSZC8GKDL6uJbR3+Lus46/vLBXyLbW/sGYydI1p4I/h5o2WrEcMeRNMntwFsgTOMiz0aqkk4CwRCe\nB3t79vK7bb/jvJnncXRl8oZRSti5FWs2ws514vAZ6WGK6vRL1ZSR1XIY0UxZBcI8Kp8h3VyGxp4B\nqgvzMhvPgTfAU63kWaSJOobGbm21hVTU71w1KQRDcj/JWbVnceKUE/nle7+MZES39vpHQlWjUQVt\nlsxJSeslHXxL8XHZPaM2T4ZyGCpq9vNP3v5JJCJMC+1Gd2E0gJxgMJh0y2JEBEO267fb3Urjnn2v\nRTZVeBzjwvS00NQ9SFVBBoJBB/8CEBlDU4rCbcRxOxkEg31cuOpYhBD817H/BcCt79yKlDK+xlAw\nRXHmZ8kBHamXFEu4DQcUDbX2hHG7JkM5DJWKfAft8g3eaX6Hr674KiV5JZrOUwvoHU7kBIPBqKu1\nVHMZJrL5OTNPVWoThf0MFQVKWYyQ1nr6KBm3Hd4AUzIxJXXsUQqqZeBfAMWUBIoGkwpNPYPkWc3k\nOzLwkehEqUdbX4kaTw1fXvZlXq1/lef3vUiXbyh+OY/aE+HAm6MKJxpFscuGEHHmQcMGGB4c51+A\nyaUxuPIGMJc9y1Gly7hw7oXJTwjT1JPhAmkCyAkGg8l3WLBbTCnbt9v7/RTkTUzzc2Z+WHEE7n8d\nUJyvwyGZUic0dXWekSkp4l/ITDA4bRYK8qypm5J6B6kscESarUwkpW47nb4Aw8HkL/HPLPgMC4oX\ncNu6n4JpYHSoajS1JyqJg20f6Dza8VjMJkpcttiCYc8axb8w45Rxu9r7lV4Sk0E4v9P7IIgAV83/\nNiahbV4OB0M09w4yJVOTapbJCQaDEUJQnm9P2fnc2jc4caukmqPB6oS9a4ERU0oqwk19CWe0Utr/\nOrgroSTlYrzjqC7Mo6k7RVNSz2D8l2qWqSpQHOhaosMsJgs/POGHdPu7sFc8N1JAbyzTw36GLJmT\nSt1xsp/3rlHKgecVjtvVpvaSmGDhvObgGrZ0v0ag/XSsoUrN5zX3DhIMSWqKcoIhxxjKPclruY+l\nvmtg4h4mi01ZTYYFQzq5DA1hwZD2SklK5YWVoX9BpbrAQWOKPobm3onPelaZWuQElOdCC4tKFnFq\n5cXYCjfQ6H8v9kGFtVAwNXKfjaY83zFeYxjoUsyWMz8c85zJUA6jL9DHLW/fwnTPLAIdH0ppLqv3\na0pOMOQYS3kaZTEOdfomdpUx81SloFlPQ1r1ktTVeUVBmpO6fRf0NWXsX1CpKnTQlIKPQUpJa6+f\nykliG1afhUOdPs3nLHdfTNBfzoMf3EFvoHf8AULA7NNh76tZqZtU5o6R/bzv30p49KzTYp4zGcph\n3LbuNjoGO/jh8T8CzClVS27oynCBNEHkBEMWUASD9lVGz8AQvYPDkVXihDDzVOXnvlcpddswidRN\nSWUeO3aL9jaco9j9kvJz1unpnT+G6sI8un1D+ALaEg07vQECwRCVk8SUVF2YhxDaNQaATm+IQNNF\ndPrb+fn6n8c+aPYZEOjLShlutcLqqCoAe9cofbyjOrZFM9HlMP518F88s+cZrlpyFauqluJxpFYt\nWdWcMw7bzjI5wZAFyvMdicsBjKG+S1kVTi2eQMFQsUjpz7B3LRaziTKPPSVTUmPPANWZxP/veknp\nv1BUm/41oqguUHMZtH0H1XleOQlyGABsFhOV+Q4OdWnXGFp6Bym2zOaKRVfw5O4nea3+tfEHzTgF\nTBbY/bKOo41NmcfOUFCO7oG+51+KVhgje9g/HKTTF5gwP0/nYCc/eutHzC+ezxeO+gKA5t7PKvVd\nPso8dhzWNBdIE0ROMGSBSNanxpXGoU5llTGhGoMQitawdy1ISWW+IyVTUmN3BsltAa/iX5jzkfTO\nj4GapKbVnDSZchhUaoryUtIYWvv8lOfb+dKyLzG7cDY/eOMHdAx0jD7IUaD00c6SYICoXIbOfdC1\nP64Z6VDnAFLCtAlYIEkpueXtW+gL9HHrSbdGyl5UpFgUs6F7An2FGZATDFkg1QqlqsYw4Q/UzFOV\nPILWupRafEopaeoZTF8w7HsNggHFzKETqWY/q0JwsmgMoCwUGlIRDL1+KjwObGYbt59yO32BPn74\n5g/HF3ScfTo0b4a+Fp1HPBpVOKvmFfauUX7Oiu14PtjpBaC2xGXouGLx7N5neenAS3xp2ZeYWzQ3\nsj3VVrcNXQOHnX8BMhQMQohiIcRLQohd4Z/j+i4KIT4shNgU9W9QCHFBeN/vhRD7ovYty2Q8k5Xy\nZHVixlDfNYDbbqHQmbg4l+HM+JDyc+/alMpi9AwM4QsE0y8lseslpfx3jEzYdFHyEbSbklp6BjGJ\nyVURs6Yoj6aeAYY05DLAiMYAMLdoLl9b+TVerX+VR3c+OvpAVQDv+Zeewx1HbYmy8j/Q7h35vPwa\nKJkd8/gDHb5R52WL/T37ueXtW1hZsZIrFl0xal95vvZqyaGQpLF78LCLSILMNYbvAq9IKecAr4T/\nHoWUco2UcpmUchlwGuADXow65FvqfinlpgzHMymJCAaNKw01ImmiY7cpnKqUZ979UkotPtWXb1or\nJSkVwTCxC7cYAAAbIUlEQVTzQ2DR76VsNZsoc9s1m5KUlp727PTb1khNsZOQRFM+xnAwFC5XPSKc\nP73g05xQfQJ3rL+Dvd17Rw6uWAKucsPNSWVuOy6bmf0dPqWq677XFG0hznN+oMOHy2amJItVSQPB\nAN9+7dvYzDZuO/m2ca06KzwOAsHQaD9JHNr6/QSCIWom0iScJpk+9auBh8O/PwxckOT4C4HnpZTa\nPWj/ARQ5bVjNgmatPoYu38Q6nqOZ/1HY9xpT85SsZy1+hkhyWzqCoX0n9BzU1YykUlWYp7leUnOv\nf1IUz4smErKqwQHd3h9AytEtPU3CxC0n3qIUgHv1GwwMh4WkyaSYk/b8y9Ay3EIIaktc7O/wQv0G\npeRKHDMSwIEOL9NKXFldIN218S7qOuu4+YSbqXSNT2RTNTAtIasRk/CRZkoCKqSUTeHfm4H4bYwU\nLgH+MmbbrUKIzUKIO4UQcZeIQohrhRAbhBAb2traMhhy9jGZBFOLnexXVegESCk51DkwsY7naOaf\nB6Fh5vUp7T612FfVmkTV6dRJ2hUOU9XR8awypdAxYt9OgpL1PLkEw0iSW3LB0NCtHDP2HpQ5y7jt\n5NvY072HW9++dWTH7DOU8hiNxirtM0pdyjyoewZM1oQLgAOdPmqzuEB69dCr/GH7H7hk3iWcNi22\nQzziL9SwyFMDBSbcV5gGSQWDEOJlIcTWGP9WRx8nFaNbXMObEKIKWAK8ELX5e8B84GigGPhOvPOl\nlPdLKVdJKVeVlZUlG/akY1aZmz1t/UmP6/QGGBgKTp6HacpKcFcypVkxM2gSDN2DWM0ivZaeu16E\nsvlQOC31c5NQVaCUxdBiH27qGZhUjmdQnLdmk9AUmbSnVVmEzCpzj9t3wpQTuPaoa3l6z9M8uetJ\nZePMDwNiJH/EIKaXOqnv8iHrnlG0BUdBzOOCIUl950DW/AsHew/yvX9/jwXFCxKW067waE/2jGT/\nT5a5nAJJBYOU8gwp5eIY/54GWsIvfPXF35rgUhcDT0opI8Y5KWWTVPADvwOOiXv2Yc6sMjf7O7xJ\ni6AdCk/6SWNKMplg/kdxHVyLnYAmB3Rj9wBVBXnju4Ylw9+v1OU3wIwEyot1YChIz0Bi+/BAIEjv\n4PCk0xgs5nAug4bs5z1t/djMprj27S8u/SLHVh7Lre/cyo7OHeAqUZrk1P1d72GPorbExTy5H9F9\nEBacF/e45t5BAsFQViKSBoYH+NraryGE4Ben/gKHJf59ryxwYDEJTdp/fdcARU4rTtvEFwBMlUxN\nSc8AasPTy4GnExx7KWPMSFFCRaD4J7LTTmoCmFnmYigok6721Ek/tXgSrTLmn4sY8nKGfbumlVJT\nz0CaZqQXlDDVuWenMcjkqM7wZOakSKjqJBMMoDwXmjSGtn5mlLowxxHOZpOZ2065jXxbPjeuuZGu\nwS5Y9HFo2QJtO/UedoQZpS7ONq9DCjPM+1jc4w50qKGqxi6QpJTc/NbN7OraxW0n30aNpybh8TaL\niemlLna1Jtf+G7oGDkvHM2QuGG4DPiKE2AWcEf4bIcQqIcRv1YOEENOBqcCrY87/sxBiC7AFKAVu\nyXA8kxZVpU9mTjoUyWGYRA/U9JPBXsC51nc1m5Kq06kxtOUx8FTpGqYajeoMTxbV0zzJsp6jqSly\nanI+727tZ3b5eDNSNKV5pfzvh/+XNl8b33j1GwzNPxcQsO0JnUY7ntoSJ+eY1tFUtFLRUuJwMByq\nanRy25/r/szf9/6dLy77IifXnKzpnDnlbnZrEQzdh2cOA2QoGKSUHVLK06WUc8Imp87w9g1Syquj\njtsvpZwipQyNOf80KeWSsGnqMill8v/tw5RZZYpKnFQwdCrqp9s+idRPiw3mnsVJofXsb41RjC2K\nYEjS3JtGcttAl+J4XvQJMBlTPqBaY/azGlU1GQXD1CInLb1+/MPxo4f8w0EOdvoiz1wijio7iptO\nuIn1zeu5/YOHFaG89QklbNgAygb2MdvUyHuuxC/hA50+rGZhaI2h1+pf444Nd3Da1NO47qjrNJ83\np8LDgQ5vwhI3Ukrquya4EGYGTJ4g7f9wCp02Slw29rYltk3WT6ZQ1WgWnIsn1EtRx7sEhuP7SVr7\nlPrzVamakrY/A6EhWKK9M1aqlLrtWM0iafntHS192CymrEbEaEV90SRK1DvQ4SMkYVYSjUHlvFnn\nccXiK/jbjr/xl+pZ0L4DWrfrMt6xiLAP46VQ7KJ5Kgc7fNQUOeOawjJlR+cOvvXqt5hXNI+fnvxT\nzY13QNEYQhL2JfAzdHoDDA6FDkvHM+QEQ1bREplU3zWJQlWjmXU6QZONM1iX8Ds0pltNcutjUDwT\nqpdnMsqEmEyCygJH0rIYdU29zKvwTKrkNhUt5bdVM0esiKR43Lj8Rk6tOZWfNr/Kyy4nbH08s4HG\no+5p9jgW8X5P4md8f4fXMDNSm6+NL7/yZdw2N3effjdOa2qfM6dC+X9N5GcYCVWdhHNZA5Pvyf8P\nZla5iz0JNIZQSCoOq8nkeFaxuxmoPZ3zzG/xQUN73MPUlWxKPobeJqUu/5KLdGnKkwg1ZDUeUkq2\nN/ayoMpj6DjSRdUmEzmg94RfWDM1mJJUzCYzP/vQz1hStoTvlJexoe5x/c1JnfugeQsHKs7gUKcv\nboSelJKDHT5DHM89/h6ue/k6egO93H3a3ZQ7y1O+xoxSFyYBu1r64h6TcaOqCSYnGLLIzFI3nd4A\nXXF6J7f0KSF6k1JjAPKOu5JS0YtMENI4ojGkYEra9iQgYbFxZiSV6oLESW6tfX46vAEWVuUbPpZ0\nqMh3YDWLhA7oPW39TCnMSzlMMs+Sxz2n3UO1vYgbnMPs3PlspsMdzXt/AmHCN/tchkMy7n3o8g3R\n5x/WPVTVN+TjS698if09+7nrw3exoGRBWtexW8xML3GxqyW+xtBwmHZuU8kJhiwyq1x50Pe2x36g\nJnumpHnO6bSYyplXH9/MsK/dS6HTiseRQgHALY9C5RIom5v82AyZU+GhoXuAbl9s4by9SXGuL5ik\ngsFsUhyyCTWGNm9K2kI0hY5CfnP6veRJybXrbh5dUykThgOw8WGYezblNUrRvHg2+kioqo6mpEAw\nwFfXfJWt7Vu545Q7OL76+IyuN7vcza7W+BpDfZcPj8NCQd4EF8JMk5xgyCKRkNXW2BNiJIdhcmoM\nmMxsLD2fRf5N0LEn5iEbD3axfOr4pu5x6dgDjRsVM1IWWD5NGdt7B7tj7t/eGBYM1ZNTMIDalyG2\nxhAKSfa0JQ9VTUR12UIeyJsPQwNc+cIV+giHumfA2wZHX8X00nCV1Y7Y3+Fgp75VVf1BP19f+3Xe\nanqLm0+4mdNrM+8KOLfCw/4OX9xAjMM5VBVygiGr1BQ5sZlN7ImjMagNeibzA9U972KGpQnvWw+O\n29fjG2JnSz8ra8dVX4/P2/cqNXOWXKzjKOOztKYQs0mw8WBXzP11Tb3UFOWRn4rGk2VqCp2RZ2Us\nzb2D+ALBlBzPsZh5/Nd4qKkZhge58oUrMxcO6x+Eohkw87RIldX4GoN+CyTfkI/rX7meV+tf5b+P\n+29Wz16d/CQNzKlwEwxJpSBgDOoP4+Q2yAmGrGI2CaaXOuNrDF0+KvIndxvA2umzeDm0EuuW/4Ph\n0YXE1JftytpibRfztit256M+BflVeg81Ji67hfmVnriCYXtT76T1L6hMLc6jvd8fswS6GjGWqWBg\n+knMLF3EQz1Kj+zP//PzbGnbkt61WrbBwTdh1ZVgMkWqrB6I81I90OGjMt+R8TzoD/TzxZe/yLrm\nddx60q1cPE+/xYeqke2M4YDuHRxiV2v/pA1g0EJOMGSZWWVu9sYJ99zV2j8hbQxTYWFVPv8XPA2b\nvws+GO2E3nCgE7NJsHRq7MJo41h3PwwPwIk3GDDS+KyYVsSmg90EQ6OjbnyBYfa1e1k4ic1IoPhJ\nAN6vH28OUyOSMjElAUp02PHXM7N1N7+ffw1Oq5OrXryKVw+NLV6ggQ0PgdkOyy+LbJpe6lT6MsTg\nYKeXaRmakZq9zXz+n59nc9tmfnbKzzh/1vkZXW8ss8rcCEFMB/TbezoIhiQnzS7V9TOzSU4wZJlZ\nZW4OdI63Tbb3+9lc382Jk/xhKnTa2O0+mg5rJbxz/6iQxncPdLGoOl9bNEzAqwiGeR+FsnkGjng8\nK2oL8QaC7Ggevdrb0dyHlJPX8axy4uxSrGbBmg/G16zc3dZPvsNCqVuH5jYLLwBPNdM3/Y0/ffRP\nzCiYwQ1rbuCRHY9ov4a/D97/Kyz+JDhHNMnpJa64Iav7O3xMz0AwbG3fyqXPXUpDfwO/Ov1XnDX9\nrLSvFQ+H1cy0YmfM0hiv727HaTOzfFoKJtVJRk4wZJmZZS6CIRlxsKms+aAVKeGMBclaWkw886oK\n+D/zBXDo7YjWMBQMselQt3b/wnt/UspgnHijgSONzcppygtqrDlJjUia7KYkt93CMTOKWbNjvGDY\n0+plVrlbn+Y2Fhscey3se5XSniZ+d9bvOKH6BH789o/54Zs/ZHBYQ9Ojd34DgX44+upRm6eXuGKG\nrHb7ArT1+dMOVX1+3/Nc8c8rsJvt/PGcP3LSlJPSuo4W5pR7YkYmvb67nWNnFGOzHL6v18N35Icp\n8YrpvVLXSmW+g0WT3IwByor67t6TCJXOgxf/G4b9bG/sZXAopE0wBIfgzbth6nEw7TjjBzyGqcV5\nlLptbDwwWjDUNfXicVgmbbhwNB+eV87Olv5x0Ul72voz9y9Es/LzSv/tN+7CaXVy92l3c82Sa3hi\n1xN87vnPcajvUPxze+rh3/8D88+FmpWjds0Ih9PWNY2uvfXsZqXv14fmptZzxTfk46Y3b+Lbr32b\nBSUL+PNH/8zsoti9pPViToWbfe3eUT24G7sH2NvmnfSafzJygiHLzIxRTG9wKMhru9o4fUH5xPd5\n1sCCqnz8IROHjv4+dO2Ddffzbvglu0qL43nDQ0r7zgnQFkBpMbl8WtF4jaGxlwVV+YfFPfjwfCVj\nN9qc1Ds4RGufP3P/QjR5RXDsdUquye5XMJvM3LDiBu45/R4a+hu46NmLeGTHI4RkjLDNF74PMgRn\n/WTcrqU1hZR57Px1/WjB8vi79cyv9KS0QPqg8wMuee4Sntj1BNcsuYYHz3qQkrz4lVv1Yk65m6Gg\nHBV2+/pupSrAyXMOv2Zi0eQEQ5bxOKxMKczj1R1tkU5ib+/twBcIHhZmJCASbbHeslJpqvPqHezY\ns48phXnJK5K21sFLP4DZH4F552RhtLFZMa2I/R0+OvqVyKpQSPJBc9+kNyOpzCx1UVvi5F9RgmFT\nODdDV40B4EPfgdK58MxXlD7NwCk1p/DIeY+wuHQxP377x1z94tUc6o16ye9dC9ufgpO/AUW14y5p\ns5j49DHTWLujLRK2uru1j02HurlwZY0m4dwf6Odn63/GJX+/hP5AP/efeT83rLgBqyk7ocZzypV5\nEF0a4/Vd7ZR57Myt0PkeZJmcYJgArvvQTN7Z18k/tzYDihkpz2rm+FnGr3L0YHqJC7vFpJgBzrwV\nGejn2AP3JjcjDfvh8avB5oYLfm14XaREqGPdGH6ZHuj04QsEDxvBIITgw/PKeXNPB4NDQYaCIW59\nro7qAgcn6P0cWR1wwb3Q16RoAWGmuKfwwEce4Kbjb6Kuo44Lnr6AO9bfQZe3Ff7xbSiaDifEjzj7\nzLHTsJgEf3zrAACPvduA2SRYvWxKwuEEQ0Ge3fMs5z91Pn/a/ic+PufjPHH+ExxXlV2z5KxyFxaT\n4Jn3G5FSEgpJ3tjdzkmzSw8LrTMRGQkGIcRFQohtQoiQECJuHV0hxNlCiB1CiN1CiO9GbZ8hhHgn\nvP1vQggdQikmP58+ZhrzKz3c8lwdA4Egr9S1cNKc0kmdvxCNxWxiXqWHV3e20WCrpX/plXwi9CJX\nBB9LfOIrN0PLVlh9D7hTL16mJ0fVFGAJJ7pJKXlpuyKkJ3tEUjQfnl+OfzjEW3s6uP+1vexo6ePm\n1YtxGdHLo2aVYvp7749K34wwQgg+OfeTPLX6KT4282P8qe5PnPPYmdwz3Ez7ad9XhEocyvMdnLOk\nikc3HKJ3cIgn36vn1LlllHli9wofCg7x5K4nWf30av7f6/+P0rxS/vzRP/PD439IoSOFbHudcNos\nfO0jc3l+azMPvr6PD5r76PAGDnv/AmSuMWwFPgG8Fu8AIYQZuAc4B1gIXCqEWBjefTtwp5RyNtAF\nXJXheA4LLGYTN52/iIbuAb7+yCYaewY5Y8HEvihT5eqTZ1Lf5eOM/3mVb3RfyJPBE1m++25Ye/v4\ng0NBeOvX8NbdSnTKPGNad6aCw2pmYXU+L2xt5ry7X+cn//iA+ZUe5lYePiaAY2cUk2c184e39vPL\nV3ZxzuJKzlhooDny1O9B2Xx47ErYPDpktcJVwc3HfI8nLbM53tvHfUUFfOS9n/L1tV/nzcY3GQrF\n7rP9+RNq6fMP881H3qel18+FK0e31pRS8kHnB/x8/c856/Gz+MGbP8BpcXLnqXfy13P/ypKyJYZ9\nXS186dRZnLWogp8+/wF3vaK0RD2c8xdUMlpaSCnrgGRq0zHAbinl3vCxfwVWCyHqgNOAT4ePexi4\nCbg3kzEdLhw3s4Tzllbz7PuNwIgz8XDh/KXVLJ9ayI//vp0Xt7fwlu16Vh81FdPanyg1cRacC5VH\nQfcBePar0LQJZp0OH/nxRA89wqraYh56Yx+1JU5+duFRfHz5FKyTsAdDPBxWMyfOLuXluhY8dgs3\nnb/I2A+02OEzjyrmwCeugZ0vwJm3wJAP+lvglZuZefBt7jzndvbPP4vHdj7GU3ue4qUDL+Gyuji2\n8liOrz6eOUVzqM2vpcRRwoppRSyeks+L21sodFo5brabPd172NK+hU2tm3i35V329+7HYrJw8pST\nuXDuhZw85eRJY6oRQvDzi5ZywT1v8MK2FmaXuydl579UyUb/yClAdOhBPXAsUAJ0SymHo7YnNi7+\nh/H/Pjqfl7e3MLfSQ7nn8HuYphY7uf9zq3htZxtDwRCmeWcppoP1Dyj/VNwVcOFDStvOSTKhAb5y\n2mxOnlPKyXNKJ2VTHi2cvqCcl+ta+PbZ86jIz8IzVDgNPv8PeP1OWPtTpcGSitmm3OfFn2A68M2j\nv8lXVnyF1+tf543GN3ij4Q3+dehfkcNdVhdOi5NAicDlCCJsPk55ZCSvwWPzsKxsGZctuIyzpp81\nIeYiLXgcVn7z2VV8/J43OP0wW+DFQ8gkzTiEEC8DlTF2fV9K+XT4mLXAN6WUG2KcfyFwttoDWgjx\nWRTBcBPwdtiMhBBiKvC8lHJxnHFcC1wLMG3atJUHDhzQ8v0mPev2dZKfZ2F+5eFj206Ktx2atyj/\ngn445lpwaCyTkSMlAsMhXt3ZxunzyzEZ1AYzLo2blOgjd7ki/MvmQ0H8tZ2UkkZvI/t69rG/Zz/1\n/fUMDg8yOOxnS0MXy6dMYUZRNeXOchYUL2Bm4cyUWm5ONF3eAC67ZVIntgkh3pVSJu6rigbBoPHD\n1hJfMBwP3CSlPCv89/fCu24D2oBKKeXw2OMSsWrVKrlhw7iPypEjR44cCdAqGLIh2tYDc8IRSDbg\nEuAZqUikNYDatuty4OksjCdHjhw5ciQg03DVjwsh6oHjgeeEEC+Et1cLIf4BEPYhXA+8ANQBj0gp\nt4Uv8R3g60KI3Sg+h/FF/nPkyJEjR1bRxZSUbXKmpBw5cuRInclkSsqRI0eOHIcROcGQI0eOHDlG\nkRMMOXLkyJFjFDnBkCNHjhw5RpETDDly5MiRYxSHZVSSEKINSDf1uRRo13E4hwtH4vc+Er8zHJnf\nO/edtVErpUzaReiwFAyZIITYoCVc6z+NI/F7H4nfGY7M7537zvqSMyXlyJEjR45R5ARDjhw5cuQY\nxZEoGO6f6AFMEEfi9z4SvzMcmd8795115IjzMeTIkSNHjsQciRpDjhw5cuRIwBElGIQQZwshdggh\ndgshvjvR4zECIcRUIcQaIcR2IcQ2IcSN4e3FQoiXhBC7wj+LJnqseiOEMAsh3hNC/D389wwhxDvh\n+/23cNn3/yiEEIVCiMeEEB8IIeqEEMf/p99rIcTXws/2ViHEX4QQjv/Eey2EeEgI0SqE2Bq1Lea9\nFQq/DH//zUKIFZl89hEjGIQQZuAe4BxgIXCpEGLhxI7KEIaBb0gpFwLHAV8Of8/vAq9IKecAr4T/\n/k/jRpTS7iq3A3eGuwR2AVdNyKiM5S7gn1LK+cBSlO///9u7v9ea4ziO4893G8WU4WKx0aaWW5O0\nIq1x4ccyF+KCSMmlXEhxIxfuJBdqN/NjSiTE/gCKG8uPXShuNGLaDyUjypSXi8/n5JzDCXPOTj7f\n96PWzvdzTn0/317r+z7n/f3ufJLN2swagQPAyrjaYw1hjZcUs74AbCgaK5XtRqA1/uwHev5lx5kp\nDMAq4LmkIUmTwBWgu8pzKjtJI5Iex8cfCSeKRsKx9sWX9QFbqzPDyjCzJmAz0Bu3DegEcosSp3jM\nc4G1xHVMJE1Kek/iWRPWqp9lZrXAbGCEBLOWdBd4VzRcKttu4KKC+0C9mS2c6r6zVBgagdd528Nx\nLFlm1gy0AQNAg6SR+NQo0FClaVXKaeAw8C1uLwDex4WiIM28WwjL456PLbReM6sj4awlvQFOAq8I\nBWECeET6WeeUyras57csFYZMMbM5wHXgoKQP+c/FZVWTuR3NzLqAcUmPqj2XaVYLrAB6JLUBnyhq\nGyWY9TzCu+MWYBFQx8/tlkyoZLZZKgxvgMV5201xLDlmNoNQFC5JuhGHx3IfLePv8WrNrwJWA1vM\n7CWhRdhJ6L3Xx3YDpJn3MDAsaSBuXyMUipSzXg+8kPRW0lfgBiH/1LPOKZVtWc9vWSoMD4DWePfC\nTMIFq/4qz6nsYm/9LPBM0qm8p/qBPfHxHuDWdM+tUiQdkdQkqZmQ621JO4E7wLb4sqSOGUDSKPDa\nzJbFoXXAUxLOmtBCajez2fFvPXfMSWedp1S2/cDueHdSOzCR13L6a5n6Bzcz20ToRdcA5ySdqPKU\nys7M1gD3gCf86LcfJVxnuAosIXwz7XZJxRe2/ntm1gEcktRlZksJnyDmA4PALklfqjm/cjOz5YQL\n7jOBIWAv4Q1fslmb2XFgB+EOvEFgH6GfnlTWZnYZ6CB8i+oYcAy4yS+yjUXyDKGt9hnYK+nhlPed\npcLgnHPu97LUSnLOOfcHvDA455wr4IXBOedcAS8MzjnnCnhhcM45V8ALg3POuQJeGJxzzhXwwuCc\nc67Ad1I7KDhuhr8nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1WxCy-sDTQ1B",
        "colab_type": "code",
        "outputId": "ee2a235e-290d-43ca-8a18-e6ca8bc687cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "res[:6,:6]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],\n",
              "        [ 0.8415,  0.3877,  0.1578,  0.0631,  0.0251,  0.0100],\n",
              "        [ 0.9093,  0.7147,  0.3117,  0.1259,  0.0502,  0.0200],\n",
              "        [ 0.1411,  0.9300,  0.4578,  0.1882,  0.0753,  0.0300],\n",
              "        [-0.7568,  0.9998,  0.5923,  0.2497,  0.1003,  0.0400],\n",
              "        [-0.9589,  0.9132,  0.7121,  0.3103,  0.1253,  0.0500]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JCRCw9SuTQ1F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TransformerEmbedding(nn.Module):\n",
        "    \"Embedding + positional encoding + dropout\"\n",
        "    def __init__(self, vocab_sz, emb_sz, inp_p=0.):\n",
        "        super().__init__()\n",
        "        self.emb_sz = emb_sz\n",
        "        self.embed = embedding(vocab_sz, emb_sz)\n",
        "        self.pos_enc = PositionalEncoding(emb_sz)\n",
        "        self.drop = nn.Dropout(inp_p)\n",
        "    \n",
        "    def forward(self, inp): \n",
        "        pos = torch.arange(0, inp.size(1), device=inp.device).float()\n",
        "        return self.drop(self.embed(inp) * math.sqrt(self.emb_sz) + self.pos_enc(pos))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gymJQzHuVEZl",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "heading_collapsed": true,
        "id": "C7y_dF25TQ1N",
        "colab_type": "text"
      },
      "source": [
        "### Feed forward"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "hidden": true,
        "id": "4G_VhmalTQ1O",
        "colab_type": "text"
      },
      "source": [
        "The feed forward cell is easy: it's just two linear layers with a skip connection and a LayerNorm."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "YuCRv4FgTQ1P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def feed_forward(d_model, d_ff, ff_p=0., double_drop=True):\n",
        "    layers = [nn.Linear(d_model, d_ff), nn.ReLU()]\n",
        "    if double_drop: layers.append(nn.Dropout(ff_p))\n",
        "    return SequentialEx(*layers, nn.Linear(d_ff, d_model), nn.Dropout(ff_p), MergeLayer(), nn.LayerNorm(d_model))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3xzPnJMGTQ1V",
        "colab_type": "text"
      },
      "source": [
        "### Multi-head attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9e3hXJyfTQ1W",
        "colab_type": "text"
      },
      "source": [
        "![Multi head attention](https://github.com/fastai/course-nlp/blob/master/images/attention.png?raw=1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GBeMpG2TQ1X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, n_heads, d_model, d_head=None, p=0., bias=True, scale=True):\n",
        "        super().__init__()\n",
        "        d_head = ifnone(d_head, d_model//n_heads)\n",
        "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
        "        self.q_wgt,self.k_wgt,self.v_wgt = [nn.Linear(\n",
        "            d_model, n_heads * d_head, bias=bias) for o in range(3)]\n",
        "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
        "        self.drop_att,self.drop_res = nn.Dropout(p),nn.Dropout(p)\n",
        "        self.ln = nn.LayerNorm(d_model)\n",
        "        \n",
        "    def forward(self, q, kv, mask=None):\n",
        "        return self.ln(q + self.drop_res(self.out(self._apply_attention(q, kv, mask=mask))))\n",
        "    \n",
        "    def create_attn_mat(self, x, layer, bs):\n",
        "        return layer(x).view(bs, x.size(1), self.n_heads, self.d_head\n",
        "                            ).permute(0, 2, 1, 3)\n",
        "    \n",
        "    def _apply_attention(self, q, kv, mask=None):\n",
        "        bs,seq_len = q.size(0),q.size(1)\n",
        "        wq,wk,wv = map(lambda o: self.create_attn_mat(*o,bs),\n",
        "                       zip((q,kv,kv),(self.q_wgt,self.k_wgt,self.v_wgt)))\n",
        "        attn_score = wq @ wk.transpose(2,3)\n",
        "        if self.scale: attn_score /= math.sqrt(self.d_head)\n",
        "        if mask is not None: \n",
        "            attn_score = attn_score.float().masked_fill(mask, -float('inf')).type_as(attn_score)\n",
        "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
        "        attn_vec = attn_prob @ wv\n",
        "        return attn_vec.permute(0, 2, 1, 3).contiguous().view(bs, seq_len, -1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "heading_collapsed": true,
        "id": "GepJ0UB4TQ1Z",
        "colab_type": "text"
      },
      "source": [
        "### Masking"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "hidden": true,
        "id": "J1Oud3g6TQ1a",
        "colab_type": "text"
      },
      "source": [
        "The attention layer uses a mask to avoid paying attention to certain timesteps. The first thing is that we don't really want the network to pay attention to the padding, so we're going to mask it. The second thing is that since this model isn't recurrent, we need to mask (in the output) all the tokens we're not supposed to see yet (otherwise it would be cheating)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "qykyASsdTQ1a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_output_mask(inp, pad_idx=1):\n",
        "    return torch.triu(inp.new_ones(inp.size(1),inp.size(1)), diagonal=1)[None,None].byte()\n",
        "#     return ((inp == pad_idx)[:,None,:,None].long() + torch.triu(inp.new_ones(inp.size(1),inp.size(1)), diagonal=1)[None,None] != 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "hidden": true,
        "id": "WyxvImz-TQ1d",
        "colab_type": "text"
      },
      "source": [
        "Example of mask for the future tokens:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "SYY_6iQxTQ1d",
        "colab_type": "code",
        "outputId": "5c4d2a6e-d013-48bf-a36d-11e99825110c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        }
      },
      "source": [
        "torch.triu(torch.ones(10,10), diagonal=1).byte()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
              "        [0, 0, 1, 1, 1, 1, 1, 1, 1, 1],\n",
              "        [0, 0, 0, 1, 1, 1, 1, 1, 1, 1],\n",
              "        [0, 0, 0, 0, 1, 1, 1, 1, 1, 1],\n",
              "        [0, 0, 0, 0, 0, 1, 1, 1, 1, 1],\n",
              "        [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],\n",
              "        [0, 0, 0, 0, 0, 0, 0, 1, 1, 1],\n",
              "        [0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
              "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 1],\n",
              "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=torch.uint8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7efe14ZyTQ1h",
        "colab_type": "text"
      },
      "source": [
        "### Encoder and decoder blocks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nKLYePTaTQ1i",
        "colab_type": "text"
      },
      "source": [
        "We are now ready to regroup these layers in the blocks we add in the model picture:\n",
        "\n",
        "![Transformer model](https://github.com/fastai/course-nlp/blob/master/images/Transformer.png?raw=1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bqPqLAeTQ1i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class EncoderBlock(nn.Module):\n",
        "    \"Encoder block of a Transformer model.\"\n",
        "    #Can't use Sequential directly cause more than one input...\n",
        "    def __init__(self, n_heads, d_model, d_head, d_inner, p=0., bias=True, scale=True, double_drop=True):\n",
        "        super().__init__()\n",
        "        self.mha = MultiHeadAttention(n_heads, d_model, d_head, p=p, bias=bias, scale=scale)\n",
        "        self.ff  = feed_forward(d_model, d_inner, ff_p=p, double_drop=double_drop)\n",
        "    \n",
        "    def forward(self, x, mask=None): return self.ff(self.mha(x, x, mask=mask))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Zu0XjT4TQ1m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderBlock(nn.Module):\n",
        "    \"Decoder block of a Transformer model.\"\n",
        "    #Can't use Sequential directly cause more than one input...\n",
        "    def __init__(self, n_heads, d_model, d_head, d_inner, p=0., bias=True, scale=True, double_drop=True):\n",
        "        super().__init__()\n",
        "        self.mha1 = MultiHeadAttention(n_heads, d_model, d_head, p=p, bias=bias, scale=scale)\n",
        "        self.mha2 = MultiHeadAttention(n_heads, d_model, d_head, p=p, bias=bias, scale=scale)\n",
        "        self.ff   = feed_forward(d_model, d_inner, ff_p=p, double_drop=double_drop)\n",
        "    \n",
        "    def forward(self, x, enc, mask_out=None): return self.ff(self.mha2(self.mha1(x, x, mask_out), enc))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "heading_collapsed": true,
        "id": "SPAxQR3JTQ1p",
        "colab_type": "text"
      },
      "source": [
        "### The whole model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "gXPPg7wcTQ1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transformer(Module):\n",
        "    def __init__(self, inp_vsz, out_vsz, n_layers=6, n_heads=8, d_model=256, d_head=32, \n",
        "                 d_inner=1024, p=0.1, bias=True, scale=True, double_drop=True, pad_idx=1):\n",
        "        self.enc_emb = TransformerEmbedding(inp_vsz, d_model, p)\n",
        "        self.dec_emb = TransformerEmbedding(out_vsz, d_model, 0.)\n",
        "        args = (n_heads, d_model, d_head, d_inner, p, bias, scale, double_drop)\n",
        "        self.encoder = nn.ModuleList([EncoderBlock(*args) for _ in range(n_layers)])\n",
        "        self.decoder = nn.ModuleList([DecoderBlock(*args) for _ in range(n_layers)])\n",
        "        self.out = nn.Linear(d_model, out_vsz)\n",
        "        self.out.weight = self.dec_emb.embed.weight\n",
        "        self.pad_idx = pad_idx\n",
        "        \n",
        "    def forward(self, inp, out):\n",
        "        mask_out = get_output_mask(out, self.pad_idx)\n",
        "        enc,out = self.enc_emb(inp),self.dec_emb(out)\n",
        "        enc = compose(self.encoder)(enc)\n",
        "        out = compose(self.decoder)(out, enc, mask_out)\n",
        "        return self.out(out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "heading_collapsed": true,
        "hidden": true,
        "id": "AODm4mqETQ1v",
        "colab_type": "text"
      },
      "source": [
        "#### Bleu metric (see dedicated notebook)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "PcicfxVATQ1w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NGram():\n",
        "    def __init__(self, ngram, max_n=5000): self.ngram,self.max_n = ngram,max_n\n",
        "    def __eq__(self, other):\n",
        "        if len(self.ngram) != len(other.ngram): return False\n",
        "        return np.all(np.array(self.ngram) == np.array(other.ngram))\n",
        "    def __hash__(self): return int(sum([o * self.max_n**i for i,o in enumerate(self.ngram)]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "Blk-LEF0TQ14",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_grams(x, n, max_n=5000):\n",
        "    return x if n==1 else [NGram(x[i:i+n], max_n=max_n) for i in range(len(x)-n+1)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "28fWpmISTQ18",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_correct_ngrams(pred, targ, n, max_n=5000):\n",
        "    pred_grams,targ_grams = get_grams(pred, n, max_n=max_n),get_grams(targ, n, max_n=max_n)\n",
        "    pred_cnt,targ_cnt = Counter(pred_grams),Counter(targ_grams)\n",
        "    return sum([min(c, targ_cnt[g]) for g,c in pred_cnt.items()]),len(pred_grams)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "hidden": true,
        "id": "1HgzsQ4_TQ1_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CorpusBLEU(Callback):\n",
        "    def __init__(self, vocab_sz):\n",
        "        self.vocab_sz = vocab_sz\n",
        "        self.name = 'bleu'\n",
        "    \n",
        "    def on_epoch_begin(self, **kwargs):\n",
        "        self.pred_len,self.targ_len,self.corrects,self.counts = 0,0,[0]*4,[0]*4\n",
        "    \n",
        "    def on_batch_end(self, last_output, last_target, **kwargs):\n",
        "        last_output = last_output.argmax(dim=-1)\n",
        "        for pred,targ in zip(last_output.cpu().numpy(),last_target.cpu().numpy()):\n",
        "            self.pred_len += len(pred)\n",
        "            self.targ_len += len(targ)\n",
        "            for i in range(4):\n",
        "                c,t = get_correct_ngrams(pred, targ, i+1, max_n=self.vocab_sz)\n",
        "                self.corrects[i] += c\n",
        "                self.counts[i]   += t\n",
        "    \n",
        "    def on_epoch_end(self, last_metrics, **kwargs):\n",
        "        precs = [c/t for c,t in zip(self.corrects,self.counts)]\n",
        "        len_penalty = exp(1 - self.targ_len/self.pred_len) if self.pred_len < self.targ_len else 1\n",
        "        bleu = len_penalty * ((precs[0]*precs[1]*precs[2]*precs[3]) ** 0.25)\n",
        "        return add_metrics(last_metrics, bleu)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VwnG-LdXTQ2B",
        "colab_type": "text"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dimocn1ATQ2B",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213
        },
        "outputId": "c37b3724-cf7a-40db-8f9e-551213c93e61"
      },
      "source": [
        "n_x_vocab,n_y_vocab = len(data.train_ds.x.vocab.itos), len(data.train_ds.y.vocab.itos)\n",
        "\n",
        "model = Transformer(n_x_vocab, n_y_vocab, d_model=256)\n",
        "learn = Learner(data, model, metrics=[accuracy, CorpusBLEU(n_y_vocab)], loss_func = CrossEntropyFlat())"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-7f900774bf5d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mn_x_vocab\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_y_vocab\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTransformer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_x_vocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_y_vocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mlearn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLearner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCorpusBLEU\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_y_vocab\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCrossEntropyFlat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2eQJgVoTQ2L",
        "colab_type": "code",
        "outputId": "0dafc847-a5a8-4fd4-8f2d-8ca55f746983",
        "colab": {}
      },
      "source": [
        "learn.lr_find()\n",
        "learn.recorder.plot()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX9//HXJ/tCEkIIEBIg7AiILAHFFde2at1oK7bWvda1VWuX77e/+u3XfmvVahdrq+KuVVt3XBH3HTAgm8q+ZoGEJSEJ2XN+f8ygaUwgCXNzZ5L38/GYBzN3zsy8D1k+Offce6455xAREWlNlN8BREQkfKlIiIhIm1QkRESkTSoSIiLSJhUJERFpk4qEiIi0SUVCRETapCIhIiJtUpEQEZE2xfgdoKP69u3rcnNz/Y4hIhJRFi1atN05l9nR10VckcjNzSU/P9/vGCIiEcXMNnXmddrdJCIibVKREBGRNqlIiIhIm1QkRESkTSoSIiLSJk+LhJn91MxWmNlnZnZNK8+bmd1hZmvNbJmZTfYyj4iIdIxnRcLMxgM/AqYBhwCnmtnIFs2+BYwM3i4F7vIqj4iIdJyX50kcBMx3zu0BMLN3gTOBW5u1OR14xAWuoTrfzHqbWZZzrjjUYVZtreClZUVEmREd9dUtJniLjo4iNsqIiY4iNtqIi44iJjoq8NzedtFRJMdHk5oQS0pCDL3iY4iJ1h47Eem+vCwSK4Dfm1kGUA2cDLQ8Cy4b2NLscUFw238UCTO7lMBIg8GDB3cqzNqSSu58ey2hvqR3Ymw0vRJiSEmIISUhlt6JsaQnxdI7KY7eSbGkJMSSEv/V8+nJsWQkx9MnOY64GBUYEQlvnhUJ59wXZnYL8DpQCSwFGlo0s9Ze2sp7zQZmA+Tl5XXq1/wpE7I4ZcIpOOdobHI0Bv9taHI0Njrqm5poaHQ0NDrqGptoaGqivsHR0NREkwtsr290VNbWU1HT8OWtsraeytoGdtc0sLu6nl176li/vZKyqnoqalt29z+lxMeQmhhLWvDWOymWfinx9EtNIDMlnn4p8fTtFU9Grzj6JMcRHxPdma6LiHSap8tyOOfuB+4HMLObCIwUmisABjV7nAMUeZnJzIiJti5Zj6ShsYmq2kYqgoVlbxHZUVXHzsrAv7ur6ykP3lZvq+DDtdvZXdN6cUlNiCEnPYlBfRIZ3CeJwX2SGN6vFyP69SKzVzxmrdVcEZHO8/R3pZn1c86VmNlg4CxgeosmLwBXmdm/gEOBci/mI/wSEx1FWlIUaUmxHXpdTX0jpRW1lFTUsL2yjp1VdeyorKWkopaCXdWsK63inVWl1DY0ffmatMRYhmcmk9U7kf4pCfRPjWdAWgKH5PRmSEaSCoiIdIrXf1A/E5yTqAeudM7tMrPLAJxzdwOvEJirWAvsAS70OE9ESIiNZlCfJAb1SWqzTVOTo6SilrUllawtqWBNSSXrS6v4omg3b+8uYU9d45dt+/aKY/LgdKYMSWfswFRGD0jRyENE2sVcqGdyPZaXl+e0Cuz+VdY2sGXnHj7dXEb+pp0s3rSLjTv2fPl8n+Q4xmalcsakbE6dkEVCrOY7RLozM1vknMvr8OtUJHqOnVV1rNy6m1VbK1i1tYKFG3ayfnsV6UmxnD11MD84dPA+Ry8iErlUJKTDnHN8vG4Hj3y8iXmfb8UBU3P7cOqELL45fgD9UhL8jigiIaIiIQekqKyap/ILeHl5Eau3VWIWKBhTc9MZPzCN8dlp5KQnah5DJEKpSEjIrN5WwcvLinn9822s2lZBY1PgeyQjOY6LjxrKRUcM1RyGSIRRkRBP1NQ3smprBcsLy3lrZQlvrSxhYFoC139jNGdMzCYqSiMLkUigIiFd4uN1O7jplS9YXljO+OxUfnPKWA4dluF3LBHZj84WCS0eJB0yfXgGc648gr/OmsjOyjrOnj2fKx9fTGFZtd/RRMQDKhLSYVFRxukTs3nzZzP46fEjeePzbRx/+zv85Y3VlFfX+x1PREJIu5vkgBWWVXPTK1/w8rJikuKimTk5h/MPH8KIfil+RxORIM1JiO9WFJbz8EcbmbO0iLqGJo4a2ZefHj+SvNw+fkcT6fFUJCRs7Kis5V+fbOGhjzZSWlHLiWP784tvjGZkf40sRPyiIiFhZ09dAw9+uJG731lHVV0DMyfncPKELKYMSSc1oWMr44rIgVGRkLC1s6qOv7+9lkc/3kRdYxNmMGZAKtNy0/ne1EGMG5jmd0SRbk9FQsLenroGlmwu45ONu8jftJP8jbuorm/kyBF9ufToYRw1sq+W/RDxiIqERJzy6noeX7CZBz/cQElFLWMGpHDhEbmcdkg2iXFa9kMklFQkJGLVNjTywpIi7nt/A6u2VZCWGMv38nI497AhDMlI9jueSLegIiERzznHwg07eeTjTcz9bCtNznHpUcO47qRRxMdoZCFyIDpbJLy+fKlIu5kZhw7L4NBhGWwtr+Gvb67hnvfW896a7fzl7ImMHqBDaEW6mpblkLA0IC2BP5x1MPedl0dpRQ3fvvMD7v9gA01NkTXyFYl0KhIS1k4Y25+51xzN0SP78ruXPufMuz5iWUGZ37FEegwVCQl7fXvFc+95efz57EMoKqvm9L9/yH89u5xdVXV+RxPp9jQnIRHBzDhzUg4nHNSfv7yxhoc+2sirK4o5f3ou5x42hMyUeL8jinRLOrpJItKqrRXcMnclb60sIS46itMmDuTCI3J19rZIG3QIrPRI60sreeijjTyVX0B1fSPDMpM54aD+HDemH1OGpBMbrT2qIqAiIT1c+Z56nl9SyBtfbGP++h3UNzrSEmOZOTmHCw7PZXBGkt8RRXylIiESVFnbwAdrSnl5+VZeXV5Mo3OccFB/LjpiKIcN66P1oaRHUpEQacXW8hr+OX8Tjy/czM6qOsZmpXLxkUP59iEDiYvRrijpOVQkRPahpr6ROUsKuf+DDazeVklmSjznTx/ChUcMJTleB/lJ96ciIdIOzjneX7Od+z/YwLurS8lJT+TWmRM4fERfv6OJeKqzRULjbelRzIyjR2Xy8EXTePqy6cRGR/H9+xbw388tp6Km3u94ImFHRUJ6rLzcPrz606P40VFDeWLhZr75l/dZsH6H37FEwoqnRcLMrjWzz8xshZk9YWYJLZ6/wMxKzWxJ8HaJl3lEWkqIjebXp4zlmcsPJy4minPunc+db63RQoIiQZ4VCTPLBn4C5DnnxgPRwKxWmv7bOTcxeLvPqzwi+zJ5cDovXn0kp04YyG3zVnP+gwspraj1O5aI77ze3RQDJJpZDJAEFHn8eSKd1is+hr/OmsgfzjqYhRt2cvId7/Pi0iIi7eAOkVDyrEg45wqB24DNQDFQ7pyb10rTmWa2zMyeNrNBXuURaQ8z45xpg5lz1RFkJMdx9ROfMvOuj1i0aZff0UR84eXupnTgdGAoMBBINrNzWzR7Ech1zk0A3gAebuO9LjWzfDPLLy0t9SqyyJfGDEjl5Z8cxS0zD2bLrmpm3vURVz2+mJKKGr+jiXQpz86TMLPvAt90zl0cfHwecJhz7oo22kcDO51z+1zGU+dJSFerqm3gnnfXcc9760lJiOWOWRN1XoVEnHA8T2IzcJiZJVlgsZzjgS+aNzCzrGYPT2v5vEg4SI6P4bqTRvPCVUeSlhjDufcv4K9vrKFRR0BJD+DlnMQC4GlgMbA8+FmzzexGMzst2OwnwUNklxI4EuoCr/KIHKjRA1J44aojOX1iNn9+YzXnP7CQ7ZU6Akq6Ny3LIdJBzjmezN/CDXM+Iz0pjr//YDJThqT7HUtkn8Jxd5NIt2RmnD11MM9eETgBb9bsj3nk4406VFa6JRUJkU4aNzCNF686kqNGZnLDnM+47sml1NQ3+h1LJKRUJEQOQFpSLPedl8fPThzF80sKufKxxTQ0NvkdSyRkVCREDlBUlHH18SO58fTxvLmyhP9+brl2PUm3oautiITIDw8bQmlFLXe8uYbMlHh+/o0xfkcSOWAqEiIhdO0JIymtqOXvb6+jb694LjxiqN+RRA6IioRICJkZ/3fGeHZU1nLjS5+TlhjLWZNz/I4l0mmakxAJsego445zJjF9WAbXP7WU5z8t9DuSSKepSIh4ICE2mvvPn8qhQzO47sklzFmiQiGRSUVCxCOJcdHcf0EeU3P7cO2/l/DiUl1ORSKPioSIh5LiYnjwwqnkDenDNf9ewmufbfU7kkiHqEiIeGxvoTg4O42rn/iUhRt2+h1JpN1UJES6QHJ8DA9cMJWc9EQuefgTVm2t8DuSSLuoSIh0kT7JcTxy0TQS46I5/4GFFJZV+x1JZL9UJES6UE56Eg9fNI2qugbOu38Bu6rq/I4ksk8qEiJdbMyAVO47L48tO6u54rHF1GtBQAljKhIiPjh0WAY3nXUwH6/fwf+99LnfcUTapGU5RHzynSk5rNq6m3vf38CYrFTOmTbY70giX6ORhIiPfvWtgzhmVCY3zFmhQ2MlLKlIiPho7zpPg9KTuPyfi3TEk4QdFQkRn6UlxnLv+XnUNTRx5WOLqWvQRLaEDxUJkTAwPLMXt3xnAku2lHHzqyv9jiPyJRUJkTBx8sFZXHB4Lg98uIG5K7TGk4QHFQmRMPLfJx/EITlp/PzppWzaUeV3HBEVCZFwEhcTxZ3fn4wBVz6+mJr6Rr8jSQ+nIiESZgb1SeL2701kReFu7n1vvd9xpIdTkRAJQyeO7c+JY/sz+/31lO3R+k7iHxUJkTD1s5NGUVnbwD0aTYiPVCREwtSYAamcdshAHvxwAyUVNX7HkR5KRUIkjF17wijqGx3/eHud31Gkh1KREAljuX2T+V5eDo8t2ETBrj1+x5EeSEVCJMxdfdxIDOOON9f4HUV6IE+LhJlda2afmdkKM3vCzBJaPB9vZv82s7VmtsDMcr3MIxKJBvZO5NzDhvD0ogLWlVb6HUd6GM+KhJllAz8B8pxz44FoYFaLZhcDu5xzI4A/A7d4lUckkl1x7HASYqO5fd4qv6NID+P17qYYINHMYoAkoKjF86cDDwfvPw0cb2bmcSaRiNO3VzyXHDWMV5ZvZemWMr/jSA/iWZFwzhUCtwGbgWKg3Dk3r0WzbGBLsH0DUA5keJVJJJL96Kih9EmO45a5K3HO+R1HeggvdzelExgpDAUGAslmdm7LZq289Gvf/WZ2qZnlm1l+aWlp6MOKRICUhFiuOnYEH63bwftrtvsdR3oIL3c3nQBscM6VOufqgWeBw1u0KQAGAQR3SaUBX7uGo3NutnMuzzmXl5mZ6WFkkfD2g8MGk5OeyC1zV9LUpNGEeM/LIrEZOMzMkoLzDMcDX7Ro8wJwfvD+d4C3nMbRIm2Kj4nmuhNH8VnRbl5aXux3HOkBvJyTWEBgMnoxsDz4WbPN7EYzOy3Y7H4gw8zWAtcBv/Iqj0h3cfrEbMYMSOH2eat0qVPxnEXaH+55eXkuPz/f7xgivnp7ZQkXPvQJvzl1LBcfOdTvOBIBzGyRcy6vo6/TGdciEWjG6EyOHpXJn19fTcluLf4n3lGREIlAZsaNp42jrrGJ/3u55VSfSOioSIhEqNy+yVx2zHBeWFrER2t1SKx4o11FwsyGm1l88P4MM/uJmfX2NpqI7M8VM4YzuE8Sv5mzQpPY4on2jiSeARrNbASBI5KGAo97lkpE2iUhNpr/PW0c60qruO8DXcFOQq+9RaIpuGzGmcBfnHPXAlnexRKR9jp2TD++Ma4/d7y5RteckJBrb5GoN7NzCJz49lJwW6w3kUSko2749jgamxwPfLDR7yjSzbS3SFwITAd+75zbYGZDgX96F0tEOiK7dyInjRvAM4sLqKlv9DuOdCPtKhLOuc+dcz9xzj0RXLgvxTl3s8fZRKQDvj9tMOXV9by6Qst1SOi09+imd8ws1cz6AEuBB83sT95GE5GOmD4sg9yMJB5fsNnvKNKNtHd3U5pzbjdwFvCgc24KgVVeRSRMREUZ50wbzCcbd7F6W4XfcaSbaG+RiDGzLOB7fDVxLSJhZuaUHGKjjScWajQhodHeInEj8Bqwzjn3iZkNA9Z4F0tEOqNvr3i+MW4AzyzSBLaERnsnrp9yzk1wzl0efLzeOTfT22gi0hnfnzaY3TUNvKLrTUgItHfiOsfMnjOzEjPbZmbPmFmO1+FEpOOmD89gaN9kTWBLSLR3d9ODBK4iNxDIBl4MbhORMGNmnDNtEPmbNIEtB669RSLTOfegc64heHsI0MWmRcLUzMk5xEQZT+Vv8TuKRLj2FontZnaumUUHb+cCO7wMJiKdl9ErnmPH9OP5JUU0NGp1WOm89haJiwgc/roVKAa+Q2CpDhEJUzMnZ1NaUcuH6/T3nHRee49u2uycO805l+mc6+ecO4PAiXUiEqaOHdOPtMRYnl1c4HcUiWAHcmW660KWQkRCLj4mmlMnZPHaZ1uprG3wO45EqAMpEhayFCLiibMm51BT38SrOmdCOulAioQLWQoR8cTkwb3JzUji2cWFfkeRCLXPImFmFWa2u5VbBYFzJkQkjJkZZ03O4eP1O3TVOumUfRYJ51yKcy61lVuKcy6mq0KKSOedOSkbgDlLinxOIpHoQHY3iUgEGNQniWm5fXh2cQHOaS+xdIyKhEgPcNbkbNaVVrG0oNzvKBJhVCREeoCTJ2SRHBfNgx9u8DuKRBgVCZEeIDUhlnMPG8KLS4vYuL3K7zgSQVQkRHqIi48cSkx0FHe/u87vKNIJxeXV1DV0/TpcKhIiPUS/1ATOzhvEM4sLKCqr9juOdIBzjuNue5db5q7s8s9WkRDpQX58zDCcg3vfX+93FOmA8up6qusbyUpL6PLP9qxImNloM1vS7LbbzK5p0WaGmZU3a3ODV3lEBHLSkzh9YjZPLNzM9spav+NIOxWV1QAwsHdil3+2Z0XCObfKOTfROTcRmALsAZ5rpen7e9s55270Ko+IBFxx7HBqG5p44AMd6RQpissDuwe71UiiheOBdc65TV30eSLShuGZvTh5fBaPfryJ8up6v+NIOxSVd8ORRAuzgCfaeG66mS01s1fNbFxrDczsUjPLN7P80tJS71KK9BBXHDucitoGHvloo99RpB22llcTE2X07RXf5Z/teZEwszjgNOCpVp5eDAxxzh0C/A14vrX3cM7Nds7lOefyMjN1aW2RAzVuYBrHj+nH/R9u0LUmIkBxWQ39UxOIjur6KzR0xUjiW8Bi59y2lk8453Y75yqD918BYs2sbxdkEunxrj5+JGV76vnnfO0FDndF5dW+zEdA1xSJc2hjV5OZDTAzC96fFsyjC/KKdIGJg3pz1Mi+3PveevbUaTQRzorLa8jyYT4CPC4SZpYEnAg822zbZWZ2WfDhd4AVZrYUuAOY5bRMpUiX+enxI9lRVcfjCzb7HUXa4JyjuLyGgT6NJDy9JoRzbg+Q0WLb3c3u3wnc6WUGEWlbXm4fpg/LYPZ76zn3sCEkxEb7HUla2FFVR11DU7fe3SQiYezq40dQUlHLk/lb/I4irdgaPPx1QFo33N0kIuFv+rAMpuamc9c766htaPQ7jrSwd52tgb01khARH5gZVx83kuLyGh6br7mJcFMcHElkaSQhIn45amRfjh6Vye3zVmmF2DBTVF5NXHQUGclxvny+ioSIYGb8/ozxNDm4Yc4KXQs7jBSX1TAgLYEoH06kAxUJEQka1CeJ604cxRtflPDqiq1+x5GgYh9PpAMVCRFp5sIjchmfncr/vPAZ5Xu0+F84KCqrUZEQkfAQEx3FzWdNYGdVHTfP/cLvOD1eU5Nj227/zrYGFQkRaWF8dhoXHzmUJxZu4d3VWnXZT9sra2locr6dbQ0qEiLSimtOGMmYASlc9ugi8jfu9DtOj1Xk8+GvoCIhIq1Iiovh0YsPJSstgQsf/ITlBeV+R+qRioOHI2f5dCIdqEiISBsyU+J57EeHkpYUyw8fWMCqrRV+R+pxNJIQkbCWlZbIY5ccSnxMFD+4bwGbd+zxO1KPUlxWTXxMFOlJsb5lUJEQkX0akpHMY5ccSl1DI9c/vZSmJp1o11WKd9cwsHciwcvu+EJFQkT2a0S/FP7fKWNZuGEn//pEq8V2leIyf0+kAxUJEWmn7+blcPjwDP7wyhds213jd5weobi8xtf5CFCREJF2MjNuOvNg6hqbuGHOCr/jdHsNjU2BE+k0khCRSJHbN5lrTxzFa59tY+6KYr/jdGslFbU0OX8PfwUVCRHpoEuOHMq4gan8Zs5nlFdrfSevFJcHLzak3U0iEklioqO4ZWZgfadfP7dcy4p7pKgseI6ERhIiEmnGZ6fxs5NG8dKyYh5fqKvZeWFrGJxIByoSItJJlx09nGNGZfK/L37OZ0VatiPUisqrSY6LJjUhxtccKhIi0ilRUcafvncI6UmxXPX4p1TWNvgdqVvZe0U6P0+kAxUJETkAGb3iuWPWJDbtqNL8RIgVl1cz0MfrSOylIiEiB+TQYRlcd+Io5iwp4sl8nY0dKkXl/p8jASoSIhICV8wYwREjMvjtC5+zrrTS7zgRr7ahke2Vtb5PWoOKhIiEQFSUcft3J5IQG8VP//UpdQ1NfkeKaMVlNTgH2ekqEiLSTQxIS+CWmRNYUbib2+et8jtORCsMXmwoR3MSItKdnDRuAOceNph73lvPB2u2+x0nYhXuChaJ9CSfk6hIiEiI/frksYzo14vrnlzCzqo6v+NEpIKyaswCozO/qUiISEglxkVzx6xJ7NpTx61zV/odJyIV7qqmf0oCcTH+/4r2LIGZjTazJc1uu83smhZtzMzuMLO1ZrbMzCZ7lUdEus7Ygan88LBcnszfwpptujZ2RxWW7QmLSWvwsEg451Y55yY65yYCU4A9wHMtmn0LGBm8XQrc5VUeEelaVx83guT4GG7RaKLDCsuqyQ6DSWvout1NxwPrnHObWmw/HXjEBcwHeptZVhdlEhEPpSfHccWMEbzxRQnz1+/wO07EaGxyFJfVdP+RRAuzgCda2Z4NND9FsyC4TUS6gQuPyCUrLYE/vPIFTU1asqM9SipqaGhyPWckYWZxwGnAU6093cq2r30nmdmlZpZvZvmlpaWhjigiHkmIjeZnJ41maUE5Ly/XlezaY+/hrz1pJPEtYLFzblsrzxUAg5o9zgGKWjZyzs12zuU55/IyMzM9iikiXjhzUjZjBqRw62srqW1o9DtO2AunE+mga4rEObS+qwngBeC84FFOhwHlzjn9uSHSjURHGb/61hi27KzmD6+s1Eqx+1EQZiMJT69mYWZJwInAj5ttuwzAOXc38ApwMrCWwNFPF3qZR0T8ccyoTC44PJeHPtpIdJTx/045yPfrJISrwrJq0pNiSYrz92JDe3mawjm3B8hose3uZvcdcKWXGUTEf2bG/3x7LAD3f7AB5+A3p6pQtKZgV3VYLMexV3iUKhHp9vYWCjN44MMNOBw3nDpWhaKFwl17GNkvxe8YX1KREJEuY2aBwoDxwIcb6BUfw89OGu13rLDhnKOwrJoZo/v5HeVLKhIi0qXMjN+cehBVtQ387a21DMtM5sxJOX7HCgs7q+qoqW8Km3MkQAv8iYgPzIzfnTGeQ4f24ZdPL2fRpp1+RwoLew9/DZcjm0BFQkR8EhcTxd3nTmFg7wQufWQRW3bu8TuS7748kU4jCRGRwPpO950/lbrGJi55OJ+Kmnq/I/nqyxPpNJIQEQkY0a8Xd/1gCmtLK7n8n4t79PWxC3ZVkxwXTVpirN9RvqQiISK+O3JkX24+62A+WLud659a2mMXAywsqyY7PTGsDgvW0U0iEha+mzeI0spabp27ir694nvkyXaFu8LnOhJ7qUiISNi4/JjhlOyu5YEPN9AvNZ7Ljhnud6QuVVhWzeQhvf2O8R9UJEQkbOw92W57ZS03v7qSAakJnDGpZ1xiprK2gfLq+rBakgNUJEQkzERFGbd/7xBKK2r5xTPLGJKRxKTB6X7H8lw4Hv4KmrgWkTAUHxPNXedOoX9qPD9+dBFby2v8juS5wrLAeSLhdCIdqEiISJjqkxzHfedNpaq2gUsfzaemvntfsGjvdSTC5WJDe6lIiEjYGj0ghT+fPZHlheX88pll3fqCRYW7qomLjqJvr3i/o/wHFQkRCWsnjRvA9SeNZs6SIq54bHG3Xb6joKyagb0TiIoKr8N+NXEtImHvihnDcc7x97fX8ebKEi4+cihXzBhOSkL4nJl8oAp3VYfdfARoJCEiEcDMuOq4kbx1/TGcenAWd72zjmNve4e3Vm7zO1pIOOcoCMMT6UBFQkQiSFZaIn86eyLPX3kE/VISuOThfB5bsMnvWAfs72+vZXtlbVge6qsiISIRZ+Kg3jx12XSOGZXJr59bwa1zV0bspPZznxZw27zVnDUpm1lTB/kd52tUJEQkIiXHx3DveXmcM20w/3hnHdf+ewklu2u+ViwamxzrSiuZu6KYtSWVPqVt3UfrtvOLp5cxfVgGN8+cEJZrVWniWkQiVkx0FDedOZ6c9ET++Noqnl9SRHJcNEMzkxncJ4mt5TV8UVxBdbNzLEb178XJB2dxysFZjOyf0iU5d1bV8ctnltHQ2MTkwelMHpJOcnwMP350EbkZydz9wynExYTn3+wWaUO0vLw8l5+f73cMEQkzS7eUsbSgjPWlVWzYXsXmnXvITIln3MBUxmalMqJfL5ZuKeOVFVv5ZONOnIOzJmVz01kHkxAb3aHPqqpt4Pv3zic9OY6Zk3M4cWz/Nt+jvLqe7987n7UllQzuk8SaZqOZfinxPHflEV0yYW1mi5xzeR1+nYqEiPQ0JbtreOTjTdz59lom5KRxzw+nkJXW/l/Uf3p9NXe8uYb+qfFs211LSkIMp07I4pxpg5mQ89UqrpW1DZx73wI+L9rN7POmMGN0P8qr61mypYyVxbs5cWx/hmX28qKLX6MiISLSQfM+28q1/15CYlwM9/xwMlOG9Nnva4rLqzn2tnc44aD+3DFrEvPX7+DpxQXMXbGVPXWN5A1J56Ijh3LkyL5c8lA+izfv4h8/mMxJ4wZ0QY/apiIhItIJq7dV8KNH8ikuq+Gq40Zw8ZFDSY5ve7r2uieX8NKyYt687hgG9flqWe/dNfU8lV9tcAyyAAAKQ0lEQVTAQx9tYMvOauJiomhobOKOcyZx6oSBXdGVfVKREBHppLI9dfzXs8t5dcVWMpLjuPLYEXz/0MFfm2dYVlDGaXd+yOUzhvPLb45p9b0amxxvfLGNJz/ZwumTsjntEP8LBKhIiIgcsE837+KPr63io3U7GJiWwOUzhvPdvEEkxEbjnOPse+azrrSSd34+I+KWBOlskdAhsCIiQZMGp/P4jw7jw7XbuW3eKn4z5zP+8sYazj88l/6p8SzcuJPfnzk+4grEgVCREBFp4YgRfTl8eAYLN+zk7nfX8afXVwOBcyzOzgu/s6K9pCIhItIKM+PQYRkcOiyDlVt386+FWzhrcjYx0eF50ptXVCRERPZjzIBUfnvaOL9j+MLTkmhmvc3saTNbaWZfmNn0Fs/PMLNyM1sSvN3gZR4REekYr0cSfwXmOue+Y2ZxQFIrbd53zp3qcQ4REekEz4qEmaUCRwMXADjn6oA6rz5PRERCz8vdTcOAUuBBM/vUzO4zs+RW2k03s6Vm9qqZ9cydfiIiYcrLIhEDTAbucs5NAqqAX7VosxgY4pw7BPgb8Hxrb2Rml5pZvpnll5aWehhZRESa87JIFAAFzrkFwcdPEygaX3LO7XbOVQbvvwLEmlnflm/knJvtnMtzzuVlZmZ6GFlERJrzrEg457YCW8xsdHDT8cDnzduY2QALXorJzKYF8+zwKpOIiHSM10c3XQ08FjyyaT1woZldBuCcuxv4DnC5mTUA1cAsF2mLSYmIdGMRt8CfmZUCm1p5Kg0o7+Tjvff3/tsX2N7JiC0/p6NtwqUf+8u5v+dD2Q/w9mvSkX60tq217M3vqx/tz7m/NupH5/sxxDnX8f31zrlucQNmd/bx3vvN/s0PVY6OtgmXfrSnL13VD6+/Jh3pR3uzqx+d78e+2qgfoe/H/m7daRGSFw/g8YtttAlFjo62CZd+tOd9emI/WtvWWvbm99WP/Wdpbxv1I/T92KeI293UFcws33Vi3fVw0136Ad2nL+pHeFE/9q87jSRCabbfAUKku/QDuk9f1I/won7sh0YSIiLSJo0kRESkTd2+SJjZA2ZWYmYrOvHaKWa23MzWmtkde0/8Cz53tZmtMrPPzOzW0KZuNUvI+2FmvzWzwmZLtZ8c+uRfy+LJ1yP4/PVm5lo7az/UPPp6/M7MlgW/FvPMbGDok38tixf9+GPw8gDLzOw5M+sd+uRfy+JFP74b/PluMjNP5y0OJH8b73e+ma0J3s5vtn2fP0Ot8uqwqXC5EViJdjKwohOvXQhMBwx4FfhWcPuxwBtAfPBxvwjtx2+B6yP96xF8bhDwGoFzaPpGYj+A1GZtfgLcHaH9OAmICd6/BbglQvtxEDAaeAfIC8f8wWy5Lbb1IXDych8gPXg/fV993det248knHPvATubbzOz4WY218wWmdn7Zjam5evMLIvAD+3HLvC/+whwRvDpy4GbnXO1wc8o8bYXnvWjy3nYjz8DvwC6ZJLNi34453Y3a5pMF/TFo37Mc841BJvOB3K87YVn/fjCObfK6+wHkr8N3wBed87tdM7tAl4HvtnZ3wXdvki0YTZwtXNuCnA98I9W2mQTWKRwr4LgNoBRwFFmtsDM3jWzqZ6mbduB9gPgquBugQfMLN27qPt0QP0ws9OAQufcUq+D7scBfz3M7PdmtgX4AeDXlRpD8X2110UE/mL1Qyj74Yf25G9NNrCl2eO9fepUX3vcNa7NrBdwOPBUs91x8a01bWXb3r/sYggM4w4DpgJPmtmwYHXuEiHqx13A74KPfwfcTuCHusscaD/MLAn4NYFdHL4J0dcD59yvgV+b2X8BVwH/E+Ko+xSqfgTf69dAA/BYKDO2Ryj74Yd95TezC4GfBreNAF4xszpgg3PuTNruU6f62uOKBIHRU5lzbmLzjWYWDSwKPnyBwC/Q5sPkHKAoeL8AeDZYFBaaWROBtVO68mIXB9wP59y2Zq+7F3jJy8BtONB+DAeGAkuDP0w5wGIzm+YCKxF3lVB8XzX3OPAyXVwkCFE/gpOlpwLHd+UfT82E+uvR1VrND+CcexB4EMDM3gEucM5tbNakAJjR7HEOgbmLAjrTVy8nY8LlBuTSbEII+Aj4bvC+AYe08bpPCIwW9k7ynBzcfhlwY/D+KAJDO4vAfmQ1a3Mt8K9I/Hq0aLORLpi49ujrMbJZm6uBpyO0H98kcFmAzK7I7/X3FV0wcd3Z/LQ9cb2BwN6O9OD9Pu3pa6u5uvKL6McNeAIoBuoJVNKLCfzlORdYGvxmvqGN1+YBK4B1wJ18dfJhHPDP4HOLgeMitB+PAsuBZQT+qsqKxH60aLORrjm6yYuvxzPB7csIrMuTHaH9WEvgD6clwVtXHKXlRT/ODL5XLbANeC3c8tNKkQhuvyj4dVgLXNiRn6GWN51xLSIibeqpRzeJiEg7qEiIiEibVCRERKRNKhIiItImFQkREWmTioR0C2ZW2cWfd5+ZjQ3RezVaYOXXFWb24v5WTTWz3mZ2RSg+W2R/dAisdAtmVumc6xXC94txXy1S56nm2c3sYWC1c+73+2ifC7zknBvfFfmkZ9NIQrotM8s0s2fM7JPg7Yjg9mlm9pGZfRr8d3Rw+wVm9pSZvQjMM7MZZvaOmT1tgesjPLZ3/f3g9rzg/crgwnxLzWy+mfUPbh8efPyJmd3YztHOx3y1cGEvM3vTzBZb4BoApwfb3AwMD44+/hhs+/Pg5ywzs/8N4X+j9HAqEtKd/RX4s3NuKjATuC+4fSVwtHNuEoGVVm9q9prpwPnOueOCjycB1wBjgWHAEa18TjIw3zl3CPAe8KNmn//X4Ofvd42c4LpCxxM4+x2gBjjTOTeZwDVMbg8WqV8B65xzE51zPzezk4CRwDRgIjDFzI7e3+eJtEdPXOBPeo4TgLHNVtFMNbMUIA142MxGElgFM7bZa153zjVf13+hc64AwMyWEFhf54MWn1PHV4sjLgJODN6fzlfr9T8O3NZGzsRm772IwPr/EFhf56bgL/wmAiOM/q28/qTg7dPg414EisZ7bXyeSLupSEh3FgVMd85VN99oZn8D3nbOnRncv/9Os6erWrxHbbP7jbT+M1Pvvprca6vNvlQ75yaaWRqBYnMlcAeBa0pkAlOcc/VmthFIaOX1BvzBOXdPBz9XZL+0u0m6s3kErskAgJntXXY5DSgM3r/Aw8+fT2A3F8Cs/TV2zpUTuGzp9WYWSyBnSbBAHAsMCTatAFKavfQ14KLgNQgws2wz6xeiPkgPpyIh3UWSmRU0u11H4BduXnAy93MCS7wD3Ar8wcw+BKI9zHQNcJ2ZLQSygPL9vcA59ymBVT9nEbhYT56Z5RMYVawMttkBfBg8ZPaPzrl5BHZnfWxmy4Gn+c8iItJpOgRWxCPBq+ZVO+ecmc0CznHOnb6/14mEE81JiHhnCnBn8IikMrr40rAioaCRhIiItElzEiIi0iYVCRERaZOKhIiItElFQkRE2qQiISIibVKREBGRNv1/RMDi/MT8mrkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5OJMQPSuTQ2N",
        "colab_type": "code",
        "outputId": "ae3a63a5-5be4-4991-fa26-d0a89efde90d",
        "colab": {}
      },
      "source": [
        "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>bleu</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>2.388738</td>\n",
              "      <td>2.521896</td>\n",
              "      <td>0.608056</td>\n",
              "      <td>0.440603</td>\n",
              "      <td>02:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.901857</td>\n",
              "      <td>1.958518</td>\n",
              "      <td>0.687528</td>\n",
              "      <td>0.504890</td>\n",
              "      <td>01:55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.565219</td>\n",
              "      <td>1.652850</td>\n",
              "      <td>0.723345</td>\n",
              "      <td>0.536689</td>\n",
              "      <td>02:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.306575</td>\n",
              "      <td>1.455323</td>\n",
              "      <td>0.747950</td>\n",
              "      <td>0.565299</td>\n",
              "      <td>02:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.137455</td>\n",
              "      <td>1.342423</td>\n",
              "      <td>0.764108</td>\n",
              "      <td>0.587329</td>\n",
              "      <td>02:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.949592</td>\n",
              "      <td>1.282115</td>\n",
              "      <td>0.774509</td>\n",
              "      <td>0.601252</td>\n",
              "      <td>02:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.808173</td>\n",
              "      <td>1.269965</td>\n",
              "      <td>0.778810</td>\n",
              "      <td>0.607137</td>\n",
              "      <td>01:58</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.767959</td>\n",
              "      <td>1.274921</td>\n",
              "      <td>0.778822</td>\n",
              "      <td>0.607116</td>\n",
              "      <td>01:51</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uw8wy9ePTQ2R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_predictions(learn, ds_type=DatasetType.Valid):\n",
        "    learn.model.eval()\n",
        "    inputs, targets, outputs = [],[],[]\n",
        "    with torch.no_grad():\n",
        "        for xb,yb in progress_bar(learn.dl(ds_type)):\n",
        "            out = learn.model(*xb)\n",
        "            for x,y,z in zip(xb[0],xb[1],out):\n",
        "                inputs.append(learn.data.train_ds.x.reconstruct(x))\n",
        "                targets.append(learn.data.train_ds.y.reconstruct(y))\n",
        "                outputs.append(learn.data.train_ds.y.reconstruct(z.argmax(1)))\n",
        "    return inputs, targets, outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47Lrw7X9TQ2U",
        "colab_type": "code",
        "outputId": "fd4e53db-105f-4f07-a993-80ab5cf7c778",
        "colab": {}
      },
      "source": [
        "inputs, targets, outputs = get_predictions(learn)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='149' class='' max='149', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      100.00% [149/149 00:28<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-s7sIobTQ2W",
        "colab_type": "code",
        "outputId": "5d712547-d729-4122-dbbd-e3f6ebd6214d",
        "colab": {}
      },
      "source": [
        "inputs[10],targets[10],outputs[10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj pendant que xxunk les activités requises pour maintenir mon xxunk physique , est - ce que je xxunk de la protection d’un régime d’assurance ou de pension ?,\n",
              " Text xxbos xxmaj while i go about maintaining this high degree of fitness , am i protected under an insurance or pension plan ?,\n",
              " Text xxbos xxmaj while i do to the my physical physical of physical , do i aware by the pension plan service plan ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekNKzPA5TQ2e",
        "colab_type": "code",
        "outputId": "9b924312-50d7-4e76-a6ff-5d015588bc28",
        "colab": {}
      },
      "source": [
        "inputs[700],targets[700],outputs[700]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelles sont les conséquences sur la recherche , la mise en pratique et les politiques en ce qui a trait à l'ac ?,\n",
              " Text xxbos xxmaj what are the xxunk for xxup kt research , practice / policy ?,\n",
              " Text xxbos xxmaj what are the implications implications research kt , , policy and policies in)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vyWvDW7ETQ2g",
        "colab_type": "code",
        "outputId": "8c893b20-4830-4042-f171-0255baddadd7",
        "colab": {}
      },
      "source": [
        "inputs[701],targets[701],outputs[701]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelle est la position des xxmaj états - xxmaj unis , du xxmaj canada et de la xxup xxunk à ce propos ?,\n",
              " Text xxbos xxmaj where do the xxup us , xxmaj canada and xxup xxunk stand ?,\n",
              " Text xxbos xxmaj what is xxmaj xxup us xxmaj xxmaj united and the xxunk fit in)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeYmYdB5TQ2k",
        "colab_type": "code",
        "outputId": "8b77e621-9602-4ea1-a797-ed812a3617e2",
        "colab": {}
      },
      "source": [
        "inputs[2500],targets[2500],outputs[2500]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quels sont les atouts particuliers du xxmaj canada en recherche sur l'obésité sur la scène internationale ?,\n",
              " Text xxbos xxmaj what are the unique xxmaj canadian strengths in obesity research that set xxmaj canada apart on an international front ?,\n",
              " Text xxbos xxmaj what are xxmaj specific strengths canada strengths in obesity - ? are up canada ? from international international stage ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Pnc6Ev_TQ2m",
        "colab_type": "code",
        "outputId": "4afdae74-e8f0-4cdd-830b-3b2c0a39e4b8",
        "colab": {}
      },
      "source": [
        "inputs[4002],targets[4002],outputs[4002]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelles sont les répercussions politiques à long terme de cette révolution scientifique mondiale ?,\n",
              " Text xxbos xxmaj what are some of the long - term policy implications of this global knowledge revolution ?,\n",
              " Text xxbos xxmaj what are the long the long - term policies implications of this global scientific ? ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dI_0yAksTQ2v",
        "colab_type": "text"
      },
      "source": [
        "### Label smoothing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qKUtfuzTQ2w",
        "colab_type": "text"
      },
      "source": [
        "They point out in the paper that using label smoothing helped getting a better BLEU/accuracy, even if it made the loss worse."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1sGOn5YTTQ2y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Transformer(len(data.train_ds.x.vocab.itos), len(data.train_ds.y.vocab.itos), d_model=256)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XXrYqT91TQ20",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn = Learner(data, model, metrics=[accuracy, CorpusBLEU(len(data.train_ds.y.vocab.itos))], \n",
        "                loss_func=FlattenedLoss(LabelSmoothingCrossEntropy, axis=-1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HHmcyQQxTQ22",
        "colab_type": "code",
        "outputId": "18f963d2-f06a-48bc-a599-55deed01d770",
        "colab": {}
      },
      "source": [
        "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>bleu</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.281034</td>\n",
              "      <td>3.357356</td>\n",
              "      <td>0.621848</td>\n",
              "      <td>0.458009</td>\n",
              "      <td>01:45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.872045</td>\n",
              "      <td>2.923340</td>\n",
              "      <td>0.690921</td>\n",
              "      <td>0.510376</td>\n",
              "      <td>01:47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.598603</td>\n",
              "      <td>2.653438</td>\n",
              "      <td>0.729291</td>\n",
              "      <td>0.545735</td>\n",
              "      <td>01:49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.407944</td>\n",
              "      <td>2.514847</td>\n",
              "      <td>0.748187</td>\n",
              "      <td>0.567057</td>\n",
              "      <td>01:46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.195246</td>\n",
              "      <td>2.403729</td>\n",
              "      <td>0.766409</td>\n",
              "      <td>0.592165</td>\n",
              "      <td>01:50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.095695</td>\n",
              "      <td>2.362098</td>\n",
              "      <td>0.776127</td>\n",
              "      <td>0.604666</td>\n",
              "      <td>01:48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.999303</td>\n",
              "      <td>2.358647</td>\n",
              "      <td>0.779535</td>\n",
              "      <td>0.609675</td>\n",
              "      <td>01:47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.923621</td>\n",
              "      <td>2.359421</td>\n",
              "      <td>0.780211</td>\n",
              "      <td>0.610871</td>\n",
              "      <td>01:47</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pHn5zAZMTQ24",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XkPSY0waTQ27",
        "colab_type": "code",
        "outputId": "28979fbb-a5a2-4f50-a6b7-108cdd74607f",
        "colab": {}
      },
      "source": [
        "print(\"Quels sont les atouts particuliers du Canada en recherche sur l'obésité sur la scène internationale ?\")\n",
        "print(\"What are Specific strengths canada strengths in obesity - ? are up canada ? from international international stage ?\")\n",
        "print(\"Quelles sont les répercussions politiques à long terme de cette révolution scientifique mondiale ?\")\n",
        "print(\"What are the long the long - term policies implications of this global scientific ? ?\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Quels sont les atouts particuliers du Canada en recherche sur l'obésité sur la scène internationale ?\n",
            "What are Specific strengths canada strengths in obesity - ? are up canada ? from international international stage ?\n",
            "Quelles sont les répercussions politiques à long terme de cette révolution scientifique mondiale ?\n",
            "What are the long the long - term policies implications of this global scientific ? ?\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pFZxoHteTQ29",
        "colab_type": "code",
        "outputId": "d5762938-bb96-49cc-e4d3-dd12efde281d",
        "colab": {}
      },
      "source": [
        "inputs[10],targets[10],outputs[10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelle distance y a - t - il entre le point le plus rapproché de la surface à xxunk et la position d’utilisation habituelle du tube radiogène ?,\n",
              " Text xxbos xxmaj what is the distance between the nearest point of the area to be shielded and the usual operational position of the x - ray tube ?,\n",
              " Text xxbos xxmaj what is the xxmaj between the xxmaj and of the xxmaj ? the ? and the most ? ? of the xxmaj - ray tube ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "urkOsvE3TQ2-",
        "colab_type": "code",
        "outputId": "a0fd7286-4f39-4e85-ce69-b9066f051bc0",
        "colab": {}
      },
      "source": [
        "inputs[700],targets[700],outputs[700]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quels types de présentations xxmaj santé xxmaj canada xxunk - t - il dans le format ectd à compter du 1er septembre ?,\n",
              " Text xxbos xxmaj what kind of submission types will xxmaj health xxmaj canada accept on xxmaj september 1 , 2004 in ectd format ?,\n",
              " Text xxbos xxmaj what is of information is of be canadian xxmaj canada take ? the canadian ? , and ? the format ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-W54rN1kTQ2_",
        "colab_type": "code",
        "outputId": "892ee5b1-f960-423a-dadd-f0fa2dbae228",
        "colab": {}
      },
      "source": [
        "inputs[701],targets[701],outputs[701]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelles sont les trois caractéristiques qui vous incitent le plus à investir dans votre région ( xxup nommez - xxup les ) ?,\n",
              " Text xxbos xxmaj what are the three most attractive features about investing in your region ( xxup name xxup it ) ?,\n",
              " Text xxbos xxmaj what is the main main important concerns of the in the country ? xxup xxunk , xxunk ) ?)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tvx_in3sTQ3C",
        "colab_type": "code",
        "outputId": "b7c2ec4a-6679-4abc-911f-d5aca2d157f9",
        "colab": {}
      },
      "source": [
        "inputs[4001],targets[4001],outputs[4001]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(Text xxbos xxmaj quelles actions avez - vous prises et quel en a été le résultat ?,\n",
              " Text xxbos xxmaj what were your actions and the outcomes ?,\n",
              " Text xxbos xxmaj what is the targets ? how main of)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBgIwI2HTQ3E",
        "colab_type": "text"
      },
      "source": [
        "### Test leakage"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lJcFA15JTQ3H",
        "colab_type": "text"
      },
      "source": [
        "If we change a token in the targets at position n, it shouldn't impact the predictions before that."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1pXPFL0TQ3H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.model.eval();"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PF8eh7yaTQ3K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xb,yb = data.one_batch(cpu=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H1s0P53bTQ3M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inp1,out1 = xb[0][:1],xb[1][:1]\n",
        "inp2,out2 = inp1.clone(),out1.clone()\n",
        "out2[0,15] = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSNuwCYwTQ3R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y1 = learn.model(inp1, out1)\n",
        "y2 = learn.model(inp2, out2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vm2hPsqSTQ3S",
        "colab_type": "code",
        "outputId": "eab18904-7513-48ff-f01e-95d7eda18675",
        "colab": {}
      },
      "source": [
        "(y1[0,:15] - y2[0,:15]).abs().mean()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0., device='cuda:0', grad_fn=<MeanBackward1>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 0
        }
      ]
    }
  ]
}