{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9f3a7ef2-798c-4645-8280-2eb5ff0e617c",
   "metadata": {},
   "source": [
    "# Une introduction à `PyTorch` et `autograd`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1fbd3dd-cc32-4e36-8569-9c817ad4d86b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_theme() \n",
    "from numpy.random import default_rng\n",
    "rng = default_rng()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46b3f93a-d957-41c2-b1dc-54009818f60a",
   "metadata": {},
   "source": [
    "# Feedforward Neural Network (perceptron multicouche)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7aa81eb8-6a39-4050-81a6-0cdbe9279469",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from matplotlib.lines import Line2D\n",
    "from matplotlib.patches import Circle, Ellipse\n",
    "\n",
    "def draw_neural_net(layer_sizes, colors, ratio, delta):\n",
    "    left, right, bottom, top = delta, 1-delta, delta, 1-delta \n",
    "    patches = []\n",
    "    n_layers = len(layer_sizes)\n",
    "    v_spacing = (top - bottom)/float(max(layer_sizes))\n",
    "    h_spacing = (right - left)/float(len(layer_sizes) - 1)\n",
    "    diameter = v_spacing/4.\n",
    "    # Nodes\n",
    "    for n, (layer_size, c) in enumerate(zip(layer_sizes, colors)):\n",
    "        layer_top = v_spacing*(layer_size - 1)/2. + (top + bottom)/2.\n",
    "        for m in range(layer_size):\n",
    "            circle = Ellipse((n*h_spacing + left, layer_top - m*v_spacing), \n",
    "                              width=diameter, height=diameter*ratio,\n",
    "                              color=c, ec='k', zorder=1)\n",
    "            patches.append(circle)\n",
    "    # Edges\n",
    "    for n, (layer_size_a, layer_size_b) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):\n",
    "        layer_top_a = v_spacing*(layer_size_a - 1)/2. + (top + bottom)/2.\n",
    "        layer_top_b = v_spacing*(layer_size_b - 1)/2. + (top + bottom)/2.\n",
    "        for m in range(layer_size_a):\n",
    "            for o in range(layer_size_b):\n",
    "                #patches.append(Arrow(0.2, 0.2, 0.3, 0.3, width=0.01))\n",
    "                line = Line2D([n*h_spacing + left + diameter/2, (n + 1)*h_spacing + left- diameter/2],\n",
    "                              [layer_top_a - m*v_spacing, layer_top_b - o*v_spacing], \n",
    "                              color='grey', lw=1, alpha=0.5, zorder=0)\n",
    "                                       #coordsA = \"data\", coordsB = \"data\",\n",
    "                                  #arrowstyle=\"-|>\")\n",
    "                patches.append(line)    \n",
    "    return patches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6ef8b20-2e19-444b-a8e2-d405232017f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8,4))\n",
    "size = fig.get_size_inches()*fig.dpi\n",
    "ratio = size[0] / size[1]\n",
    "delta = 0.03\n",
    "patches = draw_neural_net([3, 8, 8, 1], ['lightgrey', 'C1', 'C1', 'C0'], ratio, delta)\n",
    "for p in patches: \n",
    "    ax.add_artist(p)\n",
    "ax.annotate(r\"Input $\\mathbf{R}^3$\", xy=(delta, 0.), xycoords=\"data\", \n",
    "            va=\"center\", ha=\"center\")\n",
    "ax.annotate(r\"Two hidden layers $\\mathbf{R}^8$\", xy=(0.5, 0.), xycoords=\"data\", \n",
    "            va=\"center\", ha=\"center\")\n",
    "ax.annotate(r\"Output layer $\\mathbf{R}$\", xy=(1-delta, 0.), xycoords=\"data\", \n",
    "            va=\"center\", ha=\"center\")\n",
    "ax.axis('off')\n",
    "ax.set_title(r\"Feedforward neural network from $\\mathbf{R}^3$ to $\\mathbf{R}$ ($L=3$)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4b6334e-62af-4b7c-aedc-55007b05f82d",
   "metadata": {},
   "source": [
    "## Module `torch.nn` et classe `nn.Module` \n",
    "\n",
    "**Lire tous les tutoriels:** https://pytorch.org/tutorials/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9521b736-b5a2-496c-a08e-8b796c9c941a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a99934e2-adb2-4fc7-a3d4-432064140013",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(dir(nn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe7c85d-f996-4a72-80a4-ee103d3f25f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(3, 8),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(8, 8),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(8, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.linear_relu_stack(x)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80766336-bcf6-4b0e-9178-cec865c2c8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(f\"Using {device} device\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d406603a-c149-4d1a-925b-553b1111b035",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from inspect import getsource\n",
    "print(getsource(nn.Linear))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5f85047-0ad1-474d-98fb-4962cdd7bb7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F \n",
    "print(type(F.linear))\n",
    "F.linear"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71294b37-6be0-4708-9f89-abd66c4a12df",
   "metadata": {},
   "source": [
    "## Création d'un objet: une fonction paramétrique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d448f414-c48b-45db-b810-c0d14b880d08",
   "metadata": {},
   "outputs": [],
   "source": [
    "Phi = NeuralNetwork()\n",
    "print(Phi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "185f1969-a187-4bc3-880d-cd928e61caac",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([0.2, 0.3, 0.4])\n",
    "Phi(x)  # appel comme une fonction de $\\R^3 \\to \\R$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e846ff-9bf5-470e-b338-6576a879969b",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 10 \n",
    "xx = x.repeat(n, 1)\n",
    "print(xx.shape)\n",
    "xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cde2a695-2602-4654-a612-d5cfba8a39eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "Phi(xx)  # appel vectoriel comme une fonction de $\\R^{n\\times3} \\to \\R^n$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82a1bcb2-777e-45b1-a153-d5aa80b71fcf",
   "metadata": {},
   "source": [
    "## Accès aux paramètres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66cca5b6-043e-44e5-a332-0fafb1920539",
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, param in Phi.named_parameters():\n",
    "    print(name, \"\\t\", param.size())\n",
    "# ou for name, param in Phi.state_dict().items():"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dec1e30-9c9b-4d0a-a0fa-48585a9c56ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "Phi.get_parameter(\"linear_relu_stack.0.weight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c6c79db-4752-4393-adba-768a5a689367",
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        print(\"Parameters initialization of\", m)\n",
    "        torch.nn.init.xavier_uniform_(m.weight)\n",
    "        m.bias.data.fill_(0.01)\n",
    "\n",
    "Phi.apply(init_weights)\n",
    "\n",
    "for name, param in Phi.named_parameters():\n",
    "    print(name, \"\\t\", param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed83a944-e757-4d0a-88e5-01b1623d01ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "Phi(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c760b881-3e8a-46b3-b80c-69f2132193d9",
   "metadata": {},
   "source": [
    "# Outil `autograd`\n",
    "\n",
    "- https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f4020c-ac16-4b8c-8a61-329c3b100801",
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import pi\n",
    "xx = torch.linspace(0, 1, 1000)\n",
    "yy = 0.1 - 0.3*xx + 0.5*xx**2  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32315436-7881-4a1a-b570-9d1522073a53",
   "metadata": {},
   "outputs": [],
   "source": [
    "yy_with_noise = yy + 0.005 * torch.randn_like(yy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e137d8-b547-4157-9923-4839f769ab67",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.plot(xx, yy, label=\"true function\")\n",
    "ax.scatter(xx, yy_with_noise, label=\"data\", color='C1', alpha=0.2)\n",
    "ax.legend() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d70e7d5e-219c-4ce6-8c82-60e922ba7a00",
   "metadata": {},
   "outputs": [],
   "source": [
    "xx_ = xx[:, None]\n",
    "bases = torch.cat([torch.ones_like(xx_), xx_, xx_**2], dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ca5a7c0-32d6-4b4f-98dc-642aa5e5d7a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "bases.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71c72095-92f4-4cec-8f16-ce52339d492c",
   "metadata": {},
   "outputs": [],
   "source": [
    "theta = torch.tensor([0., 0., 0.], requires_grad=True)\n",
    "loss = ((yy - bases @ theta)**2).mean()\n",
    "print(loss)\n",
    "loss.backward()   # différentiation automatique \"backward\" pour calculer le gradient\n",
    "theta.grad        # on récupère le gradient dans le champ \"grad\" de la variable paramètre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a511f7c1-35cf-4d90-98b1-ce9aae0edc57",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15bda240-beb4-4d67-84ed-0469962c8b9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# attention: juste pour illustrer la syntaxe! ce n'est pas un gradient stochastique\n",
    "\n",
    "theta = torch.tensor([0., 0., 0.], requires_grad=True)\n",
    "for n in range(10000):\n",
    "    # loss = ((yy - bases @ theta)**2).mean() # sans bruit\n",
    "    loss = ((yy_with_noise - bases @ theta)**2).mean() # avec bruit\n",
    "    loss.backward()\n",
    "    with torch.no_grad():  # on impose à pytorch de pouvoir faire des opérations sans le gradient \n",
    "        theta -= .1/(n+1) * theta.grad\n",
    "    if (n % 1000 == 0):\n",
    "        print(n, loss.item(), theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "918b6a4b-be8a-4f59-b210-57f32624f847",
   "metadata": {},
   "source": [
    "## Une fonctionnalité utile moins connue\n",
    "\n",
    "https://pytorch.org/tutorials/beginner/basics/autogradqs_tutorial.html#optional-reading-tensor-gradients-and-jacobian-products\n",
    "\n",
    "On peut calculer la dérivée d'une fonction en plusieurs points simultanément, il suffit de voir la fonction comme une application de $\\mathbf{R}^d$ dans $\\mathbf{R}^d$ et de calculer son jacobien. Il faut ajouter un argument dans l'appel de la fonction `backward`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae93b322-a71b-44f8-891a-e8475aa4e129",
   "metadata": {},
   "outputs": [],
   "source": [
    "xx = torch.linspace(0, 2*pi, 1000, requires_grad=True)\n",
    "yy = torch.sin(xx)\n",
    "yy.backward(torch.ones_like(yy), retain_graph=True)\n",
    "with torch.no_grad():\n",
    "    dyy = xx.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3d34692-eeef-47db-8f53-9519f31049f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "with torch.no_grad():\n",
    "    ax.plot(xx, yy, label=\"cosinus\")\n",
    "    ax.plot(xx, dyy, label=\"sinus via autograd\")\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19729c6f-9e6e-4c81-9223-6ced7398c27d",
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = torch.eye(4, 5, requires_grad=True)\n",
    "out = (inp+1).pow(2).t()\n",
    "out.backward(torch.ones_like(out), retain_graph=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5b58315-b4da-4f47-8fae-7b164d386733",
   "metadata": {},
   "outputs": [],
   "source": [
    "inp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5875080e-f527-4ad0-bb7a-69b5eab02991",
   "metadata": {},
   "outputs": [],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c534374-19a6-4c32-b9f8-8b37a8e8f834",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.ones_like(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}