{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Imports\n", "Les packages nécéssaires doivent déja être installé sur la machine fournie mais si ce n'est pas le cas, il faut les installer.\n", "Attention en installant pytorch, il faut choisir la bonne versio en fonction de la version de CUDa supportée par la carte graphique." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import torch\n", "from torch.autograd import Variable\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import cv2\n", "import matplotlib\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import random\n", "import math\n", "import pickle\n", "import random\n", "from PIL import Image\n", "import sys\n", "from glob import glob\n", "from IPython.display import clear_output\n", "from datetime import datetime\n", "from time import time\n", "import json\n", "import os" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Functions utiles\n", "Principalement des fonctions d'imports et d'exports, de transformation d'objects et degénération de données." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "#Les fonctions dans ce bloc ne sont pas utilisées par le réseau, mais certaines fonctions d'outils\n", "\n", "\n", "def tensor_imshow(im_tensor,cannel):\n", " b,c,h,w=im_tensor.shape\n", " if c==1:\n", " plt.imshow(im_tensor.squeeze().detach().numpy())\n", " else:\n", " plt.imshow(im_tensor.squeeze().detach().numpy()[cannel,:])\n", "\n", "# Obtenez des données d'entraînement\n", "# frag,vt=get_training_fragment(frag_size,image)\n", "# frag est un patch carrée de taille (frag_size*frag_size) a partir du image(Son emplacement est aléatoire)\n", "# vt est la vérité terrain de la forme Dirac.\n", "def get_training_fragment(frag_size,im):\n", " \"\"\"Permet de générer des fragments et leur vérité terrain. \n", " Dépréciée au profit d'un entrainement à partir de fragments déjà générés (voir load_training_fragment et le fichier gen_frags)\n", " \"\"\"\n", " h,w,c=im.shape\n", " n=random.randint(0,int(h/frag_size)-1)\n", " m=random.randint(0,int(w/frag_size)-1) \n", " shape=frag_size/4\n", " vt_h=math.ceil((h+1)/shape)\n", " vt_w=math.ceil((w+1)/shape)\n", " vt=np.zeros([vt_h,vt_w])\n", " vt_h_po=round((vt_h-1)*(n*frag_size/(h-1)+(n+1)*frag_size/(h-1))/2)\n", " vt_w_po=round((vt_w-1)*(m*frag_size/(w-1)+(m+1)*frag_size/(w-1))/2)\n", " vt[vt_h_po,vt_w_po]=1\n", " vt = np.float32(vt)\n", " vt=torch.from_numpy(vt.reshape(1,1,vt_h,vt_w))\n", " \n", " return im[n*frag_size:(n+1)*frag_size,m*frag_size:(m+1)*frag_size,:],vt\n", "\n", "def load_training_fragment(fragment_path,vt_path):\n", " \"\"\"Charge un fragment de la base de test et génère la tableau de la vérité terrain.\n", " \"\"\"\n", " # Load fragment\n", " frag = cv2.imread(fragment_path)\n", " \n", " # Load vt data\n", " with open(vt_path,'r') as f:\n", " data_vt_raw = f.readlines()\n", " data_vt = [int(d.rstrip('\\r\\n')) for d in data_vt_raw]\n", " \n", " #facteur de taille entre la fresque et la vt/carte de correlation.\n", " # Si l'entrainement se fait sur des fragments 32x32, la carte de correlation\n", " # finale sera de taille 126x126 (si initialement à 1000x1000). Le facteur de \n", " # division est alors 8.\n", " # Si par contre l'entrainement se fait sur des fragments de 16x16, il y aura\n", " # une étape de division (maxpooling) en moins donc la taille de la carte finale\n", " # sera 256x256 et la division sera donc 4\n", " div = 4 \n", " \n", " # Construct vt\n", " vt = np.zeros((int(data_vt[0]/div)+1,int(data_vt[1]/div)+1))\n", " vt[int(round(data_vt[2]/div,0)),int(round(data_vt[3]/div,0))] = 1\n", " vt = np.float32(vt)\n", " vt = torch.from_numpy(vt.reshape(1,1,int(data_vt[0]/div)+1,int(data_vt[1]/div)+1))\n", " \n", " return(frag,vt)\n", "\n", "def img2tensor(im):\n", " \"\"\"Conversion d'une image en tenseur pytorch.\n", " Pour rappel: Tensor.shape=[Batch,Channel,Height,Width]\"\"\"\n", " im=np.array(im,dtype=\"float32\")\n", " tensor_cv = torch.from_numpy(np.transpose(im, (2, 0, 1)))\n", " im_tensor=tensor_cv.unsqueeze(0)\n", " return im_tensor\n", "\n", "\n", "def show_coordonnee(position_pred):\n", " \"\"\"Trouve les coordonnées du maximum dans la carte de correlation.\"\"\"\n", " map_corre=position_pred.squeeze().detach().cpu().numpy()\n", " h,w=map_corre.shape\n", " max_value=map_corre.max()\n", " coordonnee=np.where(map_corre==max_value)\n", " return coordonnee[0].mean(),coordonnee[1].mean()\n", "\n", "def test_fragment32_32(frag,seuillage):\n", " \"\"\"Vérifie que le fragment ne contient pas trop de pixels noir (fond).\"\"\"\n", " a=frag[:,:,0]+frag[:,:,1]+frag[:,:,2]\n", " mask = (a == 0)\n", " arr_new = a[mask]\n", " if arr_new.size/a.size<=(1-seuillage):\n", " return True\n", " else:\n", " return False\n", " \n", "def save_net(file_path,net):\n", " \"\"\"Sauvegarde le modèle à l'aide de Pickle (fichier binaire).\"\"\"\n", " pkl_file = open(file_path, 'wb')\n", " pickle.dump(net,pkl_file)\n", " pkl_file.close()\n", "def load_net(file_path):\n", " \"\"\"Charge le modèle sauvegardé à l'aide de Pickle.\"\"\"\n", " pkl_file = open(file_path, 'rb')\n", " net= pickle.load(pkl_file)\n", " pkl_file.close()\n", " return net" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Paramètres du réseau\n", "Ces fonctions permettent de décrire et d'initialiser le réseau et ses couches" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ " def ini():\n", " \"\"\"Créer un poids de type DeepMatching comme valeur initiale de Conv1 (non obligatoire)\"\"\"\n", " kernel=torch.zeros([8,3,3,3])\n", " array_0=np.array([[1,2,1],[0,0,0],[-1,-2,-1]],dtype='float32')\n", " array_1=np.array([[2,1,0],[1,0,-1],[0,-1,-2]],dtype='float32')\n", " array_2=np.array([[1,0,-1],[2,0,-2],[1,0,-1]],dtype='float32')\n", " array_3=np.array([[0,-1,-2],[1,0,-1],[2,1,0]],dtype='float32')\n", " array_4=np.array([[-1,-2,-1],[0,0,0],[1,2,1]],dtype='float32')\n", " array_5=np.array([[-2,-1,0],[-1,0,1],[0,1,2]],dtype='float32')\n", " array_6=np.array([[-1,0,1],[-2,0,2],[-1,0,1]],dtype='float32')\n", " array_7=np.array([[0,1,2],[-1,0,1],[-2,-1,0]],dtype='float32')\n", " for i in range(3):\n", " kernel[0,i,:]=torch.from_numpy(array_0)\n", " kernel[1,i,:]=torch.from_numpy(array_1)\n", " kernel[2,i,:]=torch.from_numpy(array_2)\n", " kernel[3,i,:]=torch.from_numpy(array_3)\n", " kernel[4,i,:]=torch.from_numpy(array_4)\n", " kernel[5,i,:]=torch.from_numpy(array_5)\n", " kernel[6,i,:]=torch.from_numpy(array_6)\n", " kernel[7,i,:]=torch.from_numpy(array_7)\n", " return torch.nn.Parameter(kernel,requires_grad=True) \n", "\n", "\n", "def kernel_add_ini(n,m):\n", " \"\"\"Calculer le poids initial de la couche convolutive add\n", " n, m signifie qu'il y a n * m sous-patches dans le patch d'entrée\n", " Par exemple, le patch d'entrée est 16 * 16, pour les patchs 4 * 4 de la première couche, n = 4, m = 4\n", " pour les patchs 8 * 8 de la deuxième couche, n = 2, m = 2\"\"\"\n", " input_canal=int(n*m)\n", " output_canal=int(n/2)*int(m/2)\n", " for i in range(int(n/2)):\n", " for j in range(int(m/2)):\n", " kernel_add=np.zeros([1,input_canal],dtype='float32')\n", " kernel_add[0,i*2*m+j*2]=1\n", " kernel_add[0,i*2*m+j*2+1]=1\n", " kernel_add[0,(i*2+1)*m+j*2]=1\n", " kernel_add[0,(i*2+1)*m+j*2+1]=1\n", " if i==0 and j==0:\n", " add=torch.from_numpy(kernel_add.reshape(1,input_canal,1,1))\n", " else:\n", " add_=torch.from_numpy(kernel_add.reshape(1,input_canal,1,1))\n", " add=torch.cat((add,add_),0)\n", " return torch.nn.Parameter(add,requires_grad=False) \n", "\n", "\n", "def kernel_shift_ini(n,m):\n", " \"\"\"Calculer le poids initial de la couche convolutive shift\n", " shift+add Peut réaliser l'étape de l'agrégation\n", " Voir ci-dessus pour les paramètres n et m. \n", " Pour des étapes plus détaillées, veuillez consulter le rapport de stage de Boyang\"\"\"\n", " input_canal=int(n*m)\n", " output_canal=int(n*m)\n", " \n", " kernel_shift=torch.zeros([output_canal,input_canal,3,3])\n", " \n", " array_0=np.array([[1,0,0],[0,0,0],[0,0,0]],dtype='float32')\n", " array_1=np.array([[0,0,1],[0,0,0],[0,0,0]],dtype='float32')\n", " array_2=np.array([[0,0,0],[0,0,0],[1,0,0]],dtype='float32')\n", " array_3=np.array([[0,0,0],[0,0,0],[0,0,1]],dtype='float32')\n", " \n", " kernel_shift_0=torch.from_numpy(array_0)\n", " kernel_shift_1=torch.from_numpy(array_1)\n", " kernel_shift_2=torch.from_numpy(array_2)\n", " kernel_shift_3=torch.from_numpy(array_3)\n", " \n", " \n", " for i in range(n):\n", " for j in range(m):\n", " if i==0 and j==0:\n", " kernel_shift[0,0,:]=kernel_shift_0\n", " else:\n", " if i%2==0 and j%2==0:\n", " kernel_shift[i*m+j,i*m+j,:]=kernel_shift_0\n", " if i%2==0 and j%2==1:\n", " kernel_shift[i*m+j,i*m+j,:]=kernel_shift_1\n", " if i%2==1 and j%2==0:\n", " kernel_shift[i*m+j,i*m+j,:]=kernel_shift_2\n", " if i%2==1 and j%2==1:\n", " kernel_shift[i*m+j,i*m+j,:]=kernel_shift_3\n", " \n", " return torch.nn.Parameter(kernel_shift,requires_grad=False) \n", "\n", "# Trouvez le petit patch(4 * 4) dans la n ème ligne et la m ème colonne du patch d'entrée\n", "# Ceci est utilisé pour calculer la convolution et obtenir la carte de corrélation\n", "def get_patch(fragment,psize,n,m):\n", " return fragment[:,:,n*psize:(n+1)*psize,m*psize:(m+1)*psize]\n", "###################################################################################################################\n", "class Net(nn.Module):\n", " def __init__(self,frag_size,psize):\n", " super(Net, self).__init__()\n", " \n", " h_fr=frag_size\n", " w_fr=frag_size\n", " \n", " n=int(h_fr/psize) # n*m patches dans le patch d'entrée\n", " m=int(w_fr/psize)\n", " \n", " # Le nombre de couches du descripteur (self.conv1) peut être changé\n", " self.conv1 = nn.Conv2d(3,8,kernel_size=3,stride=1,padding=1)\n", " # Si vous souhaitez initialiser Conv1 avec les poids de DeepMatch, exécutez la ligne suivante\n", " #self.conv1.weight=ini()\n", " self.Relu = nn.ReLU(inplace=True)\n", " self.maxpooling=nn.MaxPool2d(3,stride=2, padding=1)\n", " \n", " self.shift1=nn.Conv2d(n*m,n*m,kernel_size=3,stride=1,padding=1)\n", " self.shift1.weight=kernel_shift_ini(n,m)\n", " self.add1 = nn.Conv2d(n*m,int(n/2)*int(m/2),kernel_size=1,stride=1,padding=0)\n", " self.add1.weight=kernel_add_ini(n,m)\n", " \n", " n=int(n/2)\n", " m=int(m/2)\n", " if n>=2 and m>=2:# Si n=m=1 Notre réseau n'a plus besoin de plus de couches pour agréger les cartes de corrélation\n", " self.shift2=nn.Conv2d(n*m,n*m,kernel_size=3,stride=1,padding=1)\n", " self.shift2.weight=kernel_shift_ini(n,m)\n", " self.add2 = nn.Conv2d(n*m,int(n/2)*int(m/2),kernel_size=1,stride=1,padding=0)\n", " self.add2.weight=kernel_add_ini(n,m)\n", " \n", " n=int(n/2)\n", " m=int(m/2)\n", " if n>=2 and m>=2:\n", " self.shift3=nn.Conv2d(n*m,n*m,kernel_size=3,stride=1,padding=1)\n", " self.shift3.weight=kernel_shift_ini(n,m)\n", " self.add3 = nn.Conv2d(n*m,int(n/2)*int(m/2),kernel_size=1,stride=1,padding=0)\n", " self.add3.weight=kernel_add_ini(n,m)\n", " \n", " def get_descripteur(self,img,using_cuda):\n", " # Utilisez Conv1 pour calculer le descripteur,\n", " descripteur_img=self.Relu(self.conv1(img))\n", " b,c,h,w=descripteur_img.shape\n", " couche_constante=0.5*torch.ones([1,1,h,w])\n", " if using_cuda:\n", " couche_constante=couche_constante.cuda()\n", " # Ajouter une couche constante pour éviter la division par 0 lors de la normalisation\n", " descripteur_img=torch.cat((descripteur_img,couche_constante),1)\n", " # la normalisation\n", " descripteur_img_norm=descripteur_img/torch.norm(descripteur_img,dim=1)\n", " return descripteur_img_norm\n", " \n", " def forward(self,img,frag,using_cuda):\n", " psize=4\n", " # Utilisez Conv1 pour calculer le descripteur,\n", " descripteur_input1=self.get_descripteur(img,using_cuda)\n", " descripteur_input2=self.get_descripteur(frag,using_cuda)\n", " \n", " b,c,h,w=frag.shape\n", " n=int(h/psize)\n", " m=int(w/psize)\n", " \n", " #######################################\n", " # Calculer la carte de corrélation par convolution pour les n*m patchs plus petit.\n", " for i in range(n):\n", " for j in range(m):\n", " if i==0 and j==0:\n", " map_corre=F.conv2d(descripteur_input1,get_patch(descripteur_input2,psize,i,j),padding=2)\n", " else:\n", " a=F.conv2d(descripteur_input1,get_patch(descripteur_input2,psize,i,j),padding=2)\n", " map_corre=torch.cat((map_corre,a),1)\n", " ########################################\n", " # Étape de polymérisation (agrégation)\n", " map_corre=self.maxpooling(map_corre)\n", " map_corre=self.shift1(map_corre)\n", " map_corre=self.add1(map_corre)\n", " \n", " #########################################\n", " # Répétez l'étape d'agrégation jusqu'à obtenir le graphique de corrélation du patch d'entrée\n", " n=int(n/2)\n", " m=int(m/2)\n", " if n>=2 and m>=2:\n", " map_corre=self.maxpooling(map_corre)\n", " map_corre=self.shift2(map_corre)\n", " map_corre=self.add2(map_corre)\n", "\n", " n=int(n/2)\n", " m=int(m/2)\n", " if n>=2 and m>=2:\n", " map_corre=self.maxpooling(map_corre)\n", " map_corre=self.shift3(map_corre)\n", " map_corre=self.add3(map_corre)\n", " \n", " \n", " \n", " \n", " b,c,h,w=map_corre.shape\n", " # Normalisation de la division par maximum\n", " map_corre=map_corre/(map_corre.max())\n", " # Normalisation SoftMax\n", " #map_corre=(F.softmax(map_corre.reshape(1,1,h*w,1),dim=2)).reshape(b,c,h,w)\n", " return map_corre" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 2, Fresque 0, fragment 117/1500 (7.8%)\n", "Temps par fragment: 0.76\n" ] } ], "source": [ "if __name__=='__main__':\n", " \n", " \n", " # Variable des données\n", " base_dir = './training_data_shift_color_16/'\n", " fresque_filename = base_dir+'fresque{}.ppm'\n", " fresque_filename_wild = base_dir+'fresque*.ppm'\n", " fragment_filename = base_dir+'fragments/fresque{}/frag_dev_{:05}.ppm'\n", " fragments_filename_wild = base_dir+'fragments/fresque{}/frag_dev_*.ppm'\n", " vt_filename = base_dir+'fragments/fresque{}/vt/frag_dev_{:05}_vt.txt'\n", " save_dir = './trained_net/'\n", " \n", " # Pour continuer l'entrainement à partir d'une modèle deja entrainé, donner \n", " # un chemin de fichier à récuperer. \n", " # Sinon laisser None\n", " net_filename = None\n", " # Plus petite taille de patch (à priori laisser 4)\n", " psize=4\n", " \n", " # Utiliser la carte graphique (attention à la version de pytorch)\n", " # Pour verifier si la carte graphique est utilisable, executer:\n", " # import torch\n", " # torch.cuda.is_available()\n", " # Dans un terminal python3\n", " using_cuda=True\n", " \n", " # Chargement du premier fragment pour obtenir la taille d'entrainement\n", " dummy_frag = cv2.imread(fragment_filename.format(0,0))\n", " frag_size = dummy_frag.shape[0]\n", " del dummy_frag\n", " \n", " # Si continuation d'entrainement, charger le modèle et spécifier les epochs.\n", " # Attention, le modèle chargé doit être de la même architecture que le modèle\n", " # décrit dans les fonctions précédentes.\n", " if net_filename:\n", " net = load_net(net_filename)\n", " epochs = [4,10]\n", " expe_id = int(net_filename.split(\"_\")[-1])\n", " else:\n", " net=Net(frag_size,psize)\n", " expe_id = 120\n", " epochs = [0,10]\n", " \n", " # Sauvegarder le modèle à la fin de l'apprentissage ou pas\n", " save = True\n", " \n", " if using_cuda:\n", " net=net.cuda()\n", " \n", " # Choisissez l'optimiseur et la fonction de coût\n", " optimizer = torch.optim.Adam(net.parameters())\n", " #loss_func = torch.nn.MSELoss()\n", " loss_func = torch.nn.SmoothL1Loss(size_average=None, reduce=None, reduction='mean', beta=20.0)\n", " \n", " # Dans le processus d'apprentissage du réseau,le changement d'erreur est placé dans loss_value=[] \n", " loss_value=[]\n", " \n", " # Detection des fresques\n", " fresques_paths = glob(fresque_filename_wild) \n", " N_fresque = len(fresques_paths)\n", " \n", " time_old = time()\n", " \n", " for epoch in range(epochs[0],epochs[1]):\n", " # Iteration sur les fresques trouvées\n", " for fresque_id,fresque_path in enumerate(fresques_paths):\n", " # Charge la fresque\n", " fresque=cv2.imread(fresque_path)\n", " h,w,c=fresque.shape\n", " fresque_tensor=img2tensor(fresque)\n", "\n", " # Si GPU, conversion de la fresque\n", " if using_cuda:\n", " fresque_tensor=fresque_tensor.cuda()\n", "\n", " # Recherche des fragments d'entrainement\n", " fragments_paths = glob(fragments_filename_wild.format(fresque_id))\n", " random.shuffle(fragments_paths)\n", " fragments_paths = fragments_paths[:1500]\n", " N_fragments = len(fragments_paths)\n", " # Itérer sur les fragments trouvés\n", " for fragment_id,fragment_path in enumerate(fragments_paths):\n", " clear_output(wait=True)\n", " print(\"Epoch {}, Fresque {}, fragment {}/{} ({:.3}%)\".format(epoch,fresque_id,fragment_id,N_fragments,(fragment_id/N_fragments)*100))\n", " print(\"Temps par fragment: {:.3}\".format(time()-time_old))\n", " time_old = time()\n", "\n", " # Chargement du fragment et de la vt\n", " frag,vt=load_training_fragment(fragment_path,vt_filename.format(fresque_id,fragment_id))\n", "\n", " # si GPU, conversion des objects\n", " frag_tensor=img2tensor(frag)\n", " if using_cuda:\n", " vt=vt.cuda()\n", " frag_tensor=frag_tensor.cuda()\n", "\n", " frag_pred=net.forward(fresque_tensor,frag_tensor,using_cuda)\n", " \n", " # Utilisez la fonction de coût pour calculer l'erreur\n", " err_=loss_func(vt,frag_pred)\n", " optimizer.zero_grad()\n", " err_.backward(retain_graph=True)\n", " optimizer.step()\n", " print(err_.tolist())\n", "\n", " loss_value.append(err_.tolist())\n", "\n", " del frag_tensor,frag_pred,err_,vt\n", " torch.cuda.empty_cache()\n", "\n", " # Sauvegarder le réseau\n", " if save:\n", " net_filename = save_dir + \"net_trainned_SLLShift_E{}_{}_{:04}\".format(epoch,datetime.now().strftime(\"%m-%d_%H-%M\"),expe_id)\n", " save_net(net_filename,net)\n", " \n", " with open(save_dir + \"loss_values_SLLShift_E{}_{}_{}\".format(epoch,datetime.now().strftime(\"%m-%d_%H-%M\"),expe_id),'w') as f:\n", " f.write(json.dumps(loss_value))\n", " \n", " print(\"Net sauvegardés dans {}\".format(net_filename))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }