predict_image_pon .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "from models.pyramid import build_pyramid_occupancy_network, build_extended_pyramid_occupancy_network\n",
    "from model import UNET\n",
    "import torchvision.transforms as transforms\n",
    "from dataset import NuScenesDataset\n",
    "from torch.utils.data import DataLoader\n",
    "import albumentations as A\n",
    "import numpy as np\n",
    "import cv2\n",
    "from experiments.ipm.ipm_utilities import ipm_transform\n",
    "import matplotlib.pyplot as plt\n",
    "from configs.config_utilities import load_config\n",
    "from utilities.torch import detect_device\n",
    "\n",
    "from logger import colorise\n",
    "import matplotlib.pyplot as plt\n",
    "from nuscenes_utilities import NUSCENES_CLASS_NAMES, flatten_labels\n",
    "from torchmetrics import classification, Precision, Recall\n",
    "import pandas as pd\n",
    "import math\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = detect_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_seperate_classes(pred: torch.Tensor, is_label:bool = True, title:str = None):\n",
    "    fig, axs = plt.subplots(nrows=2, ncols=7, figsize=(20, 6))\n",
    "    fig.suptitle(title, fontweight='bold', fontsize=18)\n",
    "    axs = axs.flatten()\n",
    "\n",
    "    if not is_label:\n",
    "        pred = pred.sigmoid()\n",
    "\n",
    "    for i, name in enumerate(NUSCENES_CLASS_NAMES):\n",
    "        class_i = pred[i]\n",
    "        axs[i].imshow(colorise(class_i, \"coolwarm\", 0, 1), origin='lower')\n",
    "        axs[i].set_title(name)\n",
    "        axs[i].axis('off')\n",
    "\n",
    "\n",
    "def plot_result(\n",
    "    img: torch.Tensor, label: torch.Tensor, mask: torch.Tensor, pred: torch.Tensor\n",
    "):\n",
    "    pred = (pred.sigmoid() >= 0.5).long().cpu()\n",
    "    img = img.cpu().permute(1, 2, 0)\n",
    "    label = label.cpu()\n",
    "    mask = (mask.cpu() == -1).long()\n",
    "\n",
    "    colorised_gt = colorise(\n",
    "        flatten_labels(label),\n",
    "        \"nipy_spectral\",\n",
    "        flatten=True,\n",
    "    ).permute(0, 2, 3, 1).squeeze(0)\n",
    "\n",
    "    colorised_pred = colorise(\n",
    "        flatten_labels(pred),\n",
    "        \"nipy_spectral\",\n",
    "        flatten=True,\n",
    "    ).permute(0, 2, 3, 1).squeeze(0)\n",
    "\n",
    "    fig, axs = plt.subplots(nrows=1, ncols=4, figsize=(20, 6))\n",
    "    axs[0].imshow(img)\n",
    "    axs[0].set_title(\"Image\")\n",
    "\n",
    "    axs[1].imshow(colorised_gt, origin=\"lower\")\n",
    "    axs[1].set_title(\"Ground truth\")\n",
    "\n",
    "    axs[2].imshow(mask[..., None]*colorised_gt, origin=\"lower\")\n",
    "    axs[2].set_title(\"Ground truth + Visible mask\")\n",
    "\n",
    "    axs[3].imshow(colorised_pred, origin=\"lower\")\n",
    "    axs[3].set_title(\"Predicted\")\n",
    "\n",
    "    for ax in axs:\n",
    "        ax.set_axis_off()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = load_config(\"configs/configs.yml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Loading NuScenes version v1.0-mini ...\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "dataset = NuScenesDataset(\n",
    "    nuscenes_dir=\"nuscenes\",\n",
    "    nuscenes_version=\"v1.0-mini\",\n",
    "    label_dir=\"labels\",\n",
    "    sample_tokens=np.loadtxt(\"configs/mini_val_sample_tokens.csv\", dtype=str),\n",
    "    image_size=(200, 112),\n",
    ")\n",
    "\n",
    "dataset_loader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=8,\n",
    "    num_workers=2,\n",
    "    pin_memory=True,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_result_to_csv(label, pred, mask, save_name):\n",
    "        iou_metric_by_class = classification.JaccardIndex(\n",
    "            task=\"multilabel\",\n",
    "            num_classes=None,\n",
    "            num_labels=14,\n",
    "            average=\"none\",\n",
    "        ).to(device)\n",
    "\n",
    "        precision = Precision(\n",
    "            task=\"multilabel\",\n",
    "            num_classes=None,\n",
    "            num_labels=14,\n",
    "            average=\"none\",\n",
    "        ).to(device)\n",
    "\n",
    "        recall = Recall(\n",
    "            task=\"multilabel\",\n",
    "            num_classes=None,\n",
    "            num_labels=14,\n",
    "            average=\"none\",\n",
    "        ).to(device)\n",
    "\n",
    "        mask = (mask == -1).long()\n",
    "        pred_with_mask= mask.unsqueeze(1).expand(-1, 14, -1, -1) * pred\n",
    "        label_with_mask = mask.unsqueeze(1).expand(-1, 14, -1, -1) * label\n",
    "\n",
    "        iou_result = iou_metric_by_class(pred, label).cpu()\n",
    "        precision_result = precision(pred, label).cpu()\n",
    "        recall_result = recall(pred, label).cpu()\n",
    "\n",
    "        iou_result_w_mask = iou_metric_by_class(pred_with_mask, label_with_mask).cpu()\n",
    "        precision_result_w_mask = precision(pred_with_mask, label_with_mask).cpu()\n",
    "        recall_result_w_mask = recall(pred_with_mask, label_with_mask).cpu()\n",
    "\n",
    "        data = {\n",
    "                \"classes\": NUSCENES_CLASS_NAMES,\n",
    "                \"iou\": iou_result,\n",
    "                \"precision\": precision_result,\n",
    "                \"recall\": recall_result,\n",
    "                \"iou_w_mask\": iou_result_w_mask,\n",
    "                \"precision_w_mask\": precision_result_w_mask,\n",
    "                \"recall_w_mask\": recall_result_w_mask\n",
    "        }\n",
    "        df = pd.DataFrame(data)\n",
    "        df.iloc[:, 1:] = df.iloc[:, 1:].applymap(lambda x: np.round(x*100, 4)).astype('float')\n",
    "        # df = df.astype('float')\n",
    "\n",
    "        df.to_csv(f\"{save_name}\", header=True, index=False)\n",
    "\n",
    "\n",
    "def save_pred_model_based(image, label, mask, pred, model_nm, n_scenes):\n",
    "    if not os.path.exists(f\"pred_img/{str(n_scenes)}scene/\"):\n",
    "        os.mkdir(f\"pred_img/{str(n_scenes)}scene/\")\n",
    "        os.mkdir(f\"pred_img/{str(n_scenes)}scene/{model_nm}\")\n",
    "        os.mkdir(f\"pred_img/{str(n_scenes)}scene/{model_nm}/pred\")\n",
    "        os.mkdir(f\"pred_img/{str(n_scenes)}scene/{model_nm}/pred_binary\")\n",
    "         \n",
    "\n",
    "\n",
    "    for i in tqdm(range(len(image))):\n",
    "        plot_result(image[i], label[i], mask[i], pred[i])\n",
    "        plt.savefig(f\"pred_img/{str(n_scenes)}scene/{model_nm}/pred/pred_img{i}.png\")\n",
    "        plt.close()\n",
    "        \n",
    "        plot_seperate_classes(pred[i])\n",
    "        plt.savefig(f\"pred_img/{str(n_scenes)}scene/{model_nm}/pred_binary/binary_img{i}.png\")\n",
    "        plt.close()\n",
    "\n",
    "\n",
    "def save_pred_image_based(image, label, mask, pred, model_nm, n_scenes):\n",
    "    pred = (pred.sigmoid() >= 0.5).long().cpu()\n",
    "    mask = (mask.cpu() == -1).long()\n",
    "\n",
    "    if not os.path.exists(f\"pred_img/{str(n_scenes)}scene_by_image/\"):\n",
    "        os.mkdir(f\"pred_img/{str(n_scenes)}scene_by_image/\")\n",
    "\n",
    "    for i in tqdm(range(len(image))):\n",
    "        if not os.path.exists(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}\"):\n",
    "            os.mkdir(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}\")\n",
    "\n",
    "        colorised_gt = colorise(\n",
    "            flatten_labels(label[i].cpu()),\n",
    "            \"nipy_spectral\",\n",
    "            flatten=True,\n",
    "        ).permute(0, 2, 3, 1).squeeze(0)\n",
    "\n",
    "\n",
    "        colorised_pred = colorise(\n",
    "            flatten_labels(pred[i].cpu()),\n",
    "            \"nipy_spectral\",\n",
    "            flatten=True,\n",
    "        ).permute(0, 2, 3, 1).squeeze(0)\n",
    "\n",
    "\n",
    "        if not (os.path.exists(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}.png\") and \\\n",
    "            os.path.exists(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_gt.png\") and \\\n",
    "            os.path.exists(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_gt_w_mask.png\")    ):\n",
    "\n",
    "            # save img\n",
    "            img = image[i].cpu().permute(1, 2, 0)\n",
    "            plt.imshow(img)\n",
    "            plt.axis('off')\n",
    "            plt.savefig(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}.png\", bbox_inches='tight', pad_inches=0)\n",
    "            plt.close()\n",
    "\n",
    "            # save gt\n",
    "            plt.imshow(colorised_gt, origin=\"lower\")\n",
    "            plt.axis('off')\n",
    "            plt.savefig(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_gt.png\", bbox_inches='tight', pad_inches=0)\n",
    "            plt.close()\n",
    "\n",
    "            # save gt + mask\n",
    "            plt.imshow(mask[i][..., None]*colorised_gt, origin=\"lower\")\n",
    "            plt.axis('off')\n",
    "            plt.savefig(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_gt_w_mask.png\", bbox_inches='tight', pad_inches=0)\n",
    "            plt.close()\n",
    "\n",
    "        # save pred\n",
    "        plt.imshow(colorised_pred, origin=\"lower\")\n",
    "        plt.axis('off')\n",
    "        plt.savefig(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_{model_nm}.png\", bbox_inches='tight', pad_inches=0)\n",
    "        plt.close()\n",
    "\n",
    "        # save pred + mask\n",
    "        plt.imshow(mask[i][..., None]*colorised_pred, origin=\"lower\")\n",
    "        plt.axis('off')\n",
    "        plt.savefig(f\"pred_img/{str(n_scenes)}scene_by_image/img_{i}/img_{i}_{model_nm}_w_mask.png\", bbox_inches='tight', pad_inches=0)\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# models = [\n",
    "#     \"Original_PON_V_multilabel_1693230110.0703723\",\n",
    "#     \"Original_Fixed_PON_V+H-reversed_multilabel_1693329350.7664964\",\n",
    "#     \"Original_Fixed_PON_V+H-stacked_multilabel_1693333195.990043\",\n",
    "# ]\n",
    "\n",
    "models = [\n",
    "    # \"Full_EPON_H-collage_1693459089.077859\",\n",
    "    \"Full_EPON_H-stack_1693466315.8951316\",\n",
    "    \"Full_PON_1693460068.9404485\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/student/miniconda3/envs/senior_project/lib/python3.8/site-packages/torch/nn/functional.py:3737: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/student/miniconda3/envs/senior_project/lib/python3.8/site-packages/torch/nn/functional.py:4236: UserWarning: Default grid_sample and affine_grid behavior has changed to align_corners=False since 1.3.0. Please specify align_corners=True if the old behavior is desired. See the documentation of grid_sample for details.\n",
      "  warnings.warn(\n",
      "100%|██████████| 122/122 [00:29<00:00,  4.17it/s]\n",
      "/home/student/miniconda3/envs/senior_project/lib/python3.8/site-packages/torch/nn/functional.py:3737: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/student/miniconda3/envs/senior_project/lib/python3.8/site-packages/torch/nn/functional.py:4236: UserWarning: Default grid_sample and affine_grid behavior has changed to align_corners=False since 1.3.0. Please specify align_corners=True if the old behavior is desired. See the documentation of grid_sample for details.\n",
      "  warnings.warn(\n",
      "100%|██████████| 122/122 [00:10<00:00, 11.91it/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for model in models:\n",
    "    if model == \"Full_PON_1693460068.9404485\":\n",
    "        network = build_pyramid_occupancy_network(config)\n",
    "    elif \"collage\" in model:\n",
    "        network = build_extended_pyramid_occupancy_network(config, htfm_method=\"collage\")\n",
    "    elif \"stack\" in model:\n",
    "        network = build_extended_pyramid_occupancy_network(config, htfm_method=\"stack\")\n",
    "\n",
    "    # match model name to model file path\n",
    "    model_path = f\"checkpoints/{model}/{model}_00299.pt\"\n",
    "\n",
    "    model_to_load = torch.load(model_path)\n",
    "    network.load_state_dict(model_to_load[\"model_state_dict\"])\n",
    "    network.to(device)\n",
    "\n",
    "    image_final = []\n",
    "    pred_final = []\n",
    "    mask_final = []\n",
    "    label_final =[]\n",
    "\n",
    "    with torch.no_grad():\n",
    "        network.eval()\n",
    "        for batch_idx, batch in enumerate(dataset_loader):\n",
    "            image, label, mask, calib = batch\n",
    "\n",
    "            image = image.to(device)\n",
    "            label = label.to(device)\n",
    "            mask = mask.to(device)\n",
    "            calib = calib.to(device)\n",
    "\n",
    "            pred = network(image, calib)\n",
    "\n",
    "            image_final.append(image)\n",
    "            pred_final.append(pred)\n",
    "            mask_final.append(mask)\n",
    "            label_final.append(label)\n",
    "\n",
    "\n",
    "    # stack all batch\n",
    "    image = torch.cat(image_final, dim=0)\n",
    "    pred = torch.cat(pred_final, dim=0)\n",
    "    mask = torch.cat(mask_final, dim=0)\n",
    "    label = torch.cat(label_final, dim=0)\n",
    "\n",
    "\n",
    "    save_result_to_csv(label, pred, mask, save_name=f\"pred_img/{model}.csv\")\n",
    "    # save_pred_model_based(image, label, mask, pred, model_nm=model, n_scenes=\"final_10_\")\n",
    "    save_pred_image_based(image, label, mask, pred, model_nm=model, n_scenes=\"final_10_\")\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classes</th>\n",
       "      <th>iou</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>iou_w_mask</th>\n",
       "      <th>precision_w_mask</th>\n",
       "      <th>recall_w_mask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>drivable_area</td>\n",
       "      <td>70.7482</td>\n",
       "      <td>80.4641</td>\n",
       "      <td>85.4210</td>\n",
       "      <td>75.4146</td>\n",
       "      <td>82.9176</td>\n",
       "      <td>89.2868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ped_crossing</td>\n",
       "      <td>37.0297</td>\n",
       "      <td>57.8653</td>\n",
       "      <td>50.7001</td>\n",
       "      <td>35.5399</td>\n",
       "      <td>54.2992</td>\n",
       "      <td>50.7076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>walkway</td>\n",
       "      <td>36.7591</td>\n",
       "      <td>56.6849</td>\n",
       "      <td>51.1176</td>\n",
       "      <td>41.8142</td>\n",
       "      <td>59.5306</td>\n",
       "      <td>58.4207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>carpark</td>\n",
       "      <td>21.7706</td>\n",
       "      <td>60.2403</td>\n",
       "      <td>25.4237</td>\n",
       "      <td>24.7612</td>\n",
       "      <td>59.9885</td>\n",
       "      <td>29.6596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>car</td>\n",
       "      <td>10.0151</td>\n",
       "      <td>24.9921</td>\n",
       "      <td>14.3191</td>\n",
       "      <td>11.7246</td>\n",
       "      <td>31.2417</td>\n",
       "      <td>15.8021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>truck</td>\n",
       "      <td>3.3748</td>\n",
       "      <td>11.1768</td>\n",
       "      <td>4.6117</td>\n",
       "      <td>4.2715</td>\n",
       "      <td>10.8514</td>\n",
       "      <td>6.5808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>bus</td>\n",
       "      <td>16.0710</td>\n",
       "      <td>31.3606</td>\n",
       "      <td>24.7913</td>\n",
       "      <td>7.5653</td>\n",
       "      <td>14.4945</td>\n",
       "      <td>13.6631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>trailer</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>construction_vehicle</td>\n",
       "      <td>2.6468</td>\n",
       "      <td>8.1281</td>\n",
       "      <td>3.7766</td>\n",
       "      <td>2.9710</td>\n",
       "      <td>10.0711</td>\n",
       "      <td>4.0438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pedestrian</td>\n",
       "      <td>0.4907</td>\n",
       "      <td>11.7647</td>\n",
       "      <td>0.5094</td>\n",
       "      <td>0.2913</td>\n",
       "      <td>11.7647</td>\n",
       "      <td>0.2978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>motorcycle</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>3.6364</td>\n",
       "      <td>0.1741</td>\n",
       "      <td>0.3115</td>\n",
       "      <td>3.6697</td>\n",
       "      <td>0.3393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bicycle</td>\n",
       "      <td>0.8467</td>\n",
       "      <td>16.9811</td>\n",
       "      <td>0.8832</td>\n",
       "      <td>1.4682</td>\n",
       "      <td>17.3077</td>\n",
       "      <td>1.5789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>traffic_cone</td>\n",
       "      <td>1.7678</td>\n",
       "      <td>27.6119</td>\n",
       "      <td>1.8537</td>\n",
       "      <td>2.5606</td>\n",
       "      <td>29.8387</td>\n",
       "      <td>2.7246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>barrier</td>\n",
       "      <td>0.6561</td>\n",
       "      <td>7.1895</td>\n",
       "      <td>0.7168</td>\n",
       "      <td>0.3551</td>\n",
       "      <td>5.6818</td>\n",
       "      <td>0.3774</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 classes      iou  precision   recall  iou_w_mask  \\\n",
       "0          drivable_area  70.7482    80.4641  85.4210     75.4146   \n",
       "1           ped_crossing  37.0297    57.8653  50.7001     35.5399   \n",
       "2                walkway  36.7591    56.6849  51.1176     41.8142   \n",
       "3                carpark  21.7706    60.2403  25.4237     24.7612   \n",
       "4                    car  10.0151    24.9921  14.3191     11.7246   \n",
       "5                  truck   3.3748    11.1768   4.6117      4.2715   \n",
       "6                    bus  16.0710    31.3606  24.7913      7.5653   \n",
       "7                trailer   0.0000     0.0000   0.0000      0.0000   \n",
       "8   construction_vehicle   2.6468     8.1281   3.7766      2.9710   \n",
       "9             pedestrian   0.4907    11.7647   0.5094      0.2913   \n",
       "10            motorcycle   0.1665     3.6364   0.1741      0.3115   \n",
       "11               bicycle   0.8467    16.9811   0.8832      1.4682   \n",
       "12          traffic_cone   1.7678    27.6119   1.8537      2.5606   \n",
       "13               barrier   0.6561     7.1895   0.7168      0.3551   \n",
       "\n",
       "    precision_w_mask  recall_w_mask  \n",
       "0            82.9176        89.2868  \n",
       "1            54.2992        50.7076  \n",
       "2            59.5306        58.4207  \n",
       "3            59.9885        29.6596  \n",
       "4            31.2417        15.8021  \n",
       "5            10.8514         6.5808  \n",
       "6            14.4945        13.6631  \n",
       "7             0.0000         0.0000  \n",
       "8            10.0711         4.0438  \n",
       "9            11.7647         0.2978  \n",
       "10            3.6697         0.3393  \n",
       "11           17.3077         1.5789  \n",
       "12           29.8387         2.7246  \n",
       "13            5.6818         0.3774  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pon = pd.read_csv(\"pred_img/Full_PON_1693460068.9404485.csv\")\n",
    "df_pon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classes</th>\n",
       "      <th>iou</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>iou_w_mask</th>\n",
       "      <th>precision_w_mask</th>\n",
       "      <th>recall_w_mask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>drivable_area</td>\n",
       "      <td>75.2112</td>\n",
       "      <td>86.4248</td>\n",
       "      <td>85.2868</td>\n",
       "      <td>76.7223</td>\n",
       "      <td>87.0786</td>\n",
       "      <td>86.5790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ped_crossing</td>\n",
       "      <td>54.7307</td>\n",
       "      <td>75.4120</td>\n",
       "      <td>66.6187</td>\n",
       "      <td>50.6781</td>\n",
       "      <td>71.3949</td>\n",
       "      <td>63.5899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>walkway</td>\n",
       "      <td>48.3228</td>\n",
       "      <td>68.6236</td>\n",
       "      <td>62.0274</td>\n",
       "      <td>51.3494</td>\n",
       "      <td>72.0786</td>\n",
       "      <td>64.0998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>carpark</td>\n",
       "      <td>35.8988</td>\n",
       "      <td>57.4729</td>\n",
       "      <td>48.8840</td>\n",
       "      <td>39.5416</td>\n",
       "      <td>59.8274</td>\n",
       "      <td>53.8355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>car</td>\n",
       "      <td>16.3792</td>\n",
       "      <td>34.1389</td>\n",
       "      <td>23.9457</td>\n",
       "      <td>18.3359</td>\n",
       "      <td>38.7155</td>\n",
       "      <td>25.8342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>truck</td>\n",
       "      <td>11.8350</td>\n",
       "      <td>32.6942</td>\n",
       "      <td>15.6473</td>\n",
       "      <td>7.7739</td>\n",
       "      <td>22.7481</td>\n",
       "      <td>10.5624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>bus</td>\n",
       "      <td>35.6363</td>\n",
       "      <td>68.2572</td>\n",
       "      <td>42.7153</td>\n",
       "      <td>18.1267</td>\n",
       "      <td>46.3292</td>\n",
       "      <td>22.9449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>trailer</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>construction_vehicle</td>\n",
       "      <td>1.9559</td>\n",
       "      <td>10.3000</td>\n",
       "      <td>2.3575</td>\n",
       "      <td>1.5251</td>\n",
       "      <td>10.2493</td>\n",
       "      <td>1.7602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pedestrian</td>\n",
       "      <td>2.3943</td>\n",
       "      <td>33.0214</td>\n",
       "      <td>2.5166</td>\n",
       "      <td>2.6767</td>\n",
       "      <td>25.4902</td>\n",
       "      <td>2.9039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>motorcycle</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bicycle</td>\n",
       "      <td>3.1034</td>\n",
       "      <td>9.4538</td>\n",
       "      <td>4.4161</td>\n",
       "      <td>4.6584</td>\n",
       "      <td>10.2041</td>\n",
       "      <td>7.8947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>traffic_cone</td>\n",
       "      <td>0.0999</td>\n",
       "      <td>22.2222</td>\n",
       "      <td>0.1002</td>\n",
       "      <td>0.1468</td>\n",
       "      <td>33.3333</td>\n",
       "      <td>0.1473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>barrier</td>\n",
       "      <td>6.2606</td>\n",
       "      <td>22.2423</td>\n",
       "      <td>8.0148</td>\n",
       "      <td>2.0518</td>\n",
       "      <td>11.1492</td>\n",
       "      <td>2.4528</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 classes      iou  precision   recall  iou_w_mask  \\\n",
       "0          drivable_area  75.2112    86.4248  85.2868     76.7223   \n",
       "1           ped_crossing  54.7307    75.4120  66.6187     50.6781   \n",
       "2                walkway  48.3228    68.6236  62.0274     51.3494   \n",
       "3                carpark  35.8988    57.4729  48.8840     39.5416   \n",
       "4                    car  16.3792    34.1389  23.9457     18.3359   \n",
       "5                  truck  11.8350    32.6942  15.6473      7.7739   \n",
       "6                    bus  35.6363    68.2572  42.7153     18.1267   \n",
       "7                trailer   0.0000     0.0000   0.0000      0.0000   \n",
       "8   construction_vehicle   1.9559    10.3000   2.3575      1.5251   \n",
       "9             pedestrian   2.3943    33.0214   2.5166      2.6767   \n",
       "10            motorcycle   0.0000     0.0000   0.0000      0.0000   \n",
       "11               bicycle   3.1034     9.4538   4.4161      4.6584   \n",
       "12          traffic_cone   0.0999    22.2222   0.1002      0.1468   \n",
       "13               barrier   6.2606    22.2423   8.0148      2.0518   \n",
       "\n",
       "    precision_w_mask  recall_w_mask  \n",
       "0            87.0786        86.5790  \n",
       "1            71.3949        63.5899  \n",
       "2            72.0786        64.0998  \n",
       "3            59.8274        53.8355  \n",
       "4            38.7155        25.8342  \n",
       "5            22.7481        10.5624  \n",
       "6            46.3292        22.9449  \n",
       "7             0.0000         0.0000  \n",
       "8            10.2493         1.7602  \n",
       "9            25.4902         2.9039  \n",
       "10            0.0000         0.0000  \n",
       "11           10.2041         7.8947  \n",
       "12           33.3333         0.1473  \n",
       "13           11.1492         2.4528  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stack = pd.read_csv(\"pred_img/Full_EPON_H-stack_1693466315.8951316.csv\")\n",
    "df_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iou_w_mask          14.932071\n",
      "precision_w_mask    27.975514\n",
      "recall_w_mask       19.534464\n",
      "dtype: float64\n",
      "iou_w_mask          19.541907\n",
      "precision_w_mask    34.899886\n",
      "recall_w_mask       24.471757\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(df_pon[['iou_w_mask', 'precision_w_mask', 'recall_w_mask']].mean())\n",
    "print(df_stack[['iou_w_mask', 'precision_w_mask', 'recall_w_mask']].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drivable_area & 75.4146  & 76.7223 \\\\\n",
      "ped_crossing & 35.5399  & 50.6781 \\\\\n",
      "walkway & 41.8142  & 51.3494 \\\\\n",
      "carpark & 24.7612  & 39.5416 \\\\\n",
      "car & 11.7246  & 18.3359 \\\\\n",
      "truck & 4.2715  & 7.7739 \\\\\n",
      "bus & 7.5653  & 18.1267 \\\\\n",
      "trailer & 0.0  & 0.0 \\\\\n",
      "construction_vehicle & 2.971  & 1.5251 \\\\\n",
      "pedestrian & 0.2913  & 2.6767 \\\\\n",
      "motorcycle & 0.3115  & 0.0 \\\\\n",
      "bicycle & 1.4682  & 4.6584 \\\\\n",
      "traffic_cone & 2.5606  & 0.1468 \\\\\n",
      "barrier & 0.3551  & 2.0518 \\\\\n"
     ]
    }
   ],
   "source": [
    "zip_result_iou = zip(NUSCENES_CLASS_NAMES, df_pon['iou_w_mask'], df_stack['iou_w_mask'])\n",
    "\n",
    "for name, pon,  stack in zip_result_iou:\n",
    "    print(f\"{name} & {pon}  & {stack}\"+\" \\\\\\\\\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drivable_area & 82.9176 & 89.2868  & 87.0786 & 86.5790 \\\\\n",
      "ped_crossing & 54.2992 & 50.7076  & 71.3949 & 63.5899 \\\\\n",
      "walkway & 59.5306 & 58.4207  & 72.0786 & 64.0998 \\\\\n",
      "carpark & 59.9885 & 29.6596  & 59.8274 & 53.8355 \\\\\n",
      "car & 31.2417 & 15.8021  & 38.7155 & 25.8342 \\\\\n",
      "truck & 10.8514 & 6.5808  & 22.7481 & 10.5624 \\\\\n",
      "bus & 14.4945 & 13.6631  & 46.3292 & 22.9449 \\\\\n",
      "trailer & 0.0000 & 0.0000  & 0.0000 & 0.0000 \\\\\n",
      "construction_vehicle & 10.0711 & 4.0438  & 10.2493 & 1.7602 \\\\\n",
      "pedestrian & 11.7647 & 0.2978  & 25.4902 & 2.9039 \\\\\n",
      "motorcycle & 3.6697 & 0.3393  & 0.0000 & 0.0000 \\\\\n",
      "bicycle & 17.3077 & 1.5789  & 10.2041 & 7.8947 \\\\\n",
      "traffic_cone & 29.8387 & 2.7246  & 33.3333 & 0.1473 \\\\\n",
      "barrier & 5.6818 & 0.3774  & 11.1492 & 2.4528 \\\\\n"
     ]
    }
   ],
   "source": [
    "zip_result_pr = zip(\n",
    "    NUSCENES_CLASS_NAMES,\n",
    "    df_pon[\"precision_w_mask\"],\n",
    "    df_pon[\"recall_w_mask\"],\n",
    "    df_stack[\"precision_w_mask\"],\n",
    "    df_stack[\"recall_w_mask\"],\n",
    ")\n",
    "\n",
    "for name, pon_p, pon_r, stack_p, stack_r in zip_result_pr:\n",
    "    print(f\"{name} & {pon_p:.4f} & {pon_r:.4f}  & {stack_p:.4f} & {stack_r:.4f}\" + \" \\\\\\\\\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(f\"-- iou score of -- \")\n",
    "# for class_name, iou in zip(NUSCENES_CLASS_NAMES, iou_result):\n",
    "#     print(f\"{class_name}, {iou*100:.4f}\")\n",
    "\n",
    "# print(f\"-- precision and recall -- \")\n",
    "# for class_name, _precision, _recall in zip(NUSCENES_CLASS_NAMES, precision_result, recall_result):\n",
    "#     print(f\"{class_name}, {_precision*100:.4f}, {_recall*100:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot_result_separate(label[image_idx], is_label=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot_result_separate(pred[image_idx], is_label=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\\begin{figure*}[h]\n",
    "    \\centering\n",
    "    \\setkeys{Gin}{width=\\linewidth}\n",
    "\\begin{subfigure}{0.25\\textwidth}\n",
    "    \\caption*{Images}\n",
    "\\includegraphics{images/pred_img/img_30/img_30.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_52/img_52.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_47/img_47.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_114/img_114.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_118/img_118.png}\n",
    "\\end{subfigure}\n",
    "\\begin{subfigure}{0.1429\\linewidth}\n",
    "    \\caption*{Ground truth}\n",
    "\\includegraphics{images/pred_img/img_30/img_30_gt_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_52/img_52_gt_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_47/img_47_gt_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_114/img_114_gt_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_118/img_118_gt_w_mask.png}\n",
    "\\end{subfigure}\n",
    "\\begin{subfigure}{0.1429\\linewidth}\n",
    "    \\caption*{PON}\n",
    "\\includegraphics{images/pred_img/img_30/img_30_Full_PON_1693460068.9404485_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_52/img_52_Full_PON_1693460068.9404485_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_47/img_47_Full_PON_1693460068.9404485_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_114/img_114_Full_PON_1693460068.9404485_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_118/img_118_Full_PON_1693460068.9404485_w_mask.png}\n",
    "\\end{subfigure}\n",
    "\\begin{subfigure}{0.1429\\linewidth}\n",
    "    \\caption*{H-PON}\n",
    "\\includegraphics{images/pred_img/img_30/img_30_Full_EPON_H-stack_1693466315.8951316_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_52/img_52_Full_EPON_H-stack_1693466315.8951316_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_47/img_47_Full_EPON_H-stack_1693466315.8951316_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_114/img_114_Full_EPON_H-stack_1693466315.8951316_w_mask.png}\\\\[3pt]\n",
    "\\includegraphics{images/pred_img/img_118/img_118_Full_EPON_H-stack_1693466315.8951316_w_mask.png}\n",
    "\\end{subfigure}\n",
    "    \\caption{Comparison of prediction results on validation samples between PON and H-PON included images and ground truths. Black occlusion masks were ignored during evaluation.}\n",
    "    \\label{fig:prediction_images}\n",
    "\\end{figure*}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "senior-project",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}