notebooks/eval_regression_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EssNet/NC-EssNet  448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "Setup configurations...\n",
      "Use device:cuda:0.\n",
      "------------ Configuration -------------\n",
      "batch_size:16\n",
      "ckpt_dir:../output/regression_models/example/ckpt\n",
      "data_root:../data/datasets_480\n",
      "datasets:['CambridgeLandmarks']\n",
      "device:cuda:0\n",
      "early_feat:False\n",
      "ess_proj:True\n",
      "feat_size:14\n",
      "incl_sces:['ShopFacade']\n",
      "log:../output/regression_models/example/test_results.txt\n",
      "network:EssNet\n",
      "num_workers:0\n",
      "odir:../output/regression_models/example\n",
      "ops:PairCompose(\n",
      "    PairResize(size=480)\n",
      "    PairCenterCropNumpy(size=448)\n",
      "    PairToTensorScaled(./255)\n",
      "    PairNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
      ")\n",
      "optimizer_dict:2\n",
      "pair_txt:test_pairs.5nn.300cm50m.vlad.minmax.txt\n",
      "pair_type:ess\n",
      "ransac_thres:[5]\n",
      "seed:1\n",
      "start_epoch:140\n",
      "test_lbl_txt:dataset_test.txt\n",
      "train_lbl_txt:dataset_train.txt\n",
      "training:False\n",
      "weights_dict:232\n",
      "weights_dir:../output/regression_models/example/ckpt/checkpoint_140_0.36m_1.97deg.pth\n",
      "with_ess:False\n",
      "----------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "Load weights dict ../output/regression_models/example/ckpt/checkpoint_140_0.36m_1.97deg.pth\n",
      "Testing pairs: test_pairs.5nn.300cm50m.vlad.minmax.txt\n",
      "PairCompose(\n",
      "    PairResize(size=480)\n",
      "    PairCenterCropNumpy(size=448)\n",
      "    PairToTensorScaled(./255)\n",
      "    PairNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
      ")\n",
      ">>>Load dataest: ['CambridgeLandmarks'] Data root: ../data/datasets_480\n",
      "Dataset: CambridgeLandmarks Scene: ['ShopFacade']\n",
      "\n",
      "Build up EssNet model: ess_proj: True\n",
      "MatchingFeatRegression: in: 14 out: 12 linear: 9216\n",
      "Load part of model parameters and Kaiming init the left\n",
      "Testing Pairs: test_pairs.5nn.300cm50m.vlad.minmax.txt Err thres: [(0.25, 2), (0.5, 5), (5, 10)]\n",
      "Evaluate on datasets: dict_keys(['ShopFacade'])\n",
      "Dataset:ShopFacade num_samples:515 total_time:44.3723 time_per_pair:0.0862\n",
      "Total prediction time: 44.373s\n",
      ">>>>Evaluate model with Ransac(iter=10, miu=1.414) Error thres:[(0.25, 2), (0.5, 5), (5, 10)])\n",
      "\n",
      ">>Ransac threshold:5\n",
      "Dataset:ShopFacade Bad/All:0/103, Rela:(15.98deg,5.77deg) Abs:(0.95m,5.18deg)/14.47deg) Pass:0.00%/15.53%/71.84%\n",
      "Avg: Rela:(15.98deg,5.77deg) Abs:(0.95m,5.18deg;14.47deg) Pass:0.00%/15.53%/71.84%\n",
      "Ransac testing time: 1.8578565120697021s\n",
      "\n",
      "Total ransac time: 1.858s, Total time: 46.231s\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import os\n",
    "import time\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "import numpy as np\n",
    "\n",
    "import networks\n",
    "from utils.common.setup_helper import lprint, make_deterministic\n",
    "from utils.common.relapose_config import RPEvalConfig\n",
    "from utils.datasets.relapose import VisualLandmarkDataset, get_datasets\n",
    "from pipeline.relapose_regressor import test\n",
    "    \n",
    "test_pair_txt='test_pairs.5nn.300cm50m.vlad.minmax.txt'\n",
    "ckpt_dir = '../output/regression_models/example/ckpt/checkpoint_140_0.36m_1.97deg.pth'\n",
    "config = RPEvalConfig(data_root='../data/datasets_480',\n",
    "                      datasets=['CambridgeLandmarks'],\n",
    "                      incl_sces=['ShopFacade'],\n",
    "                      pair_txt=test_pair_txt,\n",
    "                      network='EssNet',\n",
    "                      resume=ckpt_dir,\n",
    "                      odir='../output/regression_models/example')\n",
    "\n",
    "\n",
    "print(config)\n",
    "# Open the log file\n",
    "log = open(config.log, 'a')\n",
    "lprint('\\n\\nLoad weights dict {}'.format(config.weights_dir), log)\n",
    "lprint('Testing pairs: {}'.format(test_pair_txt), log)\n",
    "\n",
    "print(config.ops)\n",
    "# Prepare dataset sources and data loaders\n",
    "test_sets = get_datasets(datasets=config.datasets, pair_txt=config.pair_txt, data_root=config.data_root, \n",
    "                         incl_sces=config.incl_sces, ops=config.ops,\n",
    "                         train_lbl_txt=config.train_lbl_txt, \n",
    "                         test_lbl_txt=config.test_lbl_txt, \n",
    "                         with_ess=config.with_ess) \n",
    "\n",
    "# Prepare dataset loaders\n",
    "test_loaders = {}        \n",
    "for test_set in test_sets:\n",
    "    test_loaders[test_set.scene] = data.DataLoader(test_set, batch_size=config.batch_size, shuffle=False,\n",
    "                                                     num_workers=config.num_workers, \n",
    "                                                     worker_init_fn=make_deterministic(config.seed))\n",
    "\n",
    "# Test model\n",
    "# Test model\n",
    "net = networks.__dict__[config.network](config)\n",
    "test(net, config, log, test_loaders, sav_res_name=None)\n",
    "log.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EssNet/RelaposeNet 224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "Setup configurations...\n",
      "Use device:cuda:0.\n",
      "------------ Configuration -------------\n",
      "batch_size:16\n",
      "ckpt_dir:../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6/ckpt\n",
      "data_root:../data/datasets_256\n",
      "datasets:['CambridgeLandmarks']\n",
      "device:cuda:0\n",
      "early_feat:False\n",
      "ess_proj:True\n",
      "feat_size:7\n",
      "incl_sces:['ShopFacade']\n",
      "log:../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6/test_results.txt\n",
      "network:RelaPoseNet\n",
      "num_workers:0\n",
      "odir:../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6\n",
      "ops:PairCompose(\n",
      "    PairResize(size=256)\n",
      "    PairCenterCropNumpy(size=224)\n",
      "    PairToTensorUnscaled()\n",
      ")\n",
      "optimizer_dict:2\n",
      "pair_txt:test_pairs.5nn.300cm50m.vlad.minmax.txt\n",
      "pair_type:relapose\n",
      "ransac_thres:[5]\n",
      "seed:1\n",
      "start_epoch:480\n",
      "test_lbl_txt:dataset_test.txt\n",
      "train_lbl_txt:dataset_train.txt\n",
      "training:False\n",
      "weights_dict:192\n",
      "weights_dir:../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6/checkpoint_480_0.94m_1.51deg.pth\n",
      "with_ess:False\n",
      "----------------------------------------\n",
      "\n",
      "---------------------------------------------\n",
      "Load weights dict ../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6/checkpoint_480_0.94m_1.51deg.pth\n",
      "Testing pairs: test_pairs.5nn.300cm50m.vlad.minmax.txt\n",
      "PairCompose(\n",
      "    PairResize(size=256)\n",
      "    PairCenterCropNumpy(size=224)\n",
      "    PairToTensorUnscaled()\n",
      ")\n",
      ">>>Load dataest: ['CambridgeLandmarks'] Data root: ../data/datasets_256\n",
      "Dataset: CambridgeLandmarks Scene: ['ShopFacade']\n",
      "\n",
      "Build up RelaPoseNet model...\n",
      "Load part of model parameters and Kaiming init the left\n",
      "Testing Pairs: test_pairs.5nn.300cm50m.vlad.minmax.txt Err thres: [(0.25, 2), (0.5, 5), (5, 10)]\n",
      "Evaluate on datasets: dict_keys(['ShopFacade'])\n",
      "Dataset:ShopFacade num_samples:515 total_time:9.1316 time_per_pair:0.0177\n",
      "Total prediction time: 9.132s\n",
      ">>>>Evaluate model with Ransac(iter=10, miu=1.414) Error thres:[(0.25, 2), (0.5, 5), (5, 10)])\n",
      "\n",
      ">>Ransac threshold:5\n",
      "Dataset:ShopFacade Bad/All:0/103, Rela:(12.33deg,4.27deg) Abs:(0.85m,4.16deg)/10.67deg) Pass:1.94%/18.45%/94.17%\n",
      "Avg: Rela:(12.33deg,4.27deg) Abs:(0.85m,4.16deg;10.67deg) Pass:1.94%/18.45%/94.17%\n",
      "Ransac testing time: 0.7059948444366455s\n",
      "\n",
      "Total ransac time: 0.706s, Total time: 9.839s\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import os\n",
    "import time\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "import numpy as np\n",
    "\n",
    "import networks\n",
    "from utils.common.setup_helper import lprint, make_deterministic\n",
    "from utils.common.relapose_config import RPEvalConfig\n",
    "from utils.datasets.relapose import VisualLandmarkDataset, get_datasets\n",
    "from pipeline.relapose_regressor import test\n",
    "\n",
    "pair_txt='test_pairs.5nn.300cm50m.vlad.minmax.txt'\n",
    "ckpt_dir = '../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6/checkpoint_480_0.94m_1.51deg.pth'\n",
    "config = RPEvalConfig(data_root='../data/datasets_256',\n",
    "                      datasets=['CambridgeLandmarks'],\n",
    "                      incl_sces=['ShopFacade'],\n",
    "                      pair_txt=pair_txt,\n",
    "                      network='RelaPoseNet',\n",
    "                      rescale=256, \n",
    "                      crop=224,\n",
    "                      normalize=False,\n",
    "                      ess_proj=True,\n",
    "                      gpu=0,\n",
    "                      resume=ckpt_dir,\n",
    "                      odir='../output/regression_models/224_unnormalize/relaposenet/cambridge/learn_beta/30nn_med_lr0.01_lrd0.8-50_wd1e-6')\n",
    "\n",
    "print(config)\n",
    "# Open the log file\n",
    "log = open(config.log, 'a')\n",
    "lprint('---------------------------------------------\\nLoad weights dict {}'.format(config.weights_dir), log)\n",
    "lprint('Testing pairs: {}'.format(pair_txt), log)\n",
    "print(config.ops)\n",
    "# Prepare dataset sources and data loaders\n",
    "test_sets = get_datasets(datasets=config.datasets, pair_txt=config.pair_txt, data_root=config.data_root, \n",
    "                         incl_sces=config.incl_sces, ops=config.ops,\n",
    "                         train_lbl_txt=config.train_lbl_txt, \n",
    "                         test_lbl_txt=config.test_lbl_txt, \n",
    "                         with_ess=config.with_ess) \n",
    "\n",
    "# Prepare dataset loaders\n",
    "test_loaders = {}        \n",
    "for test_set in test_sets:\n",
    "    test_loaders[test_set.scene] = data.DataLoader(test_set, batch_size=config.batch_size, shuffle=False,\n",
    "                                                     num_workers=config.num_workers, \n",
    "                                                     worker_init_fn=make_deterministic(config.seed))\n",
    "\n",
    "# Test model\n",
    "# Test model\n",
    "net = networks.__dict__[config.network](config)\n",
    "test(net, config, log, test_loaders, sav_res_name=None)\n",
    "log.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeploc]",
   "language": "python",
   "name": "conda-env-deeploc-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}