notebooks/version_05/1_intro_to_automl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# AutoML solution vs single model\n",
    "#### FEDOT version = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install fedot==0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is an example of running an Auto ML solution for a classification problem.\n",
    "## Description of the task and dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30.59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60.89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.233810</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0.036050</td>\n",
       "      <td>3300.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.907239</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>0.024926</td>\n",
       "      <td>63588.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0   0                              0.766127   45   \n",
       "1   1                              0.957151   40   \n",
       "2   2                              0.658180   38   \n",
       "3   3                              0.233810   30   \n",
       "4   4                              0.907239   49   \n",
       "\n",
       "   NumberOfTime30.59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "3                                     0   0.036050         3300.0   \n",
       "4                                     1   0.024926        63588.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "3                                5                        0   \n",
       "4                                7                        0   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60.89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "3                             0                                     0   \n",
       "4                             1                                     0   \n",
       "\n",
       "   NumberOfDependents  target  \n",
       "0                 2.0       1  \n",
       "1                 1.0       0  \n",
       "2                 0.0       0  \n",
       "3                 0.0       0  \n",
       "4                 0.0       0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Input data from csv files \n",
    "train_data_path = '../data/scoring_train.csv'\n",
    "test_data_path = '../data/scoring_test.csv'\n",
    "df = pd.read_csv(train_data_path)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline model\n",
    "\n",
    "Let's use the api features to solve the classification problem. First, we create a pipeline with a single model \"xgboost\". \n",
    "To do this, we will substitute the appropriate name in the predefined_model field.\n",
    "\n",
    "Attention!\n",
    "\"predefined_model\" - is not an initial assumption for the AutoML algorithm. It's just a single model without AutoML part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from fedot.api.main import Fedot\n",
    "\n",
    "# task selection, initialisation of the framework\n",
    "baseline_model = Fedot(problem='classification')\n",
    "\n",
    "# fit model without optimisation - single XGBoost node is used \n",
    "xgb_pipeline = baseline_model.fit(features=train_data_path, target='target', predefined_model='xgboost')\n",
    "\n",
    "# evaluate the prediction with test data\n",
    "xgb_predict = baseline_model.predict_proba(features=test_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8328294111806176"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fedot.core.data.data import InputData\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# Read data from csv file as InputData\n",
    "test_data = InputData.from_csv(test_data_path)\n",
    "roc_auc_baseline = roc_auc_score(test_data.target, xgb_predict)\n",
    "roc_auc_baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    },
    "tags": []
   },
   "source": [
    "## FEDOT AutoML for classification\n",
    "\n",
    "We can identify the model using an evolutionary algorithm built into the core of the FEDOT framework.\n",
    "\n",
    "Here are some parameters that you can specify when initializing a class:\n",
    "* problem - the name of modelling problem to solve:\n",
    "        - classification\n",
    "        - regression\n",
    "        - ts_forecasting\n",
    "        - clustering\n",
    "* seed - value for fixed random seed\n",
    "* verbose_level - level of the output detailing\n",
    "        - -1 - nothing\n",
    "        - 0 - errors\n",
    "        - 1 - messages\n",
    "        - 2 - warnings and info\n",
    "        - 3-4 - basic and detailed debug\n",
    "* timeout - time for model design (in minutes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# new instance to be used as AutoML tool\n",
    "auto_model = Fedot(problem='classification', seed=42, verbose_level=0, timeout=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generations:   0%|                                                                          | 1/10000 [01:03<?, ?gen/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyperparameters optimization start\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1%|▎                                           | 6/1000 [02:42<7:29:04, 27.11s/trial, best loss: -0.8543467023733671]\n",
      "Hyperparameters optimization finished\n",
      "Return tuned pipeline due to the fact that obtained metric 0.854 equal or bigger than initial (- 5% deviation) 0.803\n"
     ]
    }
   ],
   "source": [
    "# run of the AutoML-based model generation\n",
    "pipeline = auto_model.fit(features=train_data_path, target='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "prediction = auto_model.predict_proba(features=test_data_path)\n",
    "\n",
    "# Calculate metric\n",
    "roc_auc_auto = roc_auc_score(test_data.target, prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline 0.83\n",
      "AutoML solution 0.85\n"
     ]
    }
   ],
   "source": [
    "# comparison with the manual pipeline\n",
    "\n",
    "print(f'Baseline {roc_auc_baseline:.2f}')\n",
    "print(f'AutoML solution {roc_auc_auto:.2f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thus, with just a few lines of code, we were able to launch the FEDOT framework and got a better result*.\n",
    "\n",
    "*Due to the stochastic nature of the algorithm, the metrics for the found solution may differ.\n",
    "\n",
    "If you want to learn more about FEDOT, you can use [this notebook](2_intro_to_fedot.ipynb)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}