Q-Learning-Frozen-Lake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import random\n",
    "import time\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"FrozenLake-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "action_space_size = env.action_space.n\n",
    "state_space_size = env.observation_space.n\n",
    "\n",
    "q_table = np.zeros((state_space_size, action_space_size))\n",
    "print(q_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_episodes = 100000\n",
    "max_steps_per_episode = 1000\n",
    "\n",
    "learning_rate = 0.1\n",
    "discount_rate = 0.99\n",
    "\n",
    "exploration_rate = 1\n",
    "max_exploration_rate = 1\n",
    "min_exploration_rate = 0.01\n",
    "exploration_decay_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Average reward per thounsand episods ====\n",
      "\n",
      "1000  :  0.05400000000000004\n",
      "2000  :  0.19600000000000015\n",
      "3000  :  0.4120000000000003\n",
      "4000  :  0.5470000000000004\n",
      "5000  :  0.6160000000000004\n",
      "6000  :  0.6730000000000005\n",
      "7000  :  0.6850000000000005\n",
      "8000  :  0.6790000000000005\n",
      "9000  :  0.6650000000000005\n",
      "10000  :  0.6860000000000005\n",
      "11000  :  0.6610000000000005\n",
      "12000  :  0.6930000000000005\n",
      "13000  :  0.7150000000000005\n",
      "14000  :  0.7000000000000005\n",
      "15000  :  0.6650000000000005\n",
      "16000  :  0.6670000000000005\n",
      "17000  :  0.6820000000000005\n",
      "18000  :  0.6800000000000005\n",
      "19000  :  0.6560000000000005\n",
      "20000  :  0.6860000000000005\n",
      "21000  :  0.6850000000000005\n",
      "22000  :  0.7030000000000005\n",
      "23000  :  0.6900000000000005\n",
      "24000  :  0.6890000000000005\n",
      "25000  :  0.6910000000000005\n",
      "26000  :  0.6590000000000005\n",
      "27000  :  0.6730000000000005\n",
      "28000  :  0.6820000000000005\n",
      "29000  :  0.7080000000000005\n",
      "30000  :  0.6690000000000005\n",
      "31000  :  0.6760000000000005\n",
      "32000  :  0.6970000000000005\n",
      "33000  :  0.6980000000000005\n",
      "34000  :  0.6660000000000005\n",
      "35000  :  0.6820000000000005\n",
      "36000  :  0.6790000000000005\n",
      "37000  :  0.6560000000000005\n",
      "38000  :  0.6740000000000005\n",
      "39000  :  0.6720000000000005\n",
      "40000  :  0.6880000000000005\n",
      "41000  :  0.6790000000000005\n",
      "42000  :  0.6830000000000005\n",
      "43000  :  0.6850000000000005\n",
      "44000  :  0.6420000000000005\n",
      "45000  :  0.6760000000000005\n",
      "46000  :  0.6760000000000005\n",
      "47000  :  0.6940000000000005\n",
      "48000  :  0.6590000000000005\n",
      "49000  :  0.6750000000000005\n",
      "50000  :  0.6790000000000005\n",
      "51000  :  0.6890000000000005\n",
      "52000  :  0.6520000000000005\n",
      "53000  :  0.6900000000000005\n",
      "54000  :  0.6560000000000005\n",
      "55000  :  0.6860000000000005\n",
      "56000  :  0.6700000000000005\n",
      "57000  :  0.6850000000000005\n",
      "58000  :  0.7050000000000005\n",
      "59000  :  0.6880000000000005\n",
      "60000  :  0.6770000000000005\n",
      "61000  :  0.6670000000000005\n",
      "62000  :  0.6660000000000005\n",
      "63000  :  0.6610000000000005\n",
      "64000  :  0.7090000000000005\n",
      "65000  :  0.6700000000000005\n",
      "66000  :  0.6900000000000005\n",
      "67000  :  0.6890000000000005\n",
      "68000  :  0.6800000000000005\n",
      "69000  :  0.6750000000000005\n",
      "70000  :  0.6730000000000005\n",
      "71000  :  0.6840000000000005\n",
      "72000  :  0.6710000000000005\n",
      "73000  :  0.7000000000000005\n",
      "74000  :  0.6680000000000005\n",
      "75000  :  0.6920000000000005\n",
      "76000  :  0.7120000000000005\n",
      "77000  :  0.6650000000000005\n",
      "78000  :  0.6730000000000005\n",
      "79000  :  0.6760000000000005\n",
      "80000  :  0.6810000000000005\n",
      "81000  :  0.6890000000000005\n",
      "82000  :  0.6800000000000005\n",
      "83000  :  0.7060000000000005\n",
      "84000  :  0.6970000000000005\n",
      "85000  :  0.6890000000000005\n",
      "86000  :  0.6880000000000005\n",
      "87000  :  0.6780000000000005\n",
      "88000  :  0.6670000000000005\n",
      "89000  :  0.7030000000000005\n",
      "90000  :  0.6830000000000005\n",
      "91000  :  0.7110000000000005\n",
      "92000  :  0.6680000000000005\n",
      "93000  :  0.6680000000000005\n",
      "94000  :  0.6820000000000005\n",
      "95000  :  0.6720000000000005\n",
      "96000  :  0.6580000000000005\n",
      "97000  :  0.6880000000000005\n",
      "98000  :  0.7050000000000005\n",
      "99000  :  0.6850000000000005\n",
      "100000  :  0.6910000000000005\n",
      "==== Updated Q table ====\n",
      "\n",
      "[[0.49022078 0.49026372 0.49031656 0.49031076]\n",
      " [0.4037955  0.34312517 0.34831836 0.47907784]\n",
      " [0.42847502 0.4299798  0.41360098 0.45883571]\n",
      " [0.39494026 0.34064632 0.40069866 0.45205474]\n",
      " [0.49389627 0.4537361  0.39604305 0.41663173]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.29586826 0.12573476 0.12223537 0.14717195]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.46110037 0.43276431 0.37325513 0.51109304]\n",
      " [0.46706495 0.57992516 0.48378705 0.41065372]\n",
      " [0.56939686 0.31196195 0.31445236 0.19558426]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.51829874 0.49502131 0.6698388  0.54662708]\n",
      " [0.71041025 0.82987016 0.74915172 0.76725042]\n",
      " [0.         0.         0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "rewards_all_episods = []\n",
    "\n",
    "for episode in range(num_episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    rewards_current_episode = 0\n",
    "    \n",
    "    for step in range(max_steps_per_episode):\n",
    "        exploration_rate_threshold = random.uniform(0,1)\n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            # exploitation\n",
    "            action = np.argmax(q_table[state, :])\n",
    "        else:\n",
    "            # exploration\n",
    "            action = env.action_space.sample()\n",
    "        \n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        \n",
    "        # updating Q table\n",
    "        q_table[state, action] = q_table[state, action] * (1 - learning_rate) + learning_rate * (reward + discount_rate * np.max(q_table[new_state, :]))\n",
    "        \n",
    "        state = new_state\n",
    "        rewards_current_episode += reward\n",
    "        \n",
    "        if done == True:\n",
    "            break\n",
    "        \n",
    "    # exploration rate decay\n",
    "    exploration_rate = min_exploration_rate + (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate * episode)\n",
    "\n",
    "    rewards_all_episods.append(rewards_current_episode)\n",
    "    \n",
    "# Average reward each 1000 episode\n",
    "reward_per_thosand_episodes = np.split(np.array(rewards_all_episods), num_episodes/1000)\n",
    "count = 1000\n",
    "print('==== Average reward per thounsand episods ====\\n')\n",
    "for r in reward_per_thosand_episodes:\n",
    "    print(count , ' : ' , str(sum(r/1000)))\n",
    "    count += 1000\n",
    "    \n",
    "print('==== Updated Q table ====\\n')\n",
    "print(q_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (Up)\n",
      "\u001b[41mS\u001b[0mFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFFG\n"
     ]
    }
   ],
   "source": [
    "for episode in range(3):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    print('========== EPISODE ',episode+1, ' ============')\n",
    "    time.sleep(1)\n",
    "    for step in range(max_steps_per_episode):\n",
    "        clear_output(wait=True)\n",
    "        env.render()\n",
    "        time.sleep(0.3)\n",
    "        \n",
    "        action = np.argmax(q_table[state,:])\n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        \n",
    "        if done:\n",
    "            clear_output(wait=True)\n",
    "            env.render()\n",
    "            if reward == 1:\n",
    "                print('YOU\\'VE REACHED THE GOAL !!!')\n",
    "            else:\n",
    "                print('YOU FELL THROUH A HOLE :(')\n",
    "            time.sleep(3)\n",
    "            clear_output(wait=True)\n",
    "            break\n",
    "        state = new_state\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}