class MDP:\n",
+ "\n",
+ " """A Markov Decision Process, defined by an initial state, transition model,\n",
+ " and reward function. We also keep track of a gamma value, for use by\n",
+ " algorithms. The transition model is represented somewhat differently from\n",
+ " the text. Instead of P(s' | s, a) being a probability number for each\n",
+ " state/state/action triplet, we instead have T(s, a) return a\n",
+ " list of (p, s') pairs. We also keep track of the possible states,\n",
+ " terminal states, and actions for each state. [page 646]"""\n",
+ "\n",
+ " def __init__(self, init, actlist, terminals, transitions = {}, reward = None, states=None, gamma=.9):\n",
+ " if not (0 < gamma <= 1):\n",
+ " raise ValueError("An MDP must have 0 < gamma <= 1")\n",
+ "\n",
+ " if states:\n",
+ " self.states = states\n",
+ " else:\n",
+ " ## collect states from transitions table\n",
+ " self.states = self.get_states_from_transitions(transitions)\n",
+ " \n",
+ " \n",
+ " self.init = init\n",
+ " \n",
+ " if isinstance(actlist, list):\n",
+ " ## if actlist is a list, all states have the same actions\n",
+ " self.actlist = actlist\n",
+ " elif isinstance(actlist, dict):\n",
+ " ## if actlist is a dict, different actions for each state\n",
+ " self.actlist = actlist\n",
+ " \n",
+ " self.terminals = terminals\n",
+ " self.transitions = transitions\n",
+ " if self.transitions == {}:\n",
+ " print("Warning: Transition table is empty.")\n",
+ " self.gamma = gamma\n",
+ " if reward:\n",
+ " self.reward = reward\n",
+ " else:\n",
+ " self.reward = {s : 0 for s in self.states}\n",
+ " #self.check_consistency()\n",
+ "\n",
+ " def R(self, state):\n",
+ " """Return a numeric reward for this state."""\n",
+ " return self.reward[state]\n",
+ "\n",
+ " def T(self, state, action):\n",
+ " """Transition model. From a state and an action, return a list\n",
+ " of (probability, result-state) pairs."""\n",
+ " if(self.transitions == {}):\n",
+ " raise ValueError("Transition model is missing")\n",
+ " else:\n",
+ " return self.transitions[state][action]\n",
+ "\n",
+ " def actions(self, state):\n",
+ " """Set of actions that can be performed in this state. By default, a\n",
+ " fixed list of actions, except for terminal states. Override this\n",
+ " method if you need to specialize by state."""\n",
+ " if state in self.terminals:\n",
+ " return [None]\n",
+ " else:\n",
+ " return self.actlist\n",
+ "\n",
+ " def get_states_from_transitions(self, transitions):\n",
+ " if isinstance(transitions, dict):\n",
+ " s1 = set(transitions.keys())\n",
+ " s2 = set([tr[1] for actions in transitions.values() \n",
+ " for effects in actions.values() for tr in effects])\n",
+ " return s1.union(s2)\n",
+ " else:\n",
+ " print('Could not retrieve states from transitions')\n",
+ " return None\n",
+ "\n",
+ " def check_consistency(self):\n",
+ " # check that all states in transitions are valid\n",
+ " assert set(self.states) == self.get_states_from_transitions(self.transitions)\n",
+ " # check that init is a valid state\n",
+ " assert self.init in self.states\n",
+ " # check reward for each state\n",
+ " #assert set(self.reward.keys()) == set(self.states)\n",
+ " assert set(self.reward.keys()) == set(self.states)\n",
+ " # check that all terminals are valid states\n",
+ " assert all([t in self.states for t in self.terminals])\n",
+ " # check that probability distributions for all actions sum to 1\n",
+ " for s1, actions in self.transitions.items():\n",
+ " for a in actions.keys():\n",
+ " s = 0\n",
+ " for o in actions[a]:\n",
+ " s += o[0]\n",
+ " assert abs(s - 1) < 0.001\n",
+ "class GridMDP(MDP):\n",
+ "\n",
+ " """A two-dimensional grid MDP, as in [Figure 17.1]. All you have to do is\n",
+ " specify the grid as a list of lists of rewards; use None for an obstacle\n",
+ " (unreachable state). Also, you should specify the terminal states.\n",
+ " An action is an (x, y) unit vector; e.g. (1, 0) means move east."""\n",
+ "\n",
+ " def __init__(self, grid, terminals, init=(0, 0), gamma=.9):\n",
+ " grid.reverse() # because we want row 0 on bottom, not on top\n",
+ " reward = {}\n",
+ " states = set()\n",
+ " self.rows = len(grid)\n",
+ " self.cols = len(grid[0])\n",
+ " self.grid = grid\n",
+ " for x in range(self.cols):\n",
+ " for y in range(self.rows):\n",
+ " if grid[y][x] is not None:\n",
+ " states.add((x, y))\n",
+ " reward[(x, y)] = grid[y][x]\n",
+ " self.states = states\n",
+ " actlist = orientations\n",
+ " transitions = {}\n",
+ " for s in states:\n",
+ " transitions[s] = {}\n",
+ " for a in actlist:\n",
+ " transitions[s][a] = self.calculate_T(s, a)\n",
+ " MDP.__init__(self, init, actlist=actlist,\n",
+ " terminals=terminals, transitions = transitions, \n",
+ " reward = reward, states = states, gamma=gamma)\n",
+ "\n",
+ " def calculate_T(self, state, action):\n",
+ " if action is None:\n",
+ " return [(0.0, state)]\n",
+ " else:\n",
+ " return [(0.8, self.go(state, action)),\n",
+ " (0.1, self.go(state, turn_right(action))),\n",
+ " (0.1, self.go(state, turn_left(action)))]\n",
+ " \n",
+ " def T(self, state, action):\n",
+ " if action is None:\n",
+ " return [(0.0, state)]\n",
+ " else:\n",
+ " return self.transitions[state][action]\n",
+ " \n",
+ " def go(self, state, direction):\n",
+ " """Return the state that results from going in this direction."""\n",
+ " state1 = vector_add(state, direction)\n",
+ " return state1 if state1 in self.states else state\n",
+ "\n",
+ " def to_grid(self, mapping):\n",
+ " """Convert a mapping from (x, y) to v into a [[..., v, ...]] grid."""\n",
+ " return list(reversed([[mapping.get((x, y), None)\n",
+ " for x in range(self.cols)]\n",
+ " for y in range(self.rows)]))\n",
+ "\n",
+ " def to_arrows(self, policy):\n",
+ " chars = {\n",
+ " (1, 0): '>', (0, 1): '^', (-1, 0): '<', (0, -1): 'v', None: '.'}\n",
+ " return self.to_grid({s: chars[a] for (s, a) in policy.items()})\n",
+ "def value_iteration(mdp, epsilon=0.001):\n",
+ " """Solving an MDP by value iteration. [Figure 17.4]"""\n",
+ " U1 = {s: 0 for s in mdp.states}\n",
+ " R, T, gamma = mdp.R, mdp.T, mdp.gamma\n",
+ " while True:\n",
+ " U = U1.copy()\n",
+ " delta = 0\n",
+ " for s in mdp.states:\n",
+ " U1[s] = R(s) + gamma * max([sum([p * U[s1] for (p, s1) in T(s, a)])\n",
+ " for a in mdp.actions(s)])\n",
+ " delta = max(delta, abs(U1[s] - U[s]))\n",
+ " if delta < epsilon * (1 - gamma) / gamma:\n",
+ " return U\n",
+ "
Value Iteration starts with arbitrary initial values for the utilities, calculates the right side of the Bellman equation and plugs it into the left hand side, thereby updating the utility of each state from the utilities of its neighbors. \n",
@@ -343,23 +749,11 @@
"As you might have noticed, `value_iteration` has an infinite loop. How do we decide when to stop iterating? \n",
"The concept of _contraction_ successfully explains the convergence of value iteration. \n",
"Refer to **Section 17.2.3** of the book for a detailed explanation. \n",
-<<<<<<< HEAD
-<<<<<<< HEAD
- "In the algorithm, we calculate a value $\\delta$ that measures the difference in the utilities of the current time step and the previous time step. \n",
-=======
"In the algorithm, we calculate a value $delta$ that measures the difference in the utilities of the current time step and the previous time step. \n",
->>>>>>> 9d5ec3c0e1d0c03cd1333afcbd6bbc35daf30c21
-=======
- "In the algorithm, we calculate a value $\\delta$ that measures the difference in the utilities of the current time step and the previous time step. \n",
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
"\n",
"$$\\delta = \\max{(\\delta, \\begin{vmatrix}U_{i + 1}(s) - U_i(s)\\end{vmatrix})}$$\n",
"\n",
"This value of delta decreases as the values of $U_i$ converge.\n",
-<<<<<<< HEAD
-<<<<<<< HEAD
-=======
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
"We terminate the algorithm if the $\\delta$ value is less than a threshold value determined by the hyperparameter _epsilon_.\n",
"\n",
"$$\\delta \\lt \\epsilon \\frac{(1 - \\gamma)}{\\gamma}$$\n",
@@ -368,25 +762,13 @@
"Hence, from the properties of contractions in general, it follows that `value_iteration` always converges to a unique solution of the Bellman equations whenever $gamma$ is less than 1.\n",
"We then terminate the algorithm when a reasonable approximation is achieved.\n",
"In practice, it often occurs that the policy $pi$ becomes optimal long before the utility function converges. For the given 4 x 3 environment with $gamma = 0.9$, the policy $pi$ is optimal when $i = 4$ (at the 4th iteration), even though the maximum error in the utility function is stil 0.46. This can be clarified from **figure 17.6** in the book. Hence, to increase computational efficiency, we often use another method to solve MDPs called Policy Iteration which we will see in the later part of this notebook. \n",
-=======
- "We terminate the algorithm if the $delta$ value is less than a threshold value determined by the hyperparameter _epsilon_.\n",
- "\n",
- "$$\\delta \\lt \\epsilon \\frac{(1 - \\gamma)}{\\gamma}$$\n",
- "\n",
- "To summarize, the Bellman update is a _contraction_ by a factor of $\\gamma$ on the space of utility vectors. \n",
- "Hence, from the properties of contractions in general, it follows that `value_iteration` always converges to a unique solution of the Bellman equations whenever $\\gamma$ is less than 1.\n",
- "We then terminate the algorithm when a reasonable approximation is achieved.\n",
- "In practice, it often occurs that the policy $\\pi$ becomes optimal long before the utility function converges. For the given 4 x 3 environment with $\gamma = 0.9$, the policy $\\pi$ is optimal when $i = 4$ (at the 4th iteration), even though the maximum error in the utility function is stil 0.46. This can be clarified from **figure 17.6** in the book. Hence, to increase computational efficiency, we often use another method to solve MDPs called Policy Iteration which we will see in the later part of this notebook. \n",
->>>>>>> 9d5ec3c0e1d0c03cd1333afcbd6bbc35daf30c21
"
For now, let us solve the **sequential_decision_environment** GridMDP using `value_iteration`."
]
},
{
-<<<<<<< HEAD
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {},
-<<<<<<< HEAD
"outputs": [
{
"data": {
@@ -409,30 +791,21 @@
"output_type": "execute_result"
}
],
-=======
- "cell_type": "raw",
- "metadata": {},
->>>>>>> 9d5ec3c0e1d0c03cd1333afcbd6bbc35daf30c21
-=======
- "outputs": [],
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
"source": [
"value_iteration(sequential_decision_environment)"
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {},
"source": [
"The pseudocode for the algorithm:"
]
},
{
-<<<<<<< HEAD
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 10,
"metadata": {},
-<<<<<<< HEAD
"outputs": [
{
"data": {
@@ -465,19 +838,12 @@
"output_type": "execute_result"
}
],
-=======
- "cell_type": "raw",
- "metadata": {},
->>>>>>> 9d5ec3c0e1d0c03cd1333afcbd6bbc35daf30c21
-=======
- "outputs": [],
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
"source": [
"pseudocode(\"Value-Iteration\")"
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {},
"source": [
"### AIMA3e\n",
@@ -501,7 +867,7 @@
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {},
"source": [
"## VALUE ITERATION VISUALIZATION\n",
@@ -510,15 +876,12 @@
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
-=======
"cell_type": "code",
- "execution_count": null,
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "execution_count": 11,
"metadata": {
"collapsed": true
},
+ "outputs": [],
"source": [
"def value_iteration_instru(mdp, iterations=20):\n",
" U_over_time = []\n",
@@ -534,22 +897,19 @@
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {},
"source": [
"Next, we define a function to create the visualisation from the utilities returned by **value_iteration_instru**. The reader need not concern himself with the code that immediately follows as it is the usage of Matplotib with IPython Widgets. If you are interested in reading more about these visit [ipywidgets.readthedocs.io](http://ipywidgets.readthedocs.io)"
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
-=======
"cell_type": "code",
- "execution_count": null,
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "execution_count": 12,
"metadata": {
"collapsed": true
},
+ "outputs": [],
"source": [
"columns = 4\n",
"rows = 3\n",
@@ -557,15 +917,12 @@
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
-=======
"cell_type": "code",
- "execution_count": null,
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "execution_count": 13,
"metadata": {
"collapsed": true
},
+ "outputs": [],
"source": [
"%matplotlib inline\n",
"from notebook import make_plot_grid_step_function\n",
@@ -574,19 +931,39 @@
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
- "metadata": {
- "scrolled": true
- },
-=======
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {
"scrolled": true
},
- "outputs": [],
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAATcAAADuCAYAAABcZEBhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADYxJREFUeJzt211oW2eex/Hf2Xpb0onWrVkm1otL\nW2SmrNaVtzS2K8jCFhJPXsbtRWcTX4zbmUBINkMYw5jmYrYwhNJuMWTjaTCYDSW5cQK9iEOcpDad\nLAREVtBEF+OwoDEyWEdxirvjelw36cScubCi1PWLvK0lnfnP9wMGHz2P4dEf8fWRnDie5wkArPmb\nah8AAMqBuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMKnm/7N5bk78dwagjDYHnGofwf88\nb11D4s4NgEnEDYBJxA2AScQNgEnEDYBJxA2AScQNgEnEDYBJxA2AScQNgEnEDYBJxA2AScQNgEnE\nDYBJxA2AScQNgEnEDYBJxA2AScQNgEnEDYBJxA2AScQNgEnEDYBJxA2AScQNgEnEDYBJxA2AScQN\ngEnEDYBJxA2AScQNgEm+jZvneerpOaJ4PKq2tueVTt9Ycd/Nm5+otbVJ8XhUPT1H5HnekvUTJ3oV\nCDianp6uxLErhvmUxoxW9zNJ35f0j6use5KOSIpKel7S1yd3WlJj4et0Gc/4Xfk2biMjlzU+nlE6\nnVFf34C6uw+tuK+7+5D6+gaUTmc0Pp7R6OiV4louN6mrV0fV0PBUpY5dMcynNGa0ujckXVlj/bKk\nTOFrQNKDyf2fpF9L+h9JqcL3fyjbKb8b38ZteHhInZ1dchxHLS1tmpmZ0dTU7SV7pqZua3Z2Vq2t\nL8lxHHV2dunixfPF9aNHu3Xs2HtyHKfSxy875lMaM1rdP0uqW2N9SFKXJEdSm6QZSbclfSRpe+Fn\nnyx8v1Ykq8m3ccvnXYXDDcXrcDiifN5dYU+keB0KPdwzPHxBoVBYTU3xyhy4wphPaczo23MlNXzt\nOlJ4bLXH/aim2gdYzTc/95C07Lfnanvm5+fV2/u2zp8fKdv5qo35lMaMvr3lU1m8i1vtcT/y1Z3b\nwMBJJRLNSiSaFQyG5LqTxTXXzSkYDC3ZHw5H5Lq54nU+v7gnmx3XxERWiURcsdjTct2ctm17QXfu\nTFXsuZQD8ymNGW2MiKTJr13nJIXWeNyPfBW3AwcOK5lMK5lMa8+eVzU4eEae5ymVuq7a2lrV1weX\n7K+vDyoQCCiVui7P8zQ4eEa7d7+iWKxJ2eynGhub0NjYhMLhiK5du6EtW+qr9Mw2BvMpjRltjA5J\nZ7R4p3ZdUq2koKR2SSNa/CPCHwrft1fpjKX49m1pe/sujYxcUjwe1aZNj6u//4PiWiLRrGQyLUk6\nfrxfBw++obt3v9T27Tu1Y8fOah25ophPacxodZ2S/lvStBbvxn4t6U+FtYOSdkm6pMV/CvK4pAeT\nq5P075K2Fq7f0tp/mKgmZ6XPHFYzN7fiW24AG2RzwK+fYPmI561rSL56WwoAG4W4ATCJuAEwibgB\nMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEw\nibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMIm4ATCJuAEwibgBMKmm2gew\nZPP3vGofwffmvnCqfQRfc8RrqJT1Tog7NwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3\nACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcA\nJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAm+TZunuepp+eI4vGo2tqeVzp9Y8V9N29+\notbWJsXjUfX0HJHneUvWT5zoVSDgaHp6uhLHrpgrV67oB889p2hjo959991l6/fu3dPeffsUbWxU\na1ubJiYmimvvvPOOoo2N+sFzz+mjjz6q4Kkri9dQKf8r6SVJj0nqXWNfVlKrpEZJeyV9VXj8XuE6\nWlifKNdBvxXfxm1k5LLGxzNKpzPq6xtQd/ehFfd1dx9SX9+A0umMxsczGh29UlzL5SZ19eqoGhqe\nqtSxK2JhYUGHf/5zXb50SbfGxjR49qxu3bq1ZM+pU6f05BNP6PeZjLp/8Qu9efSoJOnWrVs6e+6c\nxn73O125fFn/dviwFhYWqvE0yo7XUCl1kvok/bLEvjcldUvKSHpS0qnC46cK178vrL9ZnmN+S76N\n2/DwkDo7u+Q4jlpa2jQzM6OpqdtL9kxN3dbs7KxaW1+S4zjq7OzSxYvni+tHj3br2LH35DhOpY9f\nVqlUStFoVM8++6weffRR7du7V0NDQ0v2DF24oNdff12S9Nprr+njjz+W53kaGhrSvr179dhjj+mZ\nZ55RNBpVKpWqxtMoO15DpXxf0lZJf7vGHk/SbyW9Vrh+XdKD+QwVrlVY/7iw3x98G7d83lU43FC8\nDocjyufdFfZEiteh0MM9w8MXFAqF1dQUr8yBK8h1XTVEHj7vSCQi13WX72lYnF9NTY1qa2v12Wef\nLXlckiLh8LKftYLX0Eb4TNITkmoK1xFJD2boSnow3xpJtYX9/lBTekt1fPNzD0nLfnuutmd+fl69\nvW/r/PmRsp2vmr7LbNbzs1bwGtoIK92JOetYqz5f3bkNDJxUItGsRKJZwWBIrjtZXHPdnILB0JL9\n4XBErpsrXufzi3uy2XFNTGSVSMQViz0t181p27YXdOfOVMWeSzlFIhFN5h4+71wup1AotHzP5OL8\n7t+/r88//1x1dXVLHpeknOsu+9m/ZLyGSjkpqbnwlV/H/r+XNCPpfuE6J+nBDCOSHsz3vqTPtfg5\nnj/4Km4HDhxWMplWMpnWnj2vanDwjDzPUyp1XbW1taqvDy7ZX18fVCAQUCp1XZ7naXDwjHbvfkWx\nWJOy2U81NjahsbEJhcMRXbt2Q1u21FfpmW2srVu3KpPJKJvN6quvvtLZc+fU0dGxZE/Hj36k06dP\nS5I+/PBDvfzyy3IcRx0dHTp77pzu3bunbDarTCajlpaWajyNsuA1VMphSenC13p+qTmS/kXSh4Xr\n05JeKXzfUbhWYf1l+enOzbdvS9vbd2lk5JLi8ag2bXpc/f0fFNcSiWYlk2lJ0vHj/Tp48A3dvful\ntm/fqR07dlbryBVTU1Oj93/zG7X/8IdaWFjQz376U8ViMb311lt68cUX1dHRof379+snXV2KNjaq\nrq5OZwcHJUmxWEz/+uMf6x9iMdXU1Ojk++/rkUceqfIzKg9eQ6VMSXpR0qwW73P+U9ItSX8naZek\n/9JiAP9D0j5Jv5L0T5L2F35+v6SfaPGfgtRJOlvBs5fmrPSZw2rm5nz0pxAf2vw9xlPK3Bf++c3u\nR4FAtU/gf563vttDX70tBYCNQtwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwA\nmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYRNwAmETcAJhE3ACY\nRNwAmETcAJhE3ACYRNwAmETcAJhE3ACYVFPtA1gy94VT7SPgL9wf/1jtE9jBnRsAk4gbAJOIGwCT\niBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOI\nGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gb\nAJN8GzfP89TTc0TxeFRtbc8rnb6x4r6bNz9Ra2uT4vGoenqOyPO8JesnTvQqEHA0PT1diWNXDPMp\njRmtzfp8fBu3kZHLGh/PKJ3OqK9vQN3dh1bc1919SH19A0qnMxofz2h09EpxLZeb1NWro2poeKpS\nx64Y5lMaM1qb9fn4Nm7Dw0Pq7OyS4zhqaWnTzMyMpqZuL9kzNXVbs7Ozam19SY7jqLOzSxcvni+u\nHz3arWPH3pPjOJU+ftkxn9KY0dqsz8e3ccvnXYXDDcXrcDiifN5dYU+keB0KPdwzPHxBoVBYTU3x\nyhy4wphPacxobdbnU1PtA6zmm+/rJS377bDanvn5efX2vq3z50fKdr5qYz6lMaO1WZ+Pr+7cBgZO\nKpFoViLRrGAwJNedLK65bk7BYGjJ/nA4ItfNFa/z+cU92ey4JiaySiTiisWeluvmtG3bC7pzZ6pi\nz6UcmE9pzGhtf03z8VXcDhw4rGQyrWQyrT17XtXg4Bl5nqdU6rpqa2tVXx9csr++PqhAIKBU6ro8\nz9Pg4Bnt3v2KYrEmZbOfamxsQmNjEwqHI7p27Ya2bKmv0jPbGMynNGa0tr+m+fj2bWl7+y6NjFxS\nPB7Vpk2Pq7//g+JaItGsZDItSTp+vF8HD76hu3e/1PbtO7Vjx85qHbmimE9pzGht1ufjrPSeejVz\nc1r/ZgAog82bta4/zfrqbSkAbBTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTi\nBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIG\nwCTiBsAk4gbAJOIGwCTiBsAk4gbAJOIGwCTH87xqnwEANhx3bgBMIm4ATCJuAEwibgBMIm4ATCJu\nAEwibgBMIm4ATCJuAEwibgBM+jPdN0cNjYpeKAAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "The installed widget Javascript is the wrong version. It must satisfy the semver range ~2.1.4.\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "77e9849e074841e49d8b0ebc8191507c"
+ }
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"import ipywidgets as widgets\n",
"from IPython.display import display\n",
@@ -605,14 +982,14 @@
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {},
"source": [
"Move the slider above to observe how the utility changes across iterations. It is also possible to move the slider using arrow keys or to jump to the value by directly editing the number with a double click. The **Visualize Button** will automatically animate the slider for you. The **Extra Delay Box** allows you to set time delay in seconds upto one second for each time step. There is also an interactive editor for grid-world problems `grid_mdp.py` in the gui folder for you to play around with."
]
},
{
- "cell_type": "raw",
+ "cell_type": "markdown",
"metadata": {
"collapsed": true
},
@@ -639,35 +1016,244 @@
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
- "metadata": {},
-=======
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {},
- "outputs": [],
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "def expected_utility(a, s, U, mdp):\n",
+ " """The expected utility of doing a in state s, according to the MDP and U."""\n",
+ " return sum([p * U[s1] for (p, s1) in mdp.T(s, a)])\n",
+ "
def policy_iteration(mdp):\n",
+ " """Solve an MDP by policy iteration [Figure 17.7]"""\n",
+ " U = {s: 0 for s in mdp.states}\n",
+ " pi = {s: random.choice(mdp.actions(s)) for s in mdp.states}\n",
+ " while True:\n",
+ " U = policy_evaluation(pi, U, mdp)\n",
+ " unchanged = True\n",
+ " for s in mdp.states:\n",
+ " a = argmax(mdp.actions(s), key=lambda a: expected_utility(a, s, U, mdp))\n",
+ " if a != pi[s]:\n",
+ " pi[s] = a\n",
+ " unchanged = False\n",
+ " if unchanged:\n",
+ " return pi\n",
+ "
Fortunately, it is not necessary to do _exact_ policy evaluation. \n",
@@ -680,46 +1266,164 @@
]
},
{
-<<<<<<< HEAD
- "cell_type": "raw",
- "metadata": {},
-=======
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 17,
"metadata": {},
- "outputs": [],
->>>>>>> 3fed6614295b7270ca1226415beff7305e387eeb
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "def policy_evaluation(pi, U, mdp, k=20):\n",
+ " """Return an updated utility mapping U from each state in the MDP to its\n",
+ " utility, using an approximation (modified policy iteration)."""\n",
+ " R, T, gamma = mdp.R, mdp.T, mdp.gamma\n",
+ " for i in range(k):\n",
+ " for s in mdp.states:\n",
+ " U[s] = R(s) + gamma * sum([p * U[s1] for (p, s1) in T(s, pi[s])])\n",
+ " return U\n",
+ "
def T(self, state, action):\n",
+ " if action is None:\n",
+ " return [(0.0, state)]\n",
+ " else:\n",
+ " return self.transitions[state][action]\n",
+ "
def to_arrows(self, policy):\n",
+ " chars = {\n",
+ " (1, 0): '>', (0, 1): '^', (-1, 0): '<', (0, -1): 'v', None: '.'}\n",
+ " return self.to_grid({s: chars[a] for (s, a) in policy.items()})\n",
+ " def to_grid(self, mapping):\n",
+ " """Convert a mapping from (x, y) to v into a [[..., v, ...]] grid."""\n",
+ " return list(reversed([[mapping.get((x, y), None)\n",
+ " for x in range(self.cols)]\n",
+ " for y in range(self.rows)]))\n",
+ "