class MDP:\n",
+ "\n",
+ " """A Markov Decision Process, defined by an initial state, transition model,\n",
+ " and reward function. We also keep track of a gamma value, for use by\n",
+ " algorithms. The transition model is represented somewhat differently from\n",
+ " the text. Instead of P(s' | s, a) being a probability number for each\n",
+ " state/state/action triplet, we instead have T(s, a) return a\n",
+ " list of (p, s') pairs. We also keep track of the possible states,\n",
+ " terminal states, and actions for each state. [page 646]"""\n",
+ "\n",
+ " def __init__(self, init, actlist, terminals, transitions={}, states=None, gamma=.9):\n",
+ " if not (0 < gamma <= 1):\n",
+ " raise ValueError("An MDP must have 0 < gamma <= 1")\n",
+ "\n",
+ " if states:\n",
+ " self.states = states\n",
+ " else:\n",
+ " self.states = set()\n",
+ " self.init = init\n",
+ " self.actlist = actlist\n",
+ " self.terminals = terminals\n",
+ " self.transitions = transitions\n",
+ " self.gamma = gamma\n",
+ " self.reward = {}\n",
+ "\n",
+ " def R(self, state):\n",
+ " """Return a numeric reward for this state."""\n",
+ " return self.reward[state]\n",
+ "\n",
+ " def T(self, state, action):\n",
+ " """Transition model. From a state and an action, return a list\n",
+ " of (probability, result-state) pairs."""\n",
+ " if(self.transitions == {}):\n",
+ " raise ValueError("Transition model is missing")\n",
+ " else:\n",
+ " return self.transitions[state][action]\n",
+ "\n",
+ " def actions(self, state):\n",
+ " """Set of actions that can be performed in this state. By default, a\n",
+ " fixed list of actions, except for terminal states. Override this\n",
+ " method if you need to specialize by state."""\n",
+ " if state in self.terminals:\n",
+ " return [None]\n",
+ " else:\n",
+ " return self.actlist\n",
+ "class GridMDP(MDP):\n",
+ "\n",
+ " """A two-dimensional grid MDP, as in [Figure 17.1]. All you have to do is\n",
+ " specify the grid as a list of lists of rewards; use None for an obstacle\n",
+ " (unreachable state). Also, you should specify the terminal states.\n",
+ " An action is an (x, y) unit vector; e.g. (1, 0) means move east."""\n",
+ "\n",
+ " def __init__(self, grid, terminals, init=(0, 0), gamma=.9):\n",
+ " grid.reverse() # because we want row 0 on bottom, not on top\n",
+ " MDP.__init__(self, init, actlist=orientations,\n",
+ " terminals=terminals, gamma=gamma)\n",
+ " self.grid = grid\n",
+ " self.rows = len(grid)\n",
+ " self.cols = len(grid[0])\n",
+ " for x in range(self.cols):\n",
+ " for y in range(self.rows):\n",
+ " self.reward[x, y] = grid[y][x]\n",
+ " if grid[y][x] is not None:\n",
+ " self.states.add((x, y))\n",
+ "\n",
+ " def T(self, state, action):\n",
+ " if action is None:\n",
+ " return [(0.0, state)]\n",
+ " else:\n",
+ " return [(0.8, self.go(state, action)),\n",
+ " (0.1, self.go(state, turn_right(action))),\n",
+ " (0.1, self.go(state, turn_left(action)))]\n",
+ "\n",
+ " def go(self, state, direction):\n",
+ " """Return the state that results from going in this direction."""\n",
+ " state1 = vector_add(state, direction)\n",
+ " return state1 if state1 in self.states else state\n",
+ "\n",
+ " def to_grid(self, mapping):\n",
+ " """Convert a mapping from (x, y) to v into a [[..., v, ...]] grid."""\n",
+ " return list(reversed([[mapping.get((x, y), None)\n",
+ " for x in range(self.cols)]\n",
+ " for y in range(self.rows)]))\n",
+ "\n",
+ " def to_arrows(self, policy):\n",
+ " chars = {\n",
+ " (1, 0): '>', (0, 1): '^', (-1, 0): '<', (0, -1): 'v', None: '.'}\n",
+ " return self.to_grid({s: chars[a] for (s, a) in policy.items()})\n",
+ "