Action Value Functions

Here’s how to work with Q-values in Python:

import numpy as np
from collections import defaultdict

# Q-values as a nested dictionary: Q[state][action] = value
Q = defaultdict(lambda: defaultdict(float))

# Set some Q-values
Q['start']['left'] = 3.2
Q['start']['right'] = 7.8
Q['start']['up'] = 2.1

def get_best_action(Q, state):
    """Return the action with highest Q-value for a state."""
    actions = Q[state]
    if not actions:
        return None
    return max(actions, key=lambda a: actions[a])

def get_state_value(Q, state, policy=None):
    """
    Compute V(s) from Q-values.

    If policy is None, use greedy (max) policy.
    Otherwise, policy[state] should be a dict of {action: probability}.
    """
    actions = Q[state]
    if not actions:
        return 0.0

    if policy is None:
        # Greedy: V(s) = max_a Q(s,a)
        return max(actions.values())
    else:
        # Stochastic: V(s) = sum_a π(a|s) * Q(s,a)
        probs = policy.get(state, {})
        return sum(probs.get(a, 0) * q for a, q in actions.items())


# Example: Greedy action selection
best = get_best_action(Q, 'start')
print(f"Best action from 'start': {best}")  # right

# Example: State value under greedy policy
v = get_state_value(Q, 'start')
print(f"V(start) under greedy policy: {v}")  # 7.8

# Example: State value under uniform policy
uniform_policy = {
    'start': {'left': 1/3, 'right': 1/3, 'up': 1/3}
}
v_uniform = get_state_value(Q, 'start', uniform_policy)
print(f"V(start) under uniform policy: {v_uniform:.2f}")  # 4.37

Just as we can estimate V-values from episodes, we can estimate Q-values:

import numpy as np
from collections import defaultdict

def estimate_q_values_mc(episodes, gamma=0.99):
    """
    Estimate Q(s,a) using Monte Carlo from episodes.

    Each episode is a list of (state, action, reward) tuples.
    """
    returns = defaultdict(list)  # returns[(s,a)] = list of observed returns

    for episode in episodes:
        # Calculate returns for each (state, action) pair
        G = 0
        # Work backward through the episode
        for t in range(len(episode) - 1, -1, -1):
            state, action, reward = episode[t]
            G = reward + gamma * G
            returns[(state, action)].append(G)

    # Average the returns for each (state, action)
    Q = defaultdict(dict)
    for (state, action), obs_returns in returns.items():
        Q[state][action] = np.mean(obs_returns)

    return Q


# Example episodes
episodes = [
    # Episode 1: start --right--> middle --right--> goal
    [('start', 'right', -1), ('middle', 'right', 10)],
    # Episode 2: start --down--> bottom --right--> goal
    [('start', 'down', -1), ('bottom', 'right', 10)],
    # Episode 3: start --right--> middle --right--> goal
    [('start', 'right', -1), ('middle', 'right', 10)],
]

Q = estimate_q_values_mc(episodes, gamma=0.9)

print("Estimated Q-values:")
for state in sorted(Q.keys()):
    for action, value in Q[state].items():
        print(f"  Q({state}, {action}) = {value:.2f}")

Output:

Estimated Q-values:
  Q(bottom, right) = 10.00
  Q(middle, right) = 10.00
  Q(start, down) = 8.00
  Q(start, right) = 8.00

Note: Q(start, right) = -1 + 0.9 * 10 = 8.0, which matches our estimate!

import numpy as np

class QTable:
    """A simple Q-table for discrete state and action spaces."""

    def __init__(self, n_states, n_actions, init_value=0.0):
        self.table = np.full((n_states, n_actions), init_value)

    def __getitem__(self, key):
        state, action = key
        return self.table[state, action]

    def __setitem__(self, key, value):
        state, action = key
        self.table[state, action] = value

    def get_best_action(self, state):
        """Return action with highest Q-value."""
        return np.argmax(self.table[state])

    def get_action_values(self, state):
        """Return Q-values for all actions in a state."""
        return self.table[state].copy()

    def visualize(self, state_names=None, action_names=None):
        """Print the Q-table nicely formatted."""
        n_states, n_actions = self.table.shape

        if state_names is None:
            state_names = [f"s{i}" for i in range(n_states)]
        if action_names is None:
            action_names = [f"a{i}" for i in range(n_actions)]

        # Header
        header = "State | " + " | ".join(f"{a:>8}" for a in action_names)
        print(header)
        print("-" * len(header))

        # Rows
        for i, state in enumerate(state_names):
            values = " | ".join(f"{self.table[i,j]:>8.2f}"
                              for j in range(n_actions))
            print(f"{state:>5} | {values}")


# Example: 4-state, 2-action Q-table
Q = QTable(4, 2)
Q[0, 0] = 5.2   # Q(state0, action0)
Q[0, 1] = 8.7   # Q(state0, action1)
Q[1, 0] = 3.1
Q[1, 1] = 3.5
Q[2, 0] = 6.0
Q[2, 1] = 5.8
Q[3, 0] = 0.0   # Goal is terminal: no future reward,
Q[3, 1] = 0.0   # so its Q-values stay 0

Q.visualize(
    state_names=['Start', 'Left', 'Right', 'Goal'],
    action_names=['Left', 'Right']
)

Output:

State |     Left |    Right
---------------------------
Start |     5.20 |     8.70
 Left |     3.10 |     3.50
Right |     6.00 |     5.80
 Goal |     0.00 |     0.00

The best action in each non-terminal state is the one with the higher value (shown by comparing columns). The terminal Goal row stays at zero: rewards for reaching it live on the transitions into it.