convlab2中强化学习方法之对话策略学习浅析

class Value(nn.Module):
    def __init__(self, s_dim, hv_dim):
        super(Value, self).__init__()

        self.net = nn.Sequential(nn.Linear(s_dim, hv_dim),
                                 nn.ReLU(),
                                 nn.Linear(hv_dim, hv_dim),
                                 nn.ReLU(),
                                 nn.Linear(hv_dim, 1))

    def forward(self, s):
        """
        :param s: [b, s_dim]
        :return:  [b, 1]
        """
        value = self.net(s)

        return value

self.net = nn.Sequential(nn.Linear(s_dim, h_dim),
                         nn.ReLU(),
                         nn.Linear(h_dim, h_dim),
                         nn.ReLU(),
                         nn.Linear(h_dim, a_dim))

def select_action(self, s, sample=True):
    """
    :param s: [s_dim]
    :return: [a_dim]
    """
    # forward to get action probs
    # [s_dim] => [a_dim]
    a_weights = self.forward(s)
    a_probs = torch.sigmoid(a_weights)
    
    # [a_dim] => [a_dim, 2]
    a_probs = a_probs.unsqueeze(1)
    a_probs = torch.cat([1-a_probs, a_probs], 1)
    a_probs = torch.clamp(a_probs, 1e-10, 1 - 1e-10)
    
    # [a_dim, 2] => [a_dim]
    a = a_probs.multinomial(1).squeeze(1) if sample else a_probs.argmax(1)
    
    return a

def est_adv(self, r, v, mask):
    """
    we save a trajectory in continuous space and it reaches the ending of current trajectory when mask=0.
    :param r: reward, Tensor, [b]
    :param v: estimated value, Tensor, [b]
    :param mask: indicates ending for 0 otherwise 1, Tensor, [b]
    :return: A(s, a), V-target(s), both Tensor
    """
    batchsz = v.size(0)

    # v_target is worked out by Bellman equation.
    v_target = torch.Tensor(batchsz).to(device=DEVICE)
    delta = torch.Tensor(batchsz).to(device=DEVICE)
    A_sa = torch.Tensor(batchsz).to(device=DEVICE)

    prev_v_target = 0
    prev_v = 0
    prev_A_sa = 0
    for t in reversed(range(batchsz)):
        # mask here indicates a end of trajectory
        # this value will be treated as the target value of value network.
        # mask = 0 means the immediate reward is the real V(s) since it's end of trajectory.
        # formula: V(s_t) = r_t + gamma * V(s_t+1)
        v_target[t] = r[t] + self.gamma * prev_v_target * mask[t]

        # please refer to : https://arxiv.org/abs/1506.02438
        # for generalized adavantage estimation
        # formula: delta(s_t) = r_t + gamma * V(s_t+1) - V(s_t)
        delta[t] = r[t] + self.gamma * prev_v * mask[t] - v[t]

        # formula: A(s, a) = delta(s_t) + gamma * lamda * A(s_t+1, a_t+1)
        # here use symbol tau as lambda, but original paper uses symbol lambda.
        A_sa[t] = delta[t] + self.gamma * self.tau * prev_A_sa * mask[t]

        # update previous
        prev_v_target = v_target[t]
        prev_v = v[t]
        prev_A_sa = A_sa[t]

    # normalize A_sa
    A_sa = (A_sa - A_sa.mean()) / A_sa.std()

    return A_sa, v_target