使用强化学习训练机械臂完成人类任务

from collections import deque
import random
import numpy as np

class ReplayBuffer(object):

      def __init__(self, buffer_size):
            self.buffer_size = buffer_size
            self.count = 0
            self.buffer = deque()

      def add(self, s, a, r, t, s2):
            experience = (s, a, r, t, s2)
            if self.count < self.buffer_size: 
      self.buffer.append(experience)
      self.count += 1
            else:
      self.buffer.popleft()
      self.buffer.append(experience)

        def size(self):
      return self.count

        def sample_batch(self, batch_size):
      '''     
      batch_size specifies the number of experiences to add 
      to the batch. If the replay buffer has less than batch_size
              elements, simply return all of the elements within the buffer.
              Generally, you'll want to wait until the buffer has at least 
              batch_size elements before beginning to sample from it.
              '''
              batch = []

              if self.count < batch_size:
                    batch = random.sample(self.buffer, self.count)
              else:
                    batch = random.sample(self.buffer, batch_size)

              s_batch = np.array([_[0] for _ in batch])
              a_batch = np.array([_[1] for _ in batch])
              r_batch = np.array([_[2] for _ in batch])
              t_batch = np.array([_[3] for _ in batch])
              s2_batch = np.array([_[4] for _ in batch])

              return s_batch, a_batch, r_batch, t_batch, s2_batch

        def clear(self):
              self.buffer.clear()
              self.count = 0

class ActorNetwork(object):

      ...

      def create_actor_network(self):
            inputs = tflearn.input_data(shape=[None, self.s_dim])
            net = tflearn.fully_connected(inputs, 400)
            net = tflearn.layers.normalization.batch_normalization(net)
            net = tflearn.activations.relu(net)
            net = tflearn.fully_connected(net, 300)
            net = tflearn.layers.normalization.batch_normalization(net)
            net = tflearn.activations.relu(net)
            # Final layer weights are init to Uniform[-3e-3, 3e-3]
            w_init = tflearn.initializations.uniform(minval=-0.003, maxval=0.003)
            out = tflearn.fully_connected(
                net, self.a_dim, activation='tanh', weights_init=w_init)
            # Scale output to -action_bound to action_bound
            scaled_out = tf.multiply(out, self.action_bound)
            return inputs, out, scaled_out


class CriticNetwork(object):

      ...

      def create_critic_network(self):
            inputs = tflearn.input_data(shape=[None, self.s_dim])
            action = tflearn.input_data(shape=[None, self.a_dim])
            net = tflearn.fully_connected(inputs, 400)
            net = tflearn.layers.normalization.batch_normalization(net)
            net = tflearn.activations.relu(net)

            # Add the action tensor in the 2nd hidden layer
            # Use two temp layers to get the corresponding weights and biases
            t1 = tflearn.fully_connected(net, 300)
            t2 = tflearn.fully_connected(action, 300)

            net = tflearn.activation(
                  tf.matmul(net, t1.W) + tf.matmul(action, t2.W) + t2.b, activation='relu')

            # linear layer connected to 1 output representing Q(s,a)
            # Weights are init to Uniform[-3e-3, 3e-3]
            w_init = tflearn.initializations.uniform(minval=-0.003, maxval=0.003)
            out = tflearn.fully_connected(net, 1, weights_init=w_init)
            return inputs, action, out

# This gradient will be provided by the critic network
self.action_gradient = tf.placeholder(tf.float32, [None, self.a_dim])

# Combine the gradients, dividing by the batch size to 
# account for the fact that the gradients are summed over the 
# batch by tf.gradients 
self.unnormalized_actor_gradients = tf.gradients(
    self.scaled_out, self.network_params, -self.action_gradient)
self.actor_gradients = list(map(lambda x: tf.div(x, self.batch_size), self.unnormalized_actor_gradients))

# Optimization Op
self.optimize = tf.train.AdamOptimizer(self.learning_rate).\
    apply_gradients(zip(self.actor_gradients, self.network_params))

# Network target (y_i) # Obtained from the target networks self.predicted_q_value = tf.placeholder(tf.float32, [None, 1]) # Define loss and optimization Op self.loss = tflearn.mean_square(self.predicted_q_value, self.out) self.optimize = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss) # Get the gradient of the net w.r.t. the action self.action_grads = tf.gradients(self.out, self.action)