ReinforceJS库（动态展示DP、TD、DQN算法运行过程）

// create environment
env = new Gridworld(); 
// create the agent, yay! Discount factor 0.9
agent = new RL.DPAgent(env, {'gamma':0.9}); 

// call this function repeatedly until convergence:
agent.learn();

// once trained, get the agent's behavior with:
var action = agent.act(); // returns the index of the chosen action

evaluatePolicy: function() {
  // perform a synchronous update of the value function
  var Vnew = zeros(this.ns); // initialize new value function array for each state
  for(var s=0;s < this.ns;s++) {
    var v = 0.0;
    var poss = this.env.allowedActions(s); // fetch all possible actions
    for(var i=0,n=poss.length;i < n;i++) {
      var a = poss[i];
      var prob = this.P[a*this.ns+s]; // probability of taking action under current policy
      var ns = this.env.nextStateDistribution(s,a); // look up the next state
      var rs = this.env.reward(s,a,ns); // get reward for s->a->ns transition
      v += prob * (rs + this.gamma * this.V[ns]);
    }
    Vnew[s] = v;
  }
  this.V = Vnew; // swap
},

updatePolicy: function() {
  // update policy to be greedy w.r.t. learned Value function
  // iterate over all states...
  for(var s=0;s < this.ns;s++) {
    var poss = this.env.allowedActions(s);
    // compute value of taking each allowed action
    var vmax, nmax;
    var vs = [];
    for(var i=0,n=poss.length;i < n;i++) {
      var a = poss[i];
      // compute the value of taking action a
      var ns = this.env.nextStateDistribution(s,a);
      var rs = this.env.reward(s,a,ns);
      var v = rs + this.gamma * this.V[ns];
      // bookeeping: store it and maintain max
      vs.push(v);
      if(i === 0 || v > vmax) { vmax = v; nmax = 1; }
      else if(v === vmax) { nmax += 1; }
    }
    // update policy smoothly across all argmaxy actions
    for(var i=0,n=poss.length;i < n;i++) {
      var a = poss[i];
      this.P[a*this.ns+s] = (vs[i] === vmax) ? 1.0/nmax : 0.0;
    }
  }
},

// agent parameter spec to play with (this gets eval()'d on Agent reset)
var spec = {}
spec.update = 'qlearn'; // 'qlearn' or 'sarsa'
spec.gamma = 0.9; // discount factor, [0, 1)
spec.epsilon = 0.2; // initial epsilon for epsilon-greedy policy, [0, 1)
spec.alpha = 0.1; // value function learning rate
spec.lambda = 0; // eligibility trace decay, [0,1). 0 = no eligibility traces
spec.replacing_traces = true; // use replacing or accumulating traces
spec.planN = 50; // number of planning steps per iteration. 0 = no planning

spec.smooth_policy_update = true; // non-standard, updates policy smoothly to follow max_a Q
spec.beta = 0.1; // learning rate for smooth policy update

// create environment
env = new Gridworld(); 
// create the agent, yay!
var spec = { alpha: 0.01 } // see full options on top of this page
agent = new RL.TDAgent(env, spec); 

setInterval(function(){ // start the learning loop
  var action = agent.act(s); // s is an integer, action is integer
  // execute action in environment and get the reward
  agent.learn(reward); // the agent improves its Q,policy,model, etc.
}, 0);

// create environment
var env = {};
env.getNumStates = function() { return 8; }
env.getMaxNumActions = function() { return 4; }
// create the agent, yay!
var spec = { alpha: 0.01 } // see full options on top of this page
agent = new RL.DQNAgent(env, spec); 

setInterval(function(){ // start the learning loop
  var action = agent.act(s); // s is an array of length 8
  // execute action in environment and get the reward
  agent.learn(reward); // the agent improves its Q,policy,model, etc. reward is a float
}, 0);

// agent parameter spec to play with (this gets eval()'d on Agent reset)
var spec = {}
spec.update = 'qlearn'; // qlearn | sarsa
spec.gamma = 0.9; // discount factor, [0, 1)
spec.epsilon = 0.2; // initial epsilon for epsilon-greedy policy, [0, 1)
spec.alpha = 0.005; // value function learning rate
spec.experience_add_every = 5; // number of time steps before we add another experience to replay memory
spec.experience_size = 10000; // size of experience
spec.learning_steps_per_iteration = 5;
spec.tderror_clamp = 1.0; // for robustness
spec.num_hidden_units = 100 // number of neurons in hidden layer