独家 | 教你用Q学习算法训练神经网络玩游戏（附源码）

require 'io/console'

class Player
  attr_accessor :y,:x

 def initialize
 @x = 0
 @y = 0
 end

 def get_input
    input = STDIN.getch
 if input == 'a'
 return :left
 elsif input == 'd'
 return :right
 elsif input == 'w'
 return :up
 elsif input == 's'
 return :down
 elsif input == 'q'
 exit
 end

 return :nothing
 end
end

require 'ruby-fann'

class QLearningPlayer
  attr_accessor :y, :x, :game

 def initialize
 @x = 0
 @y = 0
 @actions = [:left, :right, :up, :down]
 @first_run = true

 @discount = 0.9
 @epsilon = 0.1
 @max_epsilon = 0.9
 @epsilon_increase_factor = 800.0

 @replay_memory_size = 500
 @replay_memory_pointer = 0
 @replay_memory = []
 @replay_batch_size = 400

 @runs = 0

 @r = Random.new
 end

def initialize_q_neural_network
 # Setup model
 # Input is the size of the map + number of actions
 # Output size is one
 @q_nn_model = RubyFann::Standard.new(
               num_inputs: @game.map_size_x*@game.map_size_y + @actions.length,
               hidden_neurons: [ (@game.map_size_x*@game.map_size_y+@actions.length) ],
               num_outputs: 1 )

 @q_nn_model.set_learning_rate(0.2)

 @q_nn_model.set_activation_function_hidden(:sigmoid_symmetric)
 @q_nn_model.set_activation_function_output(:sigmoid_symmetric)

end

def get_input
 # Pause to make sure humans can follow along
 # Increase pause with the number of runs
 sleep 0.05 + 0.01*(@runs/400.0)
 @runs += 1

 if @first_run
 # If this is first run initialize the Q-neural network
    initialize_q_neural_network
 @first_run = false
 else

# If this is not the first 
# Evaluate what happened on last action and calculate reward
r = 0 # default is 0
if !@game.new_game and @old_score < @game.score
  r = 1 # reward is 1 if our score increased
elsif !@game.new_game and @old_score > @game.score
  r = -1 # reward is -1 if our score decreased
elsif !@game.new_game
  r = -0.1
end

# Capture current state
# Set input to network map_size_x * map_size_y + actions length vector with a 1 on the player position
input_state = Array.new(@game.map_size_x*@game.map_size_y + @actions.length, 0)
input_state[@x + (@game.map_size_x*@y)] = 1

# Add reward, old_state and input state to memory
@replay_memory[@replay_memory_pointer] = {reward: r, old_input_state: @old_input_state, input_state: input_state}
# Increment memory pointer
@replay_memory_pointer = (@replay_memory_pointer<@replay_memory_size) ? @replay_memory_pointer+1 : 0

 # If replay memory is full train network on a batch of states from the memory
 if @replay_memory.length > @replay_memory_size
 # Randomly sample a batch of actions from the memory and train network with these actions
 @batch = @replay_memory.sample(@replay_batch_size)
    training_x_data = []
    training_y_data = []

 # For each batch calculate new q_value based on current network and reward
 @batch.each do |m|
 # To get entire q table row of the current state run the network once for every posible action
      q_table_row = []
 @actions.length.times do |a|
 # Create neural network input vector for this action
        input_state_action = m[:input_state].clone
 # Set a 1 in the action location of the input vector
        input_state_action[(@game.map_size_x*@game.map_size_y) + a] = 1
 # Run the network for this action and get q table row entry
        q_table_row[a] = @q_nn_model.run(input_state_action).first
 end

 # Update the q value
      updated_q_value = m[:reward] + @discount * q_table_row.max

 # Add to training set
      training_x_data.push(m[:old_input_state])
      training_y_data.push([updated_q_value])
 end

 # Train network with batch
    train = RubyFann::TrainData.new( :inputs=> training_x_data, :desired_outputs=>training_y_data );
 @q_nn_model.train_on_data(train, 1, 1, 0.01)
 end
end

# Capture current state and score
# Set input to network map_size_x * map_size_y vector with a 1 on the player position
input_state = Array.new(@game.map_size_x*@game.map_size_y + @actions.length, 0)
input_state[@x + (@game.map_size_x*@y)] = 1
# Chose action based on Q value estimates for state
# If a random number is higher than epsilon we take a random action
# We will slowly increase @epsilon based on runs to a maximum of @max_epsilon - this encourages early exploration
epsilon_run_factor = (@runs/@epsilon_increase_factor) > (@max_epsilon-@epsilon) ? (@max_epsilon-@epsilon) : (@runs/@epsilon_increase_factor)
if @r.rand > (@epsilon + epsilon_run_factor)
 # Select random action
 @action_taken_index = @r.rand(@actions.length)
else
 # To get the entire q table row of the current state run the network once for every posible action
  q_table_row = []
 @actions.length.times do |a|
 # Create neural network input vector for this action
    input_state_action = input_state.clone
 # Set a 1 in the action location of the input vector
    input_state_action[(@game.map_size_x*@game.map_size_y) + a] = 1
 # Run the network for this action and get q table row entry
    q_table_row[a] = @q_nn_model.run(input_state_action).first
 end
 # Select action with highest posible reward
 @action_taken_index = q_table_row.each_with_index.max[1]
end