通过图像训练强化学习模型
通过图像识别observation及reward
在gym中运行atari环境的时候可以选择同一个游戏的内存方式或者图像方式，内存方式直接返回游戏的状态，图像方式返回当前游戏的画面
之前的文章都是在MATLAB或者simulink中直接获取observation及reward，这里我们让环境返回图像，通过神经网络识别图像中的信息
%% 读取环境
ccc
env = rlPredefinedEnv('SimplePendulumWithImage-Continuous');
obsInfo = getObservationInfo(env);
actInfo = getActionInfo(env);
rng(0)
%%
hiddenLayerSize1 = 400;
hiddenLayerSize2 = 300;
%% 初始化agent
imgPath = [
imageInputLayer(obsInfo(1).Dimension,'Normalization','none','Name',obsInfo(1).Name)
convolution2dLayer(10,2,'Name','conv1','Stride',5,'Padding',0)
reluLayer('Name','relu1')
fullyConnectedLayer(2,'Name','fc1')
concatenationLayer(3,2,'Name','cat1')
fullyConnectedLayer(hiddenLayerSize1,'Name','fc2')
reluLayer('Name','relu2')
fullyConnectedLayer(hiddenLayerSize2,'Name','fc3')
additionLayer(2,'Name','add')
reluLayer('Name','relu3')
fullyConnectedLayer(1,'Name','fc4')
];
dthetaPath = [
imageInputLayer(obsInfo(2).Dimension,'Normalization','none','Name',obsInfo(2).Name)
fullyConnectedLayer(1,'Name','fc5','BiasLearnRateFactor',0,'Bias',0)
];
actPath =[
imageInputLayer(actInfo(1).Dimension,'Normalization','none','Name','action')
fullyConnectedLayer(hiddenLayerSize2,'Name','fc6','BiasLearnRateFactor',0,'Bias',zeros(hiddenLayerSize2,1))
];
criticNetwork = layerGraph(imgPath);
criticNetwork = addLayers(criticNetwork,dthetaPath);
criticNetwork = addLayers(criticNetwork,actPath);
criticNetwork = connectLayers(criticNetwork,'fc5','cat1/in2');
criticNetwork = connectLayers(criticNetwork,'fc6','add/in2');
%%
figure
plot(criticNetwork)
%%
criticOptions = rlRepresentationOptions('LearnRate',1e-03,'GradientThreshold',1);
% criticOptions.UseDevice = 'gpu';
critic = rlRepresentation(criticNetwork,obsInfo,actInfo,...
'Observation',{'pendImage','angularRate'},'Action',{'action'},criticOptions);
%%
imgPath = [
imageInputLayer(obsInfo(1).Dimension,'Normalization','none','Name',obsInfo(1).Name)
convolution2dLayer(10,2,'Name','conv1','Stride',5,'Padding',0)
reluLayer('Name','relu1')
fullyConnectedLayer(2,'Name','fc1')
concatenationLayer(3,2,'Name','cat1')
fullyConnectedLayer(hiddenLayerSize1,'Name','fc2')
reluLayer('Name','relu2')
fullyConnectedLayer(hiddenLayerSize2,'Name','fc3')
reluLayer('Name','relu3')
fullyConnectedLayer(1,'Name','fc4')
tanhLayer('Name','tanh1')
scalingLayer('Name','scale1','Scale',max(actInfo.UpperLimit))
];
dthetaPath = [
imageInputLayer(obsInfo(2).Dimension,'Normalization','none','Name',obsInfo(2).Name)
fullyConnectedLayer(1,'Name','fc5','BiasLearnRateFactor',0,'Bias',0)
];
%%
actorNetwork = layerGraph(imgPath);
actorNetwork = addLayers(actorNetwork,dthetaPath);
actorNetwork = connectLayers(actorNetwork,'fc5','cat1/in2');
%%
actorOptions = rlRepresentationOptions('LearnRate',1e-04,'GradientThreshold',1);
% actorOptions.UseDevice = 'gpu';
actor = rlRepresentation(actorNetwork,obsInfo,actInfo,'Observation',{'pendImage','angularRate'},'Action',{'scale1'},actorOptions);
figure
plot(actorNetwork)
%%
agentOptions = rlDDPGAgentOptions(...
'SampleTime',env.Ts,...
'TargetSmoothFactor',1e-3,...
'ExperienceBufferLength',1e6,...
'DiscountFactor',0.99,...
'MiniBatchSize',128);
agentOptions.NoiseOptions.Variance = 0.6;
agentOptions.NoiseOptions.VarianceDecayRate = 1e-6;
agent = rlDDPGAgent(actor,critic,agentOptions);
%% 设置训练参数
maxepisodes = 5000;
maxsteps = 400;
trainingOptions = rlTrainingOptions(...
'MaxEpisodes',maxepisodes,...
'MaxStepsPerEpisode',maxsteps,...
'Plots','training-progress',...
'StopTrainingCriteria','AverageReward',...
'StopTrainingValue',-740);
plot(env)
%% 并行学习设置
trainingOptions.UseParallel = true;
trainingOptions.ParallelizationOptions.Mode = "async";
trainingOptions.ParallelizationOptions.DataToSendFromWorkers = "Experiences";
trainingOptions.ParallelizationOptions.StepsUntilDataIsSent = -1;
%% 训练
trainingStats = train(agent,env,trainingOptions);
%% 结果展示
simOptions = rlSimulationOptions('MaxSteps',500);
experience = sim(env,agent,simOptions);
% bdclose(mdl)