问RL -使用PyTorch- DQN的稳定基线:为什么CustomModel不学习？

EN

import gym
from gym import spaces
from stable_baselines3 import DQN
from stable_baselines3.dqn import MultiInputPolicy
import numpy as np
import torch.nn as nn
import torch


class CustomEnv(gym.Env):
    """Custom Environment that follows gym interface"""
    metadata = {'render.modes': ['human']}

    def __init__(self):
        super(CustomEnv, self).__init__()
        self.action_space = spaces.Discrete(10)
        self.observation_space = spaces.Dict({
            "vector1": spaces.Box(low=0, high=10, shape=(10,), dtype=np.float32),
            "vector2": spaces.Box(low=0, high=10, shape=(10,), dtype=np.float32)
        })

    def obs(self):
        return dict({
            "vector1": 5*np.ones(10),
            "vector2": 5*np.ones(10)})

    def step(self, action):
        if action == 2:
            reward = 20
        else:
            reward = 0
        return self.obs(), reward, False, dict({})

    def reset(self):
        return self.obs()

    def render(self, mode='human'):
        return None

    def close(self):
        pass

env = CustomEnv()

class CustomCombinedExtractor(MultiInputPolicy):
    def __init__(self, observation_space, action_space, lr_schedule):
        super().__init__(observation_space, action_space, lr_schedule)

        extractors = {}

        total_concat_size = 0
        for key, subspace in observation_space.spaces.items():
            elif key == "vector"1:
                extractors[key] = nn.Linear(subspace.shape[0], 64)
                total_concat_size += 64
            elif key == "vector2":
                extractors[key] = nn.Linear(subspace.shape[0], 64)
                total_concat_size += 64

        self.extractors = nn.ModuleDict(extractors)
        self._features_dim = 1
        self.features_dim = 1

    def forward(self, observations):
        encoded_tensor_list = []

        x = self.extractors["vector"](observations["vector"])
        return x.T


def lr_schedule(x): return 1/x
policy_kwargs = dict(
    features_extractor_class=CustomCombinedExtractor,
    features_extractor_kwargs=dict(
        action_space=spaces.Discrete(10), lr_schedule=lr_schedule),
)

model = DQN(MultiInputPolicy, env, verbose=1,
            buffer_size=1000, policy_kwargs=policy_kwargs)

model.learn(total_timesteps=25000)
model.save("ppo_cartpole")

del model  # remove to demonstrate saving and loading

model = DQN.load("ppo_cartpole")

obs = env.reset()
while True:
    action, _states = model.predict(obs)
    print(action)
    obs, rewards, dones, info = env.step(action)
    env.render()