深度学习（七）深度强化学习：融合创新的智能之路（7/10）

import gymnasium as gym
import torch
import torch.nn as nn
import torch.nn.functional as F
import random

class ReplayMemory:
    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []

    def push(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))
        if len(self.memory) > self.capacity:
            self.memory.pop(0)

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

class DQN(nn.Module):
    def __init__(self, input_size, output_size):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(input_size, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, output_size)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        return self.fc3(x)

class Agent:
    def __init__(self, state_size, action_size, device):
        self.state_size = state_size
        self.action_size = action_size
        self.device = device
        self.q_network = DQN(state_size, action_size).to(device)
        self.optimizer = torch.optim.Adam(self.q_network.parameters(), lr=0.001)

    def act(self, state):
        state = torch.tensor(state, dtype=torch.float).unsqueeze(0).to(self.device)
        return self.q_network(state).max(1)[1].item()

    def learn(self, states, actions, rewards, next_states, dones):
        states = torch.stack(states).to(self.device)
        actions = torch.tensor(actions).to(self.device)
        rewards = torch.tensor(rewards).to(self.device)
        next_states = torch.stack(next_states).to(self.device)
        dones = torch.tensor(dones).to(self.device)
        q_values = self.q_network(states).gather(1, actions.unsqueeze(-1)).squeeze(-1)
        next_q_values = self.q_network(next_states).max(1)[0].detach()
        targets = rewards + (1 - dones) * 0.99 * next_q_values
        loss = torch.mean((q_values - targets)**2)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()
        return loss.item()

def train_dqn(agent, num_episodes, memory, batch_size):
    rewards = []
    for episode in range(num_episodes):
        state, _ = env.reset()
        state = torch.tensor(state, dtype=torch.float).unsqueeze(0)
        done = False
        total_reward = 0
        while not done:
            action = agent.act(state)
            next_state, reward, done, _, _ = env.step(action)
            next_state = torch.tensor(next_state, dtype=torch.float).unsqueeze(0)
            memory.push(state, action, reward, next_state, done)
            state = next_state
            total_reward += reward
        if len(memory) > batch_size:
            states, actions, rewards, next_states, dones = memory.sample(batch_size)
            loss = agent.learn(states, actions, rewards, next_states, dones)
        rewards.append(total_reward)
    return rewards

env = gym.make("CartPole-v1")
n_observations = env.observation_space.shape[0]
n_actions = env.action_space.n
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
agent = Agent(n_observations, n_actions, device)
memory = ReplayMemory(10000)
num_episodes = 500
batch_size = 32
rewards = train_dqn(agent, num_episodes, memory, batch_size)

import gym
import torch
import torch.nn as nn
import torch.optim as optim

class PolicyNetwork(nn.Module):
    def __init__(self, input_size, output_size):
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(input_size, 64)
        self.fc2 = nn.Linear(64, output_size)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        return torch.softmax(self.fc2(x), dim=-1)

env = gym.make("CartPole-v1")
observation_space = env.observation_space.shape[0]
action_space = env.action_space.n

policy_net = PolicyNetwork(observation_space, action_space)
optimizer = optim.Adam(policy_net.parameters(), lr=0.01)

def select_action(state):
    state_tensor = torch.FloatTensor(state)
    action_probs = policy_net(state_tensor)
    return torch.multinomial(action_probs, 1).item()

def update_policy(rewards):
    discounted_rewards = []
    R = 0
    for r in reversed(rewards):
        R = r + 0.99 * R
        discounted_rewards.insert(0, R)
    discounted_rewards = torch.FloatTensor(discounted_rewards)
    discounted_rewards = (discounted_rewards - discounted_rewards.mean()) / (discounted_rewards.std() + 1e-5)
    optimizer.zero_grad()
    for i, log_prob in enumerate(log_probs):
        loss = -log_prob * discounted_rewards[i]
        loss.backward()
    optimizer.step()

for episode in range(1000):
    state = env.reset()[0]
    log_probs = []
    rewards = []
    done = False
    while not done:
        action = select_action(state)
        next_state, reward, done, _, _ = env.step(action)
        log_prob = torch.log(policy_net(torch.FloatTensor(state))[action])
        log_probs.append(log_prob)
        rewards.append(reward)
        state = next_state
    update_policy(rewards)

import gym
import numpy as np

env = gym.make("CartPole-v1")
num_states = env.observation_space.shape[0]
num_actions = env.action_space.n
gamma = 0.99
learning_rate = 0.1

value_function = np.zeros((num_states, num_actions))

def select_action(state):
    return np.argmax(value_function[state])

for episode in range(1000):
    state = env.reset()[0]
    done = False
    while not done:
        action = select_action(state)
        next_state, reward, done, _, _ = env.step(action)
        value_function[state][action] += learning_rate * (reward + gamma * np.max(value_function[next_state]) - value_function[state][action])
        state = next_state

2.游戏关卡设计

3.游戏平衡调整

4.游戏推荐系统

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>猜数字</title>
</head>
<body>
    <h3>请输入要猜的数字: </h3>
    <input type="text">
    <div>已经猜的次数: <span id="guessCount">0</span></div>
    <div>结果是: <span id="result"></span></div>
    <button>猜</button>
    <script>
        //0. 获取到需要的元素
        let input = document.querySelector('input');
        let button = document.querySelector('button');
        let guessCount = document.querySelector('#guessCount');
        let result = document.querySelector('#result');
        // 1. 在页面加载的时候, 要生成 1 - 100 之间随机的整数
        let toGuess = parseInt(Math.random()*100)+1;
        console.log(toGuess);
        //2. 给 猜 这个按钮加上个点击事件. button.onclick=function(){
        // a) 获取到输入框里的值. let value = parseInt(input.value);
        // b) 和当前的生成的数字进行对比. if(value < toGuess){
            result.innerHTML ='低了';
        }else if(value > toGuess){
            result.innerHTML ='高了';
        }else{
            result.innerHTML ='猜对了!';
        }
        // c) 每点击一次提交, 就让 猜 的次数, 加1!
        let guessCountValue = parseInt(guessCount.innerHTML);
        guessCount.innerHTML = guessCountValue +1;
    </script>
</body>
</html>

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>表白墙</title>
</head>
<body>
    <style>
       .container{
            width: 400px;
            /* margin 外边距. 第一个数字上下外边距, 第二个数字表示水平外边距. 如果水平设置成 auto 表示元素就水平居中~~ */
            margin:0 auto;
        }
        h1{
            text-align: center;
        }
        p{
            text-align: center;color: #666;
        }
       .row{
            height: 40px;
            display: flex;
            /* 水平居中 */
            justify-content: center;
            /* 垂直居中 */
            align-items: center;
        }
       .row span{
            width: 100px;
        }
       .row input{
            width: 200px;
            height: 30px;
        }
       .row button{
            width: 306px;
            height: 40px;
            color: white;
            background: orange;
            border: none;
        }
       .row button:active{
            background-color: #666;
        }
    </style>
    <div class="container">
        <h1>表白墙</h1>
        <p>输入后点击提交, 就会把信息显示在表格中</p>
        <div class="row">
            <span>谁: </span><input type="text">
        </div>
        <div class="row">
            <span>对谁: </span><input type="text">
        </div>
        <div class="row">
            <span>说: </span><input type="text">
        </div>
        <div class="row">
            <button>提交</button>
        </div>
    </div>
    <script>
        let container = document.querySelector('.container');
        let button = document.querySelector('button');
        button.onclick=function(){
            //1. 获取到输入框的内容
            let inputs = document.querySelectorAll('input');
            let from = inputs[0].value;
            let to = inputs[1].value;
            let message = inputs[2].value;
            if(from ==''|| to ==''|| message ==''){
                alert('当前输入框内容为空!');
                return;
            }
            console.log(from +", "+ to +", "+ message);
            //2. 能够构造出新的 div, 用来保存用户提交的内容
            let rowDiv = document.createElement('div');
            rowDiv.className ='row';
            rowDiv.innerHTML = from +" 对 "+ to +" 说: "+ message;
            container.appendChild(rowDiv);
            //3. 提交完之后, 清空输入框的内容
            for(let i =0; i < inputs.length; i++){
                inputs[i].value ='';
            }
        }
    </script>
</body>
</html>

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
    <input type="text">
    <button>添加待办事项</button>
    <ul></ul>
    <script>
        let input = document.querySelector('input');
        let button = document.querySelector('button');
        let ul = document.querySelector('ul');
        button.onclick = function() {
            if(input.value === '') {
                return;
            }
            let li = document.createElement('li');
            li.textContent = input.value;
            ul.appendChild(li);
            input.value = '';
        }
    </script>
</body>
</html>

  通过利用这些学习资料，学习者可以更加系统地学习深度学习的知识和技术，提高自己的实践能力和创新能力。同时，也可以与其他学习者进行交流和互动，共同推动深度学习领域的发展。