我正在尝试遵循有关QLearning的pytorch tutorial。我保留了核心，但是尝试以不同的方式实现它。但是，我偶然发现了问题。在最佳情况下，该算法似乎无法解决环境问题，而在最坏情况下，则会发生预测QValue爆炸。我找不到根本原因
我试图调整大多数超参数，降低学习率，然后我检查了几次代码，以查看我的代码和pytorch网站上的代码之间是否存在差异。
有人知道什么可能导致此行为吗？还是看到代码中的错误？我从堆栈溢出中检查了大多数帖子，其中大多数都是关于学习率或目标网络的。我的代码中有一个目标网络，我尝试降低LR，但不幸的是它似乎无法解决问题
import gym
import torch
import random
import numpy as np
from collections import deque
from copy import deepcopy
import torch.nn as nn
import torch.optim as optim
from itertools import count
import torch.nn.functional as F
from collections import namedtuple
GAMMA = 0.99
EPSILON_END = 0.05
EPSILON_START = 0.9
EPSILON_DECAY = 500
LEARNING_RATE = 0.0001
ACTION_SPACE = 4
OBSERVATION_SPACE = 8
BATCH_SIZE = 32
MEMORY_CAPACITY = 10000
TARGET_NETWORK_UPDATE_FREQUENCY = 2
Experience = namedtuple('Experience', ['state', 'action', 'reward', 'next_state'])
class Network(nn.Module):
def __init__(self, input_features, output_features):
super().__init__()
self.fc_1 = nn.Linear(input_features, 100)
self.bn_1 = nn.LayerNorm(100)
self.fc_2 = nn.Linear(100, 100)
self.bn_2 = nn.LayerNorm(100)
self.fc_3 = nn.Linear(100, output_features)
def forward(self, x):
x = F.relu(self.bn_1(self.fc_1(x)))
x = F.relu(self.bn_2(self.fc_2(x)))
return self.fc_3(x)
class Memory(object):
def __init__(self, capacity):
self.capacity = capacity
self.memory = []
self.position = 0
def push(self, experience):
if len(self.memory) < self.capacity:
self.memory.append(None)
self.memory[self.position] = experience
self.position = (self.position + 1) % self.capacity
def sample(self, batch_size):
return random.sample(self.memory, batch_size)
def __len__(self):
return len(self.memory)
class Agent:
def __init__(self, policy_network, action_space):
self.policy_network = policy_network
self.actions = list(range(action_space))
def __call__(self, observation, step_id=None):
if np.random.random() > get_epsilon(step_id):
with torch.no_grad():
predictions = self.policy_network(torch.tensor(observation)).detach().numpy()
return np.argmax(predictions)
else:
return np.random.choice(self.actions)
class QLearning:
def __init__(self, agent: Agent, target_network: nn.Module, optimizer, memory_capacity, batch_size):
self.step = 0
self.agent = agent
self.target_network = target_network
self.optimizer = optimizer
self.batch_size = batch_size
self.memory = Memory(memory_capacity)
def train(self, num_episodes):
env = gym.make('LunarLander-v2')
recent_scores = deque(maxlen=100)
for episode_id in range(num_episodes):
observation = env.reset()
score = 0
for t in count():
action = self.agent(observation, self.step)
observation_, reward, done, info = env.step(action.item())
observation_ = None if done else observation_
experience = self.preprocess_experience(observation, action, reward, observation_)
self.memory.push(experience)
q_values = self.learn()
self.step += 1
score += reward
if done is True:
break
recent_scores.append(score)
if episode_id % TARGET_NETWORK_UPDATE_FREQUENCY == 0:
self.update_target_network()
if episode_id % 10 == 0:
print(episode_id, np.mean(recent_scores), t) # TODO Tensorboard
print('sample of the state_action_values :', list(q_values))
def learn(self):
if len(self.memory) < self.batch_size:
return
experiences = self.memory.sample(self.batch_size)
batch = Experience(*zip(*experiences))
states = torch.cat(batch.state)
actions = torch.cat(batch.action)
rewards = torch.cat(batch.reward)
state_action_values = self.agent.policy_network(states).gather(1, actions)
non_terminal_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), dtype=torch.bool)
non_terminal_next_states = torch.cat([s for s in batch.next_state if s is not None])
next_state_values = torch.zeros(BATCH_SIZE)
next_state_values[non_terminal_mask] = self.target_network(non_terminal_next_states).max(1)[0].detach()
expected_state_action_values = (next_state_values * GAMMA) + rewards
expected_state_action_values = expected_state_action_values.unsqueeze(1)
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values)
self.optimizer.zero_grad()
loss.backward()
for param in self.agent.policy_network.parameters():
param.grad.data.clamp_(-1, 1)
self.optimizer.step()
return state_action_values.detach().tolist()[:3] # This is just to monitor if there is an issue with the values predicted
def update_target_network(self):
self.target_network.load_state_dict(self.agent.policy_network.state_dict())
@staticmethod
def preprocess_experience(state, action, reward, next_state):
state = torch.tensor(state).unsqueeze(0)
next_state = torch.tensor(next_state).unsqueeze(0) if next_state is not None else None
action = torch.tensor(action, dtype=torch.int64).view(1, 1)
reward = torch.tensor(reward, dtype=torch.float32).view(1)
return Experience(state, action, reward, next_state)
def get_networks_and_optimizer(learning_rate, observation_space, action_space):
policy_network = Network(observation_space, action_space)
target_network = deepcopy(policy_network)
target_network.eval()
policy_network_optimizer = optim.RMSprop(policy_network.parameters(), learning_rate)
return policy_network, target_network, policy_network_optimizer
def get_epsilon(step):
if step is None:
return EPSILON_END
return EPSILON_END + (EPSILON_START - EPSILON_END) * np.exp(-1. * step / EPSILON_DECAY)
if __name__ == '__main__':
policy_network, target_network, policy_network_optimizer = get_networks_and_optimizer(LEARNING_RATE, OBSERVATION_SPACE, ACTION_SPACE)
agent = Agent(policy_network, ACTION_SPACE)
trainer = QLearning(agent, target_network, policy_network_optimizer, MEMORY_CAPACITY, BATCH_SIZE)
trainer.train(10000)