对于冗长的帖子，我感到抱歉，只是想提前提供实施细节。
此外，对不起代码，我知道那是一团糟。
我一直使用PyTorch和VRep的组合为差动驱动机器人设置环境，以学习操纵迷宫的方法。该机器人沿其边缘仅有6个IC接近传感器，可测量10-80cm的距离。作为第一个简单版本，我创建了仅包含狭窄走廊的VRep环境（走廊的宽度为〜2 * robot_width）。机器人的起始位置在走廊的中间，目标点在走廊的尽头，目标速度为〜0 m / s。我的想法是，我学过的特工同时处理导航和低级机器人控制。
现在，我一直在使用适用于Gym环境的预先存在的DDPG实现，但是对于我的情况，它似乎并没有收敛。我以为仅使用6个接近传感器是问题的一部分，所以我已经介绍了状态向量的位置和速度读数以及所需状态的误差（如某些论文所建议的那样）。
我将不胜感激。
ddpg_agent
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -2, 2)
def reset(self):
self.noise.reset()
def start_learn(self):
if len(self.memory) > MIN_BUFFER_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)).detach()
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
# torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
def save_samples(self):
fileSamples = open('samples.obj', 'w')
pickle.dump(self.memory, fileSamples)
def load_samples(self):
fileSamples = open('samples.obj', 'r')
return pickle.load(fileSamples)
class OUNoise:
"""Ornstein-Uhlenbeck process."""
def __init__(self, size, seed, mu=0., theta=0.15, sigma=0.2):
"""Initialize parameters and noise process."""
self.mu = mu * np.ones(size)
self.theta = theta
self.sigma = sigma
self.seed = random.seed(seed)
self.reset()
def reset(self):
"""Reset the internal state (= noise) to mean (mu)."""
self.state = copy.copy(self.mu)
def sample(self):
"""Update internal state and return it as a noise sample."""
x = self.state
dx = self.theta * (self.mu - x) + self.sigma * np.array([np.random.randn() for i in range(len(x))])
self.state = x + dx
return self.state
class ReplayBuffer:
"""Fixed-size buffer to store experience tuples."""
def __init__(self, action_size, buffer_size, batch_size, seed):
"""Initialize a ReplayBuffer object.
Params
======
buffer_size (int): maximum size of buffer
batch_size (int): size of each training batch
"""
self.action_size = action_size
self.memory = deque(maxlen=buffer_size) # internal memory (deque)
self.batch_size = batch_size
self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
self.seed = random.seed(seed)
def add(self, state, action, reward, next_state, done):
"""Add a new experience to memory."""
e = self.experience(state, action, reward, next_state, done)
self.memory.append(e)
def sample(self):
"""Randomly sample a batch of experiences from memory."""
experiences = random.sample(self.memory, k=self.batch_size)
states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)
actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).float().to(device)
rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)
next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)
dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)
return (states, actions, rewards, next_states, dones)
def __len__(self):
"""Return the current size of internal memory."""
return len(self.memory)
型号：
class Actor(nn.Module):
"""Actor (Policy) Model."""
def __init__(self, state_size, action_size, seed, fc1_units=600, fc2_units=400, fc3_units=300):
"""Initialize parameters and build model.
Params
======
state_size (int): Dimension of each state
action_size (int): Dimension of each action
seed (int): Random seed
fc1_units (int): Number of nodes in first hidden layer
fc2_units (int): Number of nodes in second hidden layer
"""
super(Actor, self).__init__()
self.seed = torch.manual_seed(seed)
self.fc1 = nn.Linear(state_size, fc1_units)
self.bn1 = nn.BatchNorm1d(fc1_units)
self.fc2 = nn.Linear(fc1_units, fc2_units)
self.fc3 = nn.Linear(fc2_units, fc3_units)
self.fc4 = nn.Linear(fc3_units, action_size)
self.reset_parameters()
def reset_parameters(self):
self.fc1.weight.data.uniform_(*hidden_init(self.fc1))
self.fc2.weight.data.uniform_(*hidden_init(self.fc2))
self.fc3.weight.data.uniform_(*hidden_init(self.fc3))
self.fc4.weight.data.uniform_(-3e-3, 3e-3)
def forward(self, state):
"""Build an actor (policy) network that maps states -> actions."""
# x = F.relu(self.bn1(self.fc1(state.unsqueeze(0))))
x = F.relu(self.fc1(state))
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
return torch.tanh(self.fc4(x))
class Critic(nn.Module):
"""Critic (Value) Model."""
def __init__(self, state_size, action_size, seed, fcs1_units=600, fc2_units=400, fc3_units=300):
"""Initialize parameters and build model.
Params
======
state_size (int): Dimension of each state
action_size (int): Dimension of each action
seed (int): Random seed
fcs1_units (int): Number of nodes in the first hidden layer
fc2_units (int): Number of nodes in the second hidden layer
"""
super(Critic, self).__init__()
self.seed = torch.manual_seed(seed)
self.fcs1 = nn.Linear(state_size, fcs1_units)
self.bn1 = nn.BatchNorm1d(fcs1_units)
self.fc2 = nn.Linear(fcs1_units+action_size, fc2_units)
self.fc3 = nn.Linear(fc2_units, fc3_units)
self.fc4 = nn.Linear(fc3_units, 1)
self.reset_parameters()
def reset_parameters(self):
self.fcs1.weight.data.uniform_(*hidden_init(self.fcs1))
self.fc2.weight.data.uniform_(*hidden_init(self.fc2))
self.fc3.weight.data.uniform_(*hidden_init(self.fc3))
self.fc4.weight.data.uniform_(-3e-3, 3e-3)
def forward(self, state, action):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# xs = F.relu(self.bn1(self.fcs1(state.unsqueeze(0))))
xs = F.relu(self.fcs1(state))
x = torch.cat((xs, action), dim=1)
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
return self.fc4(x)
主要：
def main():
vrepHeadlessMode = True
state_dim = 18 # x, y, yaw, vx, vy, v_yaw, e_x, e_y, e_yaw, e_vx, e_vy, e_v_yaw, prox 0 ... prox5
action_dim = 2
action_space = np.array([[-2, 2], [-2, 2]])
action_lim = [-2.0, 2.0] # 2 o/sec is the max angular speed of each motor, max. linear velocity is 0.5 m/s
learn_every = 1 # number of steps after which the network update occurs [20]
num_learn = 1 # number of network updates done in a row [10]
episodes = 10000
steps = 500
desiredState = [-1.4, 0.3, -np.pi, 0.0, 0.0, 0.0] # x, y, yawAngle, vx, vy, yawVelocity
mobRob = MobRob(['MobRob'],
['leftMotor', 'rightMotor'],
['proximitySensor0', 'proximitySensor1', 'proximitySensor2', 'proximitySensor3', 'proximitySensor4',
'proximitySensor5'])
env = LabEnv(mobRob, vrepHeadlessMode)
random_seed = 7
mobRob = Agent(state_dim, action_dim, random_seed)
total_num_of_steps = 0
actions = np.zeros((episodes, steps+1, action_dim), dtype=np.float)
total_rewards = []
save_rewards = []
durations = []
for episode in range(episodes):
cur_state = env.restart(desiredState)
mobRob.reset()
start_time = time.time()
reason = ''
episode_rewards = []
for step in range(steps+1):
total_num_of_steps += 1
action = mobRob.act(cur_state)
actions[episode][step] = action
# print(action)
new_state, reward, done = env.step(action, desiredState)
mobRob.step(cur_state, action, reward, new_state, done)
cur_state = new_state
episode_rewards.append(reward)
if step % learn_every == 0:
for _ in range(num_learn):
mobRob.start_learn()
if step < steps and done and ~env.collision:
reason = 'COMPLETED'
break
if step == steps: # time budget for episode was overstepped
reason = 'TIMEOUT '
break
if env.collision:
reason = 'COLLISION'
break
mean_score = np.mean(episode_rewards)
min_score = np.min(episode_rewards)
max_score = np.max(episode_rewards)
total_rewards.append(mean_score)
duration = time.time() - start_time
durations.append(duration)
save_rewards.append([total_rewards[episode], episode])
eta = np.mean(durations)*(episodes-episode) / 60 / 60
if eta < 1.0:
etaString = str(np.round(eta * 60, 2)) + " min"
else:
etaString = str(np.round(eta, 2)) + " h"
print(
'\rEpisode {}\t{}\tMean episode reward: {:.2f}\tMin: {:.2f}\tMax: {:.2f}\tDuration: {:.2f}\tETA: {}'
.format(episode, reason, mean_score, min_score, max_score, duration, etaString))
gc.collect()
torch.save(mobRob.actor_local.state_dict(), './actor.pth')
torch.save(mobRob.critic_local.state_dict(), './critic.pth')
np.save('mean_episode_rewards', save_rewards)
if __name__ == "__main__":
main()
典型输出
第3179次超时平均情节奖励：0.36最小值：0.00最大值：0.69持续时间：36.86预计到达时间：134.16小时
事件3180超时平均情节奖励：0.22最低：0.00最高：0.72持续时间：37.39预计到达时间：134.12小时
Episode 3181 COLLISION平均情节奖励：0.26最小值：-49.50最大值：0.54持续时间：29.11预计时间：134.08 h
Episode 3182 COLLISION平均情节奖励：-0.39最小值：-50.00最大值：0.21持续时间：9.50 ETA：134.02 h
第3183集冲突平均情节奖励：-0.06最小值：-50.00最大值：0.32持续时间：27.10预计到达时间：133.98小时
Episode 3184 COLLISION平均情节奖励：0.38最低：-49.32最高：0.69持续时间：37.90预计到达时间：133.94小时
Episode 3185 COLLISION平均情节奖励：-0.52最小值：-50.00最大值：0.21持续时间：7.28预计到达时间：133.88小时
位置已确定！
达到速度！
第3186集完成的平均情节奖励：0.39最低：0.00最高：80.72持续时间：37.73预计到达时间：133.84小时
第3187集冲突平均情节奖励：0.36最小值：-49.36最大值：0.68持续时间：34.68预计到达时间：133.8小时
Episode 3188 COLLISION平均情节奖励：0.32最低：-49.43最高：0.62持续时间：35.35预计到达时间：133.76小时
第3189集冲突平均情节奖励：-0.23最小值：-50.00最大值：0.21持续时间：16.59预计到达时间：133.71 h
事件3190超时平均情节奖励：0.39最小值：0.00最大值：0.65持续时间：38.15预计时间：133.67小时
第3191次超时平均情节奖励：0.35最低：0.00最高：0.67持续时间：37.76预计到达时间：133.63小时
位置已确定！
第3192集完成平均情节奖励：0.41分钟：0.00最大值：30.71持续时间：36.36预计到达时间：133.59小时
第3193次超时平均情节奖励：0.35最低：0.00最高：0.72持续时间：36.98预计到达时间：133.55小时
即使经过大量的插曲，我也看不到奖励的明显增加。