Question

我正尝试训练一个深入的Q学习Keras模型来玩CartPole-v1。但是，它似乎并没有变得更好。我不相信这是一个错误，但是我缺乏如何正确使用Keras和OpenAI Gym的知识。我正在关注本教程（https://adventuresinmachinelearning.com/reinforcement-learning-tutorial-python-keras/），该教程显示了如何训练机器人玩NChain-v0（我能够遵循），但是现在我试图将学到的知识应用到更复杂的环境中：购物车v1。这是下面的代码：

###import libraries
import gym
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam


###prepare environment
env = gym.make('CartPole-v1') #our environment is CartPole-v1


###make model
model = Sequential()
model.add(Dense(128, input_shape=(env.observation_space.shape[0],), activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(env.action_space.n, activation='linear'))
model.compile(loss='mse', optimizer=Adam(), metrics=['mae'])


###train model
def train_model(n_episodes=500, epsilon=0.5, decay_factor=0.999, gamma=0.95):
    G_array = []
    for episode in range(n_episodes):
        observation = env.reset()
        observation = observation.reshape(-1, env.observation_space.shape[0])
        epsilon *= decay_factor
        G = 0
        done = False
        while done != True:
            if np.random.random() < epsilon:
                action = env.action_space.sample()
            else:
                action = np.argmax(model.predict(observation))
            new_observation, reward, done, info = env.step(action) #It keeps going left! Why though?
            new_observation = new_observation.reshape(-1, env.observation_space.shape[0])
            target = reward + gamma*np.max(model.predict(new_observation))
            target_vector = model.predict(observation)[0]
            target_vector[action] = target
            model.fit(observation, target_vector.reshape(-1, env.action_space.n), epochs=1, verbose=0)
            observation = new_observation
            G += reward
        G_array.append(G)

    return G_array

G_array = train_model()
print(G_array)

“ G_array”（每个游戏的总奖励）的输出如下：

[14.0, 16.0, 18.0, 12.0, 16.0, 14.0, 17.0, 11.0, 11.0, 12.0, 11.0, 15.0, 13.0, 12.0, 12.0, 19.0, 13.0, 9.0, 10.0, 10.0, 11.0, 11.0, 14.0, 11.0, 10.0, 9.0, 10.0, 10.0, 12.0, 9.0, 15.0, 19.0, 11.0, 11.0, 10.0, 11.0, 13.0, 12.0, 13.0, 16.0, 12.0, 14.0, 9.0, 12.0, 20.0, 10.0, 12.0, 11.0, 9.0, 13.0, 13.0, 11.0, 13.0, 11.0, 24.0, 12.0, 11.0, 9.0, 9.0, 11.0, 10.0, 16.0, 10.0, 9.0, 9.0, 19.0, 10.0, 11.0, 13.0, 11.0, 11.0, 14.0, 23.0, 8.0, 13.0, 12.0, 15.0, 14.0, 11.0, 24.0, 9.0, 11.0, 11.0, 11.0, 10.0, 12.0, 11.0, 11.0, 10.0, 13.0, 18.0, 10.0, 17.0, 11.0, 13.0, 14.0, 12.0, 16.0, 13.0, 10.0, 10.0, 12.0, 22.0, 13.0, 11.0, 14.0, 10.0, 11.0, 11.0, 14.0, 14.0, 12.0, 18.0, 17.0, 9.0, 13.0, 12.0, 11.0, 11.0, 9.0, 16.0, 9.0, 18.0, 15.0, 12.0, 16.0, 13.0, 10.0, 13.0, 13.0, 17.0, 11.0, 11.0, 9.0, 9.0, 12.0, 9.0, 10.0, 9.0, 10.0, 18.0, 9.0, 11.0, 12.0, 10.0, 10.0, 10.0, 12.0, 12.0, 20.0, 13.0, 19.0, 9.0, 14.0, 14.0, 13.0, 19.0, 10.0, 18.0, 11.0, 11.0, 11.0, 8.0, 10.0, 14.0, 11.0, 16.0, 11.0, 13.0, 13.0, 9.0, 16.0, 11.0, 12.0, 13.0, 12.0, 11.0, 10.0, 11.0, 21.0, 12.0, 22.0, 12.0, 10.0, 13.0, 15.0, 19.0, 11.0, 10.0, 10.0, 11.0, 22.0, 11.0, 9.0, 26.0, 13.0, 11.0, 13.0, 13.0, 10.0, 10.0, 11.0, 12.0, 18.0, 9.0, 11.0, 13.0, 12.0, 13.0, 13.0, 12.0, 10.0, 11.0, 12.0, 12.0, 17.0, 11.0, 13.0, 13.0, 21.0, 12.0, 9.0, 14.0, 10.0, 15.0, 12.0, 12.0, 14.0, 11.0, 10.0, 14.0, 12.0, 12.0, 11.0, 8.0, 24.0, 9.0, 13.0, 10.0, 14.0, 10.0, 12.0, 13.0, 12.0, 13.0, 13.0, 14.0, 9.0, 17.0, 16.0, 9.0, 16.0, 14.0, 11.0, 9.0, 10.0, 15.0, 11.0, 9.0, 14.0, 12.0, 10.0, 13.0, 10.0, 10.0, 16.0, 15.0, 11.0, 8.0, 9.0, 9.0, 10.0, 9.0, 21.0, 13.0, 13.0, 10.0, 10.0, 11.0, 27.0, 13.0, 15.0, 11.0, 11.0, 12.0, 9.0, 10.0, 16.0, 10.0, 13.0, 13.0, 12.0, 12.0, 11.0, 17.0, 14.0, 9.0, 15.0, 26.0, 9.0, 9.0, 13.0, 9.0, 8.0, 12.0, 9.0, 10.0, 11.0, 9.0, 10.0, 9.0, 11.0, 9.0, 10.0, 12.0, 13.0, 13.0, 11.0, 11.0, 10.0, 15.0, 11.0, 11.0, 13.0, 10.0, 10.0, 12.0, 10.0, 10.0, 12.0, 9.0, 15.0, 29.0, 11.0, 9.0, 18.0, 11.0, 13.0, 13.0, 16.0, 13.0, 15.0, 10.0, 11.0, 18.0, 9.0, 9.0, 11.0, 15.0, 11.0, 11.0, 10.0, 25.0, 10.0, 9.0, 11.0, 15.0, 15.0, 11.0, 11.0, 11.0, 13.0, 9.0, 11.0, 9.0, 13.0, 12.0, 12.0, 14.0, 11.0, 14.0, 8.0, 10.0, 13.0, 10.0, 10.0, 10.0, 9.0, 13.0, 9.0, 12.0, 10.0, 11.0, 9.0, 11.0, 12.0, 20.0, 9.0, 10.0, 14.0, 9.0, 12.0, 13.0, 11.0, 11.0, 11.0, 10.0, 15.0, 14.0, 14.0, 12.0, 13.0, 12.0, 11.0, 10.0, 12.0, 12.0, 9.0, 11.0, 9.0, 11.0, 13.0, 10.0, 11.0, 11.0, 11.0, 12.0, 13.0, 13.0, 12.0, 8.0, 11.0, 13.0, 9.0, 12.0, 10.0, 10.0, 15.0, 12.0, 11.0, 10.0, 17.0, 10.0, 14.0, 9.0, 10.0, 10.0, 10.0, 12.0, 10.0, 10.0, 12.0, 10.0, 15.0, 10.0, 10.0, 9.0, 10.0, 10.0, 10.0, 19.0, 9.0, 10.0, 11.0, 10.0, 11.0, 11.0, 13.0, 10.0, 11.0, 12.0, 11.0, 12.0, 13.0, 11.0, 8.0, 12.0, 12.0, 14.0, 14.0, 11.0, 9.0, 11.0, 9.0, 12.0, 9.0, 8.0, 9.0, 12.0, 8.0, 10.0, 11.0, 13.0, 12.0, 12.0, 10.0, 11.0, 12.0, 10.0, 12.0, 13.0, 9.0, 9.0, 10.0, 15.0, 14.0, 16.0, 8.0, 19.0, 10.0]

这显然意味着该模型在所有500集中都没有改善。如果我是使用Keras和OpenAI Gym（尤其是Keras）的完整入门者，请问。任何帮助表示赞赏。谢谢。

更新：通过一些调试，我最近注意到该模型通常在大多数情况下倾向于左移或选择操作0。这是否意味着我应该做出一些if陈述来修改奖励系统（例如，如果极角小于5度，则增加奖励）？实际上，我现在正在执行此操作，但到目前为止仍无济于事。

Answer 1

强化学习非常嘈杂，您的批量大小为1，这使其变得更加嘈杂。您可以尝试使用过去更新的情节/更新的内存缓冲区。您可以使用集合中类似deque（）的缓冲区。然后，您可以根据给定的批量从该内存缓冲区中随机采样。我发现此存储库非常有用（根据需要，它包括重播/内存缓冲区和RL代理） https://github.com/udacity/deep-reinforcement-learning/tree/master/dqn 尽管如此，RL需要花费很长时间才能收敛，这与传统的深度学习不同，传统的深度学习在开始时损失很快就减少了，而RL中的奖励不会长时间增加，然后突然开始增加。

玩CartPole时Keras Q学习模型的性能没有提高

1 个答案: