我正在使用tensorflow实现我的第一个强化深度学习模型，我正在实施cartpole problem。
我使用六层深度神经网络训练随机生成的数据集，其得分高于阈值。问题是模型没有收敛，最终得分平均保持在10分左右。
正如在阅读某些帖子后建议我应用正规化和辍学以减少可能发生的任何过度拟合但仍然没有运气。我也试过降低学习率。
在训练一批之后，精确度也保持在.60左右，尽管每次迭代中损失都在减少，我认为即使在这些之后它也会记忆。 虽然这种模型适用于简单的深度学习任务。
这是我的代码：
import numpy as np
import tensorflow as tf
import gym
import os
import random
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
model_path = "C:/Users/sanka/codes/cart pole problem/tf_save3"
env = gym.make("CartPole-v0")
env.reset()
def train_set(): #training set generation function
try:
tx = np.load("final_trainx.npy")
ty = np.load("final_trainy.npy")
return tx,ty
except:
tx = []
ty = []
for _ in range(10000):
env.reset()
score = 0
moves = []
obs = []
p = []
for _ in range(500):
action = np.random.randint(0, 2)
observation, reward, done, info = env.step(action)
if (len(p)==0):
p = observation
else:
moves += [action]
obs += [observation]
p = observation
score += reward
if done:
break
if (score > 50):
tx+=obs
for i in range(len(moves)):
ac = moves[i]
if (ac == 1):
ty.append([0, 1])
else:
ty.append([1, 0])
tx=np.array(tx)
ty=np.array(ty)
np.save("final_trainx.npy",tx)
np.save("final_trainy.npy",ty)
return tx, ty
weights = {
1: tf.Variable(tf.truncated_normal([4, 128]), dtype=tf.float32),
2: tf.Variable(tf.truncated_normal([128, 256]), dtype=tf.float32),
3: tf.Variable(tf.truncated_normal([256, 512]), dtype=tf.float32),
4: tf.Variable(tf.truncated_normal([512, 256]), dtype=tf.float32),
5: tf.Variable(tf.truncated_normal([256, 128]), dtype=tf.float32),
6: tf.Variable(tf.truncated_normal([128, 2]), dtype=tf.float32)
}
biases = {
1: tf.Variable(tf.truncated_normal([128]), dtype=tf.float32),
2: tf.Variable(tf.truncated_normal([256]), dtype=tf.float32),
3: tf.Variable(tf.truncated_normal([512]), dtype=tf.float32),
4: tf.Variable(tf.truncated_normal([256]), dtype=tf.float32),
5: tf.Variable(tf.truncated_normal([128]), dtype=tf.float32),
6: tf.Variable(tf.truncated_normal([2]), dtype=tf.float32)
}
def neural_network(x):
x = tf.nn.relu(tf.add(tf.matmul(x, weights[1]), biases[1]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[2]), biases[2]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[3]), biases[3]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[4]), biases[4]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[5]), biases[5]))
x = tf.nn.dropout(x, 0.8)
x = tf.add(tf.matmul(x, weights[6]), biases[6])
return x
def test_nn(x):
x = tf.nn.relu(tf.add(tf.matmul(x, weights[1]), biases[1]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[2]), biases[2]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[3]), biases[3]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[4]), biases[4]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[5]), biases[5]))
x = tf.nn.softmax(tf.add(tf.matmul(x, weights[6]), biases[6]))
return x
def train_nn():
prediction = neural_network(x)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
lo=tf.nn.l2_loss(weights[1])+tf.nn.l2_loss(weights[2])+tf.nn.l2_loss(weights[3])+tf.nn.l2_loss(weights[4])+tf.nn.l2_loss(weights[5])+tf.nn.l2_loss(weights[6])
loss=tf.reduce_mean(loss+0.01*lo)
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss)
test_pred = test_nn(x)
correct = tf.equal(tf.argmax(test_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, dtype=tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
epoches = 5
batch_size = 100
for j in range(epoches):
ep_loss=0
for i in range(0,len(train_x),batch_size):
epoch_x=train_x[i:min(i+batch_size,len(train_x))]
epoch_y = train_y[i:min(i + batch_size, len(train_y))]
_,c=sess.run([optimizer,loss],feed_dict={x:epoch_x,y:epoch_y})
ep_loss+=c
#print("Accuracy is {0}".format(sess.run(accuracy, feed_dict={x: epoch_x, y: epoch_y})))
print("epoch {0} completed out of {1} with loss {2}".format(j,epoches,ep_loss))
print("Accuracy is {0}".format(sess.run(accuracy,feed_dict={x:train_x,y:train_y})))
scores = []
choices = []
for each_game in range(10):
print("game ", each_game)
score = 0
game_memory = []
prev_obs = []
env.reset()
for _ in range(500):
env.render()
if (len(prev_obs) == 0):
action = random.randrange(0, 2)
else:
x1 = np.array([prev_obs]).reshape(-1,4)
a = tf.argmax(test_pred, 1)
action = sess.run(a, feed_dict={x: x1})
action=action[0]
choices.append(action)
new_observation, reward, done, info = env.step(action)
prev_obs = new_observation
game_memory.append([new_observation, action])
score += reward
if done:
break
scores.append(score)
print('Average Score:', sum(scores) / len(scores))
print('choice 1:{} choice 0:{}'.format(choices.count(1) / len(choices), choices.count(0) / len(choices)))
train_x,train_y=train_set()
print(train_x.shape)
print(train_y.shape)
x=tf.placeholder(tf.float32,[None,4])
y=tf.placeholder(tf.int32,[None,2])
train_nn()
答案 0 :(得分：1)
因此，您首先收集或多或少做得很好的随机试验示例，然后根据这些示例训练您的模型？
在某种程度上，实际上并不是强化学习。您假设随机代理所采取的行动是好的并且正在学习模仿它。因此，如果您考虑一下，您的模型实际上会在60％的时间内预测随机代理的行为。考虑到这些行为是随机的，你的行为超过了50％，你实际上还是很富裕。
您只能获得50％以上，因为您只选择意外超过50分的随机游戏，因此它是游戏的非随机子集。 如果你提高标准只考虑超过100分的随机游戏或类似的东西，你应该得到更好的结果。通过这种方式，您将选择游戏玩法，而不是糟糕的游戏。
如果你想以更强化的学习方式来解决这个问题，那就是学习，而不是从别人的游戏中学习。我建议你看看Q-Learning或政策学习。
要记住的主要事项是，通常不会采取正确操作。也许不同的行为会导致相同的结果。因此，在给定状态的情况下，您应该尝试预测某个动作的预期结果，而不是试图预测某个状态下哪个动作是正确的。然后选择具有最佳预期结果的动作。