编辑:尝试了几件事后,我在代码中添加了以下内容:
with tf.Session(graph=self.graph) as session:
session.run(tf.initialize_all_variables())
try:
session.run(tf.assert_variables_initialized())
except tf.errors.FailedPreconditionError:
raise RuntimeError("Not all variables initialized!")
现在,偶尔会失败,即tf.assert_variables_initialized()
会引发FailedPreconditionError,即使在它之前,tf.initialize_all_variables()
也已执行。有谁知道这会怎么样?
原始问题:
背景
我在通过Tensorflow创建的基本神经网络上使用GradientDescentOptimizer运行交叉验证(CV)超参数搜索。在看似随机的时刻,我得到了一个FailedPreconditionError,用于不同的变量。例如(帖子末尾的完整堆栈跟踪):
FailedPreconditionError: Attempting to use uninitialized value Variable_5
[[Node: Variable_5/read = Identity[T=DT_FLOAT, _class=["loc:@Variable_5"], _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_5)]]
有些跑步失败的速度相当快,有些跑步失败了 - 现在已经跑了15个小时没有问题。我在多个GPU上并行运行 - 不是优化本身,而是每个CV折叠。
我检查了什么
从this和this帖子我了解到在尝试使用尚未使用tf.initialize_all_variables()
初始化的变量时会发生此错误。但是,我99%肯定我正在这样做(如果没有,我希望总是失败) - 我会在下面发布代码。
运行操作时最常引发此异常 在初始化之前读取一个tf.Variable。
“最常见”表示它也可以在不同场景中提升。所以,现在主要的问题是:
问题: 是否存在可能引发此异常的其他情况,它们是什么?
代码
MLP课程:
class MLP(object):
def __init__(self, n_in, hidden_config, n_out, optimizer, f_transfer=tf.nn.tanh, f_loss=mean_squared_error,
f_out=tf.identity, seed=None, global_step=None, graph=None, dropout_keep_ratio=1):
self.graph = tf.Graph() if graph is None else graph
# all variables defined below
with self.graph.as_default():
self.X = tf.placeholder(tf.float32, shape=(None, n_in))
self.y = tf.placeholder(tf.float32, shape=(None, n_out))
self._init_weights(n_in, hidden_config, n_out, seed)
self._init_computations(f_transfer, f_loss, f_out)
self._init_optimizer(optimizer, global_step)
def fit_validate(self, X, y, val_X, val_y, val_f, iters=100, val_step=1):
[snip]
with tf.Session(graph=self.graph) as session:
VAR INIT HERE-->tf.initialize_all_variables().run() #<-- VAR INIT HERE
for i in xrange(iters):
[snip: get minibatch here]
_, l = session.run([self.optimizer, self.loss], feed_dict={self.X:X_batch, self.y:y_batch})
# validate
if i % val_step == 0:
val_yhat = self.validation_yhat.eval(feed_dict=val_feed_dict, session=session)
如您所见,在完成任何其他操作之前,始终会调用tf.init_all_variables().run()
。网络初始化为:
def estimator_getter(params):
[snip]
graph = tf.Graph()
with graph.as_default():
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(params.get('learning_rate',0.1), global_step, decay_steps, decay_rate)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
net = MLP(config_num_inputs[config_id], hidden, 1, optimizer, seed=params.get('seed',100), global_step=global_step, graph=graph, dropout_keep_ratio=dropout)
完整示例堆栈跟踪:
FailedPreconditionError: Attempting to use uninitialized value Variable_5
[[Node: Variable_5/read = Identity[T=DT_FLOAT, _class=["loc:@Variable_5"], _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_5)]]
Caused by op u'Variable_5/read', defined at:
File "tf_paramsearch.py", line 373, in <module>
randomized_search_params(int(sys.argv[1]))
File "tf_paramsearch.py", line 356, in randomized_search_params
hypersearch.fit()
File "/home/centos/ODQ/main/python/odq/cv.py", line 430, in fit
return self._fit(sampled_params)
File "/home/centos/ODQ/main/python/odq/cv.py", line 190, in _fit
for train_key, test_key in self.cv)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 766, in __call__
n_jobs = self._initialize_pool()
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 537, in _initialize_pool
self._pool = MemmapingPool(n_jobs, **poolargs)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 580, in __init__
super(MemmapingPool, self).__init__(**poolargs)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 418, in __init__
super(PicklingPool, self).__init__(**poolargs)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 159, in __init__
self._repopulate_pool()
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 223, in _repopulate_pool
w.start()
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 130, in start
self._popen = Popen(self)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/forking.py", line 126, in __init__
code = process_obj._bootstrap()
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 113, in worker
result = (True, func(*args, **kwds))
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 130, in __call__
return self.func(*args, **kwargs)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 72, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "/home/centos/ODQ/main/python/odq/cv.py", line 131, in _fold_runner
estimator = estimator_getter(parameters)
File "tf_paramsearch.py", line 264, in estimator_getter
net = MLP(config_num_inputs[config_id], hidden, 1, optimizer, seed=params.get('seed',100), global_step=global_step, graph=graph, dropout_keep_ratio=dropout)
File "tf_paramsearch.py", line 86, in __init__
self._init_weights(n_in, hidden_config, n_out, seed)
File "tf_paramsearch.py", line 105, in _init_weights
self.out_weights = tf.Variable(tf.truncated_normal([hidden_config[-1], n_out], stddev=stdev))
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 206, in __init__
dtype=dtype)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 275, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 523, in identity
return _op_def_lib.apply_op("Identity", input=input, name=name)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
op_def=op_def)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2117, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
self._traceback = _extract_stack()
答案 0 :(得分:5)
好的,我发现了问题。在我的代码中有一个罕见的情况导致其中一个隐藏层用形状(0,N)创建,即没有输入。在这种情况下,Tensorflow显然无法初始化与该层有关的变量。
虽然这是有道理的,但Tensorflow在这种情况下记录警告消息可能很有用(顺便说一下,我也尝试将Tensorflow日志记录设置为调试模式,但无法找到 - tf.logging.set_verbosity()
如何做好像有效果。)
答案 1 :(得分:1)
tf.reset_default_graph()
a = tf.constant(1)
<add more operations to your graph>
b = tf.Variable(1)
init_op = tf.initialize_all_variables()
tf.get_default_graph().finalize()
sess = tf.InteractiveSession()
sess.run(init_op)
sess.run(compute_op)
finalize
阻止您在当前版本中运行缓慢的运行之间修改图形。另外,因为有一个会话/一个图表,所以您不需要with
块。
答案 2 :(得分:0)
对我来说,解决方案是
with sess.as_default():
result = compute_fn([seed_input,1])
检查FailedPreconditionError: Attempting to use uninitialized in Tensorflow 其他选项和我的解释
奇怪session.run()
与使用sess.as_default()
运行函数不同,我试过了。