我正在尝试在anaconda中使用keras-gpu，在模型上可以运行：
内部错误：Blas SGEMM启动失败
我尝试了来自先前查询的解决方案，但均出现相同的错误，但无济于事。在使用GPU的同一节点上没有其他程序正在运行。
在运行jupyter笔记本的终端中
[E tensorflow/stream_executor/cuda/cuda_blas.cc:472] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_FAILED
I tensorflow/stream_executor/stream.cc:3813] stream 0x558aa7477d00 did not memzero GPU location; source: 0x2aabe36090d0]
在jupyter笔记本中：
Epoch 1/3
---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1021 try:
-> 1022 return fn(*args)
1023 except errors.OpError as e:
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
1003 feed_dict, fetch_list, target_list,
-> 1004 status, run_metadata)
1005
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
87 try:
---> 88 next(self.gen)
89 except StopIteration:
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status()
465 compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466 pywrap_tensorflow.TF_GetCode(status))
467 finally:
InternalError: Blas SGEMM launch failed : a.shape=(10, 512), b.shape=(10, 512), m=512, n=512, k=10
[[Node: training/SGD/gradients/model_2/dense_4/MatMul_grad/MatMul_1 = MatMul[T=DT_FLOAT, _class=["loc:@model_2/dense_4/MatMul"], transpose_a=true, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](model_2/dropout_5/cond/Merge, training/SGD/gradients/model_2/dense_4/Relu_grad/ReluGrad)]]
During handling of the above exception, another exception occurred:
InternalError Traceback (most recent call last)
in ()
----> 1 model_vgg_reg1.fit_generator(gen_va, epochs=3, steps_per_epoch=10000)
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name +
90 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
2222 outs = self.train_on_batch(x, y,
2223 sample_weight=sample_weight,
-> 2224 class_weight=class_weight)
2225
2226 if not isinstance(outs, list):
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1881 ins = x + y + sample_weights
1882 self._make_train_function()
-> 1883 outputs = self.train_function(ins)
1884 if len(outputs) == 1:
1885 return outputs[0]
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2476 session = get_session()
2477 updated = session.run(fetches=fetches, feed_dict=feed_dict,
-> 2478 **self.session_kwargs)
2479 return updated[:len(self.outputs)]
2480
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
765 try:
766 result = self._run(None, fetches, feed_dict, options_ptr,
--> 767 run_metadata_ptr)
768 if run_metadata:
769 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
963 if final_fetches or final_targets:
964 results = self._do_run(handle, final_targets, final_fetches,
--> 965 feed_dict_string, options, run_metadata)
966 else:
967 results = []
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1013 if handle is None:
1014 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1015 target_list, options, run_metadata)
1016 else:
1017 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1033 except KeyError:
1034 pass
-> 1035 raise type(e)(node_def, op, message)
1036
1037 def _extend_graph(self):
InternalError: Blas SGEMM launch failed : a.shape=(10, 512), b.shape=(10, 512), m=512, n=512, k=10
[[Node: training/SGD/gradients/model_2/dense_4/MatMul_grad/MatMul_1 = MatMul[T=DT_FLOAT, _class=["loc:@model_2/dense_4/MatMul"], transpose_a=true, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](model_2/dropout_5/cond/Merge, training/SGD/gradients/model_2/dense_4/Relu_grad/ReluGrad)]]
Caused by op 'training/SGD/gradients/model_2/dense_4/MatMul_grad/MatMul_1', defined at:
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 478, in start
self.io_loop.start()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/zmq/eventloop/ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tornado/ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2728, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2856, in run_ast_nodes
if self.run_code(code, result):
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2910, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 1, in
model_vgg_reg1.fit_generator(gen_va, epochs=3, steps_per_epoch=10000)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/training.py", line 2080, in fit_generator
self._make_train_function()
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/training.py", line 990, in _make_train_function
loss=self.total_loss)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/optimizers.py", line 173, in get_updates
grads = self.get_gradients(loss, params)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/optimizers.py", line 78, in get_gradients
grads = K.gradients(loss, params)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2515, in gradients
return tf.gradients(loss, variables, colocate_gradients_with_ops=True)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 482, in gradients
in_grads = grad_fn(op, *out_grads)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py", line 768, in _MatMulGrad
op.inputs[0], grad, transpose_a=True))
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1765, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1454, in _mat_mul
transpose_b=transpose_b, name=name)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1226, in __init__
self._traceback = _extract_stack()
...which was originally created as op 'model_2/dense_4/MatMul', defined at:
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
[elided 16 identical lines from previous traceback]
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2728, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2850, in run_ast_nodes
if self.run_code(code, result):
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2910, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 1, in
final_output = top_model(vgg_full.output)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/topology.py", line 619, in __call__
output = self.call(inputs, **kwargs)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/topology.py", line 2085, in call
output_tensors, _, _ = self.run_internal_graph(inputs, masks)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/engine/topology.py", line 2236, in run_internal_graph
output_tensors = _to_list(layer.call(computed_tensor, **kwargs))
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/layers/core.py", line 855, in call
output = K.dot(inputs, self.kernel)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 1075, in dot
out = tf.matmul(x, y)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1765, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1454, in _mat_mul
transpose_b=transpose_b, name=name)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/pylon5/ir5fpcp/amogh112/faus_dl/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1226, in __init__
self._traceback = _extract_stack()
InternalError (see above for traceback): Blas SGEMM launch failed : a.shape=(10, 512), b.shape=(10, 512), m=512, n=512, k=10
[[Node: training/SGD/gradients/model_2/dense_4/MatMul_grad/MatMul_1 = MatMul[T=DT_FLOAT, _class=["loc:@model_2/dense_4/MatMul"], transpose_a=true, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](model_2/dropout_5/cond/Merge, training/SGD/gradients/model_2/dense_4/Relu_grad/ReluGrad)]]