我正在Keras中编写一个自定义层（注意层）。该层如下所示：
class MultiHeadAdditiveSelfAttention(Layer):
def __init__(self, n_heads, alignment_vector_size, window_size = None, return_sequence=False, mode="add", **kwargs):
self.n_heads = n_heads
self.alignment_vector_size = alignment_vector_size
self.return_sequence = return_sequence
self.window_size = window_size
self.mode = mode
super(MultiHeadAdditiveSelfAttention, self).__init__(**kwargs)
def build(self, input_shape):
self.input_shape_ = input_shape
self.alignment_vector = self.add_weight(name="alignment_vector", shape=(self.n_heads, self.alignment_vector_size,), initializer='uniform', trainable=True)
self.kernel = self.add_weight(name="kernel", shape=(self.alignment_vector_size, input_shape[2]), initializer='uniform', trainable=True)
self.bias = self.add_weight(name="bias", shape=(self.alignment_vector_size,), initializer='uniform', trainable=True)
self.attention_weights = self.add_weight(name="attention_weights", shape=(self.n_heads, input_shape[1]), initializer='uniform', trainable=False)
super(MultiHeadAdditiveSelfAttention, self).build(input_shape)
def call(self, hidden_state_sequence):
hidden_state_sequence.set_shape(self.input_shape_)
if self.window_size is not None and self.input_shape_[1] > self.window_size:
hidden_state_sequence = hidden_state_sequence[:, -self.window_size:]
u = K.tanh( K.squeeze(K.dot(hidden_state_sequence, K.expand_dims(self.kernel)), axis=-1) + self.bias )
score = K.squeeze(K.dot(u, K.expand_dims(self.alignment_vector)), axis=-1)
self.attention_weights = K.softmax( score, axis=1 )
self.attention_weights = K.permute_dimensions(self.attention_weights, (0,2,1))
if self.return_sequence:
print(K.expand_dims(self.attention_weights,axis=3).shape)
print(hidden_state_sequence.shape)
context_vector = K.expand_dims(self.attention_weights, axis=3)* hidden_state_sequence
print(context_vector.shape)
context_vector = K.permute_dimensions(context_vector, (0,2,1,3))
context_vector = K.reshape(context_vector, (-1, context_vector.shape[1], context_vector.shape[2]*context_vector.shape[3]))
else:
context_vector = K.batch_dot(self.attention_weights, hidden_state_sequence)
context_vector = K.reshape(context_vector, (-1, context_vector.shape[1]*context_vector.shape[2]))
return context_vector
def compute_output_shape(self, input_shape):
if self.return_sequence:
if self.window_size is not None and self.input_shape_[1] > self.window_size:
return (input_shape[0], self.window_size, input_shape[2])
return (input_shape[0], input_shape[1], input_shape[2]*self.n_heads)
else:
return (input_shape[0], input_shape[2]*self.n_heads)
K.expand_dims（self.attention_weights，axis = 3）的形状为（？，8，80，1），形状为（batch_size，n_heads，sequence_length）
hidden_state_sequenc的形状为（？，80，256），形状为（batch_size，n_rnn_neurons）
重塑前和context_vector的形状为（？，8，80，256），重塑后为（？，80，2048）
这样使用（与imdb数据集一起使用）：
inputs = Input(shape=(maxlen,))
x = Embedding(max_features, 128)(inputs)
x = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))(x)
x = MultiHeadAdditiveSelfAttention(8, 10, return_sequence=True, name="Attention")(x)
x = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2))(x)
predictions = Dense(1, activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
print(model.summary())
一切都可以编译，但是当我想训练模型时，出现形状错误：
Epoch 1/6
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1321 try:
-> 1322 return fn(*args)
1323 except errors.OpError as e:
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1306 return self._call_tf_sessionrun(
-> 1307 options, feed_dict, fetch_list, target_list, run_metadata)
1308
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1408 self._session, options, feed_dict, fetch_list, target_list,
-> 1409 run_metadata)
1410 else:
InvalidArgumentError: Incompatible shapes: [32,8,80,1] vs. [32,80,256]
[[Node: Attention_85/mul = Mul[T=DT_FLOAT, _class=["loc:@training_14/RMSprop/gradients/AddN_23"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](Attention_85/ExpandDims_3, bidirectional_112/concat)]]
[[Node: metrics_28/acc/Mean_1/_1241 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_7813_metrics_28/acc/Mean_1", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-268-96d01a665879> in <module>()
----> 1 model.fit(x_train, y_train, batch_size=batch_size, epochs=6, validation_data=(x_test, y_test))
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1703 initial_epoch=initial_epoch,
1704 steps_per_epoch=steps_per_epoch,
-> 1705 validation_steps=validation_steps)
1706
1707 def evaluate(self, x=None, y=None,
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
1234 ins_batch[i] = ins_batch[i].toarray()
1235
-> 1236 outs = f(ins_batch)
1237 if not isinstance(outs, list):
1238 outs = [outs]
C:\ProgramData\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py in __call__(self, inputs)
2480 session = get_session()
2481 updated = session.run(fetches=fetches, feed_dict=feed_dict,
-> 2482 **self.session_kwargs)
2483 return updated[:len(self.outputs)]
2484
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
898 try:
899 result = self._run(None, fetches, feed_dict, options_ptr,
--> 900 run_metadata_ptr)
901 if run_metadata:
902 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1133 if final_fetches or final_targets or (handle and feed_dict_tensor):
1134 results = self._do_run(handle, final_targets, final_fetches,
-> 1135 feed_dict_tensor, options, run_metadata)
1136 else:
1137 results = []
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1314 if handle is None:
1315 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1316 run_metadata)
1317 else:
1318 return self._do_call(_prun_fn, handle, feeds, fetches)
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1333 except KeyError:
1334 pass
-> 1335 raise type(e)(node_def, op, message)
1336
1337 def _extend_graph(self):
InvalidArgumentError: Incompatible shapes: [32,8,80,1] vs. [32,80,256]
[[Node: Attention_85/mul = Mul[T=DT_FLOAT, _class=["loc:@training_14/RMSprop/gradients/AddN_23"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](Attention_85/ExpandDims_3, bidirectional_112/concat)]]
[[Node: metrics_28/acc/Mean_1/_1241 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_7813_metrics_28/acc/Mean_1", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
Caused by op 'Attention_85/mul', defined at:
File "C:\ProgramData\Anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\ProgramData\Anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "C:\ProgramData\Anaconda3\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 478, in start
self.io_loop.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2728, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2850, in run_ast_nodes
if self.run_code(code, result):
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2910, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-267-95c219d983f6>", line 6, in <module>
x = MultiHeadAdditiveSelfAttention(8, 10, return_sequence=True, name="Attention")(x)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\topology.py", line 619, in __call__
output = self.call(inputs, **kwargs)
File "<ipython-input-264-34a6413646ed>", line 91, in call
context_vector = K.expand_dims(self.attention_weights, axis=3)* hidden_state_sequence
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_ops.py", line 979, in binary_op_wrapper
return func(x, y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1211, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 5066, in mul
"Mul", x=x, y=y, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Incompatible shapes: [32,8,80,1] vs. [32,80,256]
[[Node: Attention_85/mul = Mul[T=DT_FLOAT, _class=["loc:@training_14/RMSprop/gradients/AddN_23"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](Attention_85/ExpandDims_3, bidirectional_112/concat)]]
[[Node: metrics_28/acc/Mean_1/_1241 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_7813_metrics_28/acc/Mean_1", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
在乘法运算中似乎有一个错误，但是我不知道为什么，因为Keras在调用过程中使用形状似乎很好。