我正在使用IMAGE_SIZE = [2, 5928, 4728, 1]
模型结构可以在下面找到。我们正在使用特斯拉K80进行训练，因此我们有12GB的VRAM。我们使用CUDA 9.0，Tensorflow 1.6.0作为Keras 2.1.5的后端。
总有一些记忆问题。我和batch_size=1达成了协议，因为否则失败了。起初，我甚至无法执行fit()更多的步骤。它因内存错误而崩溃。然后我将内存使用限制在0.9，并且它训练好了一段时间。现在我试图从之前的训练中加载权重，然后再次出现内存错误。
有没有办法以某种方式调试它？加载砝码时为什么会失败？我想了解在这种情况下该怎么做。
提前致谢。
def make_model():
img_1_inp = Input(shape=IMAGE_SIZE)
img_2_inp = Input(shape=IMAGE_SIZE)
conv_1 = Conv2D(32, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_1(img_1_inp)
img_2 = conv_1(img_2_inp)
batch_norm_1 = BatchNormalization()
img_1 = batch_norm_1(img_1)
img_2 = batch_norm_1(img_2)
activation_1 = Activation('relu')
img_1 = activation_1(img_1)
img_2 = activation_1(img_2)
pooling_1 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_1(img_1)
img_2 = pooling_1(img_2)
conv_2_1 = Conv2D(64, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_2_1(img_1)
img_2 = conv_2_1(img_2)
batch_norm_2_1 = BatchNormalization()
img_1 = batch_norm_2_1(img_1)
img_2 = batch_norm_2_1(img_2)
activation_2_1 = Activation('relu')
img_1 = activation_2_1(img_1)
img_2 = activation_2_1(img_2)
conv_2_2 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_2(img_1)
img_2 = conv_2_2(img_2)
batch_norm_2_2 = BatchNormalization()
img_1 = batch_norm_2_2(img_1)
img_2 = batch_norm_2_2(img_2)
activation_2_2 = Activation('relu')
img_1 = activation_2_2(img_1)
img_2 = activation_2_2(img_2)
conv_2_3 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_3(img_1)
img_2 = conv_2_3(img_2)
batch_norm_2_3 = BatchNormalization()
img_1 = batch_norm_2_3(img_1)
img_2 = batch_norm_2_3(img_2)
activation_2_3 = Activation('relu')
img_1 = activation_2_3(img_1)
img_2 = activation_2_3(img_2)
pooling_2 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_2(img_1)
img_2 = pooling_2(img_2)
conv_3_1 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_1(img_1)
img_2 = conv_3_1(img_2)
batch_norm_3_1 = BatchNormalization()
img_1 = batch_norm_3_1(img_1)
img_2 = batch_norm_3_1(img_2)
activation_3_1 = Activation('relu')
img_1 = activation_3_1(img_1)
img_2 = activation_3_1(img_2)
conv_3_2 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_2(img_1)
img_2 = conv_3_2(img_2)
batch_norm_3_2 = BatchNormalization()
img_1 = batch_norm_3_2(img_1)
img_2 = batch_norm_3_2(img_2)
activation_3_2 = Activation('relu')
img_1 = activation_3_2(img_1)
img_2 = activation_3_2(img_2)
conv_3_3 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_3(img_1)
img_2 = conv_3_3(img_2)
batch_norm_3_3 = BatchNormalization()
img_1 = batch_norm_3_3(img_1)
img_2 = batch_norm_3_3(img_2)
activation_3_3 = Activation('relu')
img_1 = activation_3_3(img_1)
img_2 = activation_3_3(img_2)
pooling_3 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_3(img_1)
img_2 = pooling_3(img_2)
conv_4_1 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_1(img_1)
img_2 = conv_4_1(img_2)
batch_norm_4_1 = BatchNormalization()
img_1 = batch_norm_4_1(img_1)
img_2 = batch_norm_4_1(img_2)
activation_4_1 = Activation('relu')
img_1 = activation_4_1(img_1)
img_2 = activation_4_1(img_2)
conv_4_2 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_2(img_1)
img_2 = conv_4_2(img_2)
batch_norm_4_2 = BatchNormalization()
img_1 = batch_norm_4_2(img_1)
img_2 = batch_norm_4_2(img_2)
activation_4_2 = Activation('relu')
img_1 = activation_4_2(img_1)
img_2 = activation_4_2(img_2)
conv_4_3 = Conv2D(256, kernel_size=(3, 3),
strides=[10, 10],
activation=None)
img_1 = conv_4_3(img_1)
img_2 = conv_4_3(img_2)
batch_norm_4_3 = BatchNormalization()
img_1 = batch_norm_4_3(img_1)
img_2 = batch_norm_4_3(img_2)
activation_4_3 = Activation('relu')
img_1 = activation_4_3(img_1)
img_2 = activation_4_3(img_2)
g_pooling = GlobalAveragePooling2D(data_format='channels_last')
img_1 = g_pooling(img_1)
img_2 = g_pooling(img_2)
# flat_1 = Flatten()
# flatten_1 = flat_1(img_1)
# flatten_2 = flat_1(img_2)
#
# merged = keras.layers.concatenate([flatten_1, flatten_2])
merged = keras.layers.concatenate([img_1, img_2])
dropout_1 = Dropout(0.2)
merged = dropout_1(merged)
dense_1 = (Dense(1024, activation='relu'))(merged)
dropout_2 = Dropout(0.2)
dense_1 = dropout_2(dense_1)
dense_2 = (Dense(num_classes, activation='softmax'))(dense_1)
model = Model(inputs=[img_1_inp, img_2_inp], output=dense_2)
return model