我遇到了一个非常简单的问题。在Keras中训练模型后，我使用save（filepath）方法保存模型。然后，当我想继续训练时，我加载了模型，开始拟合模型，损失跳至420！ （从〜5起），我真的找不到原因。根据Keras的文档，save（）方法应该保存所有内容，体系结构，优化程序状态和权重。
#preprocessing function
def get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255, pixel_level=False):
def eraser(input_img):
img_h, img_w, img_c = input_img.shape
p_1 = np.random.rand()
if p_1 > p:
return norm(input_img)
while True:
s = np.random.uniform(s_l, s_h) * img_h * img_w
r = np.random.uniform(r_1, r_2)
w = int(np.sqrt(s / r))
h = int(np.sqrt(s * r))
left = np.random.randint(0, img_w)
top = np.random.randint(0, img_h)
if left + w <= img_w and top + h <= img_h:
break
if pixel_level:
c = np.random.uniform(v_l, v_h, (h, w, img_c))
else:
c = np.random.uniform(v_l, v_h)
input_img[top:top + h, left:left + w, :] = c
input_img = norm(input_img)
input_img = random_crop(input_img, (50, 50))
return input_img
return eraser
def norm(img):
return img / 127.5 - 1.
def random_crop(img, random_crop_size):
# Note: image_data_format is 'channel_last'
assert img.shape[2] == 3
height, width = img.shape[0], img.shape[1]
dy, dx = random_crop_size
x = np.random.randint(0, width - dx + 1)
y = np.random.randint(0, height - dy + 1)
crop = img[y:(y+dy), x:(x+dx), :]
return cv2.resize(crop, (height, width), cv2.INTER_LANCZOS4)
model = mn.MobileNetV2(input_shape=None, alpha=1.0, include_top=False, weights='imagenet', classes=179)
model.summary()
l = model.layers
for layer in l:
print(layer.get_config(), '\n')
if 'kernel_regularizer' in layer.get_config():
print('found kernel regularizer')
layer.kernel_regularizer=l2(l=0.1)
print('kernel regularizer', layer.kernel_regularizer)
if 'bias_regularizer' in layer.get_config():
print('found kernel regularizer')
layer.bias_regularizer=l2(l=0.1)
print('bias regularizer', layer.bias_regularizer)
x = Dropout(0.7)(l[-1].output)
x = Conv2D(179, (1,1), activation='linear')(x)
x = ReLU()(x)
x = GlobalAveragePooling2D()(x)
x = Softmax()(x)
model_mod = Model(inputs=model.input, outputs=x)
gen_t = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
rotation_range=45,
width_shift_range=0.3,
height_shift_range=0.3,
shear_range = 0.3,
zoom_range = 0.3,
preprocessing_function=get_random_eraser(s_l=0, s_h=0.8),
validation_split=0.1
)
gen_v = ImageDataGenerator(
preprocessing_function=norm,
validation_split=0.1
)
early_stop = EarlyStopping(patience=10, restore_best_weights=True, verbose=True)
tb = TensorBoard(batch_size=32)
mc = ModelCheckpoint('mobilenetv2_combined.hdf5', monitor='val_loss', save_best_only=True, verbose=True)
train_generator = gen_t.flow_from_directory(os.path.join(DATA_FOLDER_PATH, 'data_mod', 'train'), target_size=(256, 256), batch_size=32, subset="training")
validation_generator = gen_v.flow_from_directory(os.path.join(DATA_FOLDER_PATH, 'data_mod', 'train'), target_size=(256, 256), batch_size=32, subset="validation")
class_weights = class_weight.compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
model_mod.compile(k.optimizers.sgd(lr=0.001, momentum=0.9, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy', 'top_k_categorical_accuracy'])
hist = model_mod.fit_generator(train_generator,validation_data=validation_generator, epochs=1, initial_epoch=0, callbacks=[early_stop, tb, mc], class_weight=class_weights)
model_mod.save('mobilenet_model_save.h5')
Found 17924 images belonging to 179 classes.
Found 1910 images belonging to 179 classes.
Epoch 1/1
561/561 [==============================] - 415s 741ms/step - loss: 4.9594 - acc: 0.0322 - top_k_categorical_accuracy: 0.1134 - val_loss: 4.4137 - val_acc: 0.0921 - val_top_k_categorical_accuracy: 0.2644
Epoch 00001: val_loss improved from inf to 4.41366, saving model to mobilenetv2_combined.hdf5
这是我正在训练的代码。现在，基本上相同的代码可以继续训练（这只是为了说明）：
gen_t = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
rotation_range=45,
width_shift_range=0.3,
height_shift_range=0.3,
shear_range = 0.3,
zoom_range = 0.3,
preprocessing_function=get_random_eraser(s_l=0, s_h=0.8),
validation_split=0.1
)
gen_v = ImageDataGenerator(
preprocessing_function=norm,
validation_split=0.1
)
early_stop = EarlyStopping(patience=10, restore_best_weights=True, verbose=True)
tb = TensorBoard(batch_size=32)
mc = ModelCheckpoint('mobilenetv2_combined.hdf5', monitor='val_loss', save_best_only=True, verbose=True)
train_generator = gen_t.flow_from_directory(os.path.join(DATA_FOLDER_PATH, 'data_mod', 'train'), target_size=(256, 256), batch_size=32, subset="training")
validation_generator = gen_v.flow_from_directory(os.path.join(DATA_FOLDER_PATH, 'data_mod', 'train'), target_size=(256, 256), batch_size=32, subset="validation")
model_mod = load_model('mobilenet_model_save.h5')
class_weights = class_weight.compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
#model_mod.compile(adam(lr=0.0001, decay=1e-6), loss='categorical_crossentropy', metrics=['accuracy', 'top_k_categorical_accuracy'])
model_mod.compile(k.optimizers.sgd(lr=0.001, momentum=0.9, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy', 'top_k_categorical_accuracy'])
hist = model_mod.fit_generator(train_generator,validation_data=validation_generator, epochs=2, initial_epoch=1, callbacks=[early_stop, tb, mc], class_weight=class_weights)
model_mod.save('mobilenet_model_save.h5')
Found 17924 images belonging to 179 classes.
Found 1910 images belonging to 179 classes.
Epoch 2/2
561/561 [==============================] - 373s 665ms/step - loss: 174.3220 - acc: 0.0815 - top_k_categorical_accuracy: 0.2320 - val_loss: 49.8441 - val_acc: 0.0110 - val_top_k_categorical_accuracy: 0.0455
Epoch 00002: val_loss improved from inf to 49.84411, saving model to mobilenetv2_combined.hdf5
有人知道发生了什么吗？我在MNIST上尝试了一个非常简单的玩具示例，似乎一切正常。我会为任何建议感到高兴。还有一件有趣的事情，那就是损失函数的值。网络的准确性与训练后保持不变，例如训练后，网络以40％的精度结束，而当我继续训练时（损失跳得很大），精度仍然是40％。
答案 0 :(得分：0)
所以我还没有弄清楚，但是我猜这可能是保存某种“自定义”（来自应用程序模块）网络问题，或者是由于使用了旧版本2.2.0（由于squeezenet错误）。
我怀疑这个问题会比过去10天得到更多的关注，所以我将结束这个问题。
我的“解决方案”是不间断地一次训练网络。
答案 1 :(得分：0)
无法发表评论，因此发布： 这里的问题不是model.save（）不会保存优化器的状态。 即学习率可能很高，这就是为什么在重新开始训练后您的损失会突然增加。