Question

我做什么：

我正在用Keras fit_generator()训练预训练的CNN。这会在每个时期之后产生评估指标（loss, acc, val_loss, val_acc）。训练模型后，我用loss, acc生成评估指标（evaluate_generator()）。

我的期望：

如果我训练模型一个时期，那么我期望通过fit_generator()和evaluate_generator()获得的度量是相同的。他们俩都应基于整个数据集得出指标。

我观察到的情况

loss和acc与fit_generator()和evaluate_generator()不同：

我不明白的地方：

为什么fit_generator()的准确度是与evaluate_generator()

我的代码：

def generate_data(path, imagesize, nBatches):
    datagen = ImageDataGenerator(rescale=1./255)
    generator = datagen.flow_from_directory\
        (directory=path,                                        # path to the target directory
         target_size=(imagesize,imagesize),                     # dimensions to which all images found will be resize
         color_mode='rgb',                                      # whether the images will be converted to have 1, 3, or 4 channels
         classes=None,                                          # optional list of class subdirectories
         class_mode='categorical',                              # type of label arrays that are returned
         batch_size=nBatches,                                   # size of the batches of data
         shuffle=True)                                          # whether to shuffle the data
    return generator

[...]

def train_model(model, nBatches, nEpochs, trainGenerator, valGenerator, resultPath):
    history = model.fit_generator(generator=trainGenerator,
                                  steps_per_epoch=trainGenerator.samples//nBatches,     # total number of steps (batches of samples)
                                  epochs=nEpochs,                   # number of epochs to train the model
                                  verbose=2,                        # verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch
                                  callbacks=None,                   # keras.callbacks.Callback instances to apply during training
                                  validation_data=valGenerator,     # generator or tuple on which to evaluate the loss and any model metrics at the end of each epoch
                                  validation_steps=
                                  valGenerator.samples//nBatches,   # number of steps (batches of samples) to yield from validation_data generator before stopping at the end of every epoch
                                  class_weight=None,                # optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function
                                  max_queue_size=10,                # maximum size for the generator queue
                                  workers=32,                       # maximum number of processes to spin up when using process-based threading
                                  use_multiprocessing=True,         # whether to use process-based threading
                                  shuffle=False,                     # whether to shuffle the order of the batches at the beginning of each epoch
                                  initial_epoch=0)                  # epoch at which to start training
    print("%s: Model trained." % datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))

    # Save model
    modelPath = os.path.join(resultPath, datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + '_modelArchitecture.h5')
    weightsPath = os.path.join(resultPath, datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + '_modelWeights.h5')
    model.save(modelPath)
    model.save_weights(weightsPath)
    print("%s: Model saved." % datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
    return history, model

[...]

def evaluate_model(model, generator):
    score = model.evaluate_generator(generator=generator,           # Generator yielding tuples
                                     steps=
                                     generator.samples//nBatches)   # number of steps (batches of samples) to yield from generator before stopping

    print("%s: Model evaluated:"
          "\n\t\t\t\t\t\t Loss: %.3f"
          "\n\t\t\t\t\t\t Accuracy: %.3f" %
          (datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
           score[0], score[1]))

[...]

def main():
    # Create model
    modelUntrained = create_model(imagesize, nBands, nClasses)

    # Prepare training and validation data
    trainGenerator = generate_data(imagePathTraining, imagesize, nBatches)
    valGenerator = generate_data(imagePathValidation, imagesize, nBatches)

    # Train and save model
    history, modelTrained = train_model(modelUntrained, nBatches, nEpochs, trainGenerator, valGenerator, resultPath)

    # Evaluate on validation data
    print("%s: Model evaluation (valX, valY):" % datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
    evaluate_model(modelTrained, valGenerator)

    # Evaluate on training data
    print("%s: Model evaluation (trainX, trainY):" % datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
    evaluate_model(modelTrained, trainGenerator)

更新

我发现了一些报告此问题的网站：

到目前为止，我一直尝试遵循他们提出的一些解决方案，但没有成功。 acc和loss与fit_generator()和evaluate_generator()仍然不同，即使使用由同一生成器生成的完全相同的数据进行训练和验证也是如此。这是我尝试过的：

静态地为整个脚本设置learning_phase，或者在将新的图层添加到预训练的图层之前

    K.set_learning_phase(0) # testing  
    K.set_learning_phase(1) # training

从预训练模型中解冻所有批次归一化层

    for i in range(len(model.layers)):
        if str.startswith(model.layers[i].name, 'bn'):
            model.layers[i].trainable=True

不将辍学或批处理规范化添加为未训练的层

    # Create pre-trained base model
    basemodel = ResNet50(include_top=False,                     # exclude final pooling and fully connected layer in the original model
                         weights='imagenet',                    # pre-training on ImageNet
                         input_tensor=None,                     # optional tensor to use as image input for the model
                         input_shape=(imagesize,                # shape tuple
                                      imagesize,
                                      nBands),
                         pooling=None,                          # output of the model will be the 4D tensor output of the last convolutional layer
                         classes=nClasses)                      # number of classes to classify images into

    # Create new untrained layers
    x = basemodel.output
    x = GlobalAveragePooling2D()(x)                             # global spatial average pooling layer
    x = Dense(1024, activation='relu')(x)                       # fully-connected layer
    y = Dense(nClasses, activation='softmax')(x)                # logistic layer making sure that probabilities sum up to 1

    # Create model combining pre-trained base model and new untrained layers
    model = Model(inputs=basemodel.input,
                  outputs=y)

    # Freeze weights on pre-trained layers
    for layer in basemodel.layers:
        layer.trainable = False

    # Define learning optimizer
    learningRate = 0.01
    optimizerSGD = optimizers.SGD(lr=learningRate,              # learning rate.
                                  momentum=0.9,                 # parameter that accelerates SGD in the relevant direction and dampens oscillations
                                  decay=learningRate/nEpochs,   # learning rate decay over each update
                                  nesterov=True)                # whether to apply Nesterov momentum
    # Compile model
    model.compile(optimizer=optimizerSGD,                       # stochastic gradient descent optimizer
                  loss='categorical_crossentropy',              # objective function
                  metrics=['accuracy'],                         # metrics to be evaluated by the model during training and testing
                  loss_weights=None,                            # scalar coefficients to weight the loss contributions of different model outputs
                  sample_weight_mode=None,                      # sample-wise weights
                  weighted_metrics=None,                        # metrics to be evaluated and weighted by sample_weight or class_weight during training and testing
                  target_tensors=None)                          # tensor model's target, which will be fed with the target data during training

使用不同的预训练CNN作为基本模型（ VGG19，InceptionV3，InceptionResNetV2，Xception ）

    from keras.applications.vgg19 import VGG19

    basemodel = VGG19(include_top=False,                        # exclude final pooling and fully connected layer in the original model
                         weights='imagenet',                    # pre-training on ImageNet
                         input_tensor=None,                     # optional tensor to use as image input for the model
                         input_shape=(imagesize,                # shape tuple
                                      imagesize,
                                      nBands),
                         pooling=None,                          # output of the model will be the 4D tensor output of the last convolutional layer
                         classes=nClasses)                      # number of classes to classify images into

请让我知道我是否还缺少其他解决方案。

Answer 1

在这种情况下，训练一个时期可能不足以提供足够的信息。同样，您的训练数据和测试数据可能也不完全相同，因为您没有为flow_from_directory方法设置随机种子。看看here。

也许，您可以设置种子，删除增强（如果有）并保存经过训练的模型权重，以便稍后加载以进行检查。

Answer 2

I now managed having the same evaluation metrics. I changed the following:

I set seed in flow_from_directory() as suggested by @Anakin

def generate_data(path, imagesize, nBatches):
        datagen = ImageDataGenerator(rescale=1./255)
        generator = datagen.flow_from_directory(directory=path,     # path to the target directory
             target_size=(imagesize,imagesize),                     # dimensions to which all images found will be resize
             color_mode='rgb',                                      # whether the images will be converted to have 1, 3, or 4 channels
             classes=None,                                          # optional list of class subdirectories
             class_mode='categorical',                              # type of label arrays that are returned
             batch_size=nBatches,                                   # size of the batches of data
             shuffle=True,                                          # whether to shuffle the data
             seed=42)                                               # random seed for shuffling and transformations
        return generator

I set use_multiprocessing=False in fit_generator() according to the warning: use_multiprocessing=True and multiple workers may duplicate your data

history = model.fit_generator(generator=trainGenerator,
                                  steps_per_epoch=trainGenerator.samples//nBatches,     # total number of steps (batches of samples)
                                  epochs=nEpochs,                   # number of epochs to train the model
                                  verbose=2,                        # verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch
                                  callbacks=callback,               # keras.callbacks.Callback instances to apply during training
                                  validation_data=valGenerator,     # generator or tuple on which to evaluate the loss and any model metrics at the end of each epoch
                                  validation_steps=
                                  valGenerator.samples//nBatches,   # number of steps (batches of samples) to yield from validation_data generator before stopping at the end of every epoch
                                  class_weight=None,                # optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function
                                  max_queue_size=10,                # maximum size for the generator queue
                                  workers=1,                        # maximum number of processes to spin up when using process-based threading
                                  use_multiprocessing=False,        # whether to use process-based threading
                                  shuffle=False,                    # whether to shuffle the order of the batches at the beginning of each epoch
                                  initial_epoch=0)                  # epoch at which to start training

I unified my python setup as suggested in the keras documentation on how to obtain reproducible results using Keras during development

import tensorflow as tf
import random as rn
from keras import backend as K

np.random.seed(42)
rn.seed(12345)
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
                              inter_op_parallelism_threads=1)
tf.set_random_seed(1234)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)

Instead of rescaling input images with datagen = ImageDataGenerator(rescale=1./255), I now generate my data with:

from keras.applications.resnet50 import preprocess_input
datagen = ImageDataGenerator(preprocessing_function=preprocess_input)

With this, I managed to have a similar accuracy and loss from fit_generator() and evaluate_generator(). Also, using the same data for training and testing now results in a similar metrics. Reasons for remaining differences are provided in the keras documentation.

Answer 3

将const wrapper = mount( <Provider...>... <ButtonA/> </Provider>); it('test click button', () => { const myButton = wrapper.find('.my-button'); //*********** Here I'm able to debug easily. myButton.simulate('click'); });级别设置为use_multiprocessing=False可以解决BUT问题，但会大大降低训练速度。更好但仍不完善的解决方法是，仅将验证生成器设置为fit_generator，因为下面的代码是从keras的use_multiprocessing=False函数修改而来的。

fit_generator

为什么fit_generator的准确性与Keras中的valuate_generator的准确性不同？

3 个答案: