Keras图像预处理

Question

我的训练图像是与其相关的HR图像的缩小版本。因此，输入图像和输出图像的尺寸不同。目前，我使用的是13张图像的手工制作样本，但最终我希望能够使用500像素的HR（高分辨率）图像数据集。但是，该数据集没有相同尺寸的图像，因此我猜测必须对其进行裁剪才能获得统一的尺寸。

我目前已经设置了以下代码：它需要一堆512x512x3图像，并应用了一些变换来增强数据（翻转）。因此，我获得了一组基本的HR形式的39张图像，然后将它们按比例缩小4倍，从而得到了由39张尺寸为128x128x3的图像组成的训练集。

import numpy as np

from keras.preprocessing.image import ImageDataGenerator

import matplotlib.image as mpimg
import skimage
from skimage import transform

from constants import data_path
from constants import img_width
from constants import img_height

from model import setUpModel


def setUpImages():

    train = []
    finalTest = []

    sample_amnt = 11
    max_amnt = 13

    # Extracting images (512x512)
    for i in range(sample_amnt):
        train.append(mpimg.imread(data_path + str(i) + '.jpg'))

    for i in range(max_amnt-sample_amnt):
        finalTest.append(mpimg.imread(data_path + str(i+sample_amnt) + '.jpg'))

    # # TODO: https://keras.io/preprocessing/image/
    # ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False,
    #                    samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=0,
    #                    width_shift_range=0.0, height_shift_range=0.0, brightness_range=None, shear_range=0.0,
    #                    zoom_range=0.0, channel_shift_range=0.0, fill_mode='nearest', cval=0.0, horizontal_flip=False,
    #                    vertical_flip=False, rescale=None, preprocessing_function=None, data_format=None,
    #                    validation_split=0.0, dtype=None)

    # Augmenting data
    trainData = dataAugmentation(train)
    testData  = dataAugmentation(finalTest)

    setUpData(trainData, testData)


def setUpData(trainData, testData):

    # print(type(trainData))                          # <class 'numpy.ndarray'>
    # print(len(trainData))                           # 64
    # print(type(trainData[0]))                       # <class 'numpy.ndarray'>
    # print(trainData[0].shape)                       # (1400, 1400, 3)
    # print(trainData[len(trainData)//2-1].shape)     # (1400, 1400, 3)
    # print(trainData[len(trainData)//2].shape)       # (350, 350, 3)
    # print(trainData[len(trainData)-1].shape)        # (350, 350, 3)

    # TODO: substract mean of all images to all images

    # Separating the training data
    Y_train = trainData[:len(trainData)//2]    # First half is the unaltered data
    X_train = trainData[len(trainData)//2:]    # Second half is the deteriorated data

    # Separating the testing data
    Y_test = testData[:len(testData)//2]  # First half is the unaltered data
    X_test = testData[len(testData)//2:]  # Second half is the deteriorated data

    # Adjusting shapes for Keras input  # TODO: make into a function ?
    X_train = np.array([x for x in X_train])
    Y_train = np.array([x for x in Y_train])
    Y_test = np.array([x for x in Y_test])
    X_test = np.array([x for x in X_test])

    # # Sanity check: display four images (2x HR/LR)
    # plt.figure(figsize=(10, 10))
    # for i in range(2):
    #     plt.subplot(2, 2, i + 1)
    #     plt.imshow(Y_train[i], cmap=plt.cm.binary)
    # for i in range(2):
    #     plt.subplot(2, 2, i + 1 + 2)
    #     plt.imshow(X_train[i], cmap=plt.cm.binary)
    # plt.show()

    setUpModel(X_train, Y_train, X_test, Y_test)


# TODO: possibly remove once Keras Preprocessing is integrated?
def dataAugmentation(dataToAugment):
    print("Starting to augment data")
    arrayToFill = []

    # faster computation with values between 0 and 1 ?
    dataToAugment = np.divide(dataToAugment, 255.)

    # TODO: switch from RGB channels to CbCrY
    # # TODO: Try GrayScale
    # trainingData = np.array(
    #     [(cv2.cvtColor(np.uint8(x * 255), cv2.COLOR_BGR2GRAY) / 255).reshape(350, 350, 1) for x in trainingData])
    # validateData = np.array(
    #     [(cv2.cvtColor(np.uint8(x * 255), cv2.COLOR_BGR2GRAY) / 255).reshape(1400, 1400, 1) for x in validateData])

    # adding the normal images   (8)
    for i in range(len(dataToAugment)):
        arrayToFill.append(dataToAugment[i])
    # vertical axis flip         (-> 16)
    for i in range(len(arrayToFill)):
        arrayToFill.append(np.fliplr(arrayToFill[i]))
    # horizontal axis flip       (-> 32)
    for i in range(len(arrayToFill)):
        arrayToFill.append(np.flipud(arrayToFill[i]))

    # downsizing by scale of 4   (-> 64 images of 128x128x3)
    for i in range(len(arrayToFill)):
        arrayToFill.append(skimage.transform.resize(
            arrayToFill[i],
            (img_width/4, img_height/4),
            mode='reflect',
            anti_aliasing=True))

    # # Sanity check: display the images
    # plt.figure(figsize=(10, 10))
    # for i in range(64):
    #     plt.subplot(8, 8, i + 1)
    #     plt.imshow(arrayToFill[i], cmap=plt.cm.binary)
    # plt.show()

    return np.array(arrayToFill)

我的问题是：就我而言，我可以使用Keras提供的预处理工具吗？理想情况下，我希望能够输入我各种尺寸的高质量图像，将它们裁剪（而不是缩小尺寸）到512x512x3，然后通过翻转和其他方式将数据放大。减去均值也将是我想要实现的目标的一部分。该集将代表我的验证集。

重新使用验证集，我想将所有图像缩小4倍，这将生成我的训练集。

然后可以适当地拆分这两个集合，以最终获得著名的X_train Y_train X_test Y_test。

我很犹豫是否要舍弃到目前为止已经完成的所有工作来预处理我的迷你样本，但是我在考虑是否可以使用单个内置函数来完成全部工作，也许我应该给去吧。

这是我的第一个ML项目，因此我对Keras的理解不是很好，并且文档并不总是最清晰。我在想我正在使用大小不同的X和Y的事实，也许此功能不适用于我的项目。

谢谢！ ：）

Answer 1

是的，您可以使用keras预处理功能。下面的一些代码片段可以帮助您...

def cropping_function(x):
    ...
    return cropped_image

X_image_gen = ImageDataGenerator(preprocessing_function = cropping_function,
                               horizontal_flip = True, 
                               vertical_flip=True)
X_train_flow = X_image_gen.flow(X_train, batch_size = 16, seed = 1)
Y_image_gen = ImageDataGenerator(horizontal_flip = True, 
                                 vertical_flip=True)
Y_train_flow = Y_image_gen.flow(y_train, batch_size = 16, seed = 1)
train_flow = zip(X_train_flow,Y_train_flow)
model.fit_generator(train_flow)

Answer 2

克里斯托夫·汉高的建议很干净也很不错。我只想提供使用imgaug的另一种方式，这是一种以许多不同方式扩充图像的便捷方式。如果您想要更多实施的增强功能，或者需要使用Keras以外的其他ML库，则很有用。

不幸的是，它没有办法那样做，但是它允许实现自定义功能。这是一个示例函数，用于从至少与所选裁切大小一样大的图像中生成设定大小的随机裁切：

from imgaug import augmenters as iaa

def random_crop(images, random_state, parents, hooks):
    crop_h, crop_w = 128, 128
    new_images = []
    for img in images:
        if (img.shape[0] >= crop_h) and (img.shape[1] >= crop_w):
            rand_h = np.random.randint(0, img.shape[0]-crop_h)
            rand_w = np.random.randint(0, img.shape[1]-crop_w)
            new_images.append(img[rand_h:rand_h+crop_h, rand_w:rand_w+crop_w])
        else:
             new_images.append(np.zeros((crop_h, crop_w, 3)))
    return np.array(new_images)

def keypoints_dummy(keypoints_on_images, random_state, parents, hooks):
    return keypoints_on_images

cropper = iaa.Lambda(func_images=random_crop, func_keypoints=keypoints_dummy)

然后您可以将此功能与任何其他内置的imgaug功能结合使用，例如您已经在使用的flip函数，如下所示：

seq = iaa.Sequential([cropper, iaa.Fliplr(0.5), iaa.Flipud(0.5)])

然后，此功能可以从每个图像生成许多不同的作物。带有可能结果的示例图像（请注意，它将产生实际的（128、128、3）图像，此处将它们合并为一个图像以进行可视化）：

您的图像集可以通过以下方式生成：

crops_per_image = 10
images = [skimage.io.imread(path) for path in glob.glob('train_data/*.jpg')]
augs = np.array([seq.augment_image(img)/255 for img in images for _ in range(crops_per_image)])

添加要应用于图像的新功能也很简单，例如您提到的remove mean函数。

Answer 3

这是另一种使用本地ImageDataGenerator和flow_from_directory进行大小调整之前的随机裁剪和中心裁剪的方法。您可以将其作为preprocess_crop.py模块添加到您的项目中。

它首先调整图像保留纵横比的大小，然后执行裁切。调整大小后的图像尺寸基于crop_fraction，该图像已硬编码但可以更改。请参见crop_fraction = 0.875行，其中0.875似乎是最常见的行，例如256像素图片可裁剪224像素。

请注意，实现是通过猴子补丁keras_preprocessing.image.utils.loag_img函数完成的，因为在重新调整大小之前，我没有找到其他方法可以执行裁剪操作，而无需重写上面的许多其他类。

由于这些限制，裁剪方法被枚举到interpolation字段中。方法用:分隔，其中第一部分是插值，第二部分是裁剪，例如lanczos:random。支持的裁剪方法为none，center，random。如果未指定裁剪方法，则假定为none。

使用方法

只需将preprocess_crop.py放到您的项目中即可进行裁剪。下面的示例显示了如何使用随机裁剪进行训练，以及如何使用中心裁剪进行验证：

import preprocess_crop
from keras.preprocessing.image import ImageDataGenerator
from keras.applications.inception_v3 import preprocess_input

#...

# Training with random crop

train_datagen = ImageDataGenerator(
    rotation_range=20,
    channel_shift_range=20,
    horizontal_flip=True,
    preprocessing_function=preprocess_input
)

train_img_generator = train_datagen.flow_from_directory(
    train_dir,
    target_size = (IMG_SIZE, IMG_SIZE),
    batch_size  = BATCH_SIZE,
    class_mode  = 'categorical',
    interpolation = 'lanczos:random', # <--------- random crop
    shuffle = True
)

# Validation with center crop

validate_datagen = ImageDataGenerator(
    preprocessing_function=preprocess_input
)

validate_img_generator = validate_datagen.flow_from_directory(
    validate_dir,
    target_size = (IMG_SIZE, IMG_SIZE),
    batch_size  = BATCH_SIZE,
    class_mode  = 'categorical',
    interpolation = 'lanczos:center', # <--------- center crop
    shuffle = False
)

这里的preprocess_crop.py文件包含在您的项目中：

import random
import keras_preprocessing.image

def load_and_crop_img(path, grayscale=False, color_mode='rgb', target_size=None,
             interpolation='nearest'):
    """Wraps keras_preprocessing.image.utils.loag_img() and adds cropping.
    Cropping method enumarated in interpolation
    # Arguments
        path: Path to image file.
        color_mode: One of "grayscale", "rgb", "rgba". Default: "rgb".
            The desired image format.
        target_size: Either `None` (default to original size)
            or tuple of ints `(img_height, img_width)`.
        interpolation: Interpolation and crop methods used to resample and crop the image
            if the target size is different from that of the loaded image.
            Methods are delimited by ":" where first part is interpolation and second is crop
            e.g. "lanczos:random".
            Supported interpolation methods are "nearest", "bilinear", "bicubic", "lanczos",
            "box", "hamming" By default, "nearest" is used.
            Supported crop methods are "none", "center", "random".
    # Returns
        A PIL Image instance.
    # Raises
        ImportError: if PIL is not available.
        ValueError: if interpolation method is not supported.
    """

    # Decode interpolation string. Allowed Crop methods: none, center, random
    interpolation, crop = interpolation.split(":") if ":" in interpolation else (interpolation, "none")  

    if crop == "none":
        return keras_preprocessing.image.utils.load_img(path, 
                                            grayscale=grayscale, 
                                            color_mode=color_mode, 
                                            target_size=target_size,
                                            interpolation=interpolation)

    # Load original size image using Keras
    img = keras_preprocessing.image.utils.load_img(path, 
                                            grayscale=grayscale, 
                                            color_mode=color_mode, 
                                            target_size=None, 
                                            interpolation=interpolation)

    # Crop fraction of total image
    crop_fraction = 0.875
    target_width = target_size[1]
    target_height = target_size[0]

    if target_size is not None:        
        if img.size != (target_width, target_height):

            if crop not in ["center", "random"]:
                raise ValueError('Invalid crop method {} specified.', crop)

            if interpolation not in keras_preprocessing.image.utils._PIL_INTERPOLATION_METHODS:
                raise ValueError(
                    'Invalid interpolation method {} specified. Supported '
                    'methods are {}'.format(interpolation,
                        ", ".join(keras_preprocessing.image.utils._PIL_INTERPOLATION_METHODS.keys())))

            resample = keras_preprocessing.image.utils._PIL_INTERPOLATION_METHODS[interpolation]

            width, height = img.size

            # Resize keeping aspect ratio
            # result shold be no smaller than the targer size, include crop fraction overhead
            target_size_before_crop = (target_width/crop_fraction, target_height/crop_fraction)
            ratio = max(target_size_before_crop[0] / width, target_size_before_crop[1] / height)
            target_size_before_crop_keep_ratio = int(width * ratio), int(height * ratio)
            img = img.resize(target_size_before_crop_keep_ratio, resample=resample)

            width, height = img.size

            if crop == "center":
                left_corner = int(round(width/2)) - int(round(target_width/2))
                top_corner = int(round(height/2)) - int(round(target_height/2))
                return img.crop((left_corner, top_corner, left_corner + target_width, top_corner + target_height))
            elif crop == "random":
                left_shift = random.randint(0, int((width - target_width)))
                down_shift = random.randint(0, int((height - target_height)))
                return img.crop((left_shift, down_shift, target_width + left_shift, target_height + down_shift))

    return img

# Monkey patch
keras_preprocessing.image.iterator.load_img = load_and_crop_img