使用TensorFlow / Keras时发现瓶颈

Question

我正在训练一个keras模型，它使用的资源比我预期的少得多。它应该使用tensorflow-gpu，但是GPU使用率几乎总是0，偶尔会有尖峰。 CPU使用率徘徊在15％左右。这种资源使用量正常吗？如果不是，如何找到瓶颈所在？

我正在从SSD本地加载图像。

import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import keras
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Convolution2D, MaxPooling2D, Dropout, Flatten, Dense
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import cv2
import pandas as pd
import ntpath
import random
import tensorboard
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import time
from keras.layers.normalization import BatchNormalization

datadir = 'D:\\BehavioralCloning\\trainingData'
columns = ['center', 'left', 'right', 'steering', 'throttle', 'reverse', 'speed']
data = pd.read_csv(os.path.join(datadir, 'trainingData.csv'), names = columns)
pd.set_option('display.max_colwidth', -1)
data.head()

b_size = len(data)  * 3

def path_leaf(path):
  head, tail = ntpath.split(path)
  return tail

data['center']  =data['center'].apply(path_leaf)
data['left']  = data['left'].apply(path_leaf)
data['right'] = data['right'].apply(path_leaf)
data.head()

num_bins = 25
samples_per_bin = 3000
hist, bins = np.histogram(data['steering'], num_bins)
center = (bins[:-1] + bins[1:]) * 0.5
plt.bar(center, hist, width = 0.05)
plt.plot((np.min(data['steering']), np.max(data['steering'])), (samples_per_bin, samples_per_bin))
plt.show()
print('total data:', len(data))
remove_list = []
for j in range(num_bins):
  list_ = []
  for i in range(len(data['steering'])):
    if data['steering'][i] >= bins[j] and data['steering'][i] <= bins[j+1]:
      list_.append(i)
  list_ = shuffle(list_)
  list_ = list_[samples_per_bin:]
  remove_list.extend(list_)

print('removed:', len(remove_list))
data.drop(data.index[remove_list], inplace=True)
print('remaining:', len(data))

hist, _ = np.histogram(data['steering'], (num_bins))
plt.bar(center, hist, width=0.05)
plt.plot((np.min(data['steering']), np.max(data['steering'])), (samples_per_bin, samples_per_bin))
plt.show()
print(data.iloc[1])
def load_img_steering(datadir, df):
  image_path = []
  steering = []
  for i in range(len(data)):
    indexed_data = data.iloc[i]
    center, left, right = indexed_data[0], indexed_data[1], indexed_data[2]
    image_path.append(os.path.join(datadir, center.strip() + '.png'))
    steering.append(float(indexed_data[3]))
    # left image append
    image_path.append(os.path.join(datadir,left.strip() + '.png'))
    steering.append(float(indexed_data[3])+0.15)
    # right image append
    image_path.append(os.path.join(datadir,right.strip() + '.png'))
    steering.append(float(indexed_data[3])-0.15)
  image_paths = np.asarray(image_path)
  steerings = np.asarray(steering)
  return image_paths, steerings

image_paths, steerings = load_img_steering(datadir + '/images', data)
X_train, X_valid, y_train, y_valid = train_test_split(image_paths, steerings, test_size=0.2, random_state=6)
print('Training Samples: {}\nValid Samples: {}'.format(len(X_train), len(X_valid)))

fig, axes = plt.subplots(1, 2, figsize=(12, 4))
axes[0].hist(y_train, bins=num_bins, width=0.05, color='blue')
axes[0].set_title('Training set')
axes[1].hist(y_valid, bins=num_bins, width=0.05, color='red')
axes[1].set_title('Validation set')
plt.show()

def zoom(image):
  zoom = iaa.Affine(scale=(1, 1.3))
  image = zoom.augment_image(image)
  return image
image = image_paths[random.randint(0, 1000)]
print(image)
original_image = mpimg.imread(image)
zoomed_image = zoom(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(zoomed_image)
axs[1].set_title('Zoomed Image')
plt.show()

def pan(image):
  pan = iaa.Affine(translate_percent= {"x" : (-0.1, 0.1), "y": (-0.1, 0.1)})
  image = pan.augment_image(image)
  return image

image = image_paths[random.randint(0, 1000)]
original_image = mpimg.imread(image)
panned_image = pan(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(panned_image)
axs[1].set_title('Panned Image')
plt.show()

def img_random_brightness(image):
    brightness = iaa.Multiply((0.2, 1.2))
    image = brightness.augment_image(image)
    return image

image = image_paths[random.randint(0, 1000)]
original_image = mpimg.imread(image)
brightness_altered_image = img_random_brightness(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(brightness_altered_image)
axs[1].set_title('Brightness altered image ')
plt.show()

def img_random_flip(image, steering_angle):
    image = cv2.flip(image,1)
    steering_angle = -steering_angle
    return image, steering_angle

random_index = random.randint(0, 1000)
image = image_paths[random_index]
steering_angle = steerings[random_index]

original_image = mpimg.imread(image)
flipped_image, flipped_steering_angle = img_random_flip(original_image, steering_angle)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image - ' + 'Steering Angle:' + str(steering_angle))

axs[1].imshow(flipped_image)
axs[1].set_title('Flipped Image - ' + 'Steering Angle:' + str(flipped_steering_angle))
plt.show()

def generate_poly_coordinates(imshape, no_of_polys=1):
    vertices_list=[]
    for index in range(no_of_polys):
        vertex=[]
        for dimensions in range(np.random.randint(3,15)): 
            vertex.append(( imshape[1]*np.random.uniform(),imshape[0]//3+imshape[0]*np.random.uniform()))
        vertices = np.array([vertex], dtype=np.int32) 
        vertices_list.append(vertices)
    return vertices_list 

def add_shadow(image,no_of_shadows=1):
    image_HLS = cv2.cvtColor(image,cv2.COLOR_RGB2HLS)
    mask = np.zeros_like(image) 
    imshape = image.shape
    vertices_list= generate_poly_coordinates(imshape, no_of_shadows) 
    for vertices in vertices_list: 
        cv2.fillPoly(mask, vertices, 255)

    image_HLS[:,:,1][mask[:,:,0]==255] = image_HLS[:,:,1][mask[:,:,0]==255]*0.5
    image_RGB = cv2.cvtColor(image_HLS,cv2.COLOR_HLS2RGB) 
    return image_RGB

original_image = mpimg.imread(image)
shadow_image = add_shadow(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(shadow_image)
axs[1].set_title('Shadow Image')
plt.show()   

def add_bright_spot(image,no_of_spots=1):
    image_HLS = cv2.cvtColor(image,cv2.COLOR_RGB2HLS) 
    mask = np.zeros_like(image) 
    imshape = image.shape
    vertices_list= generate_poly_coordinates(imshape, no_of_spots) 
    for vertices in vertices_list: 
        cv2.fillPoly(mask, vertices, 255) 

    image_HLS[:,:,1][mask[:,:,0]==255] = image_HLS[:,:,1][mask[:,:,0]==255]*1.75 
    image_RGB = cv2.cvtColor(image_HLS,cv2.COLOR_HLS2RGB) 
    return image_RGB

original_image = mpimg.imread(image)
bright_image = add_bright_spot(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(bright_image)
axs[1].set_title('Bright Image')
plt.show()   


def normalize(image):
    return image / np.mean(image)

original_image = mpimg.imread(image)
normal_image = normalize(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(original_image)
axs[0].set_title('Original Image')

axs[1].imshow(normal_image)
axs[1].set_title('Normal Image')
plt.show()      

def random_augment(image, steering_angle):
    image = mpimg.imread(image)
    image = normalize(image)
    if np.random.rand() < 0.5:
      image = pan(image)
    if np.random.rand() < 0.5:
      image = zoom(image)
    if np.random.rand() < 0.5:
      image = img_random_brightness(image)
    if np.random.rand() < 0.5:
      image, steering_angle = img_random_flip(image, steering_angle)
    if np.random.rand() < 0.5:
        aug_img = add_shadow(image)
    if np.random.rand() < 0.5:
        aug_img = add_bright_spot(image)
    return image, steering_angle

ncol = 2
nrow = 10

fig, axs = plt.subplots(nrow, ncol, figsize=(15, 50))
fig.tight_layout()

for i in range(10):
  randnum = random.randint(0, len(image_paths) - 1)
  random_image = image_paths[randnum]
  random_steering = steerings[randnum]

  original_image = mpimg.imread(random_image)
  augmented_image, steering = random_augment(random_image, random_steering)

  axs[i][0].imshow(original_image)
  axs[i][0].set_title("Original Image")

  axs[i][1].imshow(augmented_image)
  axs[i][1].set_title("Augmented Image")
plt.show()


def img_preprocess(img):
    img = normalize(img)
    img = img[60:144,:,:]
    img = cv2.cvtColor(img, cv2.COLOR_RGB2YUV)
    img = cv2.GaussianBlur(img,  (3, 3), 0)
    img = cv2.resize(img, (200, 66))
    #img = img/255
    return img

image = image_paths[75]
original_image = mpimg.imread(image)
preprocessed_image = img_preprocess(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('Original Image')
axs[1].imshow(preprocessed_image)
axs[1].set_title('Preprocessed Image')
plt.show()

def batch_generator(image_paths, steering_ang, batch_size, istraining):

  while True:
    batch_img = []
    batch_steering = []

    for i in range(batch_size):
      random_index = random.randint(0, len(image_paths) - 1)

      if istraining:
        im, steering = random_augment(image_paths[random_index], steering_ang[random_index])

      else:
        im = mpimg.imread(image_paths[random_index])
        steering = steering_ang[random_index]

      im = img_preprocess(im)
      batch_img.append(im)
      batch_steering.append(steering)
    yield (np.asarray(batch_img), np.asarray(batch_steering))

x_train_gen, y_train_gen = next(batch_generator(X_train, y_train, 1, 1))
x_valid_gen, y_valid_gen = next(batch_generator(X_valid, y_valid, 1, 0))

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(x_train_gen[0])
axs[0].set_title('Training Image')

axs[1].imshow(x_valid_gen[0])
axs[1].set_title('Validation Image')
plt.show()

def plot_history(history):
    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]
    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]
    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]
    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]

    if len(loss_list) == 0:
        print('Loss is missing in history')
        return 

    ## As loss always exists
    epochs = range(1,len(history.history[loss_list[0]]) + 1)

    ## Loss
    plt.figure(1)
    for l in loss_list:
        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))
    for l in val_loss_list:
        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))

    plt.title('Loss')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()

    ## Accuracy
    plt.figure(2)
    for l in acc_list:
        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')
    for l in val_acc_list:    
        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')

    plt.title('Accuracy')
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.legend()
    plt.show()

def nvidia_model():
  model = Sequential()
  model.add(Convolution2D(24, (5, 5), strides=(2, 2), input_shape=(66, 200, 3), activation='relu'))
  model.add(BatchNormalization())

  model.add(Convolution2D(36, (5, 5), strides=(2, 2), activation='relu'))
  model.add(BatchNormalization())

  model.add(Convolution2D(48, (5, 5), strides=(2, 2), activation='relu'))
  model.add(BatchNormalization())
  model.add(Convolution2D(64, (3, 3), activation='relu'))
  model.add(BatchNormalization())

  model.add(Convolution2D(64, (3, 3), activation='relu'))
  model.add(BatchNormalization())
#   model.add(Dropout(0.5))


  model.add(Flatten())

  model.add(Dense(1164, activation = 'relu'))
  model.add(BatchNormalization())
#   model.add(Dropout(0.5))

  model.add(Dense(200, activation = 'relu'))
  model.add(BatchNormalization())
#   model.add(Dropout(0.5))

  model.add(Dense(50, activation = 'relu'))
  model.add(BatchNormalization())
#   model.add(Dropout(0.5))

  model.add(Dense(10, activation = 'relu'))
  model.add(BatchNormalization())
#   model.add(Dropout(0.5))

  model.add(Dense(1))

  optimizer = Adam(lr=0.001)
  model.compile(loss='mse', optimizer=optimizer)
  return model

model = nvidia_model()
print(model.summary())

callbacks = [EarlyStopping(monitor='val_loss', patience=2),
              ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True),
               ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=1e-10),
               keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None, update_freq='epoch')]
start_time = time.time()
history = model.fit_generator(batch_generator(X_train, y_train, 500, 1),
                                  steps_per_epoch = 300, 
                                  epochs=5000,
                                  callbacks=callbacks,
                                  validation_data=batch_generator(X_valid, y_valid, 500, 0),
                                  validation_steps=200,
                                  verbose=1,
                                  shuffle = 1)

elapsed_time = time.time() - start_time
time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("Training time: %s" % time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
print(history.history.keys())
print(type(history.history))
plot_history(history)
# plt.plot(history.history['loss'])
# plt.plot(history.history['val_loss'])
# plt.plot(history.history['acc'])
# plt.plot(history.history['val_acc'])
# plt.legend(['training', 'validation'])
# plt.title('Loss')
# plt.xlabel('Epoch')
# plt.show()

# fig, axs = plt.subplots(1, 2, figsize=(15, 10))
# fig.tight_layout()

# axs[0].plot(history.history['loss'])
# axs[0].plot(history.history['val_loss'])
# axs[0].set_title('Loss')

# axs[1].plt.plot(history.history['acc'])
# axs[1].plot(history.history['val_acc'])
# axs[1].set_title('Accuracy')
# plt.show()

model.save('model.h5')