我正在尝试在fit_generator函数中使用use_multiprocessing = True运行我的CNN python代码,但出现错误,它在单个进程中也能正常工作,但CPU负载:20%,GPU:8%。
我正在使用Tensorflow后端在具有Windows 10核心i7-7820HK CPU和NVIDIA GTX 1080的MSI笔记本电脑上运行
这是我的代码:
# Part 1 - Building the CNN
# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.preprocessing.image import ImageDataGenerator
# Initialising the CNN
classifier = Sequential()
# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape=(64, 64, 3), activation='relu'))
# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size=(2, 2)))
# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))
# Step 3 - Flattening
classifier.add(Flatten())
# Step 4 - Full connection
classifier.add(Dense(units=128, activation='relu'))
classifier.add(Dense(units=1, activation='sigmoid'))
# Compiling the CNN
classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
# Part 2 - Fitting the CNN to the images
train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('dataset\\training_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
test_set = test_datagen.flow_from_directory('dataset\\test_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
if __name__ == '__main__':
classifier.fit_generator(training_set,
workers=8,
max_queue_size=100,
use_multiprocessing=True,
steps_per_epoch=(8000 / 32),
epochs=25,
validation_data=test_set,
validation_steps=(2000 / 32))
我得到这个错误:
Using TensorFlow backend. Found 8000 images belonging to 2 classes. Found 2000 images belonging to 2 classes. Epoch 1/25 Exception in thread Thread-24: Traceback (most recent call last): File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\threading.py", line 916, in
_bootstrap_inner
self.run() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\site-packages\keras\utils\data_utils.py", line 548, in _run
with closing(self.executor_fn(_SHARED_SEQUENCES)) as executor: File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\site-packages\keras\utils\data_utils.py", line 522, in <lambda>
initargs=(seqs,)) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\context.py", line 119, in Pool
context=self.get_context()) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\pool.py", line 174, in __init__
self._repopulate_pool() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\pool.py", line 239, in _repopulate_pool
w.start() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj) TypeError: can't pickle _thread.lock objects
Exception in thread Thread-23: Traceback (most recent call last): File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\threading.py", line 916, in _bootstrap_inner
self.run() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\site-packages\keras\utils\data_utils.py", line 548, in _run
with closing(self.executor_fn(_SHARED_SEQUENCES)) as executor: File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\site-packages\keras\utils\data_utils.py", line 522, in <lambda>
initargs=(seqs,)) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\context.py", line 119, in Pool
context=self.get_context()) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\pool.py", line 174, in __init__
self._repopulate_pool() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\pool.py", line 239, in _repopulate_pool
w.start() File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child) File "C:\Users\MSI-GT75\Anaconda3\envs\cnn\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj) TypeError: can't pickle _thread.lock objects
更新所有软件包后,将显示此错误,而不是上面的错误:
ValueError: Using a generator with `use_multiprocessing=True` is not supported on Windows (no marshalling of generators across process boundaries). Instead, use single thread/process or multithreading.