Question

我想保存CASIA-Webface数据集，其中包含超过500000张由pickle.dump调整为160x160尺寸的面部图像，以进行面部识别训练，并且在os.walk之后，我的代码无法将其写入pickle文件：

这是我的全部工作：

import cv2
import glob
import os
import random
from tqdm import tqdm
import numpy as np
from imutils import paths
from sklearn.preprocessing import LabelEncoder
import pickle

def read_image(image_path):
    img = cv2.imread(image_path)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    return gray

images = []
#labels = []
#names = []
#count = 0

for path, dirs, files in os.walk('/home/face_recognition/datasets/casia_aligned_160'):
    for d in tqdm(dirs):
        class_image = []
        for ext in ('jpg', 'jpeg', 'png'):
            for f in glob.glob(os.path.join(path, d, '*.' + ext)):
                try:
                    img = read_image(f)
                except:
                    continue
                if img is None:
                    continue
                class_image.append(img)

        if len(class_image) > 1:
            #names.append(d)
            for img in class_image:
                images.append(img)
                #labels.append(count)
            #count += 1


print("Write to pkl: ")
with open('/home/isadmin/anhtmn/face_recognition/jsons/X_train_triplet_casia160.pkl', 'wb') as f:
    pickle.dump(images, f)
"""with open('/home/isadmin/anhtmn/face_recognition/jsons/y_train_triplet_casia160.pkl', 'wb') as f:
    piickle.dump(labels, f)
with open('/home/isadmin/anhtmn/face_recognition/jsons/name_map_casia160.pkl', 'wb') as f:
    pickle.dump(names, f)"""

print("Done preprocessing")

追加到数组后，抛出错误：

Write to pkl:
MemoryError

我想知道是否有像这样的简单方法来存储训练数据。任何对我的问题有帮助的人，我将非常感谢！

带有数组追加的Pickle MemoryError

0 个答案: