我的代码卡在了这一行：-
prefetch_queue.put((idx, dummy_data), block=True)
错误已结束
idx为0
和dummy_data包含以下内容：-
data:array([[[-1., -1., -1.],\n [-1., -1., -1.],\n [-1.,
-1., -1.],\n ...,\n [-1., -1., -1.],\n [-1., -1.,
-1.],\n [-1., -1., -1.]],\n\n [[-1., -1., -1.],\n
[-1., -1., -1.],\n [-1., -1., -1.],\n ...,\n [-1.,
-1., -1.],\n [-1., -1., -1.],\n [-1., -1.,
-1.]],\n\n [[-1., -1., -1.],\n [-1., -1., -1.],\n
[-1., -1., -1.],\n ...,\n [-1., -1., -1.],\n [-1.,
-1., -1.],\n [-1., -1., -1.]],\n\n ...,\n\n [[-1., -
-1., -1.],\n [-1., -1., -1.],\n [-1., -1.,
-1.],\n ...,\n [-1., -1., -1.],\n [-1., -1., -1.],\n
[-1., -1., -1.]],\n\n [[-1., -1., -1.],\n [-1., -1., -1.],\n
[-1., -1., -1.],\n ...,\n [-1., -1., -1.],\n [-1.,
-1., -1.],\n [-1., -1.,
-1.]],\n\n [[-1., -1., -1.],\n [-1., -1., -1.],\n
[-1., -1., -1.],\n ...,\n [-1., -1., -1.],\n [-1.,
-1., -1.],\n [-1., -1., -1.]]]) [0:375] :[array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), array([[-1.,
-1., -1...1., -1.]]), array([[-1., -1., -1...1., -1.]]), ...]
dtype:dtype('float64') max:'ndarray too big, calculating max would slow
down debugging' min:'ndarray too big, calculating min would slow down
debugging' shape:(375, 1242, 3) size:1397250
__internals__:{'T': array([[[-1., -1., -...., -1.]]]), 'base': None,
'ctypes': <numpy.core._interna...184E30CC0>, 'data': <memory at
0x000001C184E7F048>, 'dtype': dtype('float64'), 'flags': C_CONTIGUOUS :
Tru...PY : False, 'flat': <numpy.flatiter obje...F9832E140>, 'imag':
array([[[0., 0., 0.]...0., 0.]]]), 'itemsize': 8, 'nbytes': 11178000,
'ndim': 3, 'real': array([[[-1.,
-1., -...., -1.]]]), 'shape': (375, 1242, 3), 'size': 1397250, ...}
'T':array([[[-1., -1., -1., ..., -1., -1., -1.],\n [-1., -1.,
-1., ..., -1., -1., -1.],\n [-1., -1., -1., ..., -1., -1., -1.],\n
...,\n [-1., -1., -1., ..., -1., -1., -1.],\n [-1., -1.,
-1., ..., -1., -1., -1.],\n [-1., -1., -1., ...,
-1., -1., -1.]],\n\n [[-1., -1., -1., ..., -1., -1., -1.],\n
[-1., -1., -1., ..., -1., -1., -1.],\n [-1., -1., -1., ...,
-1., -1., -1.],\n ...,\n [-1., -1., -1., ..., -1., -1.,
-1.],\n [-1., -1., -1., ..., -1., -1., -1.],\n [-1., -1.,
-1., ..., -1., -1., -1.]],\n\n [[-1., -1., -1., ..., -1., -1.,
-1.],\n [-1., -1., -1., ..., -1., -1., -1.],\n [-1., -1.,
-1., ..., -1., -1., -1.],\n ...,\n [-1., -1.,
-1., ..., -1., -1., -1.],\n [-1., -1., -1., ..., -1., -1., -1.],\n
[-1., -1., -1., ..., -1., -1., -1.]]]) 'base':None 'ctypes':
<numpy.core._internal._ctypes object at 0x000001C184E30CC0> 'data':
<memory at 0x000001C184E7F048> 'dtype':dtype('float64') 'flags':
C_CONTIGUOUS : True\n F_CONTIGUOUS : False\n OWNDATA : True\n
WRITEABLE : True\n ALIGNED : True\n WRITEBACKIFCOPY : False\n
UPDATEIFCOPY : False 'flat':<numpy.flatiter object at 0x000001BF9832E140>
'imag':array([[[0., 0., 0.],\n [0., 0.,
0.],\n [0., 0., 0.],\n ...,\n [0., 0., 0.],\n
[0., 0., 0.],\n [0., 0., 0.]],\n\n [[0., 0., 0.],\n
[0., 0., 0.],\n [0., 0., 0.],\n ...,\n [0., 0.,
0.],\n [0., 0., 0.],\n [0., 0., 0.]],\n\n [[0., 0.,
0.],\n [0., 0., 0.],\n [0., 0., 0.],\n ...,\n
[0., 0., 0.],\n [0., 0., 0.],\n [0., 0., 0.]],\n\n
...,\n\n [[0., 0., 0.],\n [0., 0., 0.],\n [0., 0.,
0.],\n ...,\n [0., 0., 0.],\n [0., 0., 0.],\n
[0., 0., 0.]],\n\n [[0., 0., 0.],\n [0., 0., 0.],\n
[0., 0., 0.],\n ...,\n [0., 0., 0.],\n [0., 0.,
0.],\n [0., 0., 0.]],\n\n [[0., 0., 0.],\n [0., 0.,
0.],\n [0., 0., 0.],\n ...,\n [0., 0., 0.],\n
[0., 0., 0.],\n [0., 0., 0.]]]) 'itemsize':8 'nbytes':11178000
我浏览了堆栈溢出的帖子，并在下面的代码中搜索>运算符，或者等于None，但找不到任何运算符。
从这里开始，代码有两行： https://github.com/liuziwei7/voxel-flow/voxel_flow_train.py
Trainingset_load_fn_frame1 = partial(TrainingSet_F1.process_func)
Trainingset_p_queue_frame1 = PrefetchQueue(Trainingset_load_fn_frame1,
data_list_frame1, FLAGS.batch_size, shuffle=False,
num_workers=num_workers)
然后输入代码 https://github.com/liuziwei7/voxel-flow/tree/master/utils/prefetch_queue_shuffle.py:-
from __future__ import print_function
import glob
import numpy as np
import os
import queue
import random
import scipy
from scipy import misc
import threading
class DummpyData(object):
def __init__(self, data):
self.data = data
def __cmp__(self, other):
return 0
def prefetch_job(load_fn, prefetch_queue, data_list, shuffle, prefetch_size):
"""
"""
data_count = 0
total_count = len(data_list)
idx = 0
while True:
if shuffle:
if data_count == 0:
random.shuffle(data_list)
data = load_fn(data_list[data_count]) #Load your data here
if type(data) is list:
for data_point in data:
idx = random.randint(0, prefetch_size)
dummy_data = DummpyData(data_point)
prefetch_queue.put((idx, dummy_data), block=True)
else:
idx = random.randint(0, prefetch_size)
dummy_data = DummpyData(data)
prefetch_queue.put((idx, dummy_data), block=True)
else:
data = load_fn(data_list[data_count]) #Load your data here
dummy_data = DummpyData(data)
prefetch_queue.put((idx, dummy_data), block=True) # it gets stuck here
idx = (idx + 1) % prefetch_size
data_count = (data_count + 1) % total_count
class PrefetchQueue(object):
def __init__(self, load_fn, data_list, batch_size, prefetch_size=None, shuffle=True, num_workers=4):
self.data_list = data_list
self.shuffle = shuffle
self.prefetch_size = prefetch_size
self.load_fn = load_fn
self.batch_size = batch_size
if prefetch_size is None:
self.prefetch_size = num_workers * batch_size # was set to 4
# Start prefetching thread
# self.prefetch_queue = Queue.Queue(maxsize=prefetch_size)
self.prefetch_queue = queue.PriorityQueue(maxsize=prefetch_size)
for k in range(num_workers):
t = threading.Thread(target=prefetch_job,
args=(self.load_fn, self.prefetch_queue, self.data_list,
self.shuffle, self.prefetch_size))
t.daemon = True
t.start()
def get_batch(self):
data_list = []
for k in range(0, self.batch_size):
# if self.prefetch_queue.empty():
# print('Prefetch Queue is empty, waiting for data to be read.')
_, data_dummy = self.prefetch_queue.get(block=True)
data = data_dummy.data
data_list.append(np.expand_dims(data,0))
return np.concatenate(data_list, axis=0)
我希望它可以运行，但是却出现以下错误：-
Exception in thread Thread-35:
Traceback (most recent call last):
File "C:\...\Python37\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "C:\...\Python37\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "c:\...\utils\prefetch_queue_shuffle.py", line 41, in prefetch_job
prefetch_queue.put((idx, dummy_data), block=True)
File "C:\...\Python37\lib\queue.py", line 133, in put
if self.maxsize > 0:
TypeError: '>' not supported between instances of 'NoneType' and 'int'
答案 0 :(得分：0)
这是因为prefetch_queue_shuffle.py中的第47行具有prefetch_size = None
PrefetchQueue类（对象）： def init （自身，load_fn，data_list，batch_size，prefetch_size = None，shuffle = True，num_workers = 4）：
self.prefetch_queue =队列.PriorityQueue（maxsize = prefetch_size）
这意味着queue.py中的第133行变为None
如果self.maxsize> 0：
但是，Python 3x对此不接受
这导致此错误
TypeError：“ DummpyData”和“ DummpyData”的实例之间不支持“ <” 这可能与prefetch_queue_shuffle.py的第15行有关： def cmp （自己，其他）： 返回0 Why can't I use the method __cmp__ in Python 3 as for Python 2?
将此更改为
从functools导入total_ordering
DummpyData类（对象）： def init （自身，数据）： self.data =数据 @total_ordering def lt （自己，其他）：＃是 cmp （自己，其他）： 返回0
这可以起作用，但是，似乎并没有提高GPU的使用效率。实际上，这似乎要慢一些……
在这种情况下，更多的工人减少了GPU工作的时间