我正在尝试使用tf.py_func（）作为队列上的阅读器创建输入管道。我的数据分布如下：
/data
/images/*.jpg
/masks/*.png
/activations/*.npz
我有一个文件，其中包含以下格式的这些数据文件的路径：
/path/to/Image_1.jpg /path/to/mask1.png /path/to/activation1.npz
/path/to/Image_2.jpg /path/to/mask2.png /path/to/activation2.npz
. . .
我正在创建三个队列，然后使用读取器和解码器来解析队列中的元素。由于tensorflow已经定义了读取器，如tf.read_file（）和解码器，如tf.image.decode_jpeg（）和tf.image.decode_png（），用于 jpg 和 png 格式，但没有 npz 格式，所以我使用python包装器来解析激活文件的队列（上面文件中的第三列）：
def load_npz(npz_file):
actv_arr = np.load(npz_file)
activation = actv_arr['actv']
return activation
现在，如果我尝试使用上面python函数周围的 tf.py_func（）构建一个包装器并将其用作读取器，我遇到了超出范围的错误：
I tensorflow/core/common_runtime/gpu/gpu_device.cc:975] Creating TensorFlow device (/gpu:0) -> (device: 0, name: TITAN X (Pascal), pci bus id: 0000:01:00.0)
Restored model parameters from ./snapshots_curriculum_5k_data/model.ckpt-10000
W tensorflow/core/framework/op_kernel.cc:975] Out of range: FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
W tensorflow/core/framework/op_kernel.cc:975] Out of range: FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
W tensorflow/core/framework/op_kernel.cc:975] Out of range: FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
W tensorflow/core/framework/op_kernel.cc:975] Out of range: FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
Traceback (most recent call last):
File "train_online.py", line 260, in <module>
main()
File "train_online.py", line 241, in main
num_steps=args.num_steps, restore_from=args.restore_from, snapshot_dir=args.snapshot_dir)
File "/home/mac/phd/arslan-repo/tensorflow-deeplab-resnet/distillation.py", line 290, in main
_, l_val = sess.run([accum_grads_op, reduced_loss], feed_dict=feed_dict)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 766, in run
run_metadata_ptr)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
Caused by op u'create_inputs/batch', defined at:
File "train_online.py", line 260, in <module>
main()
File "train_online.py", line 241, in main
num_steps=args.num_steps, restore_from=args.restore_from, snapshot_dir=args.snapshot_dir)
File "/home/mac/phd/arslan-repo/tensorflow-deeplab-resnet/distillation.py", line 94, in main
image_batch, label_batch, activation_batch = reader.dequeue(BATCH_SIZE)
File "/home/mac/phd/arslan-repo/tensorflow-deeplab-resnet/deeplab_resnet/image_reader_distill.py", line 195, in dequeue
num_elements)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/input.py", line 692, in batch
dequeued = queue.dequeue_many(batch_size, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/data_flow_ops.py", line 458, in dequeue_many
self._queue_ref, n=n, component_types=self._dtypes, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 1099, in _queue_dequeue_many
timeout_ms=timeout_ms, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
self._traceback = _extract_stack()
OutOfRangeError (see above for traceback): FIFOQueue '_3_create_inputs/batch/fifo_queue' is closed and has insufficient elements (requested 1, current size 0)
[[Node: create_inputs/batch = QueueDequeueMany[_class=["loc:@create_inputs/batch/fifo_queue"], component_types=[DT_FLOAT, DT_UINT8, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](create_inputs/batch/fifo_queue, create_inputs/batch/n)]]
以下是在输入队列上调用不同的读取器和解码器：
def read_data_from_disk(input_queue):
img_contents = tf.read_file(input_queue[0])
label_contents = tf.read_file(input_queue[1])
img = tf.image.decode_jpeg(img_contents, channels=3)
label = tf.image.decode_png(label_contents, channels=1)
activation = tf.py_func(load_npz, [input_queue[2]], [tf.float32])
activation = tf.to_float(tf.reshape(activation, [tf.shape(img)[0], tf.shape(img)[1], 21]))
return img, label, activation
ImageReader定义为：
class ImageReader(object):
self.data_dir = data_dir
self.actv_dir = actv_dir
self.data_list = data_list
self.input_size = input_size
self.coord = coord
self.image_list, self.label_list, self.actv_list = read_labeled_image_list(self.data_dir, self.actv_dir, self.data_list)
self.images = tf.convert_to_tensor(self.image_list, dtype=tf.string)
self.labels = tf.convert_to_tensor(self.label_list, dtype=tf.string)
self.activations = tf.convert_to_tensor(self.actv_list, dtype=tf.string)
self.queue = tf.train.slice_input_producer([self.images, self.labels, self.activations],\
shuffle=input_size is not None) # not shuffling if it is val
self.image, self.label, self.activation = read_images_from_disk(self.queue, self.input_size, random_scale, random_mirror)
def dequeue(self, num_elements):
image_batch, label_batch, activation_batch = tf.train.batch([self.image, self.label, self.activation],
num_elements)
return image_batch, label_batch, activation_batch
当我尝试调用ImageReader.dequeue（1）时，我遇到了超出范围的错误。我想我在这个设置中没有正确使用python包装器。有什么建议吗？
以下是重现问题的示例代码：
#! /usr/env/bin python
import tensorflow as tf
import numpy as np
DATA_DIR = './data'
DATA_LIST = './list.txt'
NUM_STEPS = 1
def load_file(file_path):
arr = np.loadtxt(file_path)
return arr
def get_file_list(data_dir, data_list):
file_list = []
f = open(data_list)
for line in f:
img = line.strip('\n')
file_list.append(data_dir + img)
return file_list
def read_files_from_disk(input_queue):
activation = tf.py_func(load_file, [input_queue[0]], [tf.float32])
activation = tf.to_float(tf.reshape(activation, [1, 2]))
return activation
class File_Reader(object):
def __init__(self, data_dir, data_list):
self.data_dir = data_dir
self.data_list = data_list
self.file_list = get_file_list(self.data_dir, self.data_list)
self.files = tf.convert_to_tensor(self.file_list, dtype=tf.string)
self.queue = tf.train.slice_input_producer([self.files], shuffle=True)
self.file = read_files_from_disk(self.queue)
def dequeue(self, num_elements):
file_batch = tf.train.batch([self.file], num_elements)
return file_batch
def main(data_dir=DATA_DIR, data_list=DATA_LIST, num_steps=NUM_STEPS):
graph = tf.Graph()
with graph.as_default():
reader = File_Reader(data_dir, data_list)
file_contents = reader.dequeue(1)
with tf.Session(graph=graph) as sess:
tf.global_variables_initializer().run()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
for step in range(num_steps):
file_content = sess.run(file_contents)
print(file_content)
coord.request_stop()
coord.join(threads)
if __name__ == '__main__':
main()
其中＆＃39; list.txt＆＃39;只需包含＆＃39; x＆＃39;的名称文件数量和＆＃39; ./ data＆＃39;包含那些文件。为简单起见，我使用了＆＃txt＆＃39;文件。