Question

我在TFRecords中有两个数据集，一个拥有大约20,000个条目，另一个拥有120万个。

当我使用带有20,000个条目的TFRecord时，此代码完美有效，但当我使用120万条时，它会超出范围错误。

def parse(serialized):

features = \
    {
        'train/image': tf.FixedLenFeature([], tf.string),
        'train/label': tf.FixedLenFeature([], tf.int64)
    }


parsed_example = tf.parse_single_example(serialized=serialized,
                                         features=features)


image_raw = parsed_example['train/image']


image = tf.decode_raw(image_raw, tf.uint8)


image = tf.cast(image, tf.float32)


label = parsed_example['train/label']


return image, label

def input_fn(filenames, train, batch_size=32, buffer_size=2048):

dataset = tf.data.TFRecordDataset(filenames=filenames)


dataset = dataset.map(parse)

if train:

    dataset = dataset.shuffle(buffer_size=buffer_size)


    num_repeat = None
else:

    num_repeat = 1


dataset = dataset.repeat(num_repeat)


dataset = dataset.batch(batch_size)


iterator = dataset.make_one_shot_iterator()


images_batch, labels_batch = iterator.get_next()


x =  {'image':images_batch}
y = labels_batch

return x, y

x,y = input_fn('train.tfrecords',False)
print(x)
with tf.Session() as sess:
   for i in range(10):
      print(sess.run(x))

错误即将到来：

  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1374, in _do_call
    raise type(e)(node_def, op, message)

OutOfRangeError: End of sequence
     [[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]

Caused by op 'IteratorGetNext_2', defined at:
  File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 268, in <module>
    main()
  File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 264, in main
    kernel.start()
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 478, in start
    self.io_loop.start()
  File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
    super(ZMQIOLoop, self).start()
  File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 888, in start
    handler_func(fd_obj, events)
  File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
    self._handle_recv()
  File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
    self._run_callback(callback, msg)
  File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
    callback(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2728, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2856, in run_ast_nodes
    if self.run_code(code, result):
  File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2910, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-3-23a4ed6f3a2e>", line 1, in <module>
    runfile('C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py', wdir='C:/Users/kakus/Desktop/landmark')
  File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
    execfile(filename, namespace)
  File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
    exec(compile(f.read(), filename, 'exec'), namespace)
  File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 84, in <module>
    x,y = input_fn('train.tfrecords',False)
  File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 76, in input_fn
    images_batch, labels_batch = iterator.get_next()
  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 330, in get_next
    name=name)), self._output_types,
  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 895, in iterator_get_next
    output_shapes=output_shapes, name=name)
  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3271, in create_op
    op_def=op_def)
  File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1650, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

OutOfRangeError (see above for traceback): End of sequence
     [[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]

Answer 1

dataset.repeat（num_epochs）将重复指定次数的数据。在训练模式下，您将其指定为none，将其更改为要训练数据集的时期数。

Tensorflow迭代器超出范围

1 个答案: