使用数据集生成器的Tensorflow model.fit()

时间:2019-03-27 10:50:59

标签: python tensorflow tensorflow-datasets tf.keras tensorflow2.0

我正在使用Dataset API生成训练数据并将其分类为NN的批次。

这是我的代码的最小工作示例:

import tensorflow as tf
import numpy as np
import random


def my_generator():
    while True:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label

def my_input_fn():
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(),
                                             output_types=(tf.float64, tf.int32))

    dataset = dataset.batch(32)
    iterator = dataset.make_one_shot_iterator()
    batch_features, batch_labels = iterator.get_next()

    return batch_features, batch_labels


if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(128, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    data_generator = my_input_fn()
    model.fit(data_generator)

代码在model.fit()调用中使用TensorFlow 1.13.1失败,出现以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 880, in fit
    validation_steps=validation_steps)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 310, in model_iteration
    ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in slice_arrays
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in <listcomp>
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 654, in _slice_helper
    name=name)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 820, in strided_slice
    shrink_axis_mask=shrink_axis_mask)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9334, in strided_slice
    _six.raise_from(_core._status_to_exception(e.code, message), None)
  File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Attr shrink_axis_mask has value 4294967295 out of range for an int32 [Op:StridedSlice] name: strided_slice/

我尝试使用TensorFlow 2.0在另一台机器上运行相同的代码(在删除tf.enable_eager_execution()行之后,因为它默认情况下会急于运行),并且出现以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 873, in fit
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 352, in model_iteration
    batch_outs = f(ins_batch)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3217, in __call__
    outputs = self._graph_fn(*converted_inputs)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 558, in __call__
    return self._call_flat(args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 627, in _call_flat
    outputs = self._inference_function.call(ctx, args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 397, in call
    (len(args), len(list(self.signature.input_arg))))
ValueError: Arguments and signature arguments do not match: 21 23 

我尝试将model.fit()更改为model.fit_generator(),但这在两个TensorFlow版本上都失败了。在TF 1.13.1上,出现以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 1426, in fit_generator
    initial_epoch=initial_epoch)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 115, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 377, in convert_to_generator_like
    num_samples = int(nest.flatten(data)[0].shape[0])
TypeError: __int__ returned non-int (type NoneType)

在TF 2.0上,出现以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1515, in fit_generator
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 140, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 477, in convert_to_generator_like
    raise ValueError('You must specify `batch_size`')
ValueError: You must specify `batch_size`

batch_size还不是fit_generator()的公认关键字。

我对这些错误消息感到困惑,如果有人可以阐明这些错误消息或指出我做错了什么,我将不胜感激。

1 个答案:

答案 0 :(得分:2)

虽然错误的根源仍然不清楚,但是我找到了使代码正常工作的解决方案。如果对类似情况的人有用,我会在这里发布。

基本上,我将my_input_fn()更改为生成器,并按如下方式使用model.fit_generator()

import tensorflow as tf
import numpy as np
import random


def my_generator(total_items):
    i = 0
    while i < total_items:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label
        i += 1

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)


    iterator = dataset.make_one_shot_iterator()
    while True:
        batch_features, batch_labels = iterator.get_next()
        yield batch_features, batch_labels

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 200
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    train_data_generator = my_input_fn(total_items, epochs)
    model.fit_generator(generator=train_data_generator, steps_per_epoch=num_batches, epochs=epochs, verbose=1)

编辑

如giser_yugang在评论中所暗示,也可以使用my_input_fn()作为返回dataset而不是单个批处理的函数来完成此操作。

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)
    return dataset

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 100
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    dataset = my_input_fn(total_items, epochs)
    model.fit_generator(dataset, epochs=epochs, steps_per_epoch=num_batches)

这两种方法之间似乎没有任何平均性能差异。