TensorFlow模型.fit()使用数据集生成器

Question

TensorFlow模型.fit()使用数据集生成器

pythontensorflowtensorflow-datasetstf.kerastensorflow2.0

8

我正在使用数据集 API 生成训练数据，并将其分批排序到神经网络中。

以下是我的代码的最小工作示例：

import tensorflow as tf
import numpy as np
import random


def my_generator():
    while True:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label

def my_input_fn():
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(),
                                             output_types=(tf.float64, tf.int32))

    dataset = dataset.batch(32)
    iterator = dataset.make_one_shot_iterator()
    batch_features, batch_labels = iterator.get_next()

    return batch_features, batch_labels


if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(128, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    data_generator = my_input_fn()
    model.fit(data_generator)

代码在使用TensorFlow 1.13.1时，调用model.fit()函数失败并产生如下错误：

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 880, in fit
    validation_steps=validation_steps)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 310, in model_iteration
    ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in slice_arrays
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in <listcomp>
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 654, in _slice_helper
    name=name)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 820, in strided_slice
    shrink_axis_mask=shrink_axis_mask)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9334, in strided_slice
    _six.raise_from(_core._status_to_exception(e.code, message), None)
  File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Attr shrink_axis_mask has value 4294967295 out of range for an int32 [Op:StridedSlice] name: strided_slice/

我尝试在另一台使用TensorFlow 2.0的计算机上运行相同的代码（删除了 tf.enable_eager_execution() 这一行，因为它默认情况下会立即运行），但是出现了以下错误：

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 873, in fit
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 352, in model_iteration
    batch_outs = f(ins_batch)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3217, in __call__
    outputs = self._graph_fn(*converted_inputs)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 558, in __call__
    return self._call_flat(args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 627, in _call_flat
    outputs = self._inference_function.call(ctx, args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 397, in call
    (len(args), len(list(self.signature.input_arg))))
ValueError: Arguments and signature arguments do not match: 21 23

我尝试将model.fit()更改为model.fit_generator()，但是这在两个TensorFlow版本上都失败了。在TF 1.13.1上，我会收到以下错误：

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 1426, in fit_generator
    initial_epoch=initial_epoch)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 115, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 377, in convert_to_generator_like
    num_samples = int(nest.flatten(data)[0].shape[0])
TypeError: __int__ returned non-int (type NoneType)

在 TF 2.0 上，我收到以下错误信息：

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1515, in fit_generator
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 140, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 477, in convert_to_generator_like
    raise ValueError('You must specify `batch_size`')
ValueError: You must specify `batch_size`

然而batch_size不是fit_generator()的一个可识别的关键字。

我对这些错误信息感到困惑，如果有人能够解释一下或指出我的错误，我将不胜感激。

- berkelem

1

你可以尝试使用 model.fit_generator(dataset, epochs=10, steps_per_epoch=30)。 - giser_yugang

谢谢，这个方法也可以，就像我下面发布的答案一样。 - berkelem

1个回答

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- berkelem · Accepted Answer

尽管错误的起源仍然模糊不清，但我找到了一种解决方案使代码可以工作。为了让其他人在类似情况下获益，我将在此处发布解决方案。基本上，我将my_input_fn()改为生成器，并使用如下代码：model.fit_generator()。

import tensorflow as tf
import numpy as np
import random


def my_generator(total_items):
    i = 0
    while i < total_items:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label
        i += 1

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)


    iterator = dataset.make_one_shot_iterator()
    while True:
        batch_features, batch_labels = iterator.get_next()
        yield batch_features, batch_labels

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 200
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    train_data_generator = my_input_fn(total_items, epochs)
    model.fit_generator(generator=train_data_generator, steps_per_epoch=num_batches, epochs=epochs, verbose=1)

编辑

正如giser_yugang在评论中所暗示的那样，也可以通过将my_input_fn()作为返回dataset而不是单个批次的函数来实现此操作。

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)
    return dataset

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 100
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    dataset = my_input_fn(total_items, epochs)
    model.fit_generator(dataset, epochs=epochs, steps_per_epoch=num_batches)

看起来这些方法之间没有任何平均表现差异。