Question

我正在尝试使用底层api tf.nn.dynamic_rnn复制tf.nn.raw_rnn的行为。为了做到这一点，我使用了相同的数据补丁，设置了随机种子，并使用了相同的hparams来创建细胞和循环神经网络。但是，从两种实现方式生成的输出都不相等。下面是数据和代码。

data和lengths：

X = np.array([[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [0.0, 0.0, 0.0]], [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [7.7, 8.8, 9.9]], [[1.1, 2.2, 3.3], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], dtype=np.float32)
X_len = np.array([2, 3, 1], dtype=np.int32)

tf.nn.dynamic_rnn实现：

tf.reset_default_graph()
tf.set_random_seed(42)

inputs = tf.placeholder(shape=(3, None, 3), dtype=tf.float32)
lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
outputs, state = tf.nn.dynamic_rnn(inputs=inputs, sequence_length=lengths, cell=lstm_cell, dtype=tf.float32, initial_state=lstm_cell.zero_state(3, dtype=tf.float32), time_major=True)
outputs_reshaped = tf.transpose(outputs, perm=[1, 0, 2])

sess = tf.Session()
sess.run(tf.initializers.global_variables())
X = np.transpose(X, (1, 0, 2))
hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: X, lengths: X_len})
print(hidden_state)

tf.nn.raw_rnn实现：

tf.reset_default_graph()
tf.set_random_seed(42)

inputs = tf.placeholder(shape=(3, None, 3),dtype=tf.float32)
lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

inputs_ta = tf.TensorArray(dtype=tf.float32, size=3)
inputs_ta = inputs_ta.unstack(inputs)

lstm_cell = tf.nn.rnn_cell.LSTMCell(5)

def loop_fn(time, cell_output, cell_state, loop_state):
    emit_output = cell_output  # == None for time == 0
    if cell_output is None:  # time == 0
        next_cell_state = lstm_cell.zero_state(3, tf.float32)
    else:
        next_cell_state = cell_state

    elements_finished = (time >= lengths)
    finished = tf.reduce_all(elements_finished)
    next_input = tf.cond(finished, true_fn=lambda: tf.zeros([3, 3], dtype=tf.float32), false_fn=lambda: inputs_ta.read(time))

    next_loop_state = None

    return (elements_finished, next_input, next_cell_state, emit_output, next_loop_state)

outputs_ta, final_state, _ = tf.nn.raw_rnn(lstm_cell, loop_fn)
outputs_reshaped = tf.transpose(outputs_ta.stack(), perm=[1, 0, 2])

sess = tf.Session()
sess.run(tf.initializers.global_variables())

X = np.transpose(X, (1, 0, 2))
hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: X, lengths: X_len})

print(hidden_state)

我确定它们之间存在一些差异，但是我无法弄清楚它的位置和含义。如果有人有想法，那就太棒了。

期待您的回答！

Answer 1

差异的原因是您的变量被初始化为不同的值。您可以通过以下方式看到此信息：

print(sess.run(tf.trainable_variables()))

初始化后。

这种差异的原因是，存在一个全局种子和每个操作种子，因此设置随机种子不会强制调用隐藏在lstm代码中的初始化程序使用相同的随机种子。请参阅this answer for more details on this。总结一下：用于任何随机变量的随机种子都是从全局种子开始的，然后取决于添加到图中的最后一个操作的ID。

知道了这一点，我们可以通过以完全相同的顺序构建图，直到构造变量为止，从而强制变量种子在两个实现中都相同：这意味着我们从相同的全局种子开始，并添加了相同的变量以相同的顺序对图进行操作直到变量，因此变量将具有相同的操作种子。我们可以这样做：

tf.reset_default_graph()
tf.set_random_seed(42)
lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
inputs_shape = (3, None, 3)
lstm_cell.build(inputs_shape)

需要使用build方法，因为这实际上是将变量添加到图形中的方法。

这是您所拥有内容的完整工作版本：

import tensorflow as tf
import numpy as np


X = np.array([[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [0.0, 0.0, 0.0]], [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [7.7, 8.8, 9.9]], [[1.1, 2.2, 3.3], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], dtype=np.float32)
X_len = np.array([2, 3, 1], dtype=np.int32)


def dynamic():
    tf.reset_default_graph()
    tf.set_random_seed(42)
    lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
    inputs_shape = (3, None, 3)
    lstm_cell.build(inputs_shape)

    inputs = tf.placeholder(shape=inputs_shape, dtype=tf.float32)
    lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

    outputs, state = tf.nn.dynamic_rnn(inputs=inputs, sequence_length=lengths, cell=lstm_cell, dtype=tf.float32,
                                       initial_state=lstm_cell.zero_state(3, dtype=tf.float32), time_major=True)
    outputs_reshaped = tf.transpose(outputs, perm=[1, 0, 2])

    sess = tf.Session()
    sess.run(tf.initializers.global_variables())
    a = np.transpose(X, (1, 0, 2))
    hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: a, lengths: X_len})
    print(hidden_state)


def replicated():
    tf.reset_default_graph()
    tf.set_random_seed(42)
    lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
    inputs_shape = (3, None, 3)
    lstm_cell.build(inputs_shape)

    inputs = tf.placeholder(shape=inputs_shape, dtype=tf.float32)
    lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

    inputs_ta = tf.TensorArray(dtype=tf.float32, size=3)
    inputs_ta = inputs_ta.unstack(inputs)


    def loop_fn(time, cell_output, cell_state, loop_state):
        emit_output = cell_output  # == None for time == 0
        if cell_output is None:  # time == 0
            next_cell_state = lstm_cell.zero_state(3, tf.float32)
        else:
            next_cell_state = cell_state

        elements_finished = (time >= lengths)
        finished = tf.reduce_all(elements_finished)
        next_input = tf.cond(finished, true_fn=lambda: tf.zeros([3, 3], dtype=tf.float32),
                             false_fn=lambda: inputs_ta.read(time))

        next_loop_state = None

        return (elements_finished, next_input, next_cell_state, emit_output, next_loop_state)

    outputs_ta, final_state, _ = tf.nn.raw_rnn(lstm_cell, loop_fn)
    outputs_reshaped = tf.transpose(outputs_ta.stack(), perm=[1, 0, 2])

    sess = tf.Session()
    sess.run(tf.initializers.global_variables())

    a = np.transpose(X, (1, 0, 2))
    hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: a, lengths: X_len})

    print(hidden_state)


if __name__ == '__main__':
    dynamic()
    replicated()

Tensorflow：使用raw_rnn复制dynamic_rnn行为

1 个答案: