GAN：如何在Tensorflow中更新批处理规范化参数

Question

批处理规范化更新

• 我正在使用Tensorflow中的 批处理规范化 来研究（生成对抗网络）。
• 如何在Tensorflow中更新批次归一化参数（移动平均值，移动方差）。
• 在鉴别器和生成器上使用batch_normalization后，如何更新图形？
• 或者，通过在鉴别器和生成器中使用batch_normalization函数来 自动更新 吗？

完整的源代码链接

• https://github.com/HeewonChung92/GAN_BatchNormalization
• data.mat
• test_batchnorm_github.py

开发环境

• Pycharm
• OS平台和发行版：Windows 10 x64
• 从以下位置安装的TensorFlow：Anaconda
• Tensorflow版本1.12.0：
• python 3.6.7：
• 移动设备：不适用
• 要复制的精确命令：N / A
• GPU型号和内存：NVIDIA GeForce CTX 1080 Ti
• CUDA / cuDNN：9.0 / 7.4

鉴别器

# discriminator
def discriminator(x , train_state, reuse = False):
    with tf.variable_scope(name_or_scope="Dis", reuse=reuse) as scope:
        dw1 = tf.get_variable(name="w1", shape=[num_input, num_hidden], initializer=myinit)
        db1 = tf.get_variable(name="b1", shape=[num_hidden], initializer=myinit)
        dw2 = tf.get_variable(name="w2", shape=[num_hidden, num_output], initializer=myinit)
        db2 = tf.get_variable(name="b2", shape=[num_output], initializer=myinit)

    fcHidden = tf.matmul(x, dw1) + db1
    bnHidden = tf.layers.batch_normalization(fcHidden, training=train_state)
    # hidden = tf.nn.leaky_relu(bnHidden)
    hidden = tf.nn.relu(bnHidden)
    logits = tf.matmul(hidden, dw2) + db2
    bnLogits = tf.layers.batch_normalization(logits, training=train_state)
    output = tf.nn.sigmoid(bnLogits)
    return output, logits

生成器

# generator
def generator(z, train_state):
    with tf.variable_scope(name_or_scope="Gen") as scope:
        gw1 = tf.get_variable(name="w1", shape=[num_noise, num_hidden], initializer=myinit)
        gb1 = tf.get_variable(name="b1", shape=[num_hidden], initializer=myinit)
        gw2 = tf.get_variable(name="w2", shape=[num_hidden, num_input], initializer=myinit)
        gb2 = tf.get_variable(name="b2", shape=[num_input], initializer=myinit)

    fcHidden = tf.matmul(z, gw1) + gb1
    bnHidden = tf.layers.batch_normalization(fcHidden, training=train_state)
    # hidden = tf.nn.leaky_relu(bnHidden)
    hidden = tf.nn.relu(bnHidden)
    logits = tf.matmul(hidden, gw2) + gb2
    bnLogits = tf.layers.batch_normalization(logits, training=train_state)
    output = tf.nn.sigmoid(bnLogits)
    return output, logits, hidden, tf.nn.leaky_relu(fcHidden)

图表

g = tf.Graph()
with g.as_default():
    X = tf.placeholder(tf.float32, [None, num_input]) # GAN 은 unsupervised learning 이므로 y(label)을 사용하지 않습니다.
    Z = tf.placeholder(tf.float32, [None, num_noise]) # Z 는 생성기의 입력값으로 사용될 noise 입니다.
    preLabel = tf.placeholder(tf.float32, [None, 1])
    trainingState = tf.placeholder(tf.bool)

    # Pre-train
    result_of_pre, logits_pre = discriminator_pre(X)
    p_loss = tf.reduce_mean(tf.square(result_of_pre - preLabel))

    # Discriminator & Generator
    fake_x, fake_logits, ghidden, gohidden  = generator(Z, trainingState)
    result_of_real, logits_real = discriminator(X, trainingState)
    result_of_fake, logits_fake = discriminator(fake_x, trainingState, True)

    # Discriminator / Generator 손실 함수를 정의합니다.
    d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_real, labels=tf.ones_like(result_of_real)))  # log(D(x))
    d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_fake, labels=tf.zeros_like(result_of_fake)))  # log(1-D(G(z)))
    d_loss = d_loss_real + d_loss_fake  # log(D(x)) + log(1-D(G(z)))
    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_fake, labels=tf.ones_like(result_of_fake)))  # log(D(G(z)))

    # Parameter
    t_vars = tf.trainable_variables() # return list
    g_vars = [var for var in t_vars if "Gen" in var.name]
    d_vars = [var for var in t_vars if "Dis" in var.name]
    p_vars = [var for var in t_vars if "Pre" in var.name]

    # Optimizer / Gradient
    p_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(p_loss, var_list=p_vars)
    g_train = tf.train.AdamOptimizer(learning_rate=learningrate_gen, beta1=0.5, beta2=0.999).minimize(g_loss, var_list=g_vars)
    d_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(d_loss, var_list=d_vars)

    # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    # with tf.control_dependencies(update_ops):
    #     g_train = tf.train.AdamOptimizer(learning_rate=learningrate_gen, beta1=0.5, beta2=0.999).minimize(g_loss, var_list=g_vars)
    #     d_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(d_loss, var_list=d_vars)

    # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    # p_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(p_loss, var_list=p_vars)
    # p_train = tf.group([p_train, update_ops])

批处理规范化更新

# Optimizer / Gradient
    # Method 1
    p_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(p_loss, var_list=p_vars)
    g_train = tf.train.AdamOptimizer(learning_rate=learningrate_gen, beta1=0.5, beta2=0.999).minimize(g_loss, var_list=g_vars)
    d_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(d_loss, var_list=d_vars)

    # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    # Method 2
    # with tf.control_dependencies(update_ops):
    #     g_train = tf.train.AdamOptimizer(learning_rate=learningrate_gen, beta1=0.5, beta2=0.999).minimize(g_loss, var_list=g_vars)
    #     d_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(d_loss, var_list=d_vars)

    # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    # Method 3
    # p_train = tf.train.AdamOptimizer(learning_rate=learningrate_dis, beta1=0.5, beta2=0.999).minimize(p_loss, var_list=p_vars)
    # p_train = tf.group([p_train, update_ops])