我尝试使用tf 2.0中的batchnormalization层创建一个类，但是它给了我一个错误，即变量不存在渐变。我尝试直接使用批处理规范化，但是它也给了我同样的错误。似乎不是在训练与批量标准化步骤相关的变量。
我尝试使用model.trainable_variables代替model.variables，但是它也不起作用。
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
import numpy as np
import matplotlib.pyplot as plt
import os
from scipy import ndimage
learning_rate = 0.001
training_epochs = 15
batch_size = 100
tf.random.set_seed(777)
cur_dir = os.getcwd()
ckpt_dir_name = 'checkpoints'
model_dir_name = 'minst_cnn_best'
checkpoint_dir = os.path.join(cur_dir, ckpt_dir_name, model_dir_name)
os.makedirs(checkpoint_dir, exist_ok=True)
checkpoint_prefix = os.path.join(checkpoint_dir, model_dir_name)
mnist = tf.keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images.astype(np.float32) /255.
test_images = test_images.astype(np.float32) /255.
print(train_images.shape, test_images.shape)
train_images = np.expand_dims(train_images, axis = -1)
test_images = np.expand_dims(test_images, axis = -1)
print(train_images.shape, test_images.shape)
train_labels = to_categorical(train_labels, 10)
test_labels = to_categorical(test_labels, 10)
train_dataset = tf.data.Dataset.from_tensor_slices((train_images,
train_labels)).shuffle(buffer_size = 100000).batch(batch_size)
test_dataset = tf.data.Dataset.from_tensor_slices((test_images,
test_labels)).batch(batch_size)
class ConvBNRelu(tf.keras.Model):
def __init__(self, filters, kernel_size=3, strides=1, padding='SAME'):
super(ConvBNRelu, self).__init__()
self.conv = keras.layers.Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
padding=padding, kernel_initializer='glorot_normal')
self.batchnorm = tf.keras.layers.BatchNormalization()
def call(self, inputs, training=False):
layer = self.conv(inputs)
layer = self.batchnorm(layer)
layer = tf.nn.relu(layer)
return layer
class DenseBNRelu(tf.keras.Model):
def __init__(self, units):
super(DenseBNRelu, self).__init__()
self.dense = keras.layers.Dense(units=units, kernel_initializer='glorot_normal')
self.batchnorm = tf.keras.layers.BatchNormalization()
def call(self, inputs, training=False):
layer = self.dense(inputs)
layer = self.batchnorm(layer)
layer = tf.nn.relu(layer)
return layer
class MNISTModel(tf.keras.Model):
def __init__(self):
super(MNISTModel, self).__init__()
self.conv1 = ConvBNRelu(filters=32, kernel_size=[3, 3], padding='SAME')
self.pool1 = keras.layers.MaxPool2D(padding='SAME')
self.conv2 = ConvBNRelu(filters=64, kernel_size=[3, 3], padding='SAME')
self.pool2 = keras.layers.MaxPool2D(padding='SAME')
self.conv3 = ConvBNRelu(filters=128, kernel_size=[3, 3], padding='SAME')
self.pool3 = keras.layers.MaxPool2D(padding='SAME')
self.pool3_flat = keras.layers.Flatten()
self.dense4 = DenseBNRelu(units=256)
self.drop4 = keras.layers.Dropout(rate=0.4)
self.dense5 = keras.layers.Dense(units=10, kernel_initializer='glorot_normal')
def call(self, inputs, training=False):
net = self.conv1(inputs)
net = self.pool1(net)
net = self.conv2(net)
net = self.pool2(net)
net = self.conv3(net)
net = self.pool3(net)
net = self.pool3_flat(net)
net = self.dense4(net)
net = self.drop4(net)
net = self.dense5(net)
return net
models = []
num_models = 5
for m in range(num_models):
models.append(MNISTModel())
def loss_fn(model, images, labels):
logits = model(images, training=True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
return loss
def grad(model, images, labels):
with tf.GradientTape() as tape:
loss = loss_fn(model, images, labels)
return tape.gradient(loss, model.variables)
def evaluate(models, images, labels):
predictions = np.zeros_like(labels)
for model in models:
logits = model(images, training=False)
predictions += logits
correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return accuracy
optimizer = keras.optimizers.Adam(learning_rate = learning_rate)
checkpoints = []
for m in range(num_models):
checkpoints.append(tf.train.Checkpoint(cnn=models[m]))
for epoch in range(training_epochs):
avg_loss = 0.
avg_train_acc = 0.
avg_test_acc = 0.
train_step = 0
test_step = 0
for images, labels in train_dataset:
for model in models:
grads = grad(model, images, labels)
optimizer.apply_gradients(zip(grads, model.variables))
loss = loss_fn(model, images, labels)
avg_loss += loss / num_models
acc = evaluate(models, images, labels)
avg_train_acc += acc
train_step += 1
avg_loss = avg_loss / train_step
avg_train_acc = avg_train_acc / train_step
for images, labels in test_dataset:
acc = evaluate(models, images, labels)
avg_test_acc += acc
test_step += 1
avg_test_acc = avg_test_acc / test_step
print('Epoch:', '{}'.format(epoch + 1), 'loss =', '{:.8f}'.format(avg_loss),
'train accuracy = ', '{:.4f}'.format(avg_train_acc),
'test accuracy = ', '{:.4f}'.format(avg_test_acc))
for idx, checkpoint in enumerate(checkpoints):
checkpoint.save(file_prefix=checkpoint_prefix+'-{}'.format(idx))
print('Learning Finished!')
W0727 20：27：05.344142 140332288718656 Optimizer_v2.py:982]变量['mnist_model / conv_bn_relu / batch_normalization / moving_mean：0'，'mnist_model / conv_bn_relu / batch_normalization / moving_istance：0'， conv_bn_relu_1 / batch_normalization_1 / moving_mean：0'，'mnist_model / conv_bn_relu_1 / batch_normalization_1 / moving_variance：0'，'mnist_model / conv_bn_relu_2 / batch_normalization_2 / moving_mean：0'，'mnist_model / conv_b2 _ / _ batch_normalization_3 / moving_mean：0'，'mnist_model / dense_bn_relu / batch_normalization_3 / moving_variance：0']，可将损失降至最低。 W0727 20：27：05.407717 140332288718656 deprecation.py:323]来自/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py:460：BaseResourceVariable.constraint（来自tensorflow。 python.ops.resource_variable_ops）已过时，并将在以后的版本中删除。 更新说明： 在优化程序更新步骤之后，手动应用约束。 W0727 20：27：05.499249 140332288718656optimizer_v2.py:982]变量['mnist_model_1 / conv_bn_relu_3 / batch_normalization_4 / moving_mean：0'，'mnist_model_1 / conv_bn_relu_3 / batch_normalization_4 / batch_normalization_4 / batch_normalization_4 / batch_normalization_4 / batch_normalization_4 / batch_normalization_4 / batch_normalization_4_b_3 / moving_mean：0'，'mnist_model_1 / conv_bn_relu_4 / batch_normalization_5 / moving_variance：0'，'mnist_model_1 / conv_bn_relu_5 / batch_normalization_6 / moving_mean：0'，'mnist_model_1 / conv_bn_normal_ize_vari_ization / bn_ance__1_con ________ ：0'，'mnist_model_1 / dense_bn_relu_1 / batch_normalization_7 / moving_variance：0']可以将损失降至最低。 ...
答案 0 :(得分：0)
您正在计算相对于model.variables的损失梯度：此集合不仅包含可训练变量（模型权重），而且还包含不可训练变量，例如通过计算的移动平均值和方差批处理规范化层。
您必须计算相对于trainable_variables的梯度。简而言之，换行
return tape.gradient(loss, model.variables)
和
optimizer.apply_gradients(zip(grads, model.variables))
到
return tape.gradient(loss, model.trainable_variables)
和
optimizer.apply_gradients(zip(grads, model.trainable_variables))