我通过以下代码不断收到此错误
config_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_config.json'
checkpoint_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_model.ckpt'
dict_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/vocab.txt'
# 将词表中的词编号转换为字典
token_dict = {}
with codecs.open(dict_path, 'r', 'utf8') as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)
# 重写tokenizer
class OurTokenizer(Tokenizer):
def _tokenize(self, text):
R = []
for c in text:
if c in self._token_dict:
R.append(c)
elif self._is_space(c):
R.append('[unused1]') # 用[unused1]来表示空格类字符
else:
R.append('[UNK]') # 不在列表的字符用[UNK]表示
return R
tokenizer = OurTokenizer(token_dict)
# 让每条文本的长度相同，用0填充
def seq_padding(X, padding=0):
L = [len(x) for x in X]
ML = max(L)
return np.array([
np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X
])
def seq_padding_2(X, padding=0):
L = [x.shape[0] for x in X]
ML = max(L)
results = []
for x in X:
if x.shape[0] < ML:
r = np.array([0] * 21128)
x_1 = list(x)
for i in range((ML-x.shape[0])):
x_1.append(r)
results.append(np.array(x_1))
else:
results.append(x)
return np.array(results)
# data_generator只是一种为了节约内存的数据方式
class data_generator:
def __init__(self, data, batch_size=32, shuffle=True):
self.data = data
self.batch_size = batch_size
self.shuffle = shuffle
self.steps = len(self.data) // self.batch_size
if len(self.data) % self.batch_size != 0:
self.steps += 1
def __len__(self):
return self.steps
def __iter__(self):
while True:
idxs = list(range(len(self.data)))
if self.shuffle:
np.random.shuffle(idxs)
X1, X2, Y = [], [], []
for i in idxs:
d = self.data[i]
text = d[0][:maxlen]
x1, x2 = tokenizer.encode(first=text)
y = d[1]
X1.append(x1)
X2.append(x2)
Y.append(y)
if len(X1) == self.batch_size or i == idxs[-1]:
X1 = seq_padding(X1)
X2 = seq_padding(X2)
Y = seq_padding_2(Y)
yield [X1, X2], Y
[X1, X2, Y] = [], [], []
# calculate sparse categorical accuracy
def acc_top2(y_true, y_pred):
return sparse_categorical_accuracy(y_true, y_pred)
# bert模型设置
def build_corrector():
bert_model_1 = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None) # load pretrain bert model
for l in bert_model_1.layers:
l.trainable = True
x1_in = Input(shape=(None,))
x2_in = Input(shape=(None,))
x1 = bert_model_1([x1_in, x2_in])
x3 = keras.layers.GRU(maxlen)(x1)
bert_model_2 = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None, training=True) # load pretrain bert model
for l in bert_model_2.layers:
l.trainable = True
x2 = bert_model_2([x1_in, x2_in, x3])[0]
model = Model([x1_in, x2_in], x2)
model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(1e-5))
return model
DATA_LIST = []
for data_row in train_df.iloc[:].itertuples():
DATA_LIST.append((data_row.testing_sentence, to_categorical(tokenizer.encode(data_row.correct_sentence)[0], 21128)))
DATA_LIST = np.array(DATA_LIST)
DATA_LIST_TEST = []
for data_row in train_df.iloc[:].itertuples():
DATA_LIST_TEST.append((data_row.testing_sentence, to_categorical(tokenizer.encode(data_row.correct_sentence)[0], 21128)))
DATA_LIST_TEST = np.array(DATA_LIST_TEST)
#print(DATA_LIST)
#print(DATA_LIST_TEST)
# 交叉验证训练和测试模型
def run_cv(nfold, data, data_labels, data_test):
kf = KFold(n_splits=nfold, shuffle=True, random_state=520).split(data)
train_model_pred = np.zeros((len(data), 2))
test_model_pred = np.zeros((len(data_test), 2))
for i, (train_fold, test_fold) in enumerate(kf):
X_train, X_valid, = data[train_fold, :], data[test_fold, :]
model = build_corrector()
#early_stopping = EarlyStopping(monitor='val_acc', patience=3) # 早停法，防止过拟合
plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5,
patience=2) # 当评价指标不在提升时，减少学习率
checkpoint = ModelCheckpoint('./bert_dump/' + str(i) + '.hdf5', monitor='val_acc', verbose=2,
save_best_only=True, mode='max', save_weights_only=True) # 保存最好的模型
train_D = data_generator(X_train, shuffle=True)
valid_D = data_generator(X_valid, shuffle=True)
test_D = data_generator(data_test, shuffle=False)
# 模型训练
model.fit_generator(
train_D.__iter__(),
steps_per_epoch=len(train_D),
epochs=5,
validation_data=valid_D.__iter__(),
validation_steps=len(valid_D),
callbacks=[plateau, checkpoint],
)
# model.load_weights('./bert_dump/' + str(i) + '.hdf5')
# return model
train_model_pred[test_fold, :] = model.predict_generator(valid_D.__iter__(), steps=len(valid_D), verbose=1)
test_model_pred += model.predict_generator(test_D.__iter__(), steps=len(test_D), verbose=1)
del model
gc.collect() # 清理内存
K.clear_session() # clear_session就是清除一个session
# break
return train_model_pred, test_model_pred
data_generator(DATA_LIST, shuffle=True).__iter__().__next__()
train_model_pred, test_model_pred = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)
test_pred = [np.argmax(x) for x in test_model_pred]
错误回溯如下
Traceback (most recent call last):
File "C:/Users/lenovo/PycharmProjects/keras_bert/venv/ks.py", line 208, in <module>
train_model_pred, test_model_pred = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)
File "C:/Users/lenovo/PycharmProjects/keras_bert/venv/ks.py", line 191, in run_cv
callbacks=[plateau, checkpoint],
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training.py", line 1732, in fit_generator
initial_epoch=initial_epoch)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training_generator.py", line 42, in fit_generator
model._make_train_function()
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training.py", line 316, in _make_train_function
loss=self.total_loss)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\optimizers.py", line 504, in get_updates
grads = self.get_gradients(loss, params)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\optimizers.py", line 93, in get_gradients
raise ValueError('An operation has `None` for gradient. '
ValueError: An operation has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.
我正在使用的bert模型包是keras-bert。我试图自己做一个软掩饰的伯特。当我使用硬掩模处理seq2seq时，我也遇到了相同的错误。 对于这个错误，我需要一些帮助，因为据我所知keras.losses.categorical_crossentropy是可区分的。