Question

我有一个约1000行的小型数据集，其中包含两个分类列[Message]，[Intent]。我想创建一个分类模型，并对看不见的新消息做出预测。

这29个独特的意图是不平衡的，范围从116到4个值计数：

intent_1                     116
intent_2                     98
intent_3                     81
intent_4                     78
intent_5                     73
intent_6                     68
intent_7                     66
intent_8                     65
intent_9                     62
intent_10                    61
intent_11                    56
intent_12                    53
intent_13                    50
intent_14                    49
intent_15                    45
intent_16                    40
intent_17                    37
intent_18                    32
intent_19                    31
intent_20                    30
intent_21                    25
intent_22                    22
intent_23                    21
intent_24                    19
intent_25                    15
intent_26                    12
intent_27                    10
intent_28                    9
intent_29                    4

我首先尝试了CNN模型，指标准确性和分类交叉熵，但结果并不理想：

在调整了批处理大小，历元，下降，激活函数，预训练单词嵌入等参数之后，还尝试了bi-lstm模型，我的模型仍然过拟合。

因此，在训练模型之前，我试图添加一个超采样步骤，以解决班级不平衡的问题，但是现在结果更加糟糕。

有人可以帮助我吗？到目前为止，这是我的代码：

#load training and validation files and concatenate them

def load_datasets(train_file, val_file):
  train = pd.read_excel(train_file)
  val = pd.read_csv(val_file, error_bad_lines=False, sep=";")
  #drop additional info column
  train.drop(train.columns[1], axis=1, inplace=True)
  #concatenate two dataframes
  frames = [train, val]
  result = pd.concat(frames)  

  intent = result["Label"]
  unique_intent = list(set(intent))
  sentences = list(result["Message"])

  return(intent, unique_intent, sentences)

intent, unique_intent, sentences = load_datasets("Training.xlsx","Validation.csv")

#define stemmer 
stemmer = SnowballStemmer("english")

#define lemmatizer
lemmatizer = WordNetLemmatizer() 

#clean the data : remove punctuation, tokenize, lowercase, lemmatize
def preprocessing(sentences):
  words = []
  for sent in sentences:
    clean = re.sub(r'[^ a-z A-Z 0-9]', " ", sent)
    w = word_tokenize(clean)
    #stemming
    words.append([lemmatizer.lemmatize(i.lower()) for i in w])

  return words

cleaned = preprocessing(sentences)


#creating tokenizer
def create_tokenizer(words, filters = '!"#$%&()*+,-./:;<=>?@[\]^_`{|}~'):
  token = Tokenizer(filters = filters)
  token.fit_on_texts(words)

  return token

#defining maximum length
def max_length(words):

  return(len(max(words, key = len)))


#show vocabulary size and maximum length 
word_tokenizer = create_tokenizer(cleaned)
vocab_size = len(word_tokenizer.word_index) + 1
max_length = max_length(cleaned)

print("Vocab Size = %d and Maximum length = %d" % (vocab_size, max_length))

#### Vocab Size = 811 and Maximum length = 45

#encoding list of words
def encoding_doc(token, words):

  return(token.texts_to_sequences(words))

encoded_doc = encoding_doc(word_tokenizer, cleaned)


#add padding to make words of equal length to use in the model
def padding_doc(encoded_doc, max_length):

  return(pad_sequences(encoded_doc, maxlen = max_length, 
                       padding =   "post"))



padded_doc = padding_doc(encoded_doc, max_length)



output_tokenizer = create_tokenizer(unique_intent,
                        filters = '!"#$%&()*+,-/:;<=>?@[\]^`{|}~')


output_tokenizer.word_index

encoded_output = encoding_doc(output_tokenizer, intent)

encoded_output = np.array(encoded_output).reshape(len(encoded_output), 1)


#one-hot encoding
def one_hot(encode):
  o = OneHotEncoder(sparse=False, categories='auto')

  return(o.fit_transform(encode))


output_one_hot = one_hot(encoded_output)


#split dataset to train (70%) and validation set (30%)
from sklearn.model_selection import train_test_split

train_X, val_X, train_Y, val_Y = train_test_split(padded_doc, output_one_hot, 
                                                   shuffle = True, 
                                                  random_state=1,
                                                  test_size = 0.3)

print("Shape of train_X = %s and train_Y = %s" % (train_X.shape, train_Y.shape))
print("Shape of val_X = %s and val_Y = %s" % (val_X.shape, val_Y.shape))

#### Shape of train_X = (929, 45) and train_Y = (929, 29)
#### Shape of val_X = (399, 45) and val_Y = (399, 29)


#Over Sample 

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import recall_score


clf_rf = RandomForestClassifier(n_estimators=10)
clf_rf.fit(x_train_res, y_train_res)


#over sampling
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import SMOTENC

x_train, x_val, y_train, y_val = train_test_split(train_X, train_Y, 
                                                   shuffle = True, 
                                                  test_size = 0.3)


sm = SMOTENC(categorical_features=[0,44], k_neighbors=2)

x_train_res, y_train_res = sm.fit_sample(x_train, y_train)


print('Validation Results')
print(clf_rf.score(x_val, y_val))
print(recall_score(y_val, clf_rf.predict(x_val),average='micro'))
print('\nTest Results')
print(clf_rf.score(val_X, val_Y))
print(recall_score(val_Y, clf_rf.predict(val_X),average='micro'))


### Validation Results
### 0.4495798319327731
### 0.4495798319327731

### Test Results
### 0.14035087719298245
### 0.14035087719298245

#Define the model

def create_model(vocab_size, max_length):
  model = Sequential()
  model.add(Embedding(vocab_size, 100, input_length=max_length, trainable=False))
  model.add(Dropout(0.2))
  model.add(Conv1D(64, 5, activation='relu'))
  model.add(MaxPooling1D(pool_size=4))
  model.add(LSTM(32))
  model.add(Dense(29, activation='softmax'))

  return model


model = create_model(vocab_size, max_length)

model.compile(loss = "categorical_crossentropy", optimizer = "adam", 
              metrics = ["accuracy"])
model.summary()


filename = 'model.h5'
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, 
                             save_best_only=True, mode='min')

hist = model.fit(x_train_res, y_train_res, epochs = 100, batch_size = 32, 
                 validation_data = (val_X, val_Y), callbacks = [checkpoint])


model = load_model("model.h5")


plt.style.use('ggplot')

def plot_history(hist):
  acc = hist.history['acc']
  val_acc = hist.history['val_acc']
  loss = hist.history['loss']
  val_loss = hist.history['val_loss']
  x = range(1, len(acc)+1)

  plt.figure(figsize=(12,5))
  plt.subplot(1, 2, 1)
  plt.plot(x, acc, 'b', label='Training acc')
  plt.plot(x, val_acc, 'r', label='Validation acc')
  plt.title('Training and validation accuracy')
  plt.legend()
  plt.subplot(1,2,2)
  plt.plot(x, loss, 'b', label='Training loss')
  plt.plot(x, val_loss, 'r', label='Validation loss')
  plt.title('Training and validation loss')
  plt.legend()


plot_history(hist)

Answer 1

一些观察结果：

您可以观察到您拥有的第一个模型（CNN + 交叉熵）已经过拟合。转向更多在过度拟合的情况下，复杂的模型完全没有帮助；在相反，您得到的结果更糟，就像您第二次看到的那样情节。
您的数据很少。甚至1000行2个完美平衡类（500 intent 1，500 intent 2）对于神经系统来说很少网络，更不用说29个意图不平衡了，但只有 1000行。

建议：

创建一个非常基本的模型。不要将双向LSTM与很多单元一起使用，而应将简单的CNN与几层单元一起使用。
使用过采样来平衡您的数据集（SMOTE（我看您已经在使用）或ADASYN等多种技术）。
创建一个具有更多行的更复杂的数据集，适用于神经网络（收集更多真实数据，而不会过采样）。否则，我将切换到经典的机器学习模型。
如果问题允许，数据增强也是一种解决方案（@Joseph Budin下面的评论）。

Keras：多类不平衡数据分类过拟合

1 个答案: