Question

所以我一直在从事这个聊天机器人项目，我正在为其SML使用SVM，我真的想使用余弦相似度作为内核。我曾尝试使用pykernel（as suggested from this post）或其他来源的其他代码，但仍然无法正常工作，我也不知道为什么...

说我有train.py这样的代码

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
import pickle, csv, json, timeit, random, os, nltk
from nltk.stem.lancaster import LancasterStemmer
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split as tts
from sklearn.preprocessing import LabelEncoder as LE
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
import my_kernel

def preprocessing(text):
    factory1 = StopWordRemoverFactory()
    StopWord = factory1.create_stop_word_remover()
    text = StopWord.remove(text)
    factory2 = StemmerFactory()
    stemmer = factory2.create_stemmer()
    return (stemmer.stem(text))

le = LE()
tfv = TfidfVectorizer(min_df=1)

file = os.path.join(os.path.dirname(os.path.abspath(__file__)),"scraping","tes.json")
svm_pickle_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"data","svm_model.pickle")

if os.path.exists(svm_pickle_path):
    os.remove(svm_pickle_path)

tit = [] # Title
cat = [] # Category
post = [] # Post

with open(file, "r") as sentences_file:
    reader = json.load(sentences_file)
    for row in reader:
        tit.append(preprocessing(row["Judul"]))
        cat.append(preprocessing(row["Kategori"]))
        post.append(preprocessing(row["Post"]))

tfv.fit(tit)
le.fit(cat)

features = tfv.transform(tit)
labels = le.transform(cat)

trainx, testx, trainy, testy = tts(features, labels, test_size=.30, random_state=42)

model = SVC(kernel=my_kernel, C=1.5)

f = open(svm_pickle_path, 'wb')
pickle.dump(model.fit(trainx, trainy), f)
f.close()

print("SVC training score:", model.score(testx, testy))

with open(svm_pickle_path, 'rb') as file:  
    pickle_model = pickle.load(file)

score = pickle_model.score(testx, testy)  
print("Test score: {0:.2f} %".format(100 * score))  
Ypredict = pickle_model.predict(testx)
print(Ypredict)

以及my_kernel.py代码：

import numpy as np
import math
from numpy import linalg as LA

def my_kernel(X, Y):
    norm = LA.norm(X) * LA.norm(Y)
    return np.dot(X, Y.T)/norm

它显示了我每次运行程序时的情况

Traceback (most recent call last):

File "F:\env\chatbot\chatbotProj\chatbotProj\train.py", line 84, in <module>
pickle.dump(model.fit(trainx, trainy), f)

File "F:\env\lib\site-packages\sklearn\svm\base.py", line 212, in fit
fit(X, y, sample_weight, solver_type, kernel, random_seed=seed)

File "F:\env\lib\site-packages\sklearn\svm\base.py", line 252, in _dense_fit
X = self._compute_kernel(X)

File "F:\env\lib\site-packages\sklearn\svm\base.py", line 380, in _compute_kernel
kernel = self.kernel(X, self.__Xfit)

File "F:\env\chatbot\chatbotProj\chatbotProj\ChatbotCode\svm.py", line 31, in my_kernel
norm = LA.norm(X) * LA.norm(Y)

File "F:\env\lib\site-packages\numpy\linalg\linalg.py", line 2359, in norm
sqnorm = dot(x, x)

File "F:\env\lib\site-packages\scipy\sparse\base.py", line 478, in __mul__
raise ValueError('dimension mismatch')

ValueError: dimension mismatch

我是python和SVM领域的新手，有人知道这是什么问题吗，或者可以推荐我如何更好，更干净地编写余弦相似性内核？

根据train_test_split sklearn，火车X的尺寸为（193，634），火车Y为（193，），测试X为（83，634），测试Y为（83，）。

Answer 1

更新：我的朋友告诉我，这是因为我的稀疏矩阵不是一个简单的数组，所以我必须将其密集化，并替换my_kernel.py代码，像这样

def my_kernel(X, Y):
    X=np.array(X.todense())
    Y=np.array(Y.todense())
    norm = LA.norm(X) * LA.norm(Y)
    return np.dot(X, Y.T)/norm

SVM的余弦相似性内核

1 个答案: