Question

我使用Gensim LDA训练了模型。训练进行得很好，但是模型评估未达到预期。当我尝试使用文件夹中的测试文件评估模型时，它将输出以下内容

<位于0x12bba2b50的gensim.interfaces.TransformedCorpus对象>

我在做什么错？下面是我的python代码：

my_path = "/path/to/files/training folder"
files = os.listdir(my_path)
doc_set = []


for file in files:
    newpath = (os.path.join(my_path, file)) 
    newpath1 = textract.process(newpath)
    newpath2 = newpath1.decode("utf-8")
    doc_set.append(newpath2)

texts = []
for i in doc_set:
    raw = i.lower()
    tokens = tokenizer.tokenize(raw)
    stopped_tokens = [i for i in tokens if not i in stopwords.words()]
    stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]
texts.append(stemmed_tokens)

dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]


ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=4, random_state=1, id2word = dictionary, passes=60)
print(ldamodel.print_topics(num_topics=4, num_words=25))

#Model evaluation

my_path1 = "/Path/to/file/testing folder"
files1 = os.listdir(my_path1)
doc_set1 = []

for file in files1:
    newpathone = (os.path.join(my_path, file)) 
    newpathtwo = textract.process(newpath)
    newpaththree = newpathtwo.decode("utf-8")
    doc_set1.append(newpaththree)

texts1 = []
for i in doc_set1:
    raw1 = i.lower()
    tokens1 = tokenizer.tokenize(raw1)
    stopped_tokens1 = [i for i in tokens1 if not i in stopwords.words()]
    stemmed_tokens1 = [p_stemmer.stem(i) for i in stopped_tokens1]
    texts1.append(stemmed_tokens1)


dictionary1 = corpora.Dictionary(texts1)
corpus1 = [dictionary1.doc2bow(text) for text in texts1]
print(ldamodel.get_document_topics(corpus1))

如何使用一组测试文档评估LDA gensim模型

0 个答案: