Question

因此，我搞砸了gensim，可以打印出前5个主题以及与该主题相关的流行名词（使用此处的示例Topic Distribution and clustering using LDA完成）。我正在处理51个文档。我一直无法接收到“列表索引超出范围”错误，因此我的最后两个群集无法正常工作。对于修复群集可以做出哪些更改，我一无所知。我尝试使用if和else条件的方法给出了不正确的第一个簇（您会发现它已被注释掉）。我到底哪里出问题了？

    from gensim import corpora, models, similarities
from itertools import chain

# list of tokenised nouns from the noun documents
nounTokens = []

for index, row in df_Data.iterrows():
    nounTokens.append(df_Data.iloc[index]['Noun Tokens'])

# create a dictionary using noun Tokens
id2word = corpora.Dictionary(nounTokens)

# creates the bag of word corpus
mm = [id2word.doc2bow(noun) for noun in nounTokens]

# trains lda models
lda = models.ldamodel.LdaModel(corpus=mm, id2word=id2word, num_topics=5, update_every=1, chunksize=10000, passes=1)

# prints the topics of the corpus
for topics in lda.print_topics():
    print(topics)
print

lda_corpus = lda[mm]

# search for scores of all the words under each topic for all documents
scores = list(chain(*[[score for topic_id, score in topic] 
                      for topic in [doc for doc in lda_corpus]]))
# calculating the avg sum of all the probabilities to ensure we have a valid threshold.
threshold = sum(scores)/len(scores)
print(threshold)
print
# cluster1 = []
# cluster2 = []
# cluster3 = []

# for i,j in zip(lda_corpus, noun_Docs):
#     if len(i) > 0:
#         if i[0][1] > threshold:
#             cluster1.append(j)
#     elif len(i)>1:
#         if i[1][1] > threshold:
#             cluster2.append(j)
#     elif len(i) > 2:
#         if i[2][1] > threshold:
#             cluster3.append(j)

cluster1 = [j for i, j in zip(lda_corpus, noun_Docs) if i[0][1] > threshold]
cluster2 = [j for i, j in zip(lda_corpus, noun_Docs) if i[1][1] > threshold]
cluster3 = [j for i, j in zip(lda_corpus, noun_Docs) if i[2][1] > threshold]
# for i,j in zip(lda_corpus, noun_Docs):
#     print(i)

print(cluster1)
# print(cluster2)
# print(cluster3)

主题分布：使用LDA将文档聚类的问题

0 个答案: