我正在使用gensim(在jupyter笔记本中)进行主题建模。我成功创建了一个模型并将其可视化。下面是代码:
import time
start_time = time.time()
import re
import spacy
import nltk
import pyLDAvis
import pyLDAvis.gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',level=logging.ERROR)
import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)
# nlp = spacy.load('en')
stop_word_list = nltk.corpus.stopwords.words('english')
stop_word_list.extend(['from','subject','re','edu','use'])
df = pd.read_csv('Topic_modeling.csv')
data = df.Articles.values.tolist()
# Remove Emails
data = [re.sub('\S*@\S*\s?','',sent) for sent in data]
# Remove new line characters
data = [re.sub('\s+',' ',sent) for sent in data]
# Remove distracting single quotes
data = [re.sub("\'","",sent) for sent in data]
def sent_to_words(sentences):
for sentence in sentences:
yield gensim.utils.simple_preprocess(str(sentence),deacc=True) # deacc=True removes punctuations
data_words = list(sent_to_words(data))
# Build the bigram and trigram models
bigram = gensim.models.Phrases(data_words,min_count=5,threshold=100) # higher threshold fewer phrases.
trigram = gensim.models.Phrases(bigram[data_words],threshold=100)
# Faster way to get a sentence clubbed as a trigram/bigram
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)
# Define functions for stopwords,bigrams,trigrams and lemmatization
def remove_stopwords(texts):
return [[word for word in simple_preprocess(str(doc)) if word not in stop_word_list] for doc in texts]
def make_bigrams(texts):
return [bigram_mod[doc] for doc in texts]
def make_trigrams(texts):
return [trigram_mod[bigram_mod[doc]] for doc in texts]
def lemmatization(texts,allowed_postags=['NOUN','ADJ','VERB','ADV']):
"""https://spacy.io/api/annotation"""
texts_out = []
for sent in texts:
doc = nlp(" ".join(sent))
texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
return texts_out
# Remove Stop Words
data_words_nostops = remove_stopwords(data_words)
# Form Bigrams
data_words_bigrams = make_bigrams(data_words_nostops)
# Initialize spacy 'en' model,keeping only tagger component (for efficiency)
# python3 -m spacy download en
nlp = spacy.load('en',disable=['parser','ner'])
# Do lemmatization keeping only noun,adj,vb,adv
data_lemmatized = lemmatization(data_words_bigrams,'ADJ'])
# Create Dictionary
id2word = corpora.Dictionary(data_lemmatized)
# Create Corpus
texts = data_lemmatized
# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]
# Build LDA model
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=id2word,num_topics= 3,random_state=100,update_every=1,chunksize=100,passes=20,alpha=0.4,eta=0.2,per_word_topics=True)
print(lda_model.print_topics())
doc_lda = lda_model[corpus]
现在,我想在每个句子中找到主要主题。所以我正在使用以下代码:
def format_topics_sentences(ldamodel=lda_model,corpus=corpus,texts=data):
# Init output
sent_topics_df = pd.DataFrame()
# Get main topic in each document
for i,row in enumerate(ldamodel[corpus]):
row = sorted(row,key=lambda x: (x[1]),reverse=True)
# Get the Dominant topic,Perc Contribution and Keywords for each document
for j,(topic_num,prop_topic) in enumerate(row):
if j == 0: # => dominant topic
wp = ldamodel.show_topic(topic_num)
topic_keywords = ",".join([word for word,prop in wp])
sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num),round(prop_topic,4),topic_keywords]),ignore_index=True)
else:
break
sent_topics_df.columns = ['Dominant_Topic','Perc_Contribution','Topic_Keywords']
# Add original text to the end of the output
contents = pd.Series(texts)
sent_topics_df = pd.concat([sent_topics_df,contents],axis=1)
return(sent_topics_df)
df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model,texts=data)
# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No','Dominant_Topic','Topic_Perc_Contrib','Keywords','Text']
# Show
df_dominant_topic.head(10)
但是,我遇到了以下错误:
TypeError跟踪(最近的呼叫 最后) 22 23 ---> 24 df_topic_sents_keywords = format_topics_sentences(ldamodel = lda_model,语料库=语料库,文本=数据) 25 26#格式
以format_topics_sentences(ldamodel, 语料,文本) 5#获取每个文档中的主要主题 对于i,在enumerate(ldamodel [corpus])中的行为6: ----> 7行=已排序(行,键= lambda x:(x [1]),reverse = True) 8#获取每个文档的主要主题,Perc贡献和关键字 对于枚举(行)中的j,(topic_num,prop_topic)为9:
TypeError:“ int”和“ tuple”的实例之间不支持“
我不明白这是什么问题。 有人可以帮忙吗?
谢谢!