from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import RegexpTokenizer
def docs_preprocessor(docs):
tokenizer = RegexpTokenizer(r'\w+')
for idx in range(len(docs)):
docs[idx] = docs[idx].lower() # Convert to lowercase.
docs[idx] = tokenizer.tokenize(docs[idx]) # Split into words.
# 删除数字，但不要删除包含数字的单词。
docs = [[token for token in doc if not token.isdigit()] for doc in docs]
# 删除仅一个字符的单词。
docs = [[token for token in doc if len(token) > 3] for doc in docs]
# 使文档中的所有单词规则化
lemmatizer = WordNetLemmatizer()
docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]
return docs
from gensim.models import Phrases
# 向文档中添加双字母组和三字母组（仅出现10次或以上的文档）。
bigram = Phrases(docs, min_count=10)
trigram = Phrases(bigram[docs])
for idx in range(len(docs)):
for token in bigram[docs[idx]]:
if '_' in token:
# Token is a bigram, add to document.
docs[idx].append(token)
for token in trigram[docs[idx]]:
if '_' in token:
# token是一个二元组，添加到文档中。
docs[idx].append(token)
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/gensim/models/phrases.py:316: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class
warnings.warn("For a faster implementation, use the gensim.models.phrases.Phraser class")
删除
In [6]:
from gensim.corpora import Dictionary
# 创建文档的字典表示
dictionary = Dictionary(docs)
print('Number of unique words in initital documents:', len(dictionary))
# 过滤掉少于10个文档或占文档20％以上的单词。
dictionary.filter_extremes(no_below=10, no_above=0.2)
print('Number of unique words after removing rare and common words:', len(dictionary))
Number of unique words in initital documents: 39534
Number of unique words after removing rare and common words: 6001
corpus1 = [dictionary.doc2bow(doc) for doc in docs1]
corpus2 = [dictionary.doc2bow(doc) for doc in docs2]
# 使用语料库LDA模型转换
lda_corpus1 = model[corpus1]
lda_corpus2 = model[corpus2]
In [15]:
from collections import OrderedDict
def get_doc_topic_dist(model, corpus, kwords=False):
'''
LDA转换，对于每个文档，仅返回权重非零的主题
此函数对主题空间中的文档进行矩阵转换
'''
top_dist =[]
keys = []
for d in corpus:
tmp = {i:0 for i in range(num_topics)}
tmp.update(dict(model[d]))
vals = list(OrderedDict(tmp).values())
top_dist += [array(vals)]
if kwords:
keys += [array(vals).argmax()]
return array(top_dist), keys
Intra similarity: cosine similarity for corresponding parts of a doc(higher is better):
0.906086532099
Inter similarity: cosine similarity between random parts (lower is better):
0.846485334252
让我们看一下每个主题中出现的单词。
In [17]:
def explore_topic(lda_model, topic_number, topn, output=True):
"""
输出topn词的列表
"""
terms = []
for term, frequency in lda_model.show_topic(topic_number, topn=topn):
terms += [term]
if output:
print(u'{:20} {:.3f}'.format(term, round(frequency, 3)))
return terms
'''
# 1.删除非字母
paper_text = re.sub("[^a-zA-Z]"," ", paper)
# 2.将单词转换为小写并拆分
words = paper_text.lower().split()
# 3. 删除停用词
words = [w for w in words if not w in stops]
# 4. 删除短词
words = [t for t in words if len(t) > 2]
# 5. 形容词
words = [nltk.stem.WordNetLemmatizer().lemmatize(t) for t in words]
top_dist =[]
for d in corpus:
tmp = {i:0 for i in range(num_topics)}
tmp.update(dict(model[d]))
vals = list(OrderedDict(tmp).values())
top_dist += [array(vals)]
In [23]:
top_dist, lda_keys= get_doc_topic_dist(model, corpus, True)
features = tvectorizer.get_feature_names()
In [24]:
top_ws = []
for n in range(len(dtm)):
inds = int0(argsort(dtm[n])[::-1][:4])
tmp = [features[i] for i in inds]
top_ws += [' '.join(tmp)]
cluster_colors = {0: 'blue', 1: 'green', 2: 'yellow', 3: 'red', 4: 'skyblue', 5:'salmon', 6:'orange', 7:'maroon', 8:'crimson', 9:'black', 10:'gray'}
p_df['colors'] = p_df['clusters'].apply(lambda l: cluster_colors[l])
In [25]:
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2)
X_tsne = tsne.fit_transform(top_dist)
,
)表示求
之间的欧式距离。计算原始高维数据X与转换后的低维数据Z的公式如下所示。
PCA主成分分析原理与水果成熟状态数据分析实例：Python中PCA-LDA与卷积神经网络CNN
R语言拟合线性混合效应模型、固定效应、随机效应参数估计可视化生物生长、发育、繁殖影响因素
R语言逻辑回归、GAM、LDA、KNN、PCA主成分分析分类预测房价及交叉验证
R语言SVM模型用大学生行为数据对助学金精准资助预测ROC可视化
