从Python：tf-idf-cosine：查找文档相似性，可以使用tf-idf余弦计算文档相似性。不导入外部库，是否有任何方法可以计算两个字符串之间的余弦相似度？
s1 = "This is a foo bar sentence ."
s2 = "This sentence is similar to a foo bar sentence ."
s3 = "What is this string ? Totally not related to the other two lines ."
cosine_sim(s1, s2) # Should give high cosine similarity
cosine_sim(s1, s3) # Shouldn't give high cosine similarity value
cosine_sim(s2, s3) # Shouldn't give high cosine similarity value
import math
import re
from collections import Counter
WORD = re.compile(r"\w+")
def get_cosine(vec1, vec2):
intersection = set(vec1.keys()) & set(vec2.keys())
numerator = sum([vec1[x] * vec2[x] for x in intersection])
sum1 = sum([vec1[x] ** 2 for x in list(vec1.keys())])
sum2 = sum([vec2[x] ** 2 for x in list(vec2.keys())])
denominator = math.sqrt(sum1) * math.sqrt(sum2)
if not denominator:
return 0.0
else:
return float(numerator) / denominator
def text_to_vector(text):
words = WORD.findall(text)
return Counter(words)
text1 = "This is a foo bar sentence ."
text2 = "This sentence is similar to a foo bar sentence ."
vector1 = text_to_vector(text1)
vector2 = text_to_vector(text2)
cosine = get_cosine(vector1, vector2)
print("Cosine:", cosine)
输出：
Cosine: 0.861640436855
theory -> analysis, concept, approach, idea, method
voice -> vocal, tone, sound, melody, singing
james -> william, john, thomas, robert, george, charles
最简单和最明显的方法是将每个单词向量相加或相乘。这会导致一个不良的副作用，即"猫追逐狗"和"狗追逐猫"在你的系统中意味着相同的含义。此外，如果你在做乘法，你必须特别小心，否则每个句子都将以[0,0,0,...,0]表示，这将使其失去意义。
进一步阅读
我不会讨论到目前为止已经提出的更复杂的组合方法。我建议你阅读Katrin Erk的 "Vector space models of word meaning and phrase meaning: a survey"。这是一个非常好的高层次调查，可以让你入门。不幸的是，在出版商的网站上不能免费获得它，请直接联系作者获取一份副本。在那篇论文中，你会找到许多更具体的方法的参考资料。其中比较易懂的是Mitchel和Lapata（2008）和Baroni和Zamparelli（2010）。
我有类似的解决方案，但可能对Pandas有用。
import math
import re
from collections import Counter
import pandas as pd
WORD = re.compile(r"\w+")
def get_cosine(vec1, vec2):
intersection = set(vec1.keys()) & set(vec2.keys())
numerator = sum([vec1[x] * vec2[x] for x in intersection])
sum1 = sum([vec1[x] ** 2 for x in list(vec1.keys())])
sum2 = sum([vec2[x] ** 2 for x in list(vec2.keys())])
denominator = math.sqrt(sum1) * math.sqrt(sum2)
if not denominator:
return 0.0
else:
return float(numerator) / denominator
def text_to_vector(text):
words = WORD.findall(text)
return Counter(words)
df=pd.read_csv('/content/drive/article.csv')
df['vector1']=df['headline'].apply(lambda x: text_to_vector(x))
df['vector2']=df['snippet'].apply(lambda x: text_to_vector(x))
df['simscore']=df.apply(lambda x: get_cosine(x['vector1'],x['vector2']),axis=1)
text_1 = """ """
text_2 = """ """
text_3 = """ """
documents = [text_1, text_2, text_3]
import pandas as pd
# Create the Document Term Matrix
count_vectorizer = CountVectorizer(stop_words='english')
count_vectorizer = CountVectorizer()
sparse_matrix = count_vectorizer.fit_transform(documents)
# OPTIONAL: Convert Sparse Matrix to Pandas Dataframe if you want to see the word frequencies.
doc_term_matrix = sparse_matrix.todense()
df = pd.DataFrame(doc_term_matrix,
columns=count_vectorizer.get_feature_names(),
index=['text_1', 'text_2', 'text_3'])
df
from sklearn.metrics.pairwise import cosine_similarity
print(cosine_similarity(df, df))
import math
import numpy as np
std_embeddings_index = {}
with open('path/to/numberbatch-en-17.06.txt') as f:
for line in f:
values = line.split(' ')
word = values[0]
embedding = np.asarray(values[1:], dtype='float32')
std_embeddings_index[word] = embedding
def cosineValue(v1,v2):
"compute cosine similarity of v1 to v2: (v1 dot v2)/{||v1||*||v2||)"
sumxx, sumxy, sumyy = 0, 0, 0
for i in range(len(v1)):
x = v1[i]; y = v2[i]
sumxx += x*x
sumyy += y*y
sumxy += x*y
return sumxy/math.sqrt(sumxx*sumyy)
def get_sentence_vector(sentence, std_embeddings_index = std_embeddings_index ):
sent_vector = 0
for word in sentence.lower().split():
if word not in std_embeddings_index :
word_vector = np.array(np.random.uniform(-1.0, 1.0, 300))
std_embeddings_index[word] = word_vector
else:
word_vector = std_embeddings_index[word]
sent_vector = sent_vector + word_vector
return sent_vector
def cosine_sim(sent1, sent2):
return cosineValue(get_sentence_vector(sent1), get_sentence_vector(sent2))
s1 = "This is a foo bar sentence ."
s2 = "This sentence is similar to a foo bar sentence ."
s3 = "What is this string ? Totally not related to the other two lines ."
print cosine_sim(s1, s2) # Should give high cosine similarity
print cosine_sim(s1, s3) # Shouldn't give high cosine similarity value
print cosine_sim(s2, s3) # Shouldn't give high cosine similarity value
0.9851735249068168
0.6570885718962608
0.6589335425458225
感谢@vpekar的实现，它帮了我很大的忙。但我发现在计算余弦相似度时缺少了tf-idf权重。
cos(q, d) = sim(q, d) = (q · d)/(|q||d|) = (sum(qi, di)/(sqrt(sum(qi2)))*(sqrt(sum(vi2))) where i = 1 to v)
请随意查看我的代码here。但首先您需要下载Anaconda软件包。它将自动在Windows中设置您的Python路径。将此Python解释器添加到Eclipse中。