474_文本相似度计算¶

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer
from sentence_transformers import SentenceTransformer, util

# ---- 1. TF-IDF 余弦相似度 ----
texts = [
    "深度学习在图像识别中表现优异",
    "卷积神经网络用于图像分类",
    "今天天气很好适合出门",
]

tfidf = TfidfVectorizer()
tfidf_matrix = tfidf.fit_transform(texts)  # 稀疏矩阵 (3, vocab_size)

# 计算第1条和其他文本的余弦相似度
sim_matrix = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix)
print("TF-IDF相似度:", sim_matrix[0])  # [1.0, 0.4x, 0.0x]

# ---- 2. Sentence-BERT 语义相似度 ----
model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
embeddings = model.encode(texts, convert_to_tensor=True)

# util.cos_sim计算余弦相似度矩阵
cosine_scores = util.cos_sim(embeddings, embeddings)
print("\nSentence-BERT相似度矩阵:")
for i in range(len(texts)):
    for j in range(len(texts)):
        print(f"  ({i},{j}): {cosine_scores[i][j]:.4f}")

# ---- 3. 语义搜索（最近邻检索）----
query = "神经网络图像处理"
query_emb = model.encode(query, convert_to_tensor=True)
corpus_emb = model.encode(texts, convert_to_tensor=True)

# 查找最相似的文档
hits = util.semantic_search(query_emb, corpus_emb, top_k=2)
for hit in hits[0]:
    idx = hit['corpus_id']
    score = hit['score']
    print(f"相似度 {score:.4f}: {texts[idx]}")

# ---- 4. 编辑距离（字符级相似）----
def levenshtein_distance(s1, s2):
    """计算两字符串的编辑距离"""
    m, n = len(s1), len(s2)
    dp = [[0] * (n + 1) for _ in range(m + 1)]
    for i in range(m + 1):
        dp[i][0] = i   # 删除操作
    for j in range(n + 1):
        dp[0][j] = j   # 插入操作
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            if s1[i-1] == s2[j-1]:
                dp[i][j] = dp[i-1][j-1]  # 字符相同，无需操作
            else:
                dp[i][j] = 1 + min(
                    dp[i-1][j],    # 删除
                    dp[i][j-1],    # 插入
                    dp[i-1][j-1]   # 替换
                )
    return dp[m][n]

print(levenshtein_distance("深度学习", "深度学院"))  # 1（学习→学院）

# ---- 5. BM25 关键词相似度（信息检索经典）----
# pip install rank-bm25
from rank_bm25 import BM25Okapi

# BM25不需要语义，但考虑词频饱和和文档长度归一化
corpus = [text.split() for text in texts]  # 简单分词（实际用jieba）
bm25 = BM25Okapi(corpus)
query_tokens = "深度学习 图像".split()
scores = bm25.get_scores(query_tokens)  # 每个文档的BM25分数
print("BM25分数:", scores)

# ---- 6. Jaccard相似度 ----
def jaccard_sim(s1, s2):
    """基于字符集合的Jaccard相似度"""
    set1, set2 = set(s1), set(s2)
    return len(set1 & set2) / len(set1 | set2)

print(jaccard_sim("深度学习", "深度神经"))  # 0.5