469_文本摘要生成¶

# 使用 HuggingFace Transformers 中的 BART 做生成式摘要

from transformers import BartTokenizer, BartForConditionalGeneration

# ---- 1. 加载 BART 摘要模型 ----
model_name = 'facebook/bart-large-cnn'  # 在CNN/DailyMail上微调的摘要模型
tokenizer = BartTokenizer.from_pretrained(model_name)
model = BartForConditionalGeneration.from_pretrained(model_name)

# ---- 2. 准备输入文本 ----
article = """
The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building,
and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side.
During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest
man-made structure in the world.
"""

# ---- 3. 分词编码 ----
inputs = tokenizer(
    article,
    max_length=1024,       # BART最大输入长度
    truncation=True,
    return_tensors='pt'
)

# ---- 4. 生成摘要（beam search解码）----
summary_ids = model.generate(
    inputs['input_ids'],
    num_beams=4,           # beam宽度，越大结果越好但越慢
    max_length=150,        # 摘要最大长度
    min_length=30,         # 摘要最小长度
    length_penalty=2.0,    # >1倾向生成更长摘要
    early_stopping=True    # 找到num_beams个完整句子后停止
)

# ---- 5. 解码输出 ----
summary = tokenizer.decode(
    summary_ids[0],
    skip_special_tokens=True  # 去掉<s></s>等特殊token
)
print("摘要:", summary)

# ---- 6. TextRank 抽取式摘要（无监督基线）----
# pip install sumy
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.text_rank import TextRankSummarizer

parser = PlaintextParser.from_string(article, Tokenizer("english"))
summarizer = TextRankSummarizer()
# 抽取2个最重要的句子
sentences = summarizer(parser.document, 2)
for s in sentences:
    print(str(s))

# ---- 7. ROUGE 评估 ----
# pip install rouge-score
from rouge_score import rouge_scorer
scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
reference = "The Eiffel Tower is 324 metres tall and the tallest structure in Paris."
scores = scorer.score(reference, summary)
print(f"ROUGE-1 F1: {scores['rouge1'].fmeasure:.4f}")
print(f"ROUGE-L F1: {scores['rougeL'].fmeasure:.4f}")