477_多语言NLP处理¶

# ---- 1. XLM-R 多语言文本分类（零样本迁移）----
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification
import torch

# 用英文数据微调，然后在其他语言上测试
tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')
model = XLMRobertaForSequenceClassification.from_pretrained(
    'xlm-roberta-base', num_labels=2
)

# 同一个模型处理不同语言（无需任何语言标识符）
multilingual_texts = [
    "This movie is excellent!",      # 英文
    "Ce film est excellent!",         # 法文
    "这部电影太好看了！",               # 中文
    "Dieser Film ist ausgezeichnet!", # 德文
]

for text in multilingual_texts:
    enc = tokenizer(text, return_tensors='pt',
                    max_length=128, truncation=True, padding='max_length')
    with torch.no_grad():
        logits = model(**enc).logits
        pred = logits.argmax(dim=-1).item()
    print(f"[{'正面' if pred==1 else '负面'}] {text}")

# ---- 2. LaBSE 跨语言句子向量（语义对齐）----
# LaBSE专门训练使不同语言的同义句靠近
from sentence_transformers import SentenceTransformer, util

labse_model = SentenceTransformer('sentence-transformers/LaBSE')

# 不同语言的相同含义句子
en_sentences = ["The weather is nice today", "I love machine learning"]
zh_sentences = ["今天天气很好", "我喜欢机器学习"]
fr_sentences = ["Le temps est beau aujourd'hui", "J'aime l'apprentissage automatique"]

en_emb = labse_model.encode(en_sentences)
zh_emb = labse_model.encode(zh_sentences)
fr_emb = labse_model.encode(fr_sentences)

# 计算英文-中文跨语言相似度
cos_sim = util.cos_sim(en_emb, zh_emb)
print("\n英文-中文跨语言相似度:")
for i, (en, zh) in enumerate(zip(en_sentences, zh_sentences)):
    print(f"  EN: {en}")
    print(f"  ZH: {zh}")
    print(f"  相似度: {cos_sim[i][i]:.4f}")  # 对角线是配对的相似度

# ---- 3. 多语言NER（XLM-R + CRF）----
from transformers import pipeline

# 使用多语言NER模型
ner_pipeline = pipeline(
    "ner",
    model="xlm-roberta-large-finetuned-conll03-english",
    aggregation_strategy="simple"  # 合并相同实体的subtoken
)

# 同一模型可处理多语言（零样本迁移能力）
texts = [
    "Apple Inc. was founded by Steve Jobs in California.",
    "阿里巴巴由马云创立于杭州。",
]
for text in texts:
    entities = ner_pipeline(text)
    print(f"\n文本: {text}")
    for ent in entities:
        print(f"  {ent['word']} → {ent['entity_group']} ({ent['score']:.3f})")

# ---- 4. 多语言机器翻译（NLLB-200）----
from transformers import pipeline

# NLLB-200：No Language Left Behind，支持200+语言对直接互译
nllb_translator = pipeline(
    "translation",
    model="facebook/nllb-200-distilled-600M",
    src_lang="zho_Hans",  # 简体中文的语言代码
    tgt_lang="fra_Latn",  # 法语
    max_length=200
)

result = nllb_translator("人工智能正在改变世界")
print(f"中→法: {result[0]['translation_text']}")

# ---- 5. 语言检测 ----
# pip install langdetect
from langdetect import detect, detect_langs

texts = ["Hello world", "你好世界", "Bonjour le monde", "こんにちは"]
for t in texts:
    lang = detect(t)              # 返回语言代码
    probs = detect_langs(t)       # 返回带概率的语言列表
    print(f"'{t}' → {lang} (置信度: {probs[0].prob:.3f})")