Jina Reader API 内容提取¶

# Jina Reader 最简单的用法：在 URL 前加前缀
# 不需要安装任何东西，不需要 API Key

# 读取模式：r.jina.ai/
curl https://r.jina.ai/https://en.wikipedia.org/wiki/Metagenomics
# 自动返回 Markdown 格式的页面主要内容

# 搜索模式：s.jina.ai/
curl "https://s.jina.ai/?q=宏基因组分析流程"
# 搜索并返回前 5 条结果的完整内容

# 事实核查模式：g.jina.ai/
curl "https://g.jina.ai/?statement=人类肠道中有超过1000种细菌"
# 核查这个说法是否正确

import requests  # HTTP 请求库

# 最简单的调用方式：直接请求 URL
response = requests.get(
    "https://r.jina.ai/https://docs.python.org/3/tutorial/"  # 前缀 + 目标 URL
)

print(response.text)  # 输出干净的 Markdown 内容

import requests

# 注册 Jina AI 获取 API Key（免费）
# https://jina.ai

headers = {
    "Authorization": "Bearer jina_xxxxxxxxxxxx",  # 你的 API Key
    "Accept": "application/json"                   # 返回 JSON 格式
}

response = requests.get(
    "https://r.jina.ai/https://example.com",
    headers=headers
)

data = response.json()          # 解析 JSON
print(data["data"]["content"])  # 提取内容
print(data["data"]["title"])    # 提取标题

// 在 Claude Desktop 或 Cursor 中配置 Jina MCP Server
{
  "mcpServers": {
    "jina": {
      "url": "https://mcp.jina.ai/sse",
      "headers": {
        "Authorization": "Bearer jina_xxxxxxxxxxxx"
      }
    }
  }
}
// 配置后，AI 助手可以直接使用 read_url、search_web 等工具

import requests

# 基本读取
url = "https://r.jina.ai/https://www.nature.com/articles/s41586-024-00001-1"
response = requests.get(url)
print(response.text)  # 论文内容变成 Markdown

# 使用 CSS 选择器提取特定内容
headers = {
    "X-Target-Selector": "article .c-article-body",  # 只提取文章正文
    "X-Remove-Selector": "nav, footer, .sidebar"      # 移除导航栏和侧边栏
}
response = requests.get(url, headers=headers)

# 等待动态内容加载
headers = {
    "X-Wait-For-Selector": ".dynamic-content"  # 等待这个元素出现再提取
}
response = requests.get(url, headers=headers)

import requests

# 搜索模式：返回搜索结果及其完整内容
response = requests.get(
    "https://s.jina.ai/",
    params={"q": "metagenome assembled genomes MAGs"},  # 搜索关键词
    headers={"Accept": "application/json"}               # JSON 格式
)

data = response.json()
for result in data["data"]:
    print(f"标题: {result['title']}")          # 搜索结果标题
    print(f"URL: {result['url']}")              # 来源 URL
    print(f"内容预览: {result['content'][:200]}")  # 内容前 200 字
    print("---")

import requests

# 事实核查模式：验证一个说法是否正确
response = requests.get(
    "https://g.jina.ai/",
    params={
        "statement": "Shannon多样性指数越高，微生物群落越多样"  # 要核查的说法
    },
    headers={"Accept": "application/json"}
)

data = response.json()
print(f"结论: {data['data']['factuality']}")     # 正确/错误/部分正确
print(f"解释: {data['data']['explanation']}")     # 详细解释
print(f"来源: {data['data']['references']}")      # 引用来源

import requests

# Jina Reader 可以直接读取在线 PDF
pdf_url = "https://r.jina.ai/https://arxiv.org/pdf/2401.12345.pdf"
response = requests.get(pdf_url)
print(response.text)  # PDF 内容转成 Markdown

# 上传本地 PDF
with open("paper.pdf", "rb") as f:
    response = requests.post(
        "https://r.jina.ai/",
        files={"file": f},                     # 上传文件
        headers={"Accept": "text/markdown"}    # Markdown 格式
    )
print(response.text)

import requests
from concurrent.futures import ThreadPoolExecutor  # 线程池

def read_url(url):
    """读取单个 URL 的内容"""
    response = requests.get(f"https://r.jina.ai/{url}")
    return {"url": url, "content": response.text}

# 要处理的 URL 列表
urls = [
    "https://docs.python.org/3/tutorial/",
    "https://bioconductor.org/packages/release/bioc/",
    "https://snakemake.readthedocs.io/en/stable/",
]

# 并发请求（3 个线程同时处理）
with ThreadPoolExecutor(max_workers=3) as executor:
    results = list(executor.map(read_url, urls))

for r in results:
    print(f"URL: {r['url']}")
    print(f"内容长度: {len(r['content'])} 字符")
    print("---")

import requests

headers = {
    "Authorization": "Bearer jina_xxx",
    "X-Set-Cookie": "lang=zh-CN",         # 设置 Cookie
    "X-With-Shadow-Dom": "true",           # 提取 Shadow DOM
    "X-With-Iframe": "true",               # 提取 iframe 内容
    "X-Run-Js": "document.querySelector('.show-more').click()"  # 执行 JS
}

response = requests.get(
    "https://r.jina.ai/https://example.com/dynamic-page",
    headers=headers
)

import requests
import openai

def rag_search_and_answer(question):
    """基于 Jina Reader 的简单 RAG 管道"""

    # 第一步：用 Jina Search 搜索相关内容
    search_response = requests.get(
        "https://s.jina.ai/",
        params={"q": question},
        headers={"Accept": "application/json"}
    )
    search_data = search_response.json()

    # 第二步：拼接上下文
    context = ""
    for item in search_data["data"][:3]:  # 取前 3 条结果
        context += f"来源: {item['url']}\n{item['content'][:1000]}\n\n"

    # 第三步：用 LLM 生成回答
    client = openai.Client(base_url="http://localhost:11434/v1", api_key="ollama")
    response = client.chat.completions.create(
        model="llama3.1:8b",
        messages=[
            {"role": "system", "content": f"根据以下搜索结果回答问题。\n\n{context}"},
            {"role": "user", "content": question}
        ]
    )
    return response.choices[0].message.content

# 使用
answer = rag_search_and_answer("什么是宏基因组 binning？")
print(answer)

# === Jina Reader 速查 ===

# 读取网页（curl）
curl https://r.jina.ai/https://example.com

# 搜索（curl）
curl "https://s.jina.ai/?q=关键词"

# 事实核查（curl）
curl "https://g.jina.ai/?statement=要核查的说法"

# 读取 PDF
curl https://r.jina.ai/https://example.com/file.pdf

# 带 API Key
curl -H "Authorization: Bearer jina_xxx" https://r.jina.ai/https://example.com

# JSON 格式输出
curl -H "Accept: application/json" https://r.jina.ai/https://example.com

# CSS 选择器提取
curl -H "X-Target-Selector: article" https://r.jina.ai/https://example.com

# 移除元素
curl -H "X-Remove-Selector: nav,footer" https://r.jina.ai/https://example.com

# Python 一行代码
import requests; print(requests.get("https://r.jina.ai/https://example.com").text)