Marker PDF 转 Markdown¶

pip install marker-pdf

# 或带 GPU 支持
pip install "marker-pdf[gpu]"

# 首次运行会自动下载模型
# 模型包括：
# - 版面检测模型
# - OCR 模型（Surya）
# - 表格识别模型
# - 公式识别模型

# 单个文件转换
marker_single input.pdf output_dir

# 指定输出格式
marker_single input.pdf output_dir --output_format markdown

# 批量转换目录下所有 PDF
marker output_dir --input_dir ./pdfs

# 带参数
marker_single paper.pdf ./output \
  --languages "Chinese,English" \
  --max_pages 50

from marker.converters.pdf import PdfConverter
from marker.models import create_model_dict

# 创建模型
models = create_model_dict()

# 创建转换器
converter = PdfConverter(artifact_dict=models)

# 转换
rendered = converter("paper.pdf")

# 获取 Markdown 文本
markdown_text = rendered.markdown

# 获取元数据
metadata = rendered.metadata
print(f"标题: {metadata.get('title', 'Unknown')}")
print(f"页数: {metadata.get('pages', 0)}")

# 保存
with open("output.md", "w", encoding="utf-8") as f:
    f.write(markdown_text)

# 保存图片
for name, image in rendered.images.items():
    image.save(f"images/{name}")

PDF 原文:
┌─────────────────────────────┐
│ Chapter 3: Data Structures  │ ← 标题
│                             │
│ Python provides several     │ ← 正文
│ built-in data structures:   │
│                             │
│ • Lists                     │ ← 列表
│ • Dictionaries              │
│ • Sets                      │
│                             │
│ | Name | Type | Mutable |   │ ← 表格
│ |------|------|---------|   │
│ | list | seq  | Yes     |   │
└─────────────────────────────┘

Marker 输出:
# Chapter 3: Data Structures

Python provides several built-in data structures:

- Lists
- Dictionaries
- Sets

| Name | Type | Mutable |
|------|------|---------|
| list | seq  | Yes     |

from marker.converters.pdf import PdfConverter
from marker.models import create_model_dict
from pathlib import Path
import json

models = create_model_dict()
converter = PdfConverter(artifact_dict=models)

input_dir = Path("./pdfs")
output_dir = Path("./markdown")
output_dir.mkdir(exist_ok=True)

results = []
for pdf_path in input_dir.glob("*.pdf"):
    print(f"Converting: {pdf_path.name}")
    try:
        rendered = converter(str(pdf_path))

        # 保存 Markdown
        md_path = output_dir / f"{pdf_path.stem}.md"
        md_path.write_text(rendered.markdown, encoding="utf-8")

        # 保存图片
        img_dir = output_dir / f"{pdf_path.stem}_images"
        if rendered.images:
            img_dir.mkdir(exist_ok=True)
            for name, img in rendered.images.items():
                img.save(img_dir / name)

        results.append({
            "file": pdf_path.name,
            "status": "success",
            "pages": rendered.metadata.get("pages", 0)
        })
    except Exception as e:
        results.append({"file": pdf_path.name, "status": "error", "error": str(e)})

# 保存转换报告
with open(output_dir / "report.json", "w") as f:
    json.dump(results, f, ensure_ascii=False, indent=2)

converter = PdfConverter(
    artifact_dict=models,
    config={
        "languages": ["Chinese", "English"],   # 文档语言
        "force_ocr": False,                     # 强制 OCR（扫描件时启用）
        "paginate_output": True,                # 添加页码分隔
        "extract_images": True,                 # 提取图片
        "max_pages": None,                      # 最大页数限制
    }
)

# 学术论文通常包含：双栏排版、公式、引用、图表

rendered = converter("arxiv_paper.pdf")

# 公式会被转为 LaTeX
# $E = mc^2$
# $$\int_{0}^{\infty} e^{-x^2} dx = \frac{\sqrt{\pi}}{2}$$

# 引用保持格式
# [1] Author et al., Title, Journal, 2024.

from marker.converters.pdf import PdfConverter
from marker.models import create_model_dict
from langchain.text_splitter import RecursiveCharacterTextSplitter

# 转换 PDF
models = create_model_dict()
converter = PdfConverter(artifact_dict=models)
rendered = converter("knowledge_base.pdf")

# 切片
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n## ", "\n### ", "\n\n", "\n", " "]
)
chunks = splitter.split_text(rendered.markdown)

# 存入向量库
from chromadb import Client
client = Client()
collection = client.create_collection("knowledge")
collection.add(
    documents=chunks,
    ids=[f"chunk_{i}" for i in range(len(chunks))]
)

import re

def clean_markdown(text: str) -> str:
    # 修复多余换行
    text = re.sub(r'\n{3,}', '\n\n', text)

    # 修复断行（PDF 中常见的行内断行）
    text = re.sub(r'(?<=[a-z,])\n(?=[a-z])', ' ', text)

    # 清理页眉页脚残留
    text = re.sub(r'^\d+\s*$', '', text, flags=re.MULTILINE)

    # 规范化标题
    text = re.sub(r'^(#+)\s*(\d+\.?\s*)', r'\1 ', text, flags=re.MULTILINE)

    return text.strip()