Modin 分布式 Pandas¶

# 使用 Ray 后端（默认推荐）
pip install "modin[ray]"

# 使用 Dask 后端
pip install "modin[dask]"

# 只需改一行！
# import pandas as pd
import modin.pandas as pd

# 之后所有代码完全一样
df = pd.read_csv("large_file.csv")
result = df.groupby("category").agg({"price": "mean", "quantity": "sum"})
filtered = df[df["score"] > 80]
merged = pd.merge(df1, df2, on="id")

# 自动使用所有 CPU 核心！

import time

# Pandas
import pandas as pd_pandas
start = time.time()
df = pd_pandas.read_csv("large.csv")
result = df.groupby("col").mean()
print(f"Pandas: {time.time()-start:.2f}s")

# Modin
import modin.pandas as pd_modin
start = time.time()
df = pd_modin.read_csv("large.csv")
result = df.groupby("col").mean()
print(f"Modin: {time.time()-start:.2f}s")
# 通常快 2-10 倍（取决于核心数和操作类型）

import os
os.environ["MODIN_BACKEND"] = "ray"   # 或 "dask"

import modin.pandas as pd

import ray
ray.init(num_cpus=8, memory=8*1024*1024*1024)  # 8 核 8GB

import modin.pandas as pd
df = pd.read_csv("data.csv")

# 某些操作 Modin 不支持时会自动回退到 Pandas
# 可以手动转换
pandas_df = df._to_pandas()  # Modin → Pandas
modin_df = pd.DataFrame(pandas_df)  # Pandas → Modin

import modin.pandas as pd

# 读取超大 CSV
df = pd.read_csv("50gb_file.csv")  # 自动分块并行读取

# 读取多个 Parquet 文件
df = pd.read_parquet("data/")  # 读取目录下所有 parquet

# 分块输出
df.to_parquet("output/", partition_cols=["year", "month"])