Pandas 2 新特性¶

pip install "pandas>=2.0" pyarrow

import pandas as pd

# 使用 Arrow 后端读取数据
df = pd.read_csv("data.csv", dtype_backend="pyarrow")
df = pd.read_parquet("data.parquet", dtype_backend="pyarrow")

# 查看数据类型
print(df.dtypes)
# name     string[pyarrow]
# age      int64[pyarrow]
# score    double[pyarrow]

# 全局设置 Arrow 后端
pd.set_option("mode.dtype_backend", "pyarrow")

# 内存对比
df_numpy = pd.read_csv("large.csv")
df_arrow = pd.read_csv("large.csv", dtype_backend="pyarrow")
print(f"NumPy: {df_numpy.memory_usage(deep=True).sum() / 1e6:.1f} MB")
print(f"Arrow: {df_arrow.memory_usage(deep=True).sum() / 1e6:.1f} MB")

# 旧版：整数列有 NaN 会变成 float64
# Pandas 2：支持 Nullable Integer
s = pd.array([1, 2, None, 4], dtype="Int64")  # 注意大写 I
print(s)  # [1, 2, <NA>, 4]
print(s.dtype)  # Int64

# Arrow 版本
s = pd.array([1, 2, None, 4], dtype="int64[pyarrow]")

# 启用 CoW（Pandas 3.0 将默认启用）
pd.set_option("mode.copy_on_write", True)

df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
df2 = df[["a"]]   # 不再是视图，修改 df2 不会影响 df
df2.iloc[0] = 100
print(df)  # 原始数据不变

import pandas as pd
import time

# 字符串操作性能对比
n = 1_000_000
df_old = pd.DataFrame({"text": ["hello world"] * n})
df_arrow = pd.DataFrame({"text": pd.array(["hello world"] * n, dtype="string[pyarrow]")})

start = time.time()
df_old["text"].str.upper()
print(f"Object dtype: {time.time()-start:.3f}s")

start = time.time()
df_arrow["text"].str.upper()
print(f"Arrow dtype: {time.time()-start:.3f}s")
# Arrow 通常快 3-10 倍

# named aggregation（命名聚合）
result = df.groupby("category").agg(
    avg_price=("price", "mean"),
    total_qty=("quantity", "sum"),
    unique_products=("product", "nunique"),
)

# 读取特定列（列式存储优势）
df = pd.read_parquet("data.parquet", columns=["name", "age"])

# 过滤读取
df = pd.read_parquet("data.parquet", 
    filters=[("age", ">", 25), ("city", "==", "Beijing")])

# 写入压缩
df.to_parquet("output.parquet", compression="zstd", index=False)

# 自动选择最佳类型
df = df.convert_dtypes()

# 降低内存
df = df.astype({
    "id": "int32[pyarrow]",
    "name": "string[pyarrow]",
    "score": "float32[pyarrow]",
    "is_active": "bool[pyarrow]",
})