Datalab 开源文档解析基准 OmniParseBench：2.9 万页 1.6 万项测试

在本教程中，我们实现 OmniParseBench，这是 Datalab 面向文档解析器的开放基准测试，它用 16,288 个小型通过/失败测试对解析器的输出进行评分，覆盖 2,937 页文档，从扫描表单和手写内容到 arXiv 数学公式和密集表格。我们没有直接信任排行榜，而是根据 Datalab 为 27 个解析器发布的判定结果重建它，在此之前先给评分器输入一个手工制作的页面，看看哪些错误会翻转判定。随后我们测量排名的确定性，按文档类型切分，测试卡片中列为协助构建测试的两个模型在模型修正过的测试上是否表现不同，并自己在 50 页样本上运行 Tesseract 和 Indra OCR。在此过程中，我们发现排名前两位的解析器在统计上并列，领先者在大多数切分上都会变化，而且一个发布数字与已发布文件不符。

复制代码已复制使用其他浏览器

import os

import re

import sys

import json

import time

import html

import shutil

import platform

import traceback

import subprocess

import collections

import urllib.request

from pathlib import Path

RESULTS = {} # step -> one-line result, for the summary table

CLAIMS = [] # (step, claim, held)

CURRENT = ["1"]

def banner(title):

print("\n" + "=" * 78)

print(title)

print("=" * 78)

def section(name):

def wrap(fn):

def run(*a, **kw):

CURRENT[0] = name.split(".")[0]

banner(name)

t0 = time.time()

try:

out = fn(*a, **kw)

RESULTS[name] = out if isinstance(out, str) else "ok"

return out

except Exception as e:

RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"

print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")

traceback.print_exc(limit=3)

return None

finally:

print(f"\n[{time.time() - t0:.1f} s]")

return run

return wrap

def claim(text, held):

CLAIMS.append((CURRENT[0], text, bool(held)))

return held

banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")

SYSTEM = platform.system()

OPB_VERSION = "1.0.0rc1" # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs

REPO = "datalab-to/omni_parse_bench"

REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a" # the dataset with the published verdicts of 27 parsers

INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",

"pypdfium2==5.14.0", "huggingface_hub==2.2.0"] # 该包自带 Indra 服务器镜像的固定版本

WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()

WORK.mkdir(parents=True, exist_ok=True)

subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)

if SYSTEM == "Linux" and shutil.which("tesseract") is None:

scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "

"hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()

subprocess.run(["apt-get", "-qq", "update"], check=True)

subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]

+ [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)

import numpy as np

import pandas as pd

import importlib.metadata as md

from huggingface_hub import snapshot_download

from omni_parse_bench import benchmark, data, harness, markdown, vocab

from omni_parse_bench.scorers import unit_tests

print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, "

f"{os.cpu_count()} CPU cores")

if SYSTEM == "Linux":

print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])

tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True)

print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0])

TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0])

print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})

print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}")

RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = (

f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")

我们安装固定为 1.0.0rc1 的 omni-parse-bench，因为直接 pip install 仍会解析到 0.2.0，那是一个 API 不同的旧版本。benchmark extra 会带来解析器适配器、Hugging Face 客户端以及用于 manifest 的 pyarrow，而在 Colab 上我们通过 apt 安装 Tesseract 及其脚本模型。我们还将数据集固定到某一个 commit，因为随着解析器的加入，仓库会发生变化。打印 families 可以展示该基准所依赖的结构：present、order 和 repeat 测试构成 text family；table_cell 构成 tables family；layout_kind 构成 layout family。

Copy CodeCopiedUse a different Browser

@section("2. 评分器如何判定：一个手工构建的页面、五种测试类型、一次只改一处")

def scorer_anatomy():

T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored",

"tags": tags, "args": args, "settings": st or {}}

exact = lambda s: {"text": s, "max_diffs": 0}

gold = [

T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}),

T("total", "present", "html", ["eng"], {"content": exact("Total due")}),

T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}),

T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}),

T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}),

T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}),

T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."),

"after": exact("Conclusion: the plan holds.")}),

T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}),

T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}),

T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}),

T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"),

"left": exact("Widget")}),

T("block_title", "layout_kind", "blocks", ["heading"],

{"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}),

T("block_total", "layout_kind", "blocks", ["eng"],

{"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}),

]

gold_file = WORK / "fixture" / "gold" / "demo-p1.json" # 金标准文件以其页面命名

gold_file.parent.mkdir(parents=True, exist_ok=True)

gold_file.write_text(json.dumps({"tests": gold}, indent=1))

page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf",

gt_path=gold_file, outputs=("blocks", "html"))

tests = unit_tests.gold(page)

table_a = ("<table><tr><th></th><th>2023</th><th>2024</th></tr><tr><th>Income</th><td>1,000</td><td>1,240</td></tr>"

"<tr><th>Costs</th><td>600</td><td>700</td></tr></table>")

table_b = "<table><tr><th>Product</th><th>Price</th></tr><tr><td>Widget</td><td>9.50</td></tr></table>"

md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |"

parts = {"title": "<h1>Quarterly Report</h1>", "total": "<p>Total due: 1,240.00 USD.</p>",

"growth": "<p>Growth was -4.8% year on year.</p>", "ratio": r"<p>Ratio: 1+ holds.</p>",

"half": r"<p>Half: 12x in the model.</p>",

"intro": "<p>Introduction: this report covers the second quarter.</p>",

"revenue": "<p>Revenue is shown in thousands. Revenue grew in the second quarter.</p>",

"table_a": table_a, "conclusion": "<p>Conclusion: the plan holds.</p>", "table_b": table_b}

blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly Report"},

{"label": "text", "bbox": [0.10, 0.12, 0.90, 0.16], "text": "Total due: 1,240.00 USD."},

{"label": "table", "bbox": [0.10, 0.40, 0.90, 0.60], "text": "Income 1,000 1,240 Costs 600 700"}]

def pred(change=None, order=None, new_blocks=None):

p = dict(parts, **(change or {}))

return {"html": "\n".join(p[k] for k in (order or list(parts))), "blocks": blocks if new_blocks is None else new_blocks}

base = {v.id: v.result for v in unit_tests.score(pred(), tests)}

print(f"base prediction: {sum(r == 'pass' for r in base.values())} of {len(tests)} tests pass\n")

swapped = ["title", "total", "growth", "ratio", "half", "conclusion", "intro", "revenue", "table_a", "table_b"]

variants = [

("flip"，一个拼写错误：'Total dve'，pred({"total": "<p>Total dve: 1,240.00 USD.</p>"})，())，

("flip"，负号丢失：4.8%，pred({"growth": "<p>Growth was 4.8% year on year.</p>"})，())，

("flip"，另一个符号：alpha_2，pred({"ratio": r"<p>Ratio: 2+ holds.</p>"})，())，

("flip"，结论出现在引言之前，pred(order=swapped))，

("flip"，第二个 'Revenue' 写成了 'Revenues'，

pred({"revenue": "<p>Revenue is shown in thousands. Revenues grew in the second quarter.</p>"})，())，

("flip"，2023 和 2024 的表头互换，pred({"table_a": table_a.replace("<th>2023</th><th>2024</th>",

"<th>2024</th><th>2023</th>")})，())，

("flip"，行标签 'Income' 写成了 'Margin'，pred({"table_a": table_a.replace("<th>Income</th>", "<th>Margin</th>")})，())，

("flip"，html 中残留了一个 markdown 表格，pred({"table_b": md_table_b})，())，

("flip"，标题的块被标记为 text，pred(new_blocks=[dict(blocks[0], label="text")] + blocks[1:])，())，

("control"，r"\dfrac12\rightarrow for \frac{1}{2}\to"，pred({"half": r"<p>Half: $\dfrac12\rightarrow x$ in the model.</p>"})，())，

("control"，'TOTAL DUE' 和 '1240.00'，pred({"total": "<p>TOTAL DUE: 1240.00 USD.</p>"})，())，

("control"，通过 markdown.to_html 处理的 markdown 表格，pred({"table_b": markdown.to_html(md_table_b)})，())，

("unsupported"，没有 blocks，声明为 unsupported，{"html": pred()["html"]}，("blocks",))，

]

as_expected = 0

for kind, name, p, unsupported in variants:

changed = [v for v in unit_tests.score(p, tests, unsupported) if v.result != base[v.id]]

if kind == "flip":

ok = len(changed) == 1 and changed[0].result == "fail"

elif kind == "control":

ok = not changed

else:

ok = bool(changed) and all(v.result == "unsupported" for v in changed)

as_expected += ok

shown = "; ".join(f"{v.id} -> {v.result}" + (f" ({v.why})" if v.result == "fail" else "") for v in changed) or "no verdict changes"

print(f"{kind:11s} {name:42s} {shown}")

s = unit_tests.summarize(unit_tests.lines(unit_tests.score(variants[0][2], tests), tests, page))

print(f"\nsummary of the typo variant: headline {s['headline']:.4f} = mean of "

+ ", ".join(f"{k} {v['score']:.3f} ({v['tests']} tests)" for k, v in s["families"].items()))

claim("每个单点改动变体都恰好让一个判定翻转为失败，且没有任何对照组发生翻转", as_expected == len(variants))

return f"{as_expected} of {len(variants)} hand-built variants behave as expected (9 flips, 3 controls, 1 unsupported)"

scorer_anatomy()

在信任一个分数之前，我们想先弄清楚一个判定意味着什么，因此我们手工编写了一个金标准文件，在一个虚构页面上放置十三项测试，并用 unit_tests.score（基准测试本身所使用的函数）对预测结果进行评分。一个基础预测通过了所有测试，而每个变体只改动一处。一个字母的拼写错误、丢失的负号、方程中不同的符号、互换的标题以及一个标注错误的块，各自都恰好让一项测试翻转为失败，并附上原因。对照组仍然通过：大写字母、被去掉的千位分隔符，以及用 dfrac 代替 frac，因为文本会被折叠，而方程是在 KaTeX 渲染之后进行比较的。

复制代码已复制使用其他浏览器

@section("3. 数据集：16,288 项测试，它们检查什么，以及每个答案是如何确立的")

def dataset_step():

global DATA, PAGES, TESTS, CARD_TEXT

snapshot_download(REPO, repo_type="dataset", revision=REV, local_dir=str(WORK / "dataset"),

allow_patterns=["manifest.parquet", "gold/*", "results.parquet", "README.md"])

DATA, PAGES = data.read_manifest(str(WORK / "dataset"))

rows = []

for p in PAGES:

for t in unit_tests.gold(p):

rows.append(dict(test_id=t.id, page_id=p.page_id, suite=p.suite, document=p.document, test_type=t.test_type,

derivation=t.derivation, tags=list(t.tags), pdf=p.doc_path.suffix == ".pdf"))

TESTS = pd.DataFrame(rows)

tags = collections.Counter(g for ts in TESTS.tags for g in ts)

langs = {g for g in tags if re.fullmatch(r"[a-z]{3}", g)}

scripts = {g for g in tags if re.fullmatch(r"[A-Z][a-z]{3}", g)}

n_pdf = sum(p.doc_path.suffix == ".pdf" for p in PAGES)

print(f"{len(TESTS):,} tests on {TESTS.page_id.nunique():,} pages from {TESTS.document.nunique():,} documents in "

f"{TESTS.suite.nunique()} suites; {n_pdf:,} PDF pages and {len(PAGES) - n_pdf:,} images")

print(f"tags: {len(langs)} language codes ({len(langs - {'und', 'zxx'})} without und and zxx), {len(scripts)} scripts")

counts = lambda col: {k: int(v) for k, v in col.value_counts().items()}

print("\n按类型统计的测试数： ", counts(TESTS.test_type))

print("按系列统计的测试数： ", counts(TESTS.test_type.map(vocab.FAMILIES)))

print("结构与角色标签：", {g: n for g, n in tags.most_common() if g not in langs | scripts})

print("\n每个测试的答案是如何确定的（推导方式）：")

for d, n in TESTS.derivation.value_counts().items():

print(f" {d:22s} {n:6,d} {', '.join(f'{k} {v}' for k, v in TESTS[TESTS.derivation == d].test_type.value_counts().items())}")

CARD_TEXT = (DATA / "README.md").read_text()

claim("如数据集卡片所述，22 个套件中的 2,343 份文档、2,937 个页面上共有 16,288 个测试",

(len(TESTS), TESTS.page_id.nunique(), TESTS.document.nunique(), TESTS.suite.nunique()) == (16288, 2937, 2343, 22))

return f"{len(TESTS):,} 个测试，{TESTS.page_id.nunique():,} 个页面，{TESTS.document.nunique():,} 份文档，{len(langs - {'und', 'zxx'})} 种语言"

dataset_step()

理解了评分器之后，我们在固定版本上 下载清单、金标准文件和已发布的判定结果，跳过 1.5 GB 的页面图像，并通过 unit_tests.gold 读取每个页面的测试。计数与数据集卡片一致，包括去掉未确定和无语言代码后的 92 种语言。有两个字段值得注意。诸如手写、旋转或表格之类的标签从不决定判定结果；它们只让我们能对结果进行切片。推导方式记录了每个答案是如何确定的，从发布者标记和文本层到模型面板和人工审核，这让我们之后可以检验测试构建是否偏向任何解析器。

复制代码已复制使用其他浏览器

COLS = ["page_id", "suite", "document", "test_id", "test_type", "output_type", "derivation", "tags", "result", "why"]

STARRED = ["Gemini 3.8 Flash", "Claude Sonnet 5.5"] # 卡片：模型“至少参与创建和验证了部分测试”

@section("4. 从 439,776 条判定结果重新计算已发布的排行榜")

def recompute_table():

global VERDICTS, TABLE

VERDICTS = pd.read_parquet(DATA / "results.parquet")

VERDICTS["output_type"] = VERDICTS.test_type.map(vocab.READS)

VERDICTS["tags"] = VERDICTS.tags.apply(list)

VERDICTS["family"] = VERDICTS.test_type.map(vocab.FAMILIES)

rows = []

for system, g in VERDICTS.groupby("system", sort=False):

s = unit_tests.summarize(g[COLS].to_dict("records"))

fam = {k: (np.nan if v["score"] is None else 100 * v["score"]) for k, v in s["families"].items()}

scored = g[g.result != "unsupported"]

rows.append(dict(system=system, headline=np.nan if s["headline"] is None else 100 * s["headline"],

text=fam.get("text"), tables=fam.get("tables"), layout=fam.get("layout"),

pooled=100 * (scored.result == "pass").mean()))

TABLE = pd.DataFrame(rows).set_index("system")

card = {}

for line in CARD_TEXT.splitlines():

cells = [c.strip().strip("*").strip() for c in line.strip().strip("|").split("|")]

if len(cells) == 6 and cells[1] in ("open", "closed"):

card[cells[0]] = [None if c in ("none", "unsupported") else float(c) for c in cells[2:]]

mismatch = []

for name, vals in card.items():

mine = [None if pd.isna(x) else round(x, 1) for x in TABLE.loc[name, ["headline", "text", "tables", "layout"]]]

mismatch += [(name, a, b) for a, b in zip(mine, vals) if a != b]

print(f"{len(VERDICTS):,} verdicts = {VERDICTS.system.nunique()} parsers x {VERDICTS.test_id.nunique():,} tests: "

f"{ {k: int(v) for k, v in VERDICTS.result.value_counts().items()} }")

numeric = sum(v is not None for vals in card.values() for v in vals)

print(f"the card's table: {len(card)} parsers, {4 * len(card)} entries ({numeric} numbers, the rest none or unsupported); "

f"recomputed with unit_tests.summarize, {len(mismatch)} differ {mismatch[:3] if mismatch else ''}\n")

show = TABLE.sort_values("headline", ascending=False, na_position="last")

f = lambda x: "none" if pd.isna(x) else f"{x:.2f}"

print(f"{'parser':24s} {'headline':>9s} {'text':>7s} {'tables':>7s} {'layout':>7s} {'pooled':>7s}")

for name, r in show.iterrows():

if name in show.index[:8] or pd.isna(r.headline):

print(f"{name + ('*' if name in STARRED else ''):24s} {f(r.headline):>9s} {f(r.text):>7s} {f(r.tables):>7s} "

f"{f(r.layout):>7s} {r.pooled:7.2f}")

a, g = TABLE.loc["Datalab accurate", "headline"], TABLE.loc["Gemini 3.8 Flash", "headline"]

ranked = TABLE.dropna(subset=["headline"])

print(f"\nDatalab accurate {a:.3f} vs Gemini 3.8 Flash {g:.3f}: both {a:.1f} at one decimal, {a - g:+.3f} points apart")

print("按 headline 排名前 3（三个家族的平均值）：", list(ranked.headline.nlargest(3).index))

print("按所有已评分测试的汇总通过率排名前 3：", list(ranked.pooled.nlargest(3).index))

claim("卡片结果表中的每个数字都从 results.parquet 复现", not mismatch)

return f"{4 * len(card) - len(mismatch)} / {4 * len(card)} 条卡片条目已复现；Datalab accurate {a:.2f}，Gemini {g:.2f}"

recompute_table()

由于 results.parquet 为每个解析器在每个测试上保存一行，我们可以把每个解析器的行传给 unit_tests.summarize 来重建排行榜。headline 是文本、表格和布局通过率的平均值，不支持的测试不计入，因此像 Tesseract 这样无法返回布局块的解析器完全没有 headline。卡片表格中的全部 108 个条目都与其一位小数值一致。重新计算还暴露出榜首的近乎平局：Datalab accurate 和 Gemini 3.8 Flash 都四舍五入到 92.9，相差 0.024 分，而汇总所有测试而非按家族平均则让 Gemini 排到第一。

复制代码已复制使用其他浏览器

def per_document(df, systems, groups):

"""对每个系统：一个数组（文档数 x 分组数 x [通过数, 已评分测试数]）；`groups` 将名称映射到行掩码。"""

docs = {d: i for i, d in enumerate(sorted(df.document.unique()))}

s = df[df.result != "unsupported"]

out = {}

for name in systems:

g = s[s.system == name]

arr = np.zeros((len(docs), len(groups), 2))

for k, mask in enumerate(groups.values()):

sel = g[mask(g)]

d = sel.document.map(docs).to_numpy()

arr[:, k, 0] = np.bincount(d, weights=(sel.result == "pass").to_numpy(), minlength=len(docs))

arr[:, k, 1] = np.bincount(d, minlength=len(docs))

out[name] = arr

return out, len(docs)

def resample(n_docs, B, seed):

"""对文档进行 B 次 bootstrap 重采样，每次表示为每个文档的计数。"""

rng = np.random.default_rng(seed)

return np.stack([np.bincount(rng.integers(0, n_docs, n_docs), minlength=n_docs) for _ in range(B)]).astype(float)

@section("5. 榜首顺序有多确定？文档级 bootstrap")

def bootstrap_step():

top = list(TABLE.headline.nlargest(6).index)

families = {f: (lambda g, f=f: g.family == f) for f in ("text", "tables", "layout")}

arrs, n_docs = per_document(VERDICTS[VERDICTS.system.isin(top)], top, families)

W = resample(n_docs, 2000, seed=20261022)

head = {}

for s in top:

tot = np.tensordot(W, arrs[s], axes=([1], [0])) # 重采样数 x 家族数 x [通过数, 测试数]

head[s] = 100 * (tot[..., 0] / tot[..., 1]).mean(axis=1)

point = {s: 100 * (arrs[s].sum(0)[:, 0] / arrs[s].sum(0)[:, 1]).mean() for s in top}

print(f"对 {n_docs:,} 份文档进行 2,000 次重采样；同一文档的测试保持在一起\n")

print(f"{'parser':22s} {'headline':>9s} {'95% interval':>15s} {'gap to ' + top[0]:>26s} {'resamples where it leads':>24s}")

leader = np.argmax(np.stack([head[x] for x in top]), axis=0)

for s in top:

lo, hi = np.percentile(head[s], [2.5, 97.5])

d = head[top[0]] - head[s]

gap = "" if s == top[0] else f"{point[top[0]] - point[s]:+.2f} [{np.percentile(d, 2.5):+.2f}, {np.percentile(d, 97.5):+.2f}]"

print(f"{s:22s} {point[s]:9.2f} {lo:6.2f} - {hi:6.2f} {gap:>26s} {100 * np.mean(leader == top.index(s)):23.1f}%")

d = head[top[0]] - head[top[1]]

lo, hi = np.percentile(d, [2.5, 97.5])

claim(f"{top[0]} 和 {top[1]} 无法区分（二者差距的 95% 区间包含零）", lo < 0 < hi)

return f"{top[0]} vs {top[1]}: {point[top[0]] - point[top[1]]:+.2f} 分，95% 区间 [{lo:+.2f}, {hi:+.2f}]"

bootstrap_step()

如此小的差距引出一个问题：这个排序中有多少是噪声。同一份文档上的测试往往一起通过或一起失败，因此我们重采样文档而非测试：2,000 次抽样中，每次有放回地抽取 2,343 份文档，并通过一次矩阵乘法从每份文档的通过计数重新计算每个 headline。排名前两位的差距范围约为 -0.9 到 +0.9 分，且各自在约一半的抽样中领先，因此诚实的解读是二者打平。Datalab balanced 落后领先者 1.35 分，与领先者明显区分开，其下每个 parser 也是如此。

Copy CodeCopiedUse a different Browser

def slice_headline(df):

"""按系统：在一部分测试上的家族通过率及其均值，即基准 headline 的构建方式。"""

s = df[df.result != "unsupported"]

fam = s.groupby(["system", "family"]).result.apply(lambda r: 100 * (r == "pass").mean()).unstack()

return fam.mean(axis=1), fam

@section("6. 不同解析器排在第一位的切片")

def slices_step():

ranked = TABLE.dropna(subset=["headline"]).sort_values("headline", ascending=False)

rank = {s: i + 1 for i, s in enumerate(ranked.index)}

V = VERDICTS[VERDICTS.system.isin(ranked.index)]

whole, _ = slice_headline(V)

assert np.allclose(whole[ranked.index], ranked.headline), "slice_headline 与 unit_tests.summarize 不一致"

print(f"{'tag':13s} {'tests':>6s} {'docs':>5s} {'first (overall rank)':34s} {'second':26s} families")

leaders = collections.Counter()

for tag in ("handwriting", "tiny", "rotated", "degraded", "math", "table", "form", "multi_column",

"Grek", "Hani", "Cyrl", "Armn", "Laoo"):

sl = V[V.tags.apply(lambda t: tag in t)]

head, fam = slice_headline(sl)

order = head.sort_values(ascending=False)

(a, x), (b, y) = list(order.items())[:2]

docs = sl.document.nunique()

leaders[a] += 1

note = (" (并列第一)" if x == y else "") + (" (10 篇或更少文档)" if docs <= 10 else "")

print(f"{tag:13s} {sl.test_id.nunique():6d} {docs:5d} {f'{a} {x:.1f} (#{rank[a]})':34s} {f'{b} {y:.1f}':26s} "

f"{'+'.join(c for c in ('text', 'tables', 'layout') if c in fam.columns)}{note}")

first = ranked.index[0]

claim("总体领先者并非在每个切片上都排第一", leaders[first] < 13)

return f"{first} 在 13 个切片中的 {leaders[first]} 个上排第一；其余切片由另外 {len(leaders) - (first in leaders)} 个解析器领先"

slices_step()

按标签切片揭示了单一头条数字所掩盖的、关于我们实际拥有文档的情况。我们在手写、旋转页面、表格、单一文字系统及其他切片上，用相同的家族均值规则重新计算头条数字，此前已通过断言确认该规则能在完整集合上复现官方头条数字。总体领先者仅在十三个切片中的三个上排第一。Gemini 3.8 Flash 在手写、数学和表单上领先；Claude Sonnet 5.5 在旋转页面上领先；Mistral OCR 在希腊文上领先。亚美尼亚文和老挝文分别只有六篇和五篇文档，因此我们将其视为提示而非排名。

复制代码已复制使用其他浏览器

@section("7. 卡片上提到的两个模型在模型修正过的测试上表现更好吗？一项双重差分检验")

def creation_check():

others = [s for s in TABLE.dropna(subset=["headline"]).index if s not in STARRED]

V = VERDICTS[VERDICTS.system.isin(STARRED + others)]

print(f"starred: {STARRED}; compared with the {len(others)} other parsers that have a headline")

print("DiD = (starred: pass rate on the split - elsewhere) - (others: the same gap); 1,000 document resamples\n")

print(f"{'tests established by':21s} {'tests':>6s} {'starred in / out':>17s} {'others in / out':>16s} {'DiD [95% interval]':>23s}")

W = None

out = {}

for deriv, note in (("model_review", "a vision model corrected the test"),

("model_panel", "independent models agreed"),

("human_review", "placebo: a person decided")):

groups = {"in": lambda g, d=deriv: g.derivation == d, "out": lambda g, d=deriv: g.derivation != d}

arrs, n_docs = per_document(V, STARRED + others, groups)

W = resample(n_docs, 1000, seed=7) if W is None else W

def gaps(weights):

r = {}

for s, a in arrs.items():

tot = np.tensordot(weights, a, axes=([-1], [0]))

with np.errstate(invalid="ignore", divide="ignore"):

r[s] = 100 * tot[..., 0] / tot[..., 1]

did = (np.nanmean([r[s][..., 0] - r[s][..., 1] for s in STARRED], axis=0)

- np.nanmean([r[s][..., 0] - r[s][..., 1] for s in others], axis=0))

return r, did

r, d = gaps(np.ones(n_docs))

_, boot = gaps(W)

lo, hi = np.nanpercentile(boot, [2.5, 97.5])

si, so = np.mean([r[s][0] for s in STARRED]), np.mean([r[s][1] for s in STARRED])

oi, oo = np.mean([r[s][0] for s in others]), np.mean([r[s][1] for s in others])

out[deriv] = (d, lo, hi)

print(f"{deriv:21s} {(TESTS.derivation == deriv).sum():6,d} {si:7.1f} / {so:6.1f} {oi:6.1f} / {oo:6.1f} "

f"{d:+6.1f} [{lo:+5.1f}, {hi:+5.1f}] {note}")

panel = TESTS[TESTS.derivation == "model_panel"]

layout = TESTS[TESTS.test_type == "layout_kind"]

print(f"\nmodel_panel tests are {100 * (panel.test_type == 'layout_kind').mean():.0f}% layout tests, and "

f"{100 * (layout.derivation == 'model_panel').mean():.0f}% of layout tests are model_panel")

review = VERDICTS[VERDICTS.derivation == "model_review"]

rate = lambda s: 100 * (review[review.system == s].result == "pass").mean()

print(f"pass rate on the {(TESTS.derivation == 'model_review').sum()} model_review tests: ")

+ ", ".join(f"{s} {rate(s):.1f}%" for s in STARRED) + f"，其余的中位数为 {np.median([rate(s) for s in others]):.1f}%")

claim("带星标的模型在 model_review 测试上的提升大于在安慰剂划分上的提升", out["model_review"][1] > out["human_review"][2])

claim("安慰剂划分未显示出差距（其区间包含零）", out["human_review"][1] < 0 < out["human_review"][2])

return (f"DiD model_review {out['model_review'][0]:+.1f}, model_panel {out['model_panel'][0]:+.1f}, "

f"placebo human_review {out['human_review'][0]:+.1f} points")

creation_check()

数据集卡片将 Gemini 3.8 Flash 和 Claude Sonnet 5.5 标记为用于创建和验证部分测试，但未说明具体是哪些测试，因此我们用双重差分提出一个更窄的问题：某个解析器在一个衍生划分上的通过率，相对于它在其他地方的通过率，对这对带星标的模型而言是否变化得不同？在由视觉模型校正的 102 个 model_review 测试上，两组模型的通过率都远低于其他地方，但带星标的那一对下降约少 25 个百分点。human_review 安慰剂在同一方向上显示出更小的差距，而 model_panel 则朝相反方向变化，因此我们报告的是一个标记，而非证据。

复制代码已复制使用其他浏览器

def predict_pages(run, pages, alive=lambda: True):

"""预测该次运行仍然缺少的每一页。失败的调用会被记录而非抛出，因此我们自行统计失败次数。"""

t0, n, failed = time.time(), 0, []

for p in pages:

if not alive():

raise RuntimeError(f"{run.provider}: 服务器在 {n} 页后停止")

if need := benchmark.needs(run, p):

got = harness.predict(run.provider, p.doc_path, need, timeout=900, **run.options)

benchmark.record(run, p.page_id, got)

n += 1

if any(c.get("failure") for c in got["calls"]):

failed.append(f"{p.page_id}: {got['errors']}")

if failed:

print(f"{n} 页中有 {len(failed)} 页失败，例如 {failed[0][:160]}")

if 2 * len(failed) >= n:

raise RuntimeError(f"{run.provider} 在 {n} 页中有 {len(failed)} 页失败")

return n, (time.time() - t0) / max(n, 1)

@section("8. 我们在覆盖全部 22 个套件的 50 页上自行运行 Tesseract")

def tesseract_step():

global SAMPLE

if TESS_MAJOR < 5:

raise RuntimeError(f"tesseract {TESS_MAJOR}.x is installed; this tutorial was verified with tesseract 5")

SAMPLE = data.select(PAGES, sample=50)

snapshot_download(REPO, repo_type="dataset", revision=REV, local_dir=str(DATA),

allow_patterns=sorted(str(p.doc_path.relative_to(DATA)) for p in SAMPLE))

n_pdf = sum(p.doc_path.suffix == ".pdf" for p in SAMPLE)

print(f"sample: {len(SAMPLE)} pages from {len({p.suite for p in SAMPLE})} suites, {n_pdf} PDF and {len(SAMPLE) - n_pdf} images, "

f"{sum(p.doc_path.stat().st_size for p in SAMPLE) / 1e6:.1f} MB, {sum(len(unit_tests.gold(p)) for p in SAMPLE)} tests")

# 适配器会向 Tesseract 请求 '-l script/<Name>'。Homebrew 将这些模型放在 script/ 文件夹中；Ubuntu 安装

# 时则是平铺的，所以在 Linux 上我们让 Tesseract 指向一个按它期望的目录结构组织的符号链接文件夹。

listing = subprocess.run(["tesseract", "--list-langs"], capture_output=True, text=True)

listing = listing.stdout + listing.stderr

if "script/Latin" not in listing:

src = Path(re.search(r'"(.+?)"', listing).group(1))

dst = WORK / "tessdata"

(dst / "script").mkdir(parents=True, exist_ok=True)

for f in src.glob("*.traineddata"):

for link in [dst / f.name] + ([dst / "script" / f.name] if f.name[0].isupper() else []):

if not link.exists():

link.symlink_to(f)

os.environ["TESSDATA_PREFIX"] = str(dst)

print(f"script models linked under {dst}")

run = benchmark.Run("tesseract", {}, WORK / "runs" / "tesseract")

benchmark.prepare(run, str(DATA), 900)

n, per_page = predict_pages(run, SAMPLE)

s = benchmark.grade(run, SAMPLE, say=lambda *_: None)

fam = s["families"]

print(f"\nTesseract: {n} pages read, {per_page:.1f} s per page")

for k, v in fam.items():

print(f" {k:7s} " + ("unsupported (no blocks output)" if v["score"] is None else f"{100 * v['score']:5.1f}% of {v['tests']} tests"))

print(f" headline {'none' if s['headline'] is None else round(100 * s['headline'], 1)} (a family it cannot produce leaves no headline)")

return f"Tesseract on 50 pages: text {100 * fam['text']['score']:.1f}%, tables {100 * fam['tables']['score']:.1f}%, {per_page:.1f} s/page"

tesseract_step()

从已公布的判定结果重建排行榜是一回事；我们自己产出判定结果是另一回事。data.select 会从全部 22 个套件中抽取固定的 50 页样本，我们只下载这些页面。循环是基准测试自带的：needs 列出某个页面仍然缺少的输出，harness.predict 运行解析器；record 保存预测结果并对文件夹评分，因此中断的运行可以恢复。在 Ubuntu 上，我们还会把 Tesseract 的脚本模型链接到适配器期望的布局中。Tesseract 在样本的文本测试中通过率为 30.4%，而在完整集上为 17.6%，这一差距取决于抽样了哪些页面。

复制代码已复制使用其他浏览器

def agreement(folder, published):

"""我们的判定结果与同一批测试的已公布判定结果对比，仅限双方都对测试评分的部分。"""

ours = pd.read_json(folder / "scores.jsonl", lines=True)[["test_id", "result", "why"]]

pub = VERDICTS[VERDICTS.system == published][["test_id", "result"]]

m = ours.merge(pub, on="test_id", suffixes=("_ours", "_pub"))

m = m[(m.result_ours != "unsupported") & (m.result_pub != "unsupported")]

return m, int((m.result_ours == m.result_pub).sum()), len(m)

@section("9. Indra OCR 在相同页面上运行，由该包自带的服务器代码在 CPU 上提供服务")

def indra_step():

if sys.version_info < (3, 12):

raise RuntimeError("Indra 的固定依赖（numpy 2.5.4）需要 Python 3.12 或更新版本")

venv = WORK / "indra-venv"

py = venv / "bin" / "python"

if not py.exists():

uv = shutil.which("uv") or str(Path(sys.executable).parent / "uv")

subprocess.run([uv, "venv", "-q", "--python", sys.executable, str(venv)], check=True)

subprocess.run([uv, "pip", "install", "-q", "--python", str(py)] + INDRA, check=True)

server = Path(harness.__file__).parent / "providers" / "oss" / "indra" / "server.py"

# server.py 用 os.sched_getaffinity 统计核心数，而 macOS 没有该函数；在 Linux 上这个 shim 不会改变任何行为

launcher = WORK / "indra_launch.py"

launcher.write_text("import os, runpy, sys\n"

"if not hasattr(os, 'sched_getaffinity'):\n"

" os.sched_getaffinity = lambda pid: set(range(os.cpu_count()))\n"

"sys.argv = ['server.py', '--port', '8001', '--workers', '2']\n"

f"runpy.run_path({str(server)!r}, run_name='__main__')\n")

log = open(WORK / "indra_server.log", "w")

proc = subprocess.Popen([str(py), str(launcher)], stdout=log, stderr=subprocess.STDOUT)

t0 = time.time()

while True:

try:

urllib.request.urlopen("http://127.0.0.1:8001/health", timeout=2)

break

except Exception:

if proc.poll() is not None or time.time() - t0 > 600:

proc.terminate()

raise RuntimeError("the Indra server did not start; see " + log.name)

time.sleep(2)

print(f"server up in {time.time() - t0:.0f} s: {INDRA[0]}, 2 worker processes")

try:

run = benchmark.Run("indra-ocr", {"base_url": "http://127.0.0.1:8001"}, WORK / "runs" / "indra-ocr")

benchmark.prepare(run, str(DATA), 900)

n, per_page = predict_pages(run, SAMPLE, alive=lambda: proc.poll() is None)

s = benchmark.grade(run, SAMPLE, say=lambda *_: None)

finally:

proc.terminate()

fam = {k: 100 * v["score"] for k, v in s["families"].items()}

print(f"Indra OCR: {n} pages, {per_page:.2f} s per page; text {fam['text']:.1f}%, tables {fam['tables']:.1f}%, "

f"layout {fam['layout']:.1f}%; headline on these pages {100 * s['headline']:.1f}\n")

ids = {p.page_id for p in SAMPLE}

fmt = lambda x: "none" if x is None else f"{100 * x:.1f}"

print(f"{'on the 50 pages':22s} {'headline':>9s} {'text':>6s} {'tables':>7s} {'layout':>7s}")

for name, folder in (("Tesseract", WORK / "runs" / "tesseract"), ("Indra OCR", WORK / "runs" / "indra-ocr")):

pub = unit_tests.summarize(VERDICTS[(VERDICTS.system == name) & VERDICTS.page_id.isin(ids)][COLS].to_dict("records"))

ours = unit_tests.summarize(benchmark.read_scores(benchmark.Run(name, {}, folder)))

for who, x in (("ours", ours), ("published", pub)):

f = x["families"]

print(f"{name + ', ' + who:22s} {fmt(x['headline']):>9s} " + " ".join(f"{fmt(f[k]['score']) if k in f else 'none':>6s}"

for k in ("text", "tables", "layout")))

print()

for name, folder in (("Tesseract", WORK / "runs" / "tesseract"), ("Indra OCR", WORK / "runs" / "indra-ocr")):

m, same, total = agreement(folder, name)

diff = m[m.result_ours != m.result_pub]

print(f"{name:10s} our verdicts equal the published ones on {same} of {total} tests ({100 * same / total:.1f}%); "

f"of the {len(diff)} that differ, {(diff.result_ours == 'pass').sum()} pass only for us and "

f"{(diff.result_ours == 'fail').sum()} 仅在已发布的运行中出现"）

if len(diff):

by_page = diff.test_id.map(TESTS.set_index("test_id").page_id).value_counts()

print(" 按类型区分的差异测试：", {k: int(v) for k, v in diff.test_id.map(TESTS.set_index("test_id").test_type).value_counts().items()},

f"| 分布在 {len(by_page)} 个页面上，其中 {by_page.iloc[0]} 个位于 {by_page.index[0]}"）

claim(f"{name}：我们的判定在样本测试中至少有 95% 与已发布结果一致", same >= 0.95 * total)

return f"Indra OCR 在 50 个页面上：总体得分 {100 * s['headline']:.1f}（文本 {fam['text']:.1f}，表格 {fam['tables']:.1f}，版面 {fam['layout']:.1f}），每页 {per_page:.2f} 秒"

indra_step()

Indra OCR 是一个开放的 CPU 模型，该软件包附带了其适配器所通信的 HTTP 服务器，因此我们将该服务器 Dockerfile 中固定版本的依赖安装到一个独立环境中，并在后台启动它。在同一批页面上对照已发布的判定结果，Tesseract 在每个计分测试上都一致，因此其样本得分与已发布结果相符。Indra 在 221 个测试中有 203 个一致，其所有表格差异都集中在一个页面上，这正好解释了其已发布表格得分 36.1 与我们 69.4 之间的全部差距。一个页面就能让某个类别偏移如此之大，这是重新采样文档最有力的理由。

复制代码已复制使用其他浏览器

@section("10. 自带解析器：用同样的代码为 PDF 文本层基线评分")

def byo_step():

import pypdfium2 as pdfium

def text_layer(path):

if path.suffix.lower() != ".pdf":

return "" # 图像没有可读取的文本层

doc = pdfium.PdfDocument(str(path))

try:

return "<pre>" + html.escape(doc[0].get_textpage().get_text_range()) + "</pre>"

finally:

doc.close()

out = WORK / "runs" / "text-layer"

(out / "predictions").mkdir(parents=True, exist_ok=True)

for p in SAMPLE:

(out / "predictions" / f"{p.page_id}.json").write_text(json.dumps({"html": text_layer(p.doc_path)}))

s = benchmark.grade(benchmark.Run("text-layer", {}, out), SAMPLE, say=lambda *_: None, settings={"unsupported": ["blocks"]})

kind = {p.page_id: "PDF" if p.doc_path.suffix == ".pdf" else "image" for p in SAMPLE}

print(f"{'parser on the same 50 pages':28s} {'text: PDF':>10s} {'images':>8s} {'all':>6s} {'tables':>8s}")

for name in ("text-layer", "tesseract", "indra-ocr"):

sc = pd.read_json(WORK / "runs" / name / "scores.jsonl", lines=True)

sc = sc[sc.result.isin(["pass", "fail"])]

txt = sc[sc.test_type.map(vocab.FAMILIES) == "text"]

by = txt.groupby(txt.page_id.map(kind)).result.apply(lambda r: 100 * (r == "pass").mean())

tab = sc[sc.test_type == "table_cell"]

print(f"{name:28s} {by.get('PDF', np.nan):10.1f} {by.get('image', np.nan):8.1f} "

f"{100 * (txt.result == 'pass').mean():6.1f} {100 * (tab.result == 'pass').mean():8.1f}")

fam = s["families"]

return f"text layer, no OCR: text {100 * fam['text']['score']:.1f}%, tables {100 * fam['tables']['score']:.1f}% on the 50 pages"

byo_step()

任何解析器都可以在没有适配器的情况下评分：我们为每页写一个预测文件，并调用 benchmark.grade，将无法产生的输出声明为不支持。我们的解析器用 pypdfium2 读取 PDF 内嵌的文本层，对图像返回空字符串，因为图像没有文本层。在 PDF 页面上，它通过的文本测试数量与 Tesseract 完全相同，各为 42.3%，且无需任何 OCR；而在图像上它一无所获，Indra OCR 仍能通过 43.1%。纯文本无法通过任何表格测试，因为通过需要单元格结构，所以只要比较两个解析器，PDF 和图像的混合比例就很重要。

复制代码已复制使用其他浏览器

@section("11. The launch claims, checked against the files")

def claims_step():

a, t = TABLE.loc["Datalab accurate"], TABLE.loc["Tesseract"]

g = TABLE.loc["Gemini 3.8 Flash", "headline"]

c = TABLE.loc["Claude Sonnet 5.5"]

tags = TESTS.tags

langs = {x for ts in tags for x in ts if re.fullmatch("[a-z]{3}", x)} - {"und", "zxx"}

pkg = Path(vocab.__file__).parent # the scorer and the modules it reads; data.py and the harness only fetch and run

scoring = sorted(pkg.glob("scorers/**/*.py")) + [pkg / f"{m}.py" for m in ("normalize", "tables", "equations", "views", "vocab", "markdown")]

clients = [f.name for f in scoring if re.search(r"^\s*(import|from)\s+(httpx|requests|urllib|huggingface_hub|openai|anthropic|google"

r"|transformers|torch|omni_parse_bench\.harness)\b", f.read_text(), re.M)]

n_eng, n_table = tags.apply(lambda x: "eng" in x).sum(), tags.apply(lambda x: "table" in x).sum()

rows = [

("16,288 项测试、2,937 个页面、2,343 份文档", f"{len(TESTS):,} / {TESTS.page_id.nunique():,} / {TESTS.document.nunique():,}",

(len(TESTS), TESTS.page_id.nunique(), TESTS.document.nunique()) == (16288, 2937, 2343)),

("92 种语言", f"{len(langs)} 个除 und 和 zxx 之外的代码", len(langs) == 92),

("10,089 项英文测试、6,767 项表格测试", f"{n_eng:,} 和 {n_table:,}", (n_eng, n_table) == (10089, 6767)),

("Datalab accurate 得分 92.7", f"results.parquet 中为 {a.headline:.2f}", round(a.headline, 1) == 92.7),

("Datalab accurate 排名第二，仅次于 Gemini", f"{a.headline:.3f} 对 {g:.3f}", a.headline < g),

("Claude Sonnet 5.5：文本 93.1、表格 95.4、版面 77.5", f"{c.text:.1f} / {c.tables:.1f} / {c.layout:.1f}",

[round(c[k], 1) for k in ("text", "tables", "layout")] == [93.1, 95.4, 77.5]),

("Tesseract：无总体得分，文本 17.6、表格 0.0",

f"{'none' if pd.isna(t.headline) else round(t.headline, 1)} / {t.text:.1f} / {t.tables:.1f}",

pd.isna(t.headline) and round(t.text, 1) == 17.6 and round(t.tables, 1) == 0.0),

("没有模型评判输出", f"{len(scoring)} 个评分模块中有 {len(clients)} 个导入了模型或 HTTP 客户端", not clients),

]

print(f"{'claim (launch article and blog)':56s} {'what the files say':38s} verdict")

for text, found, ok in rows:

print(f"{text:56s} {found:38s} {'holds' if ok else 'does not hold'}")

claim(text, ok)

print("\nnot checkable from these files: latency, price per 1,000 pages, the other benchmarks' test counts")

return f"{sum(bool(ok) for *_, ok in rows)} of {len(rows)} launch claims hold against the files"

claims_step()

我们没有重复发布时的说法，而是逐条对照文件进行核查。各项计数完全吻合，从测试和页面到语言和子集，Claude 和 Tesseract 两行也一致。Datalab 自己那一行则不然：发布文章和博客给 Datalab 的准确值是 92.7，并将其排在 Gemini 之后位列第二，而发布的判定结果给出的是 92.92 对 92.89，这是一个四舍五入后数值相同的平局。无评判器这一说法对评分器本身成立，其模块不导入任何模型或 HTTP 客户端；KaTeX 在嵌入式 JavaScript 引擎中运行以比较公式。延迟和价格无法从这些文件中核查。

复制代码已复制使用其他浏览器

banner("SUMMARY")

for name, res in RESULTS.items():

print(f" {name:<100s} {res}")

print("\nchecks decided in this run:")

for step, text, held in CLAIMS:

print(f" [{'held' if held else 'did not hold'}] step {step}: {text}")

failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")]

print(f"\n{sum(h for *_, h in CLAIMS)} of {len(CLAIMS)} checks held; steps failed or skipped: {failed or 'none'}")

print(f"omni-parse-bench {md.version('omni-parse-bench')}, dataset @ {REV[:7]}, Python {platform.python_version()}, {SYSTEM}")

print("not run here: the hosted parsers (paid API keys), the GPU models, latency and cost, all 2,937 pages")

摘要会打印每一步的一行结果，以及本次运行判定的每一项检查，并标出哪些成立、哪些不成立：17 项中有 13 项成立，而不成立的 4 项是发现而非失败，即安慰剂差距、Indra 的复现以及 Datalab 的两个发布数字。它还会列出我们在此未运行的内容：需要付费密钥的托管解析器、GPU 模型、延迟和成本，以及全部 2,937 个页面。

总之，OmniParseBench 是一个我们可以端到端审计的基准测试：评分器是开放代码，不调用任何模型，判定结果已公开，卡片上的每一条记录都可以据此重建。排名前两位的解析器在统计上并列，用合并测试而非按家族分组会改变排序，而在手写、旋转页面、表格和多种文字上，排在第一的是另一个解析器。卡片上提到的两个模型在由模型纠正过的测试上落后得比其他模型少，我们将其作为一项提示而非证据来报告。Tesseract 已发布的判定结果在我们的样本上完全复现，而 Indra OCR 的判定结果在 92% 的测试上一致。实际的经验教训是：把基准测试切分到我们自己的文档上，报告置信区间，并固定每一个版本。

查看 Notebook 和 Repo。所有功劳归于本项目的研究者。另外，欢迎在 Twitter 上关注我们，别忘了加入我们 150k+ 的 ML SubReddit 并订阅我们的 Newsletter。等等！你在 Telegram 上吗？现在你也可以在 Telegram 上加入我们。

需要与我们合作推广你的 GitHub Repo 或 Hugging Face 页面或产品发布或网络研讨会等？联系我们

文章《A Developer’s Guide to OmniParseBench: Leaderboard Benchmarks and OCR Testing》首发于 MarkTechPost。