在本教程中，我们使用 Underdog Saluki 27B，这是 Qwen3.8-27B 的一个 7.9 GB 构建版本，其制作者称它在每个权重约两比特的情况下仍保持工具调用能力完整，甚至在他们自己的 120 项任务基准测试中略胜完整的 54 GB 模型。Saluki 基于 ISTA-DASLab 对同一模型的 GSQ-RCO 量化构建，因此我们将两个文件并排放在原版 llama.cpp 中。我们从文件本身开始：对两个 GGUF 头进行逐张量差异比较，然后进行字节级比较，以显示 Saluki 是 ISTA 比特的重新混合还是更改过的模型。然后我们在单个 GPU 上分别提供每个文件，读取其内存和速度，并在 Berkeley Function Calling Leaderboard 的相同种子 120 项任务上对两者评分，使用 BFCL 自己的检查器，采用严格和宽松解析。
复制代码已复制使用其他浏览器
import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# notebook 下载的所有内容都按大小和 SHA-256 固定校验。
LLAMA_TAG = "b11565" # llama.cpp 2026-10-12 的发布版本，commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 数据和 BFCL 自带的检查器代码，固定在一个 gorilla commit 上。
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047"
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0"
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3"
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538"
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a"
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd"
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd"
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3"
}
# Underdog Bench 的五类任务，每类 24 个；由于它的任务列表未公开，我们使用自己设定种子的样本。
KINDS = [("simple_python", "调用唯一正确的函数"), ("multiple", "从多个函数中选出正确的那个"),
("irrelevance", "知道何时没有工具适用"), ("parallel", "同一函数，多次调用"),
("parallel_multiple", "多个不同的函数")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server 自身的默认值是 temperature 0.8 和 top_k 40，因此每个请求都会声明其采样器。
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""使用 curl 进行可断点续传的下载，然后校验大小和 SHA-256。返回下载所花费的秒数。"""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# llama.cpp 的 CUDA 12.8 构建版本不附带 CUDA 运行时；Colab 的镜像以 pip wheel 的形式提供了它。
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
notebook 涉及的一切都在任何代码运行之前被固定下来。setup 单元会按大小和 SHA-256 记录 llama.cpp 的发布版本、两个模型文件和十四个 BFCL 文件，并为每个请求固定采样器，因为 llama-server 否则会回退到我们并未选择的默认值。它会检测平台，即 Linux 上的 NVIDIA GPU 或 Apple 芯片的 Mac，并在 Colab 上确保预构建的 llama.cpp 所需的 CUDA 12 运行时库可以被导入。
Copy CodeCopiedUse a different Browser
@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
我们下载 llama.cpp b11565 的 CUDA 构建版本、两个 2-bit 文件以及 BFCL 数据，并逐字节对照其固定版本校验。Hugging Face 的下载支持断点续传，且哈希值是 Hub 自身在固定 revision 上的 LFS 摘要，因此被静默替换的文件会让 notebook 停止运行，而不是改变其结果。BFCL 的任务文件、答案文件和检查器代码均来自同一个 gorilla commit。该单元格会打印下载耗时，这是 Colab 上整个运行过程中最慢的部分。
复制代码已复制使用其他浏览器
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # 名称、每块的值数、每块的字节数（ggml-common.h）
def read_gguf(path):
"""GGUF v3 头部：元数据、张量表（名称、维度、类型、偏移量）以及张量数据的起始位置。"""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _ in range(n_kv):
k = s()
kv[k] = val(u("I"))
tensors = []
for _ in range(n_tensors):
name = s()
dims = [u("Q") for _ in range(u("I"))]
ttype, off = u("I"), u("Q")
n = math.prod(dims)
_, per, nbytes = GGML_TYPES[ttype]
tensors.append(dict(name=name, dims=dims, type=GGML_TYPES[ttype][0], n=n, bytes=n // per * nbytes, off=off))
align = kv.get("general.alignment", 32)
data_start = (f.tell() + align - 1) // align * align
return kv, tensors, data_start
def tensor_class(name):
if not name.startswith("blk."):
return name.split(".")[0] # token_embd、output、output_norm
kind = name.split(".")[2]
if kind in ("attn_q", "attn_k", "attn_v", "attn_output"):
return "full-attention " + kind[5:]
if kind in ("attn_qkv", "attn_gate", "ssm_out", "ssm_conv1d"):
return "linear-attention " + kind
if kind in ("ssm_alpha", "ssm_beta"):
return "linear-attention gates (alpha, beta)"
if kind in ("ffn_gate", "ffn_up", "ffn_down"):
return kind
return "norms and small vectors"
@section("3. Saluki 改了什么：两个 GGUF 文件头的逐张量差异")
def header_diff():
global HDR
HDR = {tag: read_gguf(os.path.join(WORK, "models", m["file"])) for tag, m in MODELS.items()}
(kv_s, t_s, start_s), (kv_i, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
assert [t["name"] for t in t_s] == [t["name"] for t in t_i] and [t["dims"] for t in t_s] == [t["dims"] for t in t_i]
n_params = sum(t["n"] for t in t_s)
print(f"两个文件：{len(t_s)} 个张量，名称和形状完全相同，{n_params:,} 个权重，"
f"架构 '{kv_s['general.architecture']}'，{kv_s['qwen35.block_count']} 个 block")
for tag, (kv, ts, start) in HDR.items():
data = sum(t["bytes"] for t in ts)
size = MODELS[tag]["size"]
print(f" {tag:7s}: 文件头 {start:,} B + 张量数据 {data:,} B = {start + data:,} B "
f"({'等于' if start + data == size else '不同于'}文件大小)；每个权重 {8 * data / n_params:.3f} bits")
claim(f"{tag}: 由类型表计算出的文件头加张量字节数恰好等于文件大小", start + data == size)
rows = collections.OrderedDict()
moves = collections.Counter()
up = down = 0
for a, b in zip(t_i, t_s):
r = rows.setdefault(tensor_class(a["name"]), dict(k=0, n=0, bi=0, bs=0))
r["k"] += 1; r["n"] += a["n"]; r["bi"] += a["bytes"]; r["bs"] += b["bytes"]
if a["type"] != b["type"]:
moves[(a["type"], b["type"])] += 1
up += b["bytes"] > a["bytes"]; down += b["bytes"] < a["bytes"]
print(f"\n{'张量类别':34s} {'数量':>5s} {'权重':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'变化 MB':>10s}")
for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
if r["n"] < 1e6:
continue
print(f"{c:34s} {r['k']:5d} {r['n'] / 1e9:8.3f}B {8 * r['bi'] / r['n']:9.3f} {8 * r['bs'] / r['n']:10.3f} {(r['bs'] - r['bi']) / 1e6:+10.1f}")
changed = sum(moves.values())
print(f"\n{len(t_s)} 个张量中有 {changed} 个改变了类型：" + ", ".join(f"{n} 个 {a}->{b}" for (a, b), n in moves.most_common()))
print(f" {down} 得到更少字节，{up} 得到更多")
claim("每个类型发生变化的张量都获得更少的比特；没有张量获得更多", up == 0 and down == changed)
untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
print(f" 完全未发生变化的类别：{', '.join(sorted(untouched))}")
proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
claim("每个注意力和 DeltaNet 投影矩阵都保持其类型（只有小的 alpha/beta 门发生变化）",
all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
print(f"\n存在差异的元数据键：{keys}")
print(f"Saluki 的头部仍然写着：general.name = {kv_s['general.name']!r}，quantized_by = {kv_s['general.quantized_by']!r}")
print(f"两个头部中都存储了采样参数：temp {kv_s['general.sampling.temp']}，top_k {kv_s['general.sampling.top_k']}，"
f"top_p {kv_s['general.sampling.top_p']:.2f}（这些是 Qwen 的思考模式设置；第 5 步展示了服务器如何处理它们）")
saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6
return f"{len(t_s)} 个张量类型中有 {changed} 个发生变化，全部为降级；小了 {saved:.0f} MB"
header_diff()
用纯 Python 读取两个 GGUF 头部，可以准确看出 Saluki 改动了什么。两个文件都有相同的 851 个张量，形状也相同，类型表可以预测每个文件的字节大小。Saluki 改变了 146 个张量的类型，而且每处改动都削减了比特：大多数落在前馈下投影和输出层上，小的 DeltaNet 门向量从 BF16 降到 Q8_0。没有任何注意力或 DeltaNet 投影矩阵发生变化，头部仍然将 ISTA DASLab 列为量化方。
复制代码已复制使用其他浏览器
@section("4. 字节级溯源：未变化的张量是 ISTA 自己的字节吗？")
def provenance():
import numpy as np
(_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r")
mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r")
q = dict(k=0, same=0, eq_bytes=0, bytes=0)
f32 = collections.defaultdict(lambda: [0, 0, []]) # kind -> [identical, differ, relative L2 changes]
for a, b in zip(t_i, t_s):
if a["type"] != b["type"]:
continue
x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]]
y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]]
if a["type"] == "F32":
kind = re.sub(r"blk\.\d+\.", "", a["name"])
u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64)
same = bool(np.array_equal(u, v))
f32[kind][0 if same else 1] += 1
if not same:
f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u))
else:
eq = int(np.count_nonzero(x == y))
q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"]
print(f"保持原类型的量化矩阵：{q['same']} / {q['k']} 个字节完全一致，但 "
f"其 {q['bytes'] / 1e9:.2f} GB 字节中有 {100 * q['eq_bytes'] / q['bytes']:.1f}% 的字节相等")
print(f"\n未量化的 F32 向量（从未重新编码，因此任何变化都意味着权重被改动）：")
print(f" {'kind':28s} {'identical':>9s} {'changed':>8s} 相对变化中位数")
for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]):
med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-"
print(f" {kind:28s} {same:9d} {diff:8d} {med}")
changed = sum(v[1] for v in f32.values())
total = sum(v[0] + v[1] for v in f32.values())
claim("Saluki 中类型未变的张量与 ISTA 逐字节一致（纯粹是类型重新混合）", q["same"] == q["k"] and changed == 0)
claim("Saluki 的 F32 norm 和 gate 向量与 ISTA 不同，因此其权重被改动过，而不只是重新编码", changed > 0)
return (f"{q['same']}/{q['k']} 个同类型矩阵逐字节一致，其字节中有 {100 * q['eq_bytes'] / q['bytes']:.1f}% 相等； "
f"{changed}/{total} 个 F32 向量发生变化")
provenance()
相同类型并不意味着相同的权重，因此我们比较了 705 个保持其类型的张量的字节。352 个量化矩阵中只有一个字节完全相同，但它们的字节有 98.4% 匹配，这与对底层权重进行小幅编辑后在同一网格上重新量化的情况一致。未量化的 norm 和 gate 向量证实了这一点：大多数变化的中位数仅为万分之几。Saluki 是一个经过轻微改动的模型，而不是对 ISTA 文件的重新编码。
复制代码已复制使用其他浏览器
SERVER = {}
def start_server(tag):
"""每个模型一个 llama-server，一个槽位，所有层都在 GPU 上，保留调试日志用于未解析的回复。"""
stop_server()
log_path = os.path.join(WORK, "runs", f"server_{tag}.log")
cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT),
"-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"]
t0 = time.time()
proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV)
while True:
try:
if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200:
break
except urllib.error.HTTPError as e:
assert e.code == 503, e
except urllib.error.URLError:
pass
assert proc.poll() is None, f"llama-server 已退出；请查看 {log_path}"
assert time.time() - t0 < 900, "服务器未在 15 分钟内变为健康状态"
time.sleep(2)
SERVER.update(tag=tag, proc=proc, log=log_path, load_s=time.time() - t0)
return SERVER["load_s"]
def stop_server():
if SERVER.get("proc") and SERVER["proc"].poll() is None:
SERVER["proc"].terminate()
SERVER["proc"].wait(timeout=60)
SERVER.clear()
def api(path, body=None, timeout=1800):
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", None if body is None else json.dumps(body).encode(),
{"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def memory_report():
"""来自服务器加载日志的缓冲区大小，以及 GPU 报告的信息。"""
text = open(SERVER["log"]).read()
out = {}
for label, what in (("weights", "model"), ("KV cache", "KV"), ("recurrent state", "RS"), ("compute", "compute")):
last = {} # llama.cpp 会先做一次 dry run 来确定每个缓冲区的大小；保留每个设备的最终值
for dev, mib in re.findall(rf"(\S+) +{what} buffer size = +([\d.]+) MiB", text):
last[dev] = float(mib)
out[label] = sum(last.values())
if ON_CUDA:
out["GPU in use (nvidia-smi)"] = float(sh(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]).stdout.split()[0])
return out
@section("5. 单 GPU 上的混合模型：启动服务器，读取其内存占用，并查看它构建的 prompt")
def architecture_and_server():
kv = HDR["Saluki"][0]
L, every = kv["qwen35.block_count"], kv["qwen35.full_attention_interval"]
full = [i for i in range(L) if (i + 1) % every == 0]
heads_kv, dk = kv["qwen35.attention.head_count_kv"], kv["qwen35.attention.key_length"]
kv_tok = len(full) * heads_kv * dk * 2 * 2 # K 和 V，f16
conv = (kv["qwen35.ssm.conv_kernel"] - 1) * (kv["qwen35.ssm.inner_size"] + 2 * kv["qwen35.ssm.group_count"] * kv["qwen35.ssm.state_size"])
state = kv["qwen35.ssm.state_size"] * kv["qwen35.ssm.inner_size"]
rs = (L - len(full)) * (conv + state) * 4
print(f"{L} 个块：{L - len(full)} 个 Gated DeltaNet（线性注意力）层和 {len(full)} 个全注意力层（3, 7, ..., {full[-1]}）")
print(f"KV 缓存：{len(full)} 个注意力层每 token 占用 {kv_tok // 1024} KiB；如果所有 "
f"{L} 层都带注意力，则需要 {L * heads_kv * dk * 4 // 1024} KiB")
print(f"循环状态：每个序列 {rs / 2**20:.1f} MiB，固定不变，与上下文长度无关")
print(f"在 -c {CTX} 下的预期值：KV {kv_tok * CTX / 2**20:.0f} MiB + 状态 {rs / 2**20:.0f} MiB")
load_s = start_server("Saluki")
mem = memory_report()
print(f"\nllama-server 在 {load_s:.0f} s 内启动（-c {CTX}, -np 1, -ngl 99, -fa on）；其日志中的缓冲区：")
for k, v in mem.items():
print(f" {k:24s} {v:8.0f} MiB")
claim(f"服务器在 -c {CTX} 下的 KV 缓冲区与每 token 64 KiB 的偏差在 2% 以内",
abs(mem["KV cache"] - kv_tok * CTX / 2**20) <= 0.02 * kv_tok * CTX / 2**20)
SERVER_MEM["Saluki"] = mem
props = api("/props")
p = props["default_generation_settings"]["params"]
g = {k: kv[f"general.sampling.{k}"] for k in ("temp", "top_k", "top_p", "min_p")}
from_file = abs(p["temperature"] - g["temp"]) < 1e-6 and p["top_k"] == g["top_k"] and abs(p["top_p"] - g["top_p"]) < 1e-6
print(f"\nserver defaults: temperature {p['temperature']:.2f}, top_k {p['top_k']}, top_p {p['top_p']:.2f}, min_p {p['min_p']:.2f}"
f" -> {'taken from the GGUF header' if from_file else 'llama.cpp built-ins, not the header'}; the card suggests 0.6 for thinking,"
f" so every request below states its own sampler")
claim("llama-server takes its default sampler from the GGUF's general.sampling keys", from_file)
caps = props.get("chat_template_caps", {})
print("template supports: " + ", ".join(k.replace("supports_", "") for k, v in caps.items() if v))
msgs = [{"role": "user", "content": "What is the weather in Oslo?"}]
tool = [{"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]
for label, kw in (("thinking off", {"enable_thinking": False}), ("thinking on, effort low", {"reasoning_effort": "low"}),
("thinking on, default", {})):
prompt = api("/apply-template", {"messages": msgs, "tools": tool, "chat_template_kwargs": kw})["prompt"]
n = len(api("/tokenize", {"content": prompt, "add_special": False})["tokens"])
print(f" {label:24s}: {n:3d} prompt tokens, ends {prompt[-28:]!r}")
return f"KV {kv_tok // 1024} KiB/token, state {rs / 2**20:.0f} MiB; server buffers {sum(mem[k] for k in ('weights', 'KV cache', 'recurrent state', 'compute')) / 1024:.1f} GiB"
SERVER_MEM = {}
architecture_and_server()
Qwen3.8-27B 是一个混合模型：其 64 个块中有 48 个是带固定循环状态的 Gated DeltaNet 线性注意力层，只有每四个块保留完整注意力。因此 KV 缓存每 token 只需 64 KiB 而非 256 KiB，另外每条序列约有 150 MiB 的状态。我们用单个槽位启动 llama-server，从日志中读取其缓冲区，检查其默认采样器来自何处，并查看聊天模板在关闭思考、开启思考以及低推理强度下构建的提示词。
Copy CodeCopiedUse a different Browser
SPEED = {}
def speed_probe(tag):
来自服务器自身计时的预填充与解码速率：两种提示长度，强制输出 128 个 token。
rng = random.Random(SEED)
words = "def class return import yield lambda tensor gradient matrix kernel cache token buffer layer".split()
rows = []
for target in (256, 1024):
text = " ".join(rng.choice(words) for _ in range(target))
ids = api("/tokenize", {"content": text, "add_special": False})["tokens"][:target]
r = api("/completion", {"prompt": ids, "n_predict": 128, "ignore_eos": True, "cache_prompt": False, **GREEDY})
t = r["timings"]
rows.append((t["prompt_n"], t["prompt_per_second"], t["predicted_n"], t["predicted_per_second"]))
print(f" {tag:7s} prompt {t['prompt_n']:5d} tok at {t['prompt_per_second']:7.1f} tok/s | "
f"decode {t['predicted_n']} tok at {t['predicted_per_second']:5.2f} tok/s")
SPEED[tag] = rows
return rows
@section("6. 由服务器测量的预填充与解码速度")
def speed_step():
print(f"hardware: {HW}")
rows = speed_probe("Saluki")
return f"Saluki on {HW}: prefill {rows[1][1]:.0f} tok/s at {rows[1][0]} tokens, decode {rows[1][3]:.1f} tok/s"
speed_step()
速度来自服务器自身的计时，而不是在请求外围掐秒表。我们发送一个 256 token 和一个 1,024 token 的提示，关闭提示缓存，并强制输出 128 个 token，从而得到每种长度下的预填充速率和解码速率。一个稠密的 27B 模型每生成一个 token 都要读取其完整的权重文件，因此解码速度主要取决于内存带宽；同样的探测稍后会在 ISTA 的文件上再跑一遍，以便进行同类对比。
复制代码已复制使用其他浏览器
def load_bfcl_checker():
"""BFCL 自带的检查器函数，从固定版本的源码中执行，其包导入已被替换。"""
src = lambda rel: open(os.path.join(WORK, "bfcl", rel.replace("/", "__"))).read()
def run(rel, ns, only=None):
tree = ast.parse(src(rel))
body = [n for n in tree.body if not isinstance(n, (ast.Import, ast.ImportFrom))]
if only:
body = [n for n in body if getattr(n, "name", None) in only]
exec(compile(ast.Module(body=body, type_ignores=[]), rel, "exec"), ns)
return ns
enums = run("constants/enums.py", {"Enum": enum.Enum})
tmap = run("constants/type_mappings.py", {})
model_cfg = collections.defaultdict(lambda: types.SimpleNamespace(underscore_to_dot=True)) # OpenAI 风格的 FC 模型
chk = run("eval_checker/ast_eval/ast_checker.py", {"re": re, "Language": enums["Language"], "MODEL_CONFIG_MAPPING": model_cfg,
"JAVA_TYPE_CONVERSION": tmap["JAVA_TYPE_CONVERSION"], "JS_TYPE_CONVERSION": tmap["JS_TYPE_CONVERSION"],
"java_type_converter": None, "js_type_converter": None})
utl = run("utils.py", {}, {"is_function_calling_format_output", "is_empty_output"})
conv = run("model_handler/utils.py", {"copy": copy, "re": re, "json": json, "ModelStyle": enums["ModelStyle"],
"GORILLA_TO_OPENAPI": tmap["GORILLA_TO_OPENAPI"]}, {"_cast_to_openai_type", "convert_to_tool"})
return types.SimpleNamespace(
ast_checker=lambda funcs, out, gt, cat: chk["ast_checker"](funcs, out, gt, enums["Language"].PYTHON, cat, "fc-model"),
is_empty=utl["is_empty_output"],
to_tools=lambda funcs: conv["convert_to_tool"](funcs, tmap["GORILLA_TO_OPENAPI"], enums["ModelStyle"].OPENAI_COMPLETIONS))
def load_tasks():
tasks = []
for kind, label in KINDS:
rows = [json.loads(l) for l in open(os.path.join(WORK, "bfcl", f"data__BFCL_v4_{kind}.json")) if l.strip()]
gt = {}
if kind != "irrelevance":
for l in open(os.path.join(WORK, "bfcl", f"data__possible_answer__BFCL_v4_{kind}.json")):
if l.strip():
d = json.loads(l); gt[d["id"]] = d["ground_truth"]
for r in sorted(random.Random(f"{SEED}-{kind}").sample(rows, N_PER_KIND), key=lambda r: int(r["id"].rsplit("_", 1)[1])):
tasks.append(dict(id=r["id"], kind=kind, messages=r["question"][0], functions=r["function"], gt=gt.get(r["id"])))
return tasks
TOOL_BLOCK = re.compile(r"<tool_call>(.*?)(?:</tool_call>|$)", re.S)
FUNC_XML = re.compile(r"<function=([^>\s]+)\s*>(.*?)(?:</function>|$)", re.S)
PARAM_XML = re.compile(r"<parameter=([^>\s]+)\s*>\n?(.*?)\n?</parameter>", re.S)
def lenient_calls(raw, functions):
"""对模型原始文本的宽松读取：任何 <function=...> 块，无论是否闭合，无论在 <tool_call> 内部还是外部。
值按函数自身的 schema（BFCL 类型）进行类型化。不属于 BFCL；仅用于 'lenient' 列。"""
raw = raw.split("</think>")[-1] # 开启思考时，推理内容位于 </think> 之前
types_ = {re.sub(r"\.", "_", f["name"]): {k: v.get("type", "string") for k, v in f["parameters"]["properties"].items()}
for f in functions} # 工具名发送时已把点号替换为下划线
calls = []
for fname, body in FUNC_XML.findall(raw):
fname = fname.strip().strip('"')
args = {}
for p, v in PARAM_XML.findall(body):
t = types_.get(re.sub(r"\.", "_", fname), {}).get(p, "string")
v = v.strip() if t != "string" else v
try:
args[p] = (int(v) if t == "integer" else float(v) if t in ("float", "number") else
v.lower() == "true" if t == "boolean" and v.lower() in ("true", "false") else
json.loads(v) if t in ("array", "tuple", "dict", "object") else v)
except ValueError:
try:
args[p] = ast.literal_eval(v)
except Exception:
args[p] = v
calls.append({fname: args})
return calls
def score(task, calls):
"""BFCL 的规则：irrelevance 类型在不发起调用时通过；其他类型走 BFCL 的 ast_checker。"""
if task["kind"] == "irrelevance":
return CHECK.is_empty(calls), "made a call" if calls else None
if not calls:
return False, "no call"
try:
r = CHECK.ast_checker(task["functions"], calls, task["gt"], task["kind"])
except Exception as e: # 检查器无法遍历的畸形参数
return False, f"checker_error:{type(e).__name__}"
return bool(r["valid"]), r.get("error_type")
def run_task(task, think=False, max_tokens=512):
body = {"messages": task["messages"], "tools": CHECK.to_tools(task["functions"]), "max_tokens": max_tokens,
"cache_prompt": False, "verbose": True, **GREEDY,
"chat_template_kwargs": {"reasoning_effort": "low"} if think else {"enable_thinking": False}}
log_pos = os.path.getsize(SERVER["log"])
t0 = time.time()
rec = dict(id=task["id"], kind=task["kind"], think=think)
try:
r = api("/v1/chat/completions", body)
msg = r["choices"][0]["message"]
rec.update(status=200, finish=r["choices"][0]["finish_reason"], raw=r["__verbose"]["content"],
prompt_n=r["timings"]["prompt_n"], out_n=r["timings"]["predicted_n"],
reasoning_chars=len(msg.get("reasoning_content") or ""))
rec["calls"] = [{tc["function"]["name"]: json.loads(tc["function"]["arguments"])} for tc in msg.get("tool_calls") or []]
except urllib.error.HTTPError as e: # stock llama.cpp 在其解析器无法读取回复时返回 500
time.sleep(0.5)
with open(SERVER["log"], errors="replace") as f:
f.seek(log_pos)
new = f.read()
m = re.findall(r"=== BEGIN ===\n(.*?)\n=== END ===", new, re.S)
rec.update(status=e.code, finish="parse error", raw=m[-1] if m else "", calls=None, prompt_n=None, out_n=None)
rec["seconds"] = time.time() - t0
rec["strict"], rec["strict_why"] = score(task, rec["calls"]) if rec["calls"] is not None else (False, "unparsed (HTTP 500)")
if rec["strict"]:
rec["lenient"] = True
else:
rec["lenient"], _ = score(task, lenient_calls(rec["raw"], task["functions"]))
return rec
def bench(tag, tasks):
out, t0 = [], time.time()
for i, task in enumerate(tasks):
out.append(run_task(task))
if (i + 1) % 30 == 0:
print(f" {tag}: {i + 1}/{len(tasks)} done, {time.time() - t0:.0f} s")
json.dump(out, open(os.path.join(WORK, "runs", f"bfcl_{tag}.json"), "w"), indent=1)
return out
def ci(k, n, alpha=0.05):
"""通过对二项分布尾部做二分法求 Clopper-Pearson 区间。"""
def cdf(x, p):
return sum(math.comb(n, i) * p ** i * (1 - p) ** (n - i) for i in range(x + 1))
def solve(f, target, rising):
lo, hi = 0.0, 1.0
for _ in range(60):
mid = (lo + hi) / 2
lo, hi = (mid, hi) if (f(mid) < target) == rising else (lo, mid)
return (lo + hi) / 2
lo = 0.0 if k == 0 else solve(lambda p: 1 - cdf(k - 1, p), alpha / 2, True)
hi = 1.0 if k == n else solve(lambda p: cdf(k, p), alpha / 2, False)
return lo, hi
def kind_table(tag, recs):
print(f"\n{tag}: {'kind':38s} {'strict':>8s} {'lenient':>8s} replies: structured / plain text / unparsed (500) / cut off")
for kind, label in KINDS:
rs = [r for r in recs if r["kind"] == kind]
st = collections.Counter("unparsed" if r["status"] != 200 else "cut off" if r["finish"] == "length"
else "structured" if r["calls"] else "plain text" for r in rs)
print(f" {label:44s} {sum(r['strict'] for r in rs):3d}/{len(rs):<4d} {sum(r['lenient'] for r in rs):3d}/{len(rs):<4d}"
f" {st['structured']:3d} / {st['plain text']:3d} / {st['unparsed']:3d} / {st['cut off']:3d}")
k, n = sum(r["strict"] for r in recs), len(recs)
lo, hi = ci(k, n)
print(f" {'total':44s} {k:3d}/{n:<4d} {sum(r['lenient'] for r in recs):3d}/{n:<4d} strict 95% CI {100 * lo:.0f}-{100 * hi:.0f}%")
@section("7. 在 120 个 BFCL v4 任务上测试工具调用，关闭思考：Saluki")
def bench_saluki():
global CHECK, TASKS, RUNS
CHECK = load_bfcl_checker()
TASKS = load_tasks()
print(f"sample: {N_PER_KIND} tasks per kind, drawn with seed {SEED}, {len(TASKS)} in all; scored with BFCL's ast_checker "
f"(gorilla@6ea5797); greedy, thinking off, max_tokens 512")
print(f"first ids per kind: " + ", ".join(t["id"] for t in TASKS[::N_PER_KIND]))
RUNS = {"Saluki": bench("Saluki", TASKS)}
kind_table("Saluki", RUNS["Saluki"])
sec = [r["seconds"] for r in RUNS["Saluki"]]
print(f"\nmedian {sorted(sec)[len(sec) // 2]:.1f} s per task on {HW}; median output {sorted(r['out_n'] or 0 for r in RUNS['Saluki'])[len(sec) // 2]} tokens")
again = [run_task(t) for t in TASKS[:: len(TASKS) // 5][:5]]
same = sum(a["calls"] == b["calls"] and a["raw"] == b["raw"] for a, b in zip(again, RUNS["Saluki"][:: len(TASKS) // 5][:5]))
print(f"greedy re-run of 5 tasks: {same} of 5 identical raw replies")
claim("greedy decoding with the prompt cache off gives the same reply on a re-run (5 tasks)", same == 5)
s = sum(r["strict"] for r in RUNS["Saluki"])
return f"Saluki {s}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['Saluki'])}/{len(TASKS)} lenient"
bench_saluki()
Underdog Bench’s 的 120 个任务并未公开，因此我们自己抽取：从同样的五个 BFCL v4 类别中各取 24 个，并使用固定随机种子。我们将每个任务以 OpenAI 工具请求的形式发送到服务器，使用 BFCL 自带的函数进行转换，关闭思考并采用贪心解码。严格评分只使用 llama.cpp 能解析的调用以及 BFCL 的 ast_checker，而宽松评分则用更宽容的解析器重新读取原始回复，我们正是以此测试厂商在严格与宽容评分之间的差距。
复制代码已复制使用其他浏览器
@section("8. 思考有帮助吗？六个多调用任务，开启推理，effort 低")
def thinking_step():
pick = [t for t in TASKS if t["kind"] in ("parallel", "parallel_multiple")]
pick = random.Random(SEED).sample(pick, min(6, len(pick)))
off = {r["id"]: r for r in RUNS["Saluki"]}
rows = []
for t in pick:
r = run_task(t, think=True, max_tokens=2048)
o = off[t["id"]]
rows.append((t["id"], o, r))
print(f" {t['id']:22s} off: {'pass' if o['strict'] else 'fail'} {o['out_n'] or 0:4d} tok {o['seconds']:5.1f} s | "
f"on: {'pass' if r['strict'] else 'fail'} {r['out_n'] or 0:4d} tok {r['seconds']:6.1f} s ({r['finish']})")
p_off, p_on = sum(o["strict"] for _, o, _ in rows), sum(r["strict"] for _, _, r in rows)
tok = sum(r["out_n"] or 0 for *_, r in rows) / max(1, sum(o["out_n"] or 0 for _, o, _ in rows))
sec = sum(r["seconds"] for *_, r in rows) / sum(o["seconds"] for _, o, _ in rows)
print(f"\nthinking off {p_off}/{len(rows)}, thinking on {p_on}/{len(rows)}; thinking used {tok:.1f}x the output tokens and {sec:.1f}x the time")
return f"thinking on (effort low): {p_on}/{len(rows)} vs {p_off}/{len(rows)} off, {tok:.1f}x tokens, {sec:.1f}x time"
thinking_step()
开启思考是卡片针对通用场景的建议，但在 27B 上每个推理 token 都要花时间。我们在最低 effort 下开启推理，重新运行六个带种子的多调用任务，并与关闭思考运行的相同任务进行对比：通过或失败、输出 token 数和秒数。六个任务无法确定思考是否能改善工具调用，因此该单元格聚焦于成本，并展示模型先思考时单次工具调用要多花多少 token。
复制代码已复制请使用其他浏览器
@section("9. 父文件：ISTA 的 IQ2_XS 通过同一服务器、同一样本和同一检查器")
def bench_ista():
load_s = start_server("ISTA")
mem = memory_report()
SERVER_MEM["ISTA"] = mem
print(f"llama-server up in {load_s:.0f} s; " + ", ".join(f"{k} {v:.0f} MiB" for k, v in mem.items()))
print(f"weights: ISTA {mem['weights']:.0f} MiB vs Saluki {SERVER_MEM['Saluki']['weights']:.0f} MiB")
speed_probe("ISTA")
s, i = SPEED["Saluki"][1], SPEED["ISTA"][1]
print(f"decode at {s[0]} prompt tokens: Saluki {s[3]:.2f} vs ISTA {i[3]:.2f} tok/s; prefill {s[1]:.0f} vs {i[1]:.0f} tok/s")
RUNS["ISTA"] = bench("ISTA", TASKS)
kind_table("ISTA", RUNS["ISTA"])
stop_server()
k = sum(r["strict"] for r in RUNS["ISTA"])
return f"ISTA IQ2_XS {k}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['ISTA'])}/{len(TASKS)} lenient"
bench_ista()
父文件遵循相同的路径。我们停止 Saluki 的服务器，在 ISTA 的 IQ2_XS 文件上用相同的参数启动一个服务器，读取其内存，重复速度探测，并用相同的检查器和解析规则在相同的 120 个任务上评分。一次运行一个模型可以让每个模型都远在 GPU 内存范围内，而对两者使用同一个测试框架可以消除提示、采样或解析方面的差异。
复制代码已复制使用其他浏览器
def mcnemar(a, b):
"""对配对的通过/失败列表进行精确的双侧 McNemar 检验。"""
b10 = sum(x and not y for x, y in zip(a, b))
b01 = sum(y and not x for x, y in zip(a, b))
n, k = b10 + b01, min(b10, b01)
p = min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.0
return b10, b01, p
@section("10. 正面交锋：相同的 120 个任务，配对比较，与厂商数字并列")
def head_to_head():
S, I = RUNS["Saluki"], RUNS["ISTA"]
assert [r["id"] for r in S] == [r["id"] for r in I]
vendor = {"simple_python": 24, "multiple": 23, "irrelevance": 18, "parallel": 12, "parallel_multiple": 11}
print(f"{'kind':38s} {'Saluki':>7s} {'ISTA':>6s} | {'lenient':>7s} {'':>5s} | vendor's Saluki (of 24)")
for kind, label in KINDS:
s = [r for r in S if r["kind"] == kind]; i = [r for r in I if r["kind"] == kind]
print(f"{label:38s} {sum(r['strict'] for r in s):7d} {sum(r['strict'] for r in i):6d} | "
f"{sum(r['lenient'] for r in s):7d} {sum(r['lenient'] for r in i):5d} | {vendor[kind]:3d}")
out = {}
for mode in ("strict", "lenient"):
a, b = [r[mode] for r in S], [r[mode] for r in I]
b10, b01, p = mcnemar(a, b)
out[mode] = (sum(a), sum(b), b10, b01, p)
print(f"{mode:8s}: Saluki {sum(a)}/120, ISTA {sum(b)}/120; 仅 Saluki 通过 {b10}, 仅 ISTA 通过 {b01}, 精确 McNemar p = {p:.3f}")
print("厂商的 Underdog Bench（不同的 120 个任务，未公开）：Saluki 88, ISTA IQ2_XS 76, 完整 Qwen3.8-27B 84")
sa, ia, b10, b01, p = out["strict"]
claim("Saluki 在我们的 120 个任务中比其父模型通过更多（严格评分）", sa > ia)
claim("差异在 5% 水平上显著（精确 McNemar）", p < 0.05)
pick_s = sum(r["strict"] for r in S if r["kind"] in ("simple_python", "multiple"))
pick_i = sum(r["strict"] for r in I if r["kind"] in ("simple_python", "multiple"))
claim("Saluki 在挑选正确的单个函数（前两种类型）上至少与其父模型一样好", pick_s >= pick_i)
return f"strict: Saluki {sa}/120 vs ISTA {ia}/120 (McNemar p = {p:.2f}); lenient: {out['lenient'][0]} vs {out['lenient'][1]}"
head_to_head()
由于两个模型回答的是同一批任务，因此比较是配对的。该单元格会在 Saluki 的厂商分项数字旁打印我们的分项得分，并对两个文件结果不一致的任务分别按严格和宽松评分运行精确 McNemar 检验。120 个任务中相差几个任务属于运行间噪声，厂商页面本身也承认这一点，因此该检验告诉我们样本是否显示出真实差异。
复制代码已复制使用其他浏览器
@section("11. 失败从何而来")
def failures():
for tag in ("Saluki", "ISTA"):
recs = RUNS[tag]
why = collections.Counter((r["strict_why"] or "?").split(":")[-1] for r in recs if not r["strict"])
print(f"{tag}: {sum(not r['strict'] for r in recs)} failures -> " + ", ".join(f"{k} {v}" for k, v in why.most_common()))
rescued = [r for r in recs if r["lenient"] and not r["strict"]]
print(f" lenient reading rescues {len(rescued)}: " + ", ".join(r["id"] for r in rescued[:8]))
ex = next((r for r in RUNS["Saluki"] if not r["strict"] and r["kind"] != "irrelevance"), None)
if ex:
task = next(t for t in TASKS if t["id"] == ex["id"])
print(f"\nexample, {ex['id']} ({ex['strict_why']}):")
print(" expected:", json.dumps(task["gt"])[:300])
print(" model :", (json.dumps(ex["calls"]) if ex["calls"] is not None else ex["raw"])[:300])
irr = [r for r in RUNS["Saluki"] if r["kind"] == "irrelevance" and not r["strict"]]
if irr:
print(f"\nirrelevance: Saluki made a call on {len(irr)} of {N_PER_KIND} prompts that no tool fits, e.g. {irr[0]['id']}: "
f"{json.dumps(irr[0]['calls'])[:160]}")
un = sum(r["status"] != 200 for tag in RUNS for r in RUNS[tag])
return f"{un} replies across both models were unparseable by the stock server"
failures()
分数掩盖了调用出错的位置，因此这个单元格按 BFCL 自身的错误类型拆解失败情况：调用次数错误、参数错误或缺失、在没有合适工具时发起调用，以及标准服务器无法解析的回复。它还列出了宽松解读所挽救的任务，并打印一个失败任务及其预期答案，这是区分模型错误与评分假象最快的方法。
复制代码已复制使用其他浏览器
banner("SUMMARY")
for name, res in RESULTS.items():
print(f" {name:<100s} {res}")
print("\n本次运行中判定的主张：")
for step, text, held in CLAIMS:
print(f" [{'held' if held else 'DID NOT HOLD' if held is False else 'undecided'}] step {step}: {text}")
failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")]
print(f"\n{sum(h is True for *_, h in CLAIMS)} of {len(CLAIMS)} claims held; steps failed or skipped: {failed or 'none'}")
print(f"run on: {HW}; llama.cpp {LLAMA_TAG}; Python {platform.python_version()}")
print("此处未测试：完整规模的 Qwen3.8-27B；Underdog Bench 自有的 120 项任务和 100 项并行集（未公开）；"
"SWE-bench、WebWalkerQA、IFEval 和 AIME；视觉附加组件；Mia 和 Ternary Bonsai 2；默认努力程度下的思考。")
摘要会打印每一步的一行结果和主张台账（成立、不成立或未定），以及硬件和 llama.cpp 版本。结尾部分说明该 notebook 未测试的内容：完整规模的 Qwen3.8-27B、厂商自有的任务列表、智能体与数学基准测试，以及视觉附加组件。
总之，Saluki 并不完全像其文件所暗示的那样。它的文件头保留了 ISTA 的 851 张量布局，只在 ISTA 花费更多的地方节省比特，但其字节和未量化向量显示出一个权重已被更改的模型，这与 Underdog 称其为预览检查点相符。这使得与其父模型的工具调用比较成为对该训练的测试，而不仅仅是量化。两个文件都在原版 llama.cpp 中运行，一次一个，BFCL 自带的检查器在相同任务上给出配对比较。小样本、贪婪解码和单次运行限制了任何单个 notebook 能得出的结论，因此置信区间与分数同样重要。
在此查看完整代码。所有功劳归于该项目的研究者。也欢迎在 Twitter 上关注我们，别忘了加入我们 150k+ 的 ML SubReddit 并订阅我们的新闻通讯。等等！你在 Telegram 上吗？现在你也可以在 Telegram 上加入我们。
需要与我们合作推广你的 GitHub 仓库、Hugging Face 页面、产品发布或网络研讨会等？联系我们
文章《How to Run Underdog Saluki 27B: 2-Bit Qwen3.8-27B Tool-Calling Guide》首发于 MarkTechPost。