R-ORD GOLD-ONLY arbitrary-cut
california_schools · dev_20251106-00000-of-00001 from https://huggingface.co/datasets/birdsql/bird_sql_dev_20251106/resolve/3c11fb193e5439b338e23677fa0aae11e8b85db9/data/dev_20251106-00000-of-00001.json (commit 3c11fb19, downloaded 2026-09-07)
Which cities have the top 5 lowest enrollment number for students in grades 1 through 12?
the hint the set supplies: K-12 refers to students in grades 1 through 12.
This question was audited without a prediction beside it, so there is nothing to compare the gold with. The probes below read the gold alone.
read by hand, 2026-09-07: wrong the gold does not answer its question on this data
three cities tie at positions 3 to 5 of 1,022 with three different names; a LIMIT 5 returns two of the three and the data does not say which
a maintainer's reading of this question, out of classification.json, copied from plans/reports/bird-dev-sqlite-260907/classification.json. It is not a verdict and nothing above it was computed from it.
SELECT T2.City FROM frpm AS T1 INNER JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode GROUP BY T2.City ORDER BY SUM(T1.`Enrollment (K-12)`) ASC LIMIT 5sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0
from evidence-gold.json, 5 rows
| CityTEXT |
|---|
| Coulterville |
| Pinecrest |
| Shaver Lake |
| Emigrant Gap |
| Hyampom |
A smell is a mechanical reason to read this gold statement again. It is a heuristic: it does not state that the statement is wrong, and a maintainer decides.
this statement orders by a text column holding only numbers, and ordering it as a number gives a different answer, so the gold may be sorting 9.5 above 10
no ORDER BY key resolves to a text column
{
"heuristic": true,
"reason": "no ORDER BY key resolves to a text column",
"keys": [
{
"key": "SUM(T1.\"Enrollment (K-12)\")",
"not_applicable": "the key is an expression and not a column"
}
]
}this statement cuts its result at a LIMIT that does not decide which rows come back, so a different but equally correct statement can return other rows and score zero
from smells.json, 3 rows
| Emigrant Gap | 5.0 |
| Hyampom | 5.0 |
| Woody | 5.0 |
{
"heuristic": true,
"cut": 5,
"offset": 0,
"distinct_kept": false,
"unbounded_sql": "SELECT T2.City, SUM(T1.\"Enrollment (K-12)\") AS attestql_ordering_key_0 FROM frpm AS T1 INNER JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode GROUP BY T2.City ORDER BY SUM(T1.\"Enrollment (K-12)\") ASC",
"unbounded_rows": 1022,
"projected_columns": [
"City"
],
"ordering_key_columns": [
"attestql_ordering_key_0"
],
"ordering_keys": [
{
"key": "SUM(T1.\"Enrollment (K-12)\")",
"direction": "asc",
"nulls": "first",
"nulls_first_in_effect": true,
"returned_rows_null_in_this_key": 0,
"fires": false
}
],
"tied_at_the_cut": {
"positions": [
3,
4,
5
],
"tied_rows": 3,
"distinct_projected_answers": 3,
"rows": [
[
{
"type": "str",
"value": "Emigrant Gap"
}
],
[
{
"type": "str",
"value": "Hyampom"
}
],
[
{
"type": "str",
"value": "Woody"
}
]
]
},
"case": "tie-at-the-cut"
}rerun over the same rows in another physical order this statement gives another answer, so its result depends on how the rows are stored and not only on the data
{
"heuristic": true,
"rule": "R-ORD",
"baseline_result_hash": "sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0",
"baseline_result": {
"columns": [
{
"name": "City",
"declared_type": "TEXT"
}
],
"row_count": 5,
"truncated": false,
"rows_shown": 5,
"rows": [
[
{
"type": "str",
"value": "Coulterville"
}
],
[
{
"type": "str",
"value": "Pinecrest"
}
],
[
{
"type": "str",
"value": "Shaver Lake"
}
],
[
{
"type": "str",
"value": "Emigrant Gap"
}
],
[
{
"type": "str",
"value": "Hyampom"
}
]
],
"result_hash": "sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0"
},
"planner_statistics": {},
"shuffle": {
"seed": "1",
"row_limit": 300000,
"tables": [
"frpm",
"schools"
],
"tables_not_shuffled": [],
"tables_skipped_for_size": {},
"tables_not_reached_by_a_copy": {}
},
"shuffled_copies": {
"run": true,
"verdict": "equal",
"differs": false,
"result_hash": "sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0",
"result": {
"columns": [
{
"name": "City",
"declared_type": "TEXT"
}
],
"row_count": 5,
"truncated": false,
"rows_shown": 5,
"rows": [
[
{
"type": "str",
"value": "Coulterville"
}
],
[
{
"type": "str",
"value": "Pinecrest"
}
],
[
{
"type": "str",
"value": "Shaver Lake"
}
],
[
{
"type": "str",
"value": "Emigrant Gap"
}
],
[
{
"type": "str",
"value": "Hyampom"
}
]
],
"result_hash": "sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0"
}
},
"plan_variant": {
"run": false,
"reason": "the plan variant was not asked for"
}
}SELECT T2.City FROM frpm AS T1 INNER JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode GROUP BY T2.City ORDER BY SUM(T1.`Enrollment (K-12)`) ASC LIMIT 5result_hash sha256:0a671f4d2d74ce3a8a569d7d96e996cdc4da39228c1775865f1fea1cc68a50b0 recomputed from this JSON: match
record_hash sha256:a4a2003773051bff2212ac7d6c8faac7ae686fa86333cf7afec5bd3ac211535c recomputed from this JSON: match
from evidence-gold.json, 5 rows
| CityTEXT |
|---|
| Coulterville |
| Pinecrest |
| Shaver Lake |
| Emigrant Gap |
| Hyampom |
re-run this statement read-only against SQLite 3.53.4 | file=/private/tmp/attestql-runs/data/dev/dev_databases/california_schools/california_schools.sqlite | size=11116544 under the session settings and over the data this record's fixture digest names, and compare the two results under R-ORD