{ "format": "attestql/audit/summary/2", "run_id": "audit-5cc2ec0f-cd89-4d64-b7f5-90a23137ee39", "backend_identity": "SQLite 3.53.4 | file=/private/tmp/attestql-runs/data/minidev/dev_databases/european_football_2/european_football_2.sqlite | size=597754880", "effective_database_role": "file", "session_settings": { "engine": "sqlite", "recorded": { "sqlite_version": "3.53.4", "encoding": "UTF-8", "reverse_unordered_selects": "0", "query_only": "1", "journal_mode": "delete", "data_version": "2", "compile_options": "ATOMIC_INTRINSICS=1,COMPILER=clang-21.0.0,DEFAULT_AUTOVACUUM,DEFAULT_CACHE_SIZE=-2000,DEFAULT_FILE_FORMAT=4,DEFAULT_JOURNAL_SIZE_LIMIT=-1,DEFAULT_MMAP_SIZE=0,DEFAULT_PAGE_SIZE=4096,DEFAULT_PCACHE_INITSZ=20,DEFAULT_RECURSIVE_TRIGGERS,DEFAULT_SECTOR_SIZE=4096,DEFAULT_SYNCHRONOUS=2,DEFAULT_WAL_AUTOCHECKPOINT=1000,DEFAULT_WAL_SYNCHRONOUS=2,DEFAULT_WORKER_THREADS=0,DIRECT_OVERFLOW_READ,ENABLE_API_ARMOR,ENABLE_COLUMN_METADATA,ENABLE_DBSTAT_VTAB,ENABLE_FTS3,ENABLE_FTS3_PARENTHESIS,ENABLE_FTS5,ENABLE_GEOPOLY,ENABLE_MATH_FUNCTIONS,ENABLE_MEMORY_MANAGEMENT,ENABLE_PERCENTILE,ENABLE_PREUPDATE_HOOK,ENABLE_RTREE,ENABLE_SESSION,ENABLE_STAT4,ENABLE_UNLOCK_NOTIFY,MALLOC_SOFT_LIMIT=1024,MAX_ATTACHED=10,MAX_COLUMN=2000,MAX_COMPOUND_SELECT=500,MAX_DEFAULT_PAGE_SIZE=8192,MAX_EXPR_DEPTH=1000,MAX_FUNCTION_ARG=1000,MAX_LENGTH=1000000000,MAX_LIKE_PATTERN_LENGTH=50000,MAX_MMAP_SIZE=0x7fff0000,MAX_PAGE_COUNT=0xfffffffe,MAX_PAGE_SIZE=65536,MAX_SQL_LENGTH=1000000000,MAX_TRIGGER_DEPTH=1000,MAX_VARIABLE_NUMBER=250000,MAX_VDBE_OP=250000000,MAX_WORKER_THREADS=8,MUTEX_PTHREADS,SYSTEM_MALLOC,TEMP_STORE=1,THREADSAFE=1,USE_URI", "collation_list": "RTRIM,NOCASE,BINARY", "case_sensitive_like": "0" } }, "parser": { "validator": "audit:sqlglot-sqlite-parse", "sqlglot": "30.18.0", "dialect": "sqlite" }, "question_set": { "path": "data/questions/mini_dev_sqlite.json", "digest": "sha256:4ba5fa8de55856222f484d380d2ba872b380bf79d825de70478e2120cb0fc43b", "origin": "https://bird-bench.oss-cn-beijing.aliyuncs.com/minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f, downloaded 2026-09-07), member minidev/MINIDEV/mini_dev_sqlite.json", "date": "2024-06-19", "entries": 500, "audited": 51, "duplicate_ids": [ 137, 138 ], "ids": [ 1025, 1028, 1029, 1030, 1031, 1032, 1035, 1036, 1037, 1039, 1040, 1042, 1044, 1048, 1057, 1058, 1068, 1076, 1078, 1079, 1080, 1084, 1088, 1091, 1092, 1094, 1096, 1098, 1102, 1103, 1105, 1107, 1110, 1113, 1114, 1115, 1116, 1122, 1124, 1130, 1133, 1134, 1135, 1136, 1139, 1141, 1144, 1145, 1146, 1147, 1148 ] }, "predictions": { "path": "data/preds-sqlite/predict_mini_dev_meta-llama-3-8b-instruct-2_sqlite.json", "digest": "sha256:e1bd066794d120ff766d010a5cf4e4118e48404c4b0168fce4dfaf362a975301", "origin": "https://raw.githubusercontent.com/bird-bench/mini_dev/b3d4bcbbae9a96934ad812551eb400c7a3b23c12/llm/exp_result/sql_output_kg/predict_mini_dev_meta-llama-3-8b-instruct-2_sqlite.json", "date": "2024-06-19", "keyed_by": "position", "statements": 500, "positions_unused": [ 487, 488 ] }, "verdicts": { "ERROR": 21, "NOT_EQUAL": 19, "EQUAL": 11 }, "smells": { "ordering-over-numeric-text": 0, "arbitrary-cut": 2, "not-a-function-of-the-data": 2, "float-aggregate-order": 0, "direction-against-question": 0 }, "smells_fired": 4, "credited_but_not_equal": { "total": 6, "by_mechanism": { "multiplicity": 6, "type": 0, "order": 0, "truncation": 0, "other": 0 }, "by_test_suite_ex": { "1": 0, "0": 6 } }, "fixture": { "depth": "counts", "cache": "runs/minidev-sqlite/meta-llama-3-8b-instruct-2/european_football_2/fixture.json", "source": { "path": "data/minidev/dev_databases/european_football_2/european_football_2.sqlite", "digest": "sha256:f72f5c9b990371417288a67b27a02e601ef904d45525c2c9f7931fb01001c536", "origin": "minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f), member minidev/MINIDEV/dev_databases, european_football_2/european_football_2.sqlite", "date": "2024-06-13" }, "schema_digest": "sha256:954a8a8d6524994d3b40baa9fb2a91c922f170e93b000994be908ae47378e5e7", "row_counts": { "main.Country": 11, "main.League": 11, "main.Match": 25979, "main.Player": 11060, "main.Player_Attributes": 183978, "main.Team": 299, "main.Team_Attributes": 1458 }, "content_digests": {}, "measured_tables": [ "Match", "League", "Team", "Team_Attributes", "Player_Attributes", "Player", "Country" ], "missing_tables": [], "unreadable_tables": [], "refused": "" }, "planner_statistics": {}, "shuffle": { "seed": 1, "row_limit": 300000, "scratch_schema": "temp", "prepared": true, "reason": "", "copied": [ "Country", "League", "Match", "Player", "Player_Attributes", "Team", "Team_Attributes" ], "skipped": {}, "not_reached_by_a_copy": {} }, "settings": { "statement_timeout_seconds": 30, "fail_on_smell": false, "experimental_s2": false, "plan_variant": false, "out": "runs/minidev-sqlite/meta-llama-3-8b-instruct-2/european_football_2", "serialization": "attestql/audit/2", "work_mem": null, "hash_mem_multiplier": null }, "data_as_of": "2026-09-08T05:04:10.210541+00:00", "data_as_of_source": "the instant the run started", "elapsed_seconds": { "fixture": 0.014, "questions": 16.218, "shuffle": 2.832 }, "errors": [ { "question_id": 1025, "side": "prediction", "step": "statement", "message": "the text does not parse: The number of provided arguments (2) is greater than the maximum number of supported arguments (1). Line 5, Col: 48. m ON l.id = m.league_id WHERE m.season = '2015/2016' ORDER BY SUM(m.home_team_goal, m.away_team_goal\u001b[4m)\u001b[0m DESC LIMIT 1;" }, { "question_id": 1029, "side": "prediction", "step": "execute", "message": "execute: no such column: t1.team_short_name" }, { "question_id": 1031, "side": "prediction", "step": "statement", "message": "the text does not parse: Expected table name but got . Line 108, Col: 24. UNION SELECT away_player_X4 \u001b[4mFROM\u001b[0m" }, { "question_id": 1037, "side": "prediction", "step": "execute", "message": "execute: no such function: MULTIPLY" }, { "question_id": 1039, "side": "prediction", "step": "execute", "message": "execute: no such column: player_name" }, { "question_id": 1068, "side": "prediction", "step": "execute", "message": "execute: no such column: t2.height" }, { "question_id": 1076, "side": "prediction", "step": "execute", "message": "execute: no such column: player_name" }, { "question_id": 1078, "side": "prediction", "step": "statement", "message": "the text does not parse: Error tokenizing 'ennon' ORDER BY birthday DESC LIMIT 1; SELECT 'A'" }, { "question_id": 1084, "side": "prediction", "step": "execute", "message": "execute: no such column: birthday" }, { "question_id": 1091, "side": "prediction", "step": "execute", "message": "execute: no such column: League.name" }, { "question_id": 1092, "side": "prediction", "step": "execute", "message": "execute: no such column: season" }, { "question_id": 1094, "side": "prediction", "step": "execute", "message": "execute: no such function: SUBTRACT" }, { "question_id": 1096, "side": "prediction", "step": "execute", "message": "execute: no such column: T1.player_name" }, { "question_id": 1103, "side": "prediction", "step": "execute", "message": "execute: no such column: player_name" }, { "question_id": 1105, "side": "prediction", "step": "execute", "message": "execute: no such column: pa.player_name" }, { "question_id": 1107, "side": "prediction", "step": "execute", "message": "execute: no such column: player_name" }, { "question_id": 1110, "side": "prediction", "step": "execute", "message": "execute: no such column: team_long_name" }, { "question_id": 1114, "side": "prediction", "step": "execute", "message": "execute: no such column: player_name" }, { "question_id": 1115, "side": "prediction", "step": "statement", "message": "the text does not parse: Expecting ). Line 1, Col: 55. SELECT (player_name = 'Landon Donovan' overall_rating \u001b[4m-\u001b[0m player_name = 'Jordan Bowery' overall_rating) / player_name = 'Landon Donovan' overall_rating * 100" }, { "question_id": 1116, "side": "prediction", "step": "execute", "message": "execute: no such column: player_id" }, { "question_id": 1148, "side": "prediction", "step": "execute", "message": "execute: no such column: pa.height" } ], "timed_out": { "gold": [], "prediction": [] }, "exit_status": 1 }