{ "format": "attestql/audit/summary/2", "run_id": "audit-49215ef1-ae35-4207-99a0-4538f9872bc4", "backend_identity": "SQLite 3.53.4 | file=/private/tmp/attestql-runs/data/minidev/dev_databases/codebase_community/codebase_community.sqlite | size=481419264", "effective_database_role": "file", "session_settings": { "engine": "sqlite", "recorded": { "sqlite_version": "3.53.4", "encoding": "UTF-8", "reverse_unordered_selects": "0", "query_only": "1", "journal_mode": "delete", "data_version": "1", "compile_options": "ATOMIC_INTRINSICS=1,COMPILER=clang-21.0.0,DEFAULT_AUTOVACUUM,DEFAULT_CACHE_SIZE=-2000,DEFAULT_FILE_FORMAT=4,DEFAULT_JOURNAL_SIZE_LIMIT=-1,DEFAULT_MMAP_SIZE=0,DEFAULT_PAGE_SIZE=4096,DEFAULT_PCACHE_INITSZ=20,DEFAULT_RECURSIVE_TRIGGERS,DEFAULT_SECTOR_SIZE=4096,DEFAULT_SYNCHRONOUS=2,DEFAULT_WAL_AUTOCHECKPOINT=1000,DEFAULT_WAL_SYNCHRONOUS=2,DEFAULT_WORKER_THREADS=0,DIRECT_OVERFLOW_READ,ENABLE_API_ARMOR,ENABLE_COLUMN_METADATA,ENABLE_DBSTAT_VTAB,ENABLE_FTS3,ENABLE_FTS3_PARENTHESIS,ENABLE_FTS5,ENABLE_GEOPOLY,ENABLE_MATH_FUNCTIONS,ENABLE_MEMORY_MANAGEMENT,ENABLE_PERCENTILE,ENABLE_PREUPDATE_HOOK,ENABLE_RTREE,ENABLE_SESSION,ENABLE_STAT4,ENABLE_UNLOCK_NOTIFY,MALLOC_SOFT_LIMIT=1024,MAX_ATTACHED=10,MAX_COLUMN=2000,MAX_COMPOUND_SELECT=500,MAX_DEFAULT_PAGE_SIZE=8192,MAX_EXPR_DEPTH=1000,MAX_FUNCTION_ARG=1000,MAX_LENGTH=1000000000,MAX_LIKE_PATTERN_LENGTH=50000,MAX_MMAP_SIZE=0x7fff0000,MAX_PAGE_COUNT=0xfffffffe,MAX_PAGE_SIZE=65536,MAX_SQL_LENGTH=1000000000,MAX_TRIGGER_DEPTH=1000,MAX_VARIABLE_NUMBER=250000,MAX_VDBE_OP=250000000,MAX_WORKER_THREADS=8,MUTEX_PTHREADS,SYSTEM_MALLOC,TEMP_STORE=1,THREADSAFE=1,USE_URI", "collation_list": "RTRIM,NOCASE,BINARY", "case_sensitive_like": "0" } }, "parser": { "validator": "audit:sqlglot-sqlite-parse", "sqlglot": "30.18.0", "dialect": "sqlite" }, "question_set": { "path": "data/questions/mini_dev_sqlite.json", "digest": "sha256:4ba5fa8de55856222f484d380d2ba872b380bf79d825de70478e2120cb0fc43b", "origin": "https://bird-bench.oss-cn-beijing.aliyuncs.com/minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f, downloaded 2026-09-07), member minidev/MINIDEV/mini_dev_sqlite.json", "date": "2024-06-19", "entries": 500, "audited": 49, "duplicate_ids": [ 137, 138 ], "ids": [ 531, 532, 533, 539, 537, 544, 547, 549, 555, 557, 563, 565, 567, 568, 571, 572, 573, 576, 578, 581, 584, 586, 587, 592, 595, 598, 604, 629, 633, 634, 637, 639, 640, 665, 669, 671, 672, 678, 682, 683, 685, 687, 694, 701, 704, 705, 707, 710, 716 ] }, "predictions": { "path": "data/preds-sqlite/predict_mini_dev_gpt-35-turbo-instruct_sqlite.json", "digest": "sha256:a4a5f5e12d19d3fa742d0f3111723bd1dca50b29887eb705438580efe6dcc04b", "origin": "https://raw.githubusercontent.com/bird-bench/mini_dev/b3d4bcbbae9a96934ad812551eb400c7a3b23c12/llm/exp_result/sql_output_kg/predict_mini_dev_gpt-35-turbo-instruct_sqlite.json", "date": "2024-06-19", "keyed_by": "position", "statements": 500, "positions_unused": [ 487, 488 ] }, "verdicts": { "NOT_EQUAL": 19, "EQUAL": 17, "ERROR": 13 }, "smells": { "ordering-over-numeric-text": 0, "arbitrary-cut": 1, "not-a-function-of-the-data": 2, "float-aggregate-order": 0, "direction-against-question": 0 }, "smells_fired": 3, "credited_but_not_equal": { "total": 0, "by_mechanism": { "multiplicity": 0, "type": 0, "order": 0, "truncation": 0, "other": 0 }, "by_test_suite_ex": { "1": 0, "0": 0 } }, "fixture": { "depth": "counts", "cache": "runs/minidev-sqlite/gpt-35-turbo-instruct/codebase_community/fixture.json", "source": { "path": "data/minidev/dev_databases/codebase_community/codebase_community.sqlite", "digest": "sha256:92101be6d2a9f6adceea59d38f6d1c556087f9eea1432432a21268cb1348036d", "origin": "minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f), member minidev/MINIDEV/dev_databases, codebase_community/codebase_community.sqlite", "date": "2024-06-13" }, "schema_digest": "sha256:486946f86da61f185ded65fa40ebd1c1d34f6761a295222563857ffb491c6d5d", "row_counts": { "main.badges": 79851, "main.comments": 174285, "main.postHistory": 303155, "main.postLinks": 11102, "main.posts": 91966, "main.tags": 1032, "main.users": 40325, "main.votes": 38930 }, "content_digests": {}, "measured_tables": [ "users", "posts", "tags", "comments", "votes", "postHistory", "badges", "postLinks" ], "missing_tables": [], "unreadable_tables": [], "refused": "" }, "planner_statistics": {}, "shuffle": { "seed": 1, "row_limit": 300000, "scratch_schema": "temp", "prepared": true, "reason": "", "copied": [ "badges", "comments", "postLinks", "posts", "tags", "users", "votes" ], "skipped": { "postHistory": 303155 }, "not_reached_by_a_copy": {} }, "settings": { "statement_timeout_seconds": 30, "fail_on_smell": false, "experimental_s2": false, "plan_variant": false, "out": "runs/minidev-sqlite/gpt-35-turbo-instruct/codebase_community", "serialization": "attestql/audit/2", "work_mem": null, "hash_mem_multiplier": null }, "data_as_of": "2026-09-08T05:00:01.024913+00:00", "data_as_of_source": "the instant the run started", "elapsed_seconds": { "fixture": 0.021, "questions": 14.561, "shuffle": 1.92 }, "errors": [ { "question_id": 587, "side": "prediction", "step": "execute", "message": "execute: no such column: Text" }, { "question_id": 592, "side": "prediction", "step": "execute", "message": "execute: misuse of aggregate: Count()" }, { "question_id": 598, "side": "prediction", "step": "execute", "message": "execute: no such function: Year" }, { "question_id": 604, "side": "prediction", "step": "execute", "message": "execute: no such column: UserId" }, { "question_id": 634, "side": "prediction", "step": "execute", "message": "execute: no such column: DisplayName" }, { "question_id": 639, "side": "prediction", "step": "execute", "message": "execute: no such column: PostId" }, { "question_id": 640, "side": "prediction", "step": "statement", "message": "the text does not parse: Invalid expression / Unexpected token. Line 1, Col: 93. SELECT SUM(ViewCount) FROM posts WHERE OwnerDisplayName = 'Mornington' - SUM(ViewCount) \u001b[4mFROM\u001b[0m posts WHERE OwnerDisplayName = 'Amos'" }, { "question_id": 665, "side": "prediction", "step": "execute", "message": "execute: no such function: YEAR" }, { "question_id": 682, "side": "prediction", "step": "execute", "message": "execute: no such column: CreationDate" }, { "question_id": 685, "side": "prediction", "step": "execute", "message": "execute: no such column: Text" }, { "question_id": 701, "side": "prediction", "step": "statement", "message": "the text does not parse: Expecting ). Line 1, Col: 39. SELECT 100 * COUNT(stats_posts.Id \u001b[4mwhere\u001b[0m Score > 50 and MAX(Reputation)) / COUNT(stats_posts.Id where MAX(Reputation)) FROM posts INNER JOIN" }, { "question_id": 707, "side": "prediction", "step": "execute", "message": "execute: no such column: Text" }, { "question_id": 716, "side": "prediction", "step": "execute", "message": "execute: no such column: UpVotes" } ], "timed_out": { "gold": [], "prediction": [] }, "exit_status": 1 }