#!/usr/bin/env python3
"""Count pinned GDPval reference-availability metadata by sector and occupation.

Sapien1 makes this original script available to download and run locally.
No additional license to modify or redistribute it is stated. The upstream
dataset's terms are separate, and its card contains a benchmark canary.
The complete table is downloaded temporarily; only aggregate counts are printed.
"""

import argparse
import ast
import hashlib
import json
from pathlib import Path
import tempfile
import urllib.request

import duckdb


DATASET_REVISION = "11e7900cdcac61bc4daf59e65feb238acda98fbf"
GBENCH_REVISION = "d3e4283162d6c34353fa933284135d2a7ef3b9fe"
BASE = f"https://huggingface.co/datasets/openai/gdpval/resolve/{DATASET_REVISION}/"
TABLE = BASE + "data/train-00000-of-00001.parquet"
TABLE_HASH = "f8422fab9b21d90c0ee5f0659842ab666d418cb8940842918f9f4b0df7ae0202"
CARD = f"https://huggingface.co/datasets/openai/gdpval/raw/{DATASET_REVISION}/README.md"
CARD_HASH = "1597a46f191bffd916d8140be98edcd5700cc9abb8877f022c869cc470aee2c5"
GBENCH = f"https://raw.githubusercontent.com/google-deepmind/gbench/{GBENCH_REVISION}/gbench/third_party/eval_suites/gdpval.py"
GBENCH_HASH = "9d9f435cec4443f46a97b2ee9d0841dd080400fbfea9b70eb288abcda3811072"


def read_small(url, expected_hash):
    request = urllib.request.Request(url, headers={"User-Agent": "Sapien1Research/1.0"})
    with urllib.request.urlopen(request, timeout=20) as response:
        data = response.read(200_001)
    if len(data) > 200_000 or hashlib.sha256(data).hexdigest() != expected_hash:
        raise ValueError("source size or hash changed")
    return data.decode("utf-8")


def verify_source_context():
    card = read_small(CARD, CARD_HASH)
    if "Canary gdpval:" not in card:
        raise ValueError("expected benchmark canary absent")
    source = read_small(GBENCH, GBENCH_HASH)
    tree = ast.parse(source)
    revisions = [ast.literal_eval(n.value) for n in tree.body if isinstance(n, ast.Assign)
                 and any(isinstance(t, ast.Name) and t.id == "_REVISION" for t in n.targets)]
    if revisions != [DATASET_REVISION]:
        raise ValueError("gbench dataset pin changed")
    funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "reference_format_reason"]
    if len(funcs) != 1:
        raise ValueError("gbench reference condition missing")
    branches = [n for n in funcs[0].body if isinstance(n, ast.If) and ast.unparse(n.test) == "not rel_paths"]
    if len(branches) != 1 or not isinstance(branches[0].body[0], ast.Return):
        raise ValueError("gbench empty-reference branch changed")
    if ast.literal_eval(branches[0].body[0].value) != "no_deliverable_shipped":
        raise ValueError("gbench empty-reference classification changed")


def download_table(path):
    request = urllib.request.Request(TABLE, headers={"User-Agent": "Sapien1Research/1.0"})
    digest = hashlib.sha256()
    total = 0
    with urllib.request.urlopen(request, timeout=30) as response, path.open("wb") as out:
        while True:
            chunk = response.read(131_072)
            if not chunk:
                break
            total += len(chunk)
            if total > 2_000_000:
                raise ValueError("table larger than expected")
            digest.update(chunk)
            out.write(chunk)
    if digest.hexdigest() != TABLE_HASH:
        raise ValueError("table hash changed")
    return total


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--scratch", help="existing directory for the short-lived source table")
    args = parser.parse_args()
    if duckdb.__version__ != "1.4.4":
        raise ValueError("this pilot was checked with DuckDB 1.4.4")
    verify_source_context()

    # The temporary Parquet file contains the full source table, including columns
    # not analyzed here. It is removed with this temporary directory on exit.
    with tempfile.TemporaryDirectory(prefix="gdpval-coverage-", dir=args.scratch) as temp:
        path = Path(temp) / "source.parquet"
        downloaded = download_table(path)
        con = duckdb.connect(database=":memory:", config={"threads": "1", "memory_limit": "128MB", "temp_directory": temp})
        try:
            totals = con.execute("""
                SELECT count(*), count(DISTINCT task_id), count(DISTINCT occupation),
                       count(*) FILTER (WHERE deliverable_files IS NULL),
                       count(*) FILTER (WHERE array_length(deliverable_files) = 0),
                       count(*) FILTER (WHERE array_length(deliverable_files) > 0)
                FROM read_parquet(?)
            """, [str(path)]).fetchone()
            by_occupation = con.execute("""
                SELECT sector, occupation, count(*) AS tasks,
                       count(*) FILTER (WHERE array_length(deliverable_files) = 0) AS missing
                FROM read_parquet(?)
                GROUP BY sector, occupation
                ORDER BY sector, occupation
            """, [str(path)]).fetchall()
        finally:
            con.close()

    total, unique_ids, unique_occupations, null_lists, missing, available = totals
    if null_lists or total != unique_ids or missing + available != total:
        raise ValueError("unexpected duplicate IDs or ambiguous deliverable presence")
    occupations = [{"sector": sector, "occupation": occupation, "tasks": n,
                    "with_deliverable": n - absent, "without_deliverable": absent}
                   for sector, occupation, n, absent in by_occupation]
    sectors = {}
    for row in occupations:
        summary = sectors.setdefault(row["sector"], {"tasks": 0, "with_deliverable": 0, "without_deliverable": 0})
        for key in summary:
            summary[key] += row[key]

    output = {
        "dataset_revision": DATASET_REVISION,
        "table_url": TABLE,
        "table_sha256": TABLE_HASH,
        "table_bytes_transferred_and_then_removed": downloaded,
        "card_url": CARD,
        "card_sha256": CARD_HASH,
        "gbench_source_url": GBENCH,
        "gbench_source_sha256": GBENCH_HASH,
        "duckdb_version": duckdb.__version__,
        "totals": {"tasks": total, "unique_task_ids": unique_ids, "distinct_occupation_labels": unique_occupations,
                   "null_deliverable_lists": null_lists, "without_deliverable": missing, "with_deliverable": available},
        "occupation_sector_groups": len(occupations),
        "occupation_groups_with_zero_deliverables": sum(row["with_deliverable"] == 0 for row in occupations),
        "occupation_groups_with_some_deliverables": sum(0 < row["with_deliverable"] < row["tasks"] for row in occupations),
        "occupation_groups_with_all_deliverables": sum(row["with_deliverable"] == row["tasks"] for row in occupations),
        "occupation_group_sizes": {str(n): sum(row["tasks"] == n for row in occupations) for n in sorted({row["tasks"] for row in occupations})},
        "sectors": sectors,
        "occupations": occupations,
        "scope": "Deliverable-list presence only; no file contents or renderability checked, and no model or benchmark run.",
    }
    print(json.dumps(output, indent=2))


if __name__ == "__main__":
    main()
