#!/usr/bin/env python3
"""Compare aggregate GDPval input URL refs and pinned repository metadata.

Sapien1 makes this script available to download and run locally. No additional
license to modify or redistribute it is stated. Upstream terms are separate.
The later pinned GDPval dataset card includes a benchmark canary; check the
applicable upstream terms before use."""

import argparse
import hashlib
import http.client
import json
from pathlib import Path
import tempfile
from urllib.parse import quote, unquote, urlsplit
import urllib.error
import urllib.request

import duckdb


OLD = "4bf642692ef091918ef20797ecca070924815ec1"
NEW = "11e7900cdcac61bc4daf59e65feb238acda98fbf"
TABLE_URL = f"https://huggingface.co/datasets/openai/gdpval/resolve/{OLD}/data/train-00000-of-00001.parquet"
TABLE_HASH = "e26aeea5542ece91356d51cfb4d2f46cdb0ac86ab311b825b8d67bc06cbe4d9f"
API_BASE = "https://huggingface.co/api/datasets/openai/gdpval/revision/"
URL_PREFIX = "/datasets/openai/gdpval/resolve/main/"


def get(url, maximum):
    req = urllib.request.Request(url, headers={"User-Agent": "Sapien1Research/1.0"})
    with urllib.request.urlopen(req, timeout=30) as response:
        raw = response.read(maximum + 1)
    if len(raw) > maximum:
        raise ValueError("source exceeded expected size")
    return raw


def repository_metadata(revision):
    doc = json.loads(get(API_BASE + revision + "?blobs=true", 2_000_000))
    if doc.get("sha") != revision:
        raise ValueError("repository revision mismatch")
    rows = {item["rfilename"]: (item.get("blobId"), item.get("size")) for item in doc["siblings"]
            if item.get("rfilename", "").startswith("reference_files/")}
    if any(not blob or not isinstance(size, int) for blob, size in rows.values()):
        raise ValueError("missing file metadata")
    # This digest makes the selected metadata auditable without retaining its paths.
    canonical = json.dumps(sorted((name, blob, size) for name, (blob, size) in rows.items()),
                           ensure_ascii=False, separators=(",", ":")).encode("utf-8")
    return rows, hashlib.sha256(canonical).hexdigest()


class HeadOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
    """Keep HTTP redirects as HEAD; never fall back to a content request."""

    def redirect_request(self, request, fp, code, message, headers, new_url):
        if (request.get_method() != "HEAD" or code not in {301, 302, 303, 307, 308}
                or urlsplit(new_url).scheme.lower() not in {"http", "https"}):
            raise urllib.error.HTTPError(request.full_url, code, "unsupported HEAD redirect", headers, fp)
        forwarded = {key: value for key, value in request.headers.items()
                     if key.lower() not in {"content-length", "content-type"}}
        return urllib.request.Request(new_url.replace(" ", "%20"), headers=forwarded,
                                      origin_req_host=request.origin_req_host,
                                      unverifiable=True, method="HEAD")

    # Python 3.10 does not supply this redirect handler, but 308 can still be HEAD-only.
    http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302


def head_file(path, revision):
    url = f"https://huggingface.co/datasets/openai/gdpval/resolve/{revision}/" + quote(path, safe="/")
    request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "Sapien1Research/1.0"})
    opener = urllib.request.build_opener(HeadOnlyRedirectHandler())
    try:
        with opener.open(request, timeout=20) as response:
            size = response.headers.get("Content-Length")
            return {"status": response.status, "content_length": int(size) if size and size.isdigit() else None}
    except urllib.error.HTTPError as error:
        return {"status": error.code, "content_length": None}
    except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError):
        return {"status": None, "content_length": None}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--scratch", help="existing directory for the short-lived source table")
    parser.add_argument("--check-current-main", action="store_true",
                        help="also HEAD one deterministically selected changed input at both revisions and current main")
    args = parser.parse_args()
    if duckdb.__version__ != "1.4.4":
        raise ValueError("this pilot was checked with DuckDB 1.4.4")
    old_meta, old_fingerprint = repository_metadata(OLD)
    new_meta, new_fingerprint = repository_metadata(NEW)
    raw = get(TABLE_URL, 1_000_000)
    if hashlib.sha256(raw).hexdigest() != TABLE_HASH:
        raise ValueError("old table hash changed")
    table_bytes = len(raw)

    # This temporary Parquet contains other source columns, including task text.
    # Only the two input-reference columns and a distinct-ID count are queried.
    with tempfile.TemporaryDirectory(prefix="gdpval-url-revisions-", dir=args.scratch) as temp:
        path = Path(temp) / "source.parquet"
        path.write_bytes(raw)
        del raw
        con = duckdb.connect(":memory:", config={"threads": "1", "memory_limit": "128MB", "temp_directory": temp})
        try:
            total, unique_ids = con.execute("SELECT count(*), count(DISTINCT task_id) FROM read_parquet(?)", [str(path)]).fetchone()
            rows = con.execute("SELECT reference_files, reference_file_urls FROM read_parquet(?)", [str(path)]).fetchall()
        finally:
            con.close()

    totals = {"task_rows": total, "distinct_task_ids": unique_ids, "rows_with_inputs": 0,
              "null_list_rows": 0, "length_mismatch_rows": 0, "input_path_entries": 0,
              "input_url_entries": 0, "main_urls_matching_input_path": 0,
              "matching_paths_in_both_revisions": 0, "matched_entries_with_changed_size": 0,
              "rows_with_changed_size_main_reference": 0}
    all_paths = set()
    changed_paths = set()
    for paths, urls in rows:
        totals["null_list_rows"] += paths is None or urls is None
        paths, urls = paths or [], urls or []
        totals["rows_with_inputs"] += bool(paths)
        totals["length_mismatch_rows"] += len(paths) != len(urls)
        totals["input_path_entries"] += len(paths)
        totals["input_url_entries"] += len(urls)
        row_changed = False
        for path, url in zip(paths, urls):
            parsed = urlsplit(url)
            matches = (parsed.scheme == "https" and parsed.hostname == "huggingface.co"
                       and parsed.path.startswith(URL_PREFIX)
                       and unquote(parsed.path[len(URL_PREFIX):]) == path)
            totals["main_urls_matching_input_path"] += matches
            if not matches:
                continue
            all_paths.add(path)
            present = path in old_meta and path in new_meta
            totals["matching_paths_in_both_revisions"] += present
            if present and old_meta[path][1] != new_meta[path][1]:
                totals["matched_entries_with_changed_size"] += 1
                changed_paths.add(path)
                row_changed = True
        totals["rows_with_changed_size_main_reference"] += row_changed

    shared = old_meta.keys() & new_meta.keys()
    output = {
        "old_revision": OLD,
        "new_revision": NEW,
        "old_table_url": TABLE_URL,
        "old_table_sha256": TABLE_HASH,
        "old_table_bytes_transferred_and_then_removed": table_bytes,
        "duckdb_version": duckdb.__version__,
        "old_reference_metadata_entries": len(old_meta),
        "new_reference_metadata_entries": len(new_meta),
        "shared_reference_metadata_paths": len(shared),
        "old_only_reference_metadata_paths": len(old_meta.keys() - new_meta.keys()),
        "new_only_reference_metadata_paths": len(new_meta.keys() - old_meta.keys()),
        "shared_reference_paths_with_changed_size": sum(old_meta[p][1] != new_meta[p][1] for p in shared),
        "shared_reference_paths_with_changed_blob_id": sum(old_meta[p][0] != new_meta[p][0] for p in shared),
        "shared_reference_paths_with_both_changed": sum(old_meta[p][1] != new_meta[p][1] and old_meta[p][0] != new_meta[p][0] for p in shared),
        "old_reference_metadata_fingerprint_sha256": old_fingerprint,
        "new_reference_metadata_fingerprint_sha256": new_fingerprint,
        "old_table_aggregate": totals,
        "unique_matching_input_paths": len(all_paths),
        "unique_matching_paths_with_changed_size": len(changed_paths),
        "scope": "Repository-reported metadata and URL patterns only; no input file contents or benchmark run.",
    }
    if args.check_current_main:
        if not changed_paths:
            raise ValueError("no changed input path available for HEAD check")
        selected = sorted(changed_paths)[0]
        current_url = "https://huggingface.co/api/datasets/openai/gdpval"
        before = json.loads(get(current_url, 2_000_000)).get("sha")
        old_head = head_file(selected, OLD)
        new_head = head_file(selected, NEW)
        main_head = head_file(selected, "main")
        after = json.loads(get(current_url, 2_000_000)).get("sha")
        output["one_changed_input_head_check"] = {
            "selection": "lexicographically first changed input path, not retained",
            "main_revision_before": before,
            "main_revision_after": after,
            "old_revision_head": old_head,
            "new_revision_head": new_head,
            "main_head": main_head,
            "old_metadata_size": old_meta[selected][1],
            "new_metadata_size": new_meta[selected][1],
        }
    print(json.dumps(output, indent=2))


if __name__ == "__main__":
    main()
