#!/usr/bin/env python3
"""Check the pinned Inspect Evals GDPval dataset and file-mapping source.

Sapien1 makes this script available to download and run locally. No additional
license to modify or redistribute it is stated. Upstream terms are separate."""

import ast
import hashlib
import json
import urllib.request


REV = "5c092d54ea8591b4680d4dd593beab637f478ffc"
BASE = "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/" + REV + "/src/inspect_evals/"
SOURCES = {
    "gdpval": {"url": BASE + "gdpval/util.py",
               "sha256": "9caa0ee39d0d467bdf0299434571a170b0ed85762a560c128ef484d71eadb18c"},
    "hf_wrapper": {"url": BASE + "utils/huggingface.py",
                   "sha256": "f14ec2928c166904bf7f4c160e4ca9e770c76a8df0d212f8da4a7fa8ce6f76a9"},
}


def parse(source):
    request = urllib.request.Request(source["url"], headers={"User-Agent": "Sapien1Research/1.0"})
    with urllib.request.urlopen(request, timeout=30) as response:
        raw = response.read(200_001)
    if len(raw) > 200_000 or hashlib.sha256(raw).hexdigest() != source["sha256"]:
        raise ValueError("source size or hash mismatch")
    return ast.parse(raw)


def one(nodes, description):
    if len(nodes) != 1:
        raise ValueError("expected one " + description)
    return nodes[0]


def function(tree, name):
    return one([node for node in tree.body if isinstance(node, ast.FunctionDef)
                and node.name == name], name + " function")


def main():
    trees = {label: parse(source) for label, source in SOURCES.items()}
    gdpval = trees["gdpval"]
    constants = {}
    for node in gdpval.body:
        if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
            if node.targets[0].id in {"HF_DATASET_PATH", "GDPVAL_DATASET_REVISION"}:
                constants[node.targets[0].id] = ast.literal_eval(node.value)
    if constants != {"HF_DATASET_PATH": "openai/gdpval",
                     "GDPVAL_DATASET_REVISION": "a3848a2a812d5d4d0f08003fac3c8eac40805962"}:
        raise ValueError("dataset path or revision changed")

    loader = function(gdpval, "load_gdpval_dataset")
    call = one([node for node in ast.walk(loader) if isinstance(node, ast.Call)
                and isinstance(node.func, ast.Name) and node.func.id == "hf_dataset"], "dataset call")
    keywords = {keyword.arg: ast.unparse(keyword.value) for keyword in call.keywords}
    if ([ast.unparse(arg) for arg in call.args] != ["HF_DATASET_PATH"] or
        keywords.get("revision") != "GDPVAL_DATASET_REVISION" or
        keywords.get("sample_fields") != "get_gdpval_record" or keywords.get("split") != "'train'"):
        raise ValueError("loader arguments changed")

    record = function(gdpval, "get_gdpval_record")
    sample = one([node for node in ast.walk(record) if isinstance(node, ast.Call)
                  and isinstance(node.func, ast.Name) and node.func.id == "Sample"], "Sample call")
    files = one([keyword.value for keyword in sample.keywords if keyword.arg == "files"], "files field")
    expected = ast.parse("{remove_middle_stem(k): v for k, v in zip(record['reference_files'], record['reference_file_urls'])}", mode="eval").body
    if ast.dump(files, include_attributes=False) != ast.dump(expected, include_attributes=False):
        raise ValueError("file mapping changed")

    wrapper = function(trees["hf_wrapper"], "hf_dataset")
    returns = [node.value for node in ast.walk(wrapper) if isinstance(node, ast.Return) and node.value]
    if len(returns) != 1 or ast.unparse(returns[0]) != "inspect_ai.dataset.hf_dataset(*args, **kwargs)":
        raise ValueError("dataset wrapper forwarding changed")
    print(json.dumps({
        "source_commit": REV,
        "sources": SOURCES,
        "dataset_path": constants["HF_DATASET_PATH"],
        "dataset_revision": constants["GDPVAL_DATASET_REVISION"],
        "dataset_split": "train",
        "record_mapper": "get_gdpval_record",
        "file_mapping": "paired reference_file_urls values passed to Sample.files",
        "wrapper_forwards_arguments": True,
        "scope": "Static source check only; no evaluator, framework, input-file download or model run.",
    }, indent=2))


if __name__ == "__main__":
    main()
