#!/usr/bin/env python3
"""Compare pinned zero-shot score reduction across NLI label orderings."""

import ast
from hashlib import sha256
from itertools import permutations
import json
from math import exp, isclose
from types import SimpleNamespace
from urllib.request import urlopen

import numpy as np

PIPELINE = "src/transformers/pipelines/zero_shot_classification.py"
RELEASES = {
    "4.55.0": (
        "06f8004e5cd9d06cfbffc3f47afb6c2b43bcb3d2",
        12_553,
        "80c7e771f22d6a64cb1d495d2dc4aa6619fd512091d8911208125f125d56ce4b",
    ),
    "5.23.0": (
        "52b080b6416a7897e2af90febeb4dfad22b84d19",
        11_976,
        "07ed3f8bb39561bc81036c28de2ce32fffc0c621175b426838f36dcb131aa804",
    ),
}
MODELS = {
    "aiface/ModernBERT-large_nli": (
        "4f4d21a8351871d2ab4b6bf466b3952d096955bf",
        1456,
        "30c427c17e22f2b88679b19f7bb864c42011bb029ab504402f0cb9139877e4a4",
    ),
    "facebook/bart-large-mnli": (
        "d7645e127eaf1aefc7862fd59a17a5aa8558b8ce",
        1154,
        "a0f9bcb245b680a96ccae0ad8d155f267ec3e3c971ef4a4937e52ea9ba368a86",
    ),
}
CLASSES = ("entailment", "contradiction", "neutral")
LOGITS = {
    "label A": dict(zip(CLASSES, (2, 0, 4))),
    "label B": dict(zip(CLASSES, (1, 2, -1))),
}


def read_pinned(url, size, digest):
    with urlopen(url, timeout=12) as response:
        raw = response.read(size + 1)
    if len(raw) != size or sha256(raw).hexdigest() != digest:
        raise ValueError(f"source bytes differ: {url}")
    return raw, {"url": url, "bytes": size, "sha256": digest}


def read_pipeline(pin):
    revision, size, digest = pin
    url = f"https://raw.githubusercontent.com/huggingface/transformers/{revision}/{PIPELINE}"
    raw, receipt = read_pinned(url, size, digest)
    source = ast.parse(raw)
    cls, = (
        node for node in source.body
        if isinstance(node, ast.ClassDef) and node.name == "ZeroShotClassificationPipeline"
    )
    methods = {"entailment_id", "postprocess"}
    cls.body = [
        node for node in cls.body if isinstance(node, ast.FunctionDef) and node.name in methods
    ]
    if {node.name for node in cls.body} != methods:
        raise ValueError("the two pinned methods were not found")
    cls.bases = []
    cls.decorator_list = []
    namespace = {"np": np}
    exec(compile(ast.Module(body=[cls], type_ignores=[]), PIPELINE, "exec"), namespace)
    return namespace[cls.name], receipt


class LogitArray:
    """Provide the float-array interface expected from model output tensors."""

    def __init__(self, values):
        self.values = np.array([values], dtype=np.float32)

    def float(self):
        return self

    def numpy(self):
        return self.values


def postprocess(pipeline, index_order, labels, multi_label):
    scorer = pipeline()
    scorer.framework = "pt"
    label_map = {name: i for i, name in enumerate(index_order)}
    scorer.model = SimpleNamespace(config=SimpleNamespace(label2id=label_map))
    outputs = [
        {
            "candidate_label": label,
            "sequence": "Fictional input",
            "logits": LogitArray([LOGITS[label][slot] for slot in index_order]),
        }
        for label in labels
    ]
    result = scorer.postprocess(outputs, multi_label=multi_label)
    return dict(zip(result["labels"], result["scores"]))


def two_class_scores(labels, negative):
    return {
        label: 1 / (1 + exp(LOGITS[label][negative] - LOGITS[label]["entailment"]))
        for label in labels
    }


def matches(observed, expected):
    return observed.keys() == expected.keys() and all(
        isclose(observed[k], v, rel_tol=1e-6, abs_tol=1e-7) for k, v in expected.items()
    )


def shown(scores):
    return [{"label": label, "score": round(score, 6)} for label, score in scores.items()]


def compare_orders(pipeline):
    labeled_scores = two_class_scores(LOGITS, "contradiction")
    neutral_scores = two_class_scores(LOGITS, "neutral")
    observations = []
    for order in permutations(CLASSES):
        observed = postprocess(pipeline, order, LOGITS, True)
        if matches(observed, labeled_scores):
            negative = "contradiction"
        elif matches(observed, neutral_scores):
            negative = "neutral"
        else:
            raise ValueError(f"unexpected two-class scores for {order}")
        observations.append({
            "index_order": order,
            "fixture_scores_match_negative_class": negative,
            "multi_label_top_candidate": next(iter(observed)),
        })
    return observations


def compare_models(pipeline, configs):
    expected_pair = two_class_scores(LOGITS, "contradiction")
    expected_pair = dict(sorted(expected_pair.items(), key=lambda kv: kv[1], reverse=True))
    cases = {}
    for model, labels in configs.items():
        order = sorted(labels, key=labels.get)
        cases[model] = {
            "label2id": labels,
            "multi_label": shown(postprocess(pipeline, order, LOGITS, True)),
            "exclusive_two_candidates": shown(postprocess(pipeline, order, LOGITS, False)),
            "default_one_candidate": shown(postprocess(pipeline, order, ("label A",), False)),
            "declared_pair_reference": shown(expected_pair),
        }
    return cases


def main():
    configs = {}
    receipts = {}
    for model, (revision, size, digest) in MODELS.items():
        raw, receipts[model] = read_pinned(
            f"https://huggingface.co/{model}/raw/{revision}/config.json", size, digest
        )
        config = json.loads(raw)
        labels = config["label2id"]
        if (set(labels) != set(CLASSES) or set(labels.values()) != {0, 1, 2}
                or config["id2label"] != {str(i): label for label, i in labels.items()}):
            raise ValueError(f"unsupported or inconsistent label map for {model}")
        configs[model] = labels
    studies = {}
    for version, pin in RELEASES.items():
        pipeline, receipt = read_pipeline(pin)
        studies[version] = {
            "pipeline_source": receipt,
            "config_cases": compare_models(pipeline, configs),
            "six_index_orders": compare_orders(pipeline),
        }
    print(json.dumps({
        "fictional_logits_by_semantic_class": LOGITS,
        "configs": receipts,
        "releases": studies,
    }, indent=2))


if __name__ == "__main__":
    main()
