cross-judge¶

def majority_vote(
    verdicts: list[Verdict],
    *,
    priority: list[str] | None = None,
    fallback: str = "UNCLEAR",
) -> tuple[str, bool]:
    """Majority vote: most common label wins.

    Args:
        verdicts: per-critic verdicts.
        priority: tiebreaker order — labels earlier in the list win ties.
        fallback: returned if no valid verdicts.

    Returns:
        (consensus_label, disagreement_bool)
    """
    counts = _label_counts(verdicts)
    if not counts:
        return fallback, False
    labels = {v.kind for v in verdicts if v.error is None}
    disagree = len(labels) > 1
    max_count = max(counts.values())
    tied = [label for label, c in counts.items() if c == max_count]
    if len(tied) == 1:
        return tied[0], disagree
    if priority:
        for p in priority:
            if p in tied:
                return p, disagree
    # else preserve insertion order
    seen: list[str] = []
    for v in verdicts:
        if v.error is None and v.kind in tied and v.kind not in seen:
            seen.append(v.kind)
    return (seen[0] if seen else tied[0]), disagree

def unanimous(
    verdicts: list[Verdict],
    *,
    fallback: str = "UNCLEAR",
) -> tuple[str, bool]:
    """Unanimous vote: return label only if all critics agree.

    Args:
        verdicts: per-critic verdicts.
        fallback: returned on any disagreement.

    Returns:
        (consensus_label, disagreement_bool)
    """
    labels = {v.kind for v in verdicts if v.error is None}
    if len(labels) == 1:
        return labels.pop(), False
    return fallback, True

def agreement_pct(verdicts: list[Verdict], consensus: str) -> float:
    """Fraction of valid critics whose kind == consensus.

    Returns:
        float in [0.0, 1.0]. 0.0 if no valid verdicts.
    """
    valid = _valid(verdicts)
    if not valid:
        return 0.0
    matching = sum(1 for v in valid if v.kind == consensus)
    return matching / len(valid)

def krippendorff_alpha(verdicts: list[Verdict]) -> float | None:
    """Krippendorff's α for nominal data.

    Treats each critic as a rater and each label as nominal.

    For the single-item, N-rater case:
        α = 1 - (D_observed / D_expected)

    where:
        D_observed = number of disagreeing pairs of critics
        D_expected = sum over (cat_i, cat_j) of n_i * n_j (i != j) / (N-1)
                     where n_i is the count of critics that voted cat_i,
                     N is the total number of critics.

    The (N-1) divisor in D_expected is the standard small-sample correction
    (Krippendorff 2011 eq. 4).

    Returns:
        α in [-1.0, 1.0]. None if fewer than 2 valid verdicts.
        1.0  → perfect agreement
        0.0  → agreement equal to chance
        <0.0 → systematic disagreement
    """
    valid = _valid(verdicts)
    n = len(valid)
    if n < 2:
        return None

    counts = _label_counts(valid)
    total = n

    # D_observed: count of disagreeing pairs out of C(N,2)
    # For each pair of critics (i, j), if their labels differ → +1
    # Equivalently: D_obs = C(N,2) - sum_c C(n_c, 2)
    total_pairs = n * (n - 1) // 2
    same_pairs = sum(c * (c - 1) // 2 for c in counts.values())
    disagreeing_pairs = total_pairs - same_pairs

    if total_pairs == 0:
        return None

    # D_observed normalized to per-pair rate
    d_observed = disagreeing_pairs / total_pairs

    # D_expected: probability two randomly chosen labels differ given marginals
    # = 1 - sum_c (n_c / N) * ((n_c - 1) / (N - 1))   (sampling without replacement)
    if total == 1:
        return None
    same_chance = sum((c / total) * ((c - 1) / (total - 1)) for c in counts.values())
    d_expected = 1.0 - same_chance

    if d_expected == 0.0:
        # All critics had to agree by structure (e.g. only 1 label observed)
        return 1.0 if d_observed == 0.0 else 0.0

    alpha = 1.0 - (d_observed / d_expected)
    # Clamp to [-1, 1] for numerical safety; α can go slightly negative for
    # systematic disagreement but should never exceed 1.0.
    return max(-1.0, min(1.0, alpha))

def get_voting_strategy(name_or_fn: str | VotingStrategy) -> VotingStrategy:
    """Resolve a voting strategy by name or pass through if already callable."""
    if callable(name_or_fn):
        return name_or_fn  # type: ignore[return-value]
    key = str(name_or_fn).lower()
    if key not in VOTING_STRATEGIES:
        raise KeyError(
            f"Unknown voting strategy '{name_or_fn}'. Known: {sorted(VOTING_STRATEGIES)}."
        )
    return VOTING_STRATEGIES[key]

def get_vendor(name: str) -> VendorConfig:
    """Look up a vendor by name. Raises KeyError on unknown vendor."""
    key = name.lower()
    if key not in VENDORS:
        raise KeyError(
            f"Unknown vendor '{name}'. Known: {sorted(VENDORS)}. "
            f"For arbitrary OpenAI-compatible endpoints, build your own client and pass it directly."
        )
    return VENDORS[key]

def make_client(vendor: str = "deepseek", api_key: str | None = None, base_url: str | None = None) -> Any:
    """Build an OpenAI-compatible client for a vendor.

    Args:
        vendor: One of 'deepseek', 'openai', 'openrouter' (default: deepseek).
        api_key: Explicit API key. If None, read from the vendor's env var.
        base_url: Override base URL (useful for custom endpoints / mock servers).

    Returns:
        An `openai.OpenAI` client instance configured for the vendor.

    Raises:
        RuntimeError: if the API key is missing.
        ImportError: if `openai` package is not installed.
    """
    try:
        from openai import OpenAI
    except ImportError as e:
        raise ImportError(
            "The 'openai' package is required for cross-judge LLM calls. "
            "Install with: pip install 'openai>=1.0'"
        ) from e

    cfg = get_vendor(vendor)
    key = api_key or os.getenv(cfg.api_key_env)
    if not key:
        raise RuntimeError(
            f"Missing API key for vendor '{vendor}'. "
            f"Set {cfg.api_key_env} env var or pass api_key=... explicitly."
        )
    return OpenAI(api_key=key, base_url=base_url or cfg.base_url)

def majority(
    verdicts: list[Verdict],
    *,
    priority: list[str] | None = None,
    fallback: str = "UNCLEAR",
) -> tuple[str, bool]:
    """Most common label wins. Ties broken by `priority` order (if given),
    else by first-seen order.

    Args:
        verdicts: per-reviewer verdicts.
        priority: tiebreaker order (labels earlier in the list win ties).
        fallback: label to return if no valid verdicts exist.
    """
    counts = _label_counts(verdicts)
    if not counts:
        return fallback, False
    disagree = _any_disagreement(verdicts)
    max_count = max(counts.values())
    tied = [label for label, c in counts.items() if c == max_count]
    if len(tied) == 1:
        return tied[0], disagree
    # tiebreaker
    if priority:
        for p in priority:
            if p in tied:
                return p, disagree
    # else preserve insertion order from `verdicts`
    seen_order: list[str] = []
    for v in verdicts:
        if v.error is None and v.verdict in tied and v.verdict not in seen_order:
            seen_order.append(v.verdict)
    return (seen_order[0] if seen_order else tied[0]), disagree

def weighted(
    verdicts: list[Verdict],
    *,
    weights: dict[str, float] | None = None,
    use_confidence: bool = True,
    fallback: str = "UNCLEAR",
) -> tuple[str, bool]:
    """Weighted vote: each verdict contributes weight = (reviewer_weight × confidence).

    Args:
        verdicts: per-reviewer verdicts.
        weights: optional per-reviewer weight overrides keyed by reviewer_id.
                 Reviewers not in the dict default to weight=1.0.
        use_confidence: if True, multiply weight by verdict.confidence.
        fallback: returned if no valid verdicts.
    """
    weights = weights or {}
    scores: dict[str, float] = {}
    for v in verdicts:
        if v.error is not None:
            continue
        w = float(weights.get(v.reviewer_id, 1.0))
        if use_confidence:
            w *= max(0.0, min(1.0, v.confidence))
        scores[v.verdict] = scores.get(v.verdict, 0.0) + w
    if not scores:
        return fallback, False
    disagree = _any_disagreement(verdicts)
    best = max(scores.items(), key=lambda kv: kv[1])
    return best[0], disagree

def first_disagreement(
    verdicts: list[Verdict],
    *,
    disagree_label: str = "DISAGREE",
    fallback: str = "UNCLEAR",
) -> tuple[str, bool]:
    """Returns `disagree_label` if any pair of reviewers differ; else the agreed label."""
    labels = {v.verdict for v in verdicts if v.error is None}
    if not labels:
        return fallback, False
    if len(labels) == 1:
        return labels.pop(), False
    return disagree_label, True

def get_strategy(name_or_fn: str | AggregationStrategy) -> AggregationStrategy:
    """Resolve a strategy by name (string) or pass through if already a callable."""
    if callable(name_or_fn):
        return name_or_fn  # type: ignore[return-value]
    key = str(name_or_fn).lower()
    if key not in STRATEGIES:
        raise KeyError(f"Unknown strategy '{name_or_fn}'. Known: {sorted(STRATEGIES)}.")
    return STRATEGIES[key]

def avg_confidence(verdicts: list[Verdict]) -> float:
    """Average confidence across valid (non-errored) verdicts."""
    valid = [v.confidence for v in verdicts if v.error is None]
    if not valid:
        return 0.0
    return sum(valid) / len(valid)