{"type": "owin.selective", "selective": {"split": "v3fresh", "cells": 33, "items": 11, "vendors": 7, "vendors_new": 6, "pages": 8, "auto_answered": "27/33", "auto_correct": "26/27", "with_sentence": 18, "not_stated": 8, "sent_to_confirm": 6, "confirm_majority_right": "2/6", "all_cells_majority_right": "28/33", "auto_misses": [{"item": "openrouter-inference-zdr", "field": "retention_days", "value": 0, "gold": null, "footnote": "Self-disclosed label error: the page says prompts are not retained unless you opt in to logging, so the label should be 0; v3 answered 0 on 3/3 runs. The label is kept as pre-registered, so this counts as a miss."}], "reference_not_clean": {"v3_old": {"auto_answered": "80/90", "auto_correct": "77/80", "sent_to_confirm": 10}, "v2_old": {"auto_answered": "77/90", "auto_correct": "72/77", "sent_to_confirm": 13}}}, "run_files": ["eval/datause/runs/20261021T221257-v3fresh-nemotron-3-super-120b-a12b.json"], "quotes_rechecked_against_source": false, "majority_all_cells": "28/33", "correct_with_sentence": 18, "correct_not_stated": 8, "pages": 8, "vendors_new": 6, "items_exact_majority": "7/11", "negative_items_held": "2/2", "baseline_items_exact": "4/11", "model": "nvidia/nemotron-3-super-120b-a12b", "per_call": {"calls": 39, "total_cost_usd": 0.11475, "mean_cost_usd": 0.00294, "cost_per_exact_run_usd": 0.0041, "mean_latency_s_exact_runs": 8.81, "p95_latency_s_exact_runs": 15.86}, "recompute": "owin eval selective --corpus datause --split v3fresh", "ladder": {"split": "v3fresh", "arms": {"super": {"model": "nvidia/nemotron-3-super-120b-a12b", "auto_answered": "27/33", "auto_correct": "26/27", "sent_to_confirm": 6, "confident_misses": 1, "negatives": "6/6", "cost_per_reading_usd": 0.00294, "mean_latency_s": 9.69, "errored_abstained": 0}, "lightning": {"model": "nvidia/Nemotron-3_5-Lightning", "auto_answered": "0/33", "auto_correct": "0/0", "sent_to_confirm": 33, "confident_misses": 0, "negatives": "6/6", "cost_per_reading_usd": 0.00206, "mean_latency_s": 27.61, "errored_abstained": 32}, "nano": {"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "auto_answered": "25/33", "auto_correct": "24/25", "sent_to_confirm": 8, "confident_misses": 1, "negatives": "6/6", "cost_per_reading_usd": 0.00115, "mean_latency_s": 44.77, "errored_abstained": 1}, "ultra": {"model": "nvidia/Nemotron-3-Ultra-550b-a55b", "auto_answered": "30/33", "auto_correct": "29/30", "sent_to_confirm": 3, "confident_misses": 1, "negatives": "6/6", "cost_per_reading_usd": 0.01357, "mean_latency_s": 10.4, "errored_abstained": 0}}, "escalation": {"auto_answered": "31/33", "auto_correct": "30/31", "sent_to_person": 2, "resolved_by_ultra": "4/4", "confident_misses": [{"item": "openrouter-inference-zdr", "field": "retention_days", "by": "super", "value": 0, "gold": null}], "features_escalated": 6, "extra_cost_per_feature_usd": 0.0352, "adopted": true}, "retraction_triggered": true, "cascade_nano_first": {"features": 11, "decided_by_nano": 5, "auto_answered": "30/33", "auto_correct": "29/30", "sent_to_person": 3, "cost_per_feature_usd": 0.00914, "super_cost_per_feature_usd": 0.0086, "confident_misses": 1}, "recompute": "owin eval ladder --corpus datause --brief"}}