{
  "protocol": "HaluEval-QA faithful, balanced, both answers scored",
  "dataset": "pminervini/HaluEval[qa]",
  "seed": 42,
  "witness": {
    "n_items": 10000,
    "n_decisions": 20000,
    "auroc_full": 0.7976,
    "calibrated_tau": 0.0004,
    "cal_balanced_acc": 0.848,
    "test_accuracy_calibrated": {
      "n": 16000,
      "tp": 7654,
      "fp": 2066,
      "fn": 346,
      "tn": 5934,
      "accuracy": 0.8492,
      "acc_ci95": 0.0055,
      "precision": 0.7874,
      "recall": 0.9567,
      "f1": 0.8639
    },
    "test_accuracy_oracle_ceiling": {
      "n": 16000,
      "tp": 7746,
      "fp": 2146,
      "fn": 254,
      "tn": 5854,
      "accuracy": 0.85,
      "acc_ci95": 0.0055,
      "precision": 0.7831,
      "recall": 0.9683,
      "f1": 0.8659
    },
    "ms_per_decision": 2.361
  },
  "witness_on_gpt_sample": {
    "auroc": 0.8132,
    "n": 1200,
    "tp": 581,
    "fp": 142,
    "fn": 19,
    "tn": 458,
    "accuracy": 0.8658,
    "acc_ci95": 0.0193,
    "precision": 0.8036,
    "recall": 0.9683,
    "f1": 0.8783
  },
  "gpt": [
    {
      "model": "gpt-3.5-turbo",
      "n": 1200,
      "errors": 0,
      "tp": 60,
      "fp": 34,
      "fn": 540,
      "tn": 566,
      "accuracy": 0.5217,
      "acc_ci95": 0.0283,
      "precision": 0.6383,
      "recall": 0.1,
      "f1": 0.1729,
      "sec": 85.5
    },
    {
      "model": "gpt-4o-mini",
      "n": 1200,
      "errors": 0,
      "tp": 479,
      "fp": 44,
      "fn": 121,
      "tn": 556,
      "accuracy": 0.8625,
      "acc_ci95": 0.0195,
      "precision": 0.9159,
      "recall": 0.7983,
      "f1": 0.8531,
      "sec": 84.4
    }
  ],
  "published_reference": {
    "gpt35_qa_accuracy_no_knowledge": 0.6259,
    "source": "HaluEval, Li et al., EMNLP 2023"
  }
}