{
  "schema_version": "mediationbench-external-readout/v2",
  "sample_id": "external-200-v1",
  "status": "partial",
  "total_cases": 200,
  "source_counts": {
    "casino": 100,
    "contractnli": 100
  },
  "difficulty_counts": {
    "easy": 68,
    "medium": 66,
    "hard": 66
  },
  "datasets": [
    {
      "id": "casino",
      "label": "CaSiNo",
      "case_count": 100,
      "difficulty_counts": {
        "easy": 34,
        "medium": 33,
        "hard": 33
      },
      "source_url": "https://aclanthology.org/2021.naacl-main.254/",
      "license": "CC-BY-4.0",
      "tasks": [
        {
          "id": "accepted",
          "label": "Recorded deal acceptance"
        },
        {
          "id": "allocation",
          "label": "Exact allocation"
        }
      ]
    },
    {
      "id": "contractnli",
      "label": "ContractNLI",
      "case_count": 100,
      "difficulty_counts": {
        "easy": 34,
        "medium": 33,
        "hard": 33
      },
      "source_url": "https://aclanthology.org/2021.findings-emnlp.164/",
      "license": "CC-BY-4.0",
      "tasks": [
        {
          "id": "nli",
          "label": "Contract entailment"
        }
      ]
    }
  ],
  "judges": [
    {
      "id": "sol",
      "label": "Sol",
      "completed": 200,
      "failed": 0,
      "pending": 0,
      "cost_usd": 2.30074,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 75,
          "incorrect": 25,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0,
          "label_agreement": 75,
          "by_reference": {
            "false": {
              "correct": 3,
              "incorrect": 3,
              "failed": 0,
              "pending": 0,
              "label_agreement": 3
            },
            "true": {
              "correct": 72,
              "incorrect": 22,
              "failed": 0,
              "pending": 0,
              "label_agreement": 72
            }
          }
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 69,
          "incorrect": 31,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0,
          "label_agreement": 69,
          "by_reference": {
            "deal": {
              "correct": 66,
              "incorrect": 28,
              "failed": 0,
              "pending": 0,
              "label_agreement": 66
            },
            "no_deal": {
              "correct": 3,
              "incorrect": 3,
              "failed": 0,
              "pending": 0,
              "label_agreement": 3
            }
          }
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 77,
          "incorrect": 23,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0,
          "label_agreement": 77,
          "by_reference": {
            "Contradiction": {
              "correct": 21,
              "incorrect": 12,
              "failed": 0,
              "pending": 0,
              "label_agreement": 21
            },
            "Entailment": {
              "correct": 30,
              "incorrect": 4,
              "failed": 0,
              "pending": 0,
              "label_agreement": 30
            },
            "NotMentioned": {
              "correct": 26,
              "incorrect": 7,
              "failed": 0,
              "pending": 0,
              "label_agreement": 26
            }
          }
        }
      ]
    },
    {
      "id": "gemini",
      "label": "Gemini",
      "completed": 198,
      "failed": 2,
      "pending": 0,
      "cost_usd": 0.7484205,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 71,
          "incorrect": 29,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0,
          "label_agreement": 71,
          "by_reference": {
            "false": {
              "correct": 4,
              "incorrect": 2,
              "failed": 0,
              "pending": 0,
              "label_agreement": 4
            },
            "true": {
              "correct": 67,
              "incorrect": 27,
              "failed": 0,
              "pending": 0,
              "label_agreement": 67
            }
          }
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 67,
          "incorrect": 33,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0,
          "label_agreement": 67,
          "by_reference": {
            "deal": {
              "correct": 63,
              "incorrect": 31,
              "failed": 0,
              "pending": 0,
              "label_agreement": 63
            },
            "no_deal": {
              "correct": 4,
              "incorrect": 2,
              "failed": 0,
              "pending": 0,
              "label_agreement": 4
            }
          }
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 82,
          "incorrect": 16,
          "failed": 2,
          "pending": 0,
          "total": 100,
          "evidence_valid": 98,
          "evidence_invalid": 2,
          "label_agreement": 84,
          "by_reference": {
            "Contradiction": {
              "correct": 24,
              "incorrect": 9,
              "failed": 0,
              "pending": 0,
              "label_agreement": 24
            },
            "Entailment": {
              "correct": 29,
              "incorrect": 3,
              "failed": 2,
              "pending": 0,
              "label_agreement": 31
            },
            "NotMentioned": {
              "correct": 29,
              "incorrect": 4,
              "failed": 0,
              "pending": 0,
              "label_agreement": 29
            }
          }
        }
      ]
    },
    {
      "id": "grok",
      "label": "Grok",
      "completed": 196,
      "failed": 4,
      "pending": 0,
      "cost_usd": 2.19349,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 69,
          "incorrect": 28,
          "failed": 3,
          "pending": 0,
          "total": 100,
          "evidence_valid": 97,
          "evidence_invalid": 3,
          "label_agreement": 71,
          "by_reference": {
            "false": {
              "correct": 4,
              "incorrect": 2,
              "failed": 0,
              "pending": 0,
              "label_agreement": 4
            },
            "true": {
              "correct": 65,
              "incorrect": 26,
              "failed": 3,
              "pending": 0,
              "label_agreement": 67
            }
          }
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 64,
          "incorrect": 33,
          "failed": 3,
          "pending": 0,
          "total": 100,
          "evidence_valid": 97,
          "evidence_invalid": 3,
          "label_agreement": 66,
          "by_reference": {
            "deal": {
              "correct": 60,
              "incorrect": 31,
              "failed": 3,
              "pending": 0,
              "label_agreement": 62
            },
            "no_deal": {
              "correct": 4,
              "incorrect": 2,
              "failed": 0,
              "pending": 0,
              "label_agreement": 4
            }
          }
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 77,
          "incorrect": 22,
          "failed": 1,
          "pending": 0,
          "total": 100,
          "evidence_valid": 99,
          "evidence_invalid": 1,
          "label_agreement": 78,
          "by_reference": {
            "Contradiction": {
              "correct": 21,
              "incorrect": 11,
              "failed": 1,
              "pending": 0,
              "label_agreement": 22
            },
            "Entailment": {
              "correct": 29,
              "incorrect": 5,
              "failed": 0,
              "pending": 0,
              "label_agreement": 29
            },
            "NotMentioned": {
              "correct": 27,
              "incorrect": 6,
              "failed": 0,
              "pending": 0,
              "label_agreement": 27
            }
          }
        }
      ]
    }
  ],
  "consensus": {
    "judges": [
      "sol",
      "gemini",
      "grok"
    ],
    "tasks": [
      {
        "dataset": "casino",
        "task": "accepted",
        "labeled_by_all": 100,
        "all_agree": 68,
        "none_agree": 24,
        "none_agree_identical": 24,
        "none_agree_by_reference": {
          "false": 2,
          "true": 22
        }
      },
      {
        "dataset": "casino",
        "task": "allocation",
        "labeled_by_all": 100,
        "all_agree": 64,
        "none_agree": 29,
        "none_agree_identical": 28,
        "none_agree_by_reference": {
          "deal": 27,
          "no_deal": 2
        }
      },
      {
        "dataset": "contractnli",
        "task": "nli",
        "labeled_by_all": 100,
        "all_agree": 73,
        "none_agree": 14,
        "none_agree_identical": 13,
        "none_agree_by_reference": {
          "Contradiction": 9,
          "Entailment": 1,
          "NotMentioned": 4
        }
      }
    ]
  }
}