{
  "schema_version": "mediationbench-external-readout/v1",
  "sample_id": "external-200-v1",
  "status": "partial",
  "total_cases": 200,
  "source_counts": {
    "casino": 100,
    "contractnli": 100
  },
  "difficulty_counts": {
    "easy": 68,
    "medium": 66,
    "hard": 66
  },
  "datasets": [
    {
      "id": "casino",
      "label": "CaSiNo",
      "case_count": 100,
      "difficulty_counts": {
        "easy": 34,
        "medium": 33,
        "hard": 33
      },
      "source_url": "https://aclanthology.org/2021.naacl-main.254/",
      "license": "CC-BY-4.0",
      "tasks": [
        {
          "id": "accepted",
          "label": "Recorded deal acceptance"
        },
        {
          "id": "allocation",
          "label": "Exact allocation"
        }
      ]
    },
    {
      "id": "contractnli",
      "label": "ContractNLI",
      "case_count": 100,
      "difficulty_counts": {
        "easy": 34,
        "medium": 33,
        "hard": 33
      },
      "source_url": "https://aclanthology.org/2021.findings-emnlp.164/",
      "license": "CC-BY-4.0",
      "tasks": [
        {
          "id": "nli",
          "label": "Contract entailment"
        }
      ]
    }
  ],
  "judges": [
    {
      "id": "sol",
      "label": "Sol",
      "completed": 200,
      "failed": 0,
      "pending": 0,
      "cost_usd": 2.390784,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 100,
          "incorrect": 0,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 98,
          "incorrect": 2,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 79,
          "incorrect": 21,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        }
      ]
    },
    {
      "id": "gemini",
      "label": "Gemini",
      "completed": 200,
      "failed": 0,
      "pending": 0,
      "cost_usd": 0.717864,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 100,
          "incorrect": 0,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 100,
          "incorrect": 0,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 82,
          "incorrect": 18,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        }
      ]
    },
    {
      "id": "grok",
      "label": "Grok",
      "completed": 114,
      "failed": 7,
      "pending": 79,
      "cost_usd": 1.552816,
      "tasks": [
        {
          "dataset": "casino",
          "task": "accepted",
          "correct": 100,
          "incorrect": 0,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "casino",
          "task": "allocation",
          "correct": 100,
          "incorrect": 0,
          "failed": 0,
          "pending": 0,
          "total": 100,
          "evidence_valid": 100,
          "evidence_invalid": 0
        },
        {
          "dataset": "contractnli",
          "task": "nli",
          "correct": 13,
          "incorrect": 1,
          "failed": 7,
          "pending": 79,
          "total": 100,
          "evidence_valid": 14,
          "evidence_invalid": 0
        }
      ]
    }
  ]
}