{
  "calibration": {
    "eligible": false,
    "max_override_error_upper_95": 0.1,
    "minimum_overrides": 40,
    "reason": "no threshold passed the finite-sample risk and non-inferiority gates",
    "threshold": null
  },
  "caveats": [
    "This benchmark measures answer-bearing paragraph retrieval, not downstream LLM answers.",
    "The local transformer is a pretrained semantic encoder, not an Entroly-trained compressor.",
    "The calibration policy uses a Wilson error bound and is not a conformal guarantee.",
    "A headline requires a positive transformer delta and paired McNemar p < 0.05 on the held-out partition."
  ],
  "claim_scope": "Answer-bearing paragraph retrieval under fixed one-of-N selection on a frozen SQuAD v2 validation subset.",
  "generated_at": "2026-07-16T03:18:50.876419+00:00",
  "headline_eligible": false,
  "implementation": {
    "git_commit": "0dc83f1f7759d7ace58cfc2d7ae19380473452f1",
    "packages": {
      "datasets": "4.8.4",
      "sentence-transformers": "5.1.1",
      "torch": "2.8.0+cpu"
    },
    "python": "3.10.0"
  },
  "model": {
    "fingerprint_sha256": "3efc859f2086261cc877fff243cddd2e4532e55e4157747861f705cd36b05a13",
    "id": "c9745ed1d9f207416be6d2e6f8de32d1f16199bf",
    "path_disclosed": false,
    "repository": "sentence-transformers/all-MiniLM-L6-v2",
    "revision": "c9745ed1d9f207416be6d2e6f8de32d1f16199bf"
  },
  "protocol": {
    "calibration_fraction": 0.5,
    "calibration_trials": 300,
    "candidates_per_trial": 16,
    "dataset": "rajpurkar/squad_v2",
    "dataset_fingerprint": "461c0301636b3e0b",
    "distractors_per_trial": 15,
    "offline_only": true,
    "seed": 73021,
    "split": "validation",
    "test_trials": 300,
    "trials": 600
  },
  "schema_version": "entroly.neural-evidence-frontier.v1",
  "test_metrics": {
    "dual_channel_guard": {
      "answer_passage_correct": 298,
      "answer_passage_recall": 0.993333,
      "average_selected_passages": 1.02,
      "disagreements_guarded": 6,
      "passage_compression_ratio": 15.686275,
      "trials": 300
    },
    "lexical_bm25": {
      "mrr": 0.993667,
      "top1_correct": 297,
      "top1_recall": 0.99,
      "top2_recall": 0.996667,
      "trials": 300
    },
    "local_transformer": {
      "mrr": 0.98697,
      "top1_correct": 293,
      "top1_recall": 0.976667,
      "top2_recall": 0.996667,
      "trials": 300
    },
    "paired": {
      "lexical_only_correct": 5,
      "mcnemar_exact_p": 0.21875,
      "transformer_only_correct": 1
    },
    "risk_gated": {
      "top1_correct": 297,
      "top1_recall": 0.99,
      "trials": 300
    }
  },
  "trials": [
    {
      "answer_sha256": "2c598436e5575a5769b69986014588d52c2698414b623e81b2e776766c30eaba",
      "candidate_ids": [
        "5726c5a9f1498d1400e8eac4",
        "57377862c3c5551400e51ef3",
        "572fcd86947a6a140053ccda",
        "57115f0a50c2381900b54aa7",
        "570d35b7b3d812140066d54d",
        "57268f05dd62a815002e8990",
        "56de52614396321400ee27fb",
        "5727ea45ff5b5019007d986e",
        "57096c95200fba1400367fbb",
        "5725bad5271a42140099d0bd",
        "56ddde6b9a695914005b9628",
        "5726431aec44d21400f3dd13",
        "5728f2e26aef051400154896",
        "57264cc6dd62a815002e80e4",
        "572a0e4b6aef051400155214",
        "57114b1a2419e31400955575"
      ],
      "candidate_sha256": [
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "ca0f7b7018127b2736fe74e8ea343545834b32796fc74beb48cde30324c80067",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707"
      ],
      "example_id": "56ddde6b9a695914005b9628",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.18067115,
      "neural_order": [
        10,
        6,
        11,
        8,
        5,
        4,
        0,
        9,
        1,
        2,
        3,
        7,
        12,
        13,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "fc616e9abd0955352e564aade946cbb549b6868cd7e5bc10e1a4ec5a187b8b4d",
      "trial_index": 0
    },
    {
      "answer_sha256": "21fc3f4f42c591959063d711a7b64684c48c60088815b5996e9ada734c239937",
      "candidate_ids": [
        "56e1a38de3433e140042305c",
        "5725fe63ec44d21400f3d7dd",
        "572fbfa504bcaa1900d76c73",
        "5726701add62a815002e84c8",
        "5733faaf4776f4190066162f",
        "57269e3bf1498d1400e8e516",
        "5733d4c8d058e614000b6353",
        "572989846aef051400154fc0",
        "57283db6ff5b5019007d9fbc",
        "5733d7cbd058e614000b63ab",
        "57107932a58dae1900cd69ca",
        "57309564069b5314008321a5",
        "57269bb8708984140094cb95",
        "572998673f37b319004784d5",
        "5725e748ec44d21400f3d733",
        "56dddf4066d3e219004dad5f"
      ],
      "candidate_sha256": [
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "58de06c671d2cd29f3b8bde136326194d51c92607fc4a624d1d5a6e90dd5cf33",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "56c5a58a20c281345ad1b156034653805b4b37502a76613ea7935761d3263ade",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760"
      ],
      "example_id": "56dddf4066d3e219004dad5f",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.1053085,
      "neural_order": [
        15,
        4,
        6,
        9,
        10,
        2,
        11,
        0,
        7,
        3,
        1,
        5,
        8,
        12,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "c09da52a275b186b522fb81b948b60d9cd35a9c2ddcd4ad0349f12fcd45f900a",
      "trial_index": 1
    },
    {
      "answer_sha256": "f748d0c7d58eaf6f47571c846fddce9528f64da78a49fbc1223af3891fbcb78d",
      "candidate_ids": [
        "572982e66aef051400154f92",
        "5727c8863acd2414000dec49",
        "572a0bafaf94a219006aa765",
        "5728fd206aef05140015494c",
        "5728eb1a3acd2414000e01c5",
        "57097b50200fba140036804d",
        "57287b322ca10214002da3be",
        "57287ee3ff5b5019007da274",
        "5726545f708984140094c2a5",
        "57294e6b1d04691400779275",
        "572871bc4b864d1900164a04",
        "571ccd9b5efbb31900334df8",
        "56dde0379a695914005b9636",
        "572a12386aef051400155234",
        "57299d1c1d04691400779581",
        "572647e2dd62a815002e805c"
      ],
      "candidate_sha256": [
        "93620703d50b4155fab9038240e64a854d1329e60d76b3887ba89b6b28625e9e",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "9e5f19ae8c4bcea47d2097bb803a6f512f53874779686d5848e1aec6bdadef81",
        "72457fa0c9150fb2a7fed021650358b9295b523f55397d61363a047683297908",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "ee6a40759a8e9342f506db55cd0c40e075a1ce7fd0ccc2da16d91a2be11ea856",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f"
      ],
      "example_id": "56dde0379a695914005b9636",
      "gated_correct": false,
      "gated_top": 0,
      "gold_index": 12,
      "lexical_correct": false,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.64325202,
      "neural_order": [
        12,
        10,
        11,
        3,
        15,
        7,
        0,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "17dbbb7a8fa5d65b2732113f1321edd0f0bf2af67fd55291458c9854a08882eb",
      "trial_index": 2
    },
    {
      "answer_sha256": "a5ccb1c538e34663a658b1be28b16455ee5285efb10e6f1d4caba1f69ec9782b",
      "candidate_ids": [
        "5728223cff5b5019007d9dc4",
        "572657d9dd62a815002e8230",
        "56de49a8cffd8e1900b4b7a7",
        "5730ac782461fd1900a9cf73",
        "5733f410d058e614000b6663",
        "571095a8a58dae1900cd6a76",
        "5725f2c838643c19005aceed",
        "56e181d9e3433e1400422fa0",
        "572fdc34a23a5019007fca93",
        "56dde0ba66d3e219004dad75",
        "57335fcad058e614000b5970",
        "570d4c3bfed7b91900d45e31",
        "5728202c4b864d19001644ec",
        "57265642f1498d1400e8dc68",
        "5733a32bd058e614000b5f32",
        "5726f96ddd62a815002e9698"
      ],
      "candidate_sha256": [
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4"
      ],
      "example_id": "56dde0ba66d3e219004dad75",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.26706948,
      "neural_order": [
        9,
        2,
        4,
        3,
        0,
        5,
        10,
        13,
        14,
        12,
        1,
        6,
        7,
        8,
        11,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "f243f17dff9a627791ca7d5ef8e2421fd66cc0c8395bd4d7cf53d8ed6e4609cc",
      "trial_index": 3
    },
    {
      "answer_sha256": "bbd334ffa6772a83ea721b4d92872832946576389460d73dc2908b67c641fa13",
      "candidate_ids": [
        "56dde1d966d3e219004dad8d",
        "57302bd0b2c2fd14005689db",
        "57287ccb2ca10214002da3da",
        "5733d4c8d058e614000b6353",
        "570966e0200fba1400367f4f",
        "572a0bafaf94a219006aa765",
        "56e1c720e3433e140042316a",
        "57293f353f37b3190047819b",
        "5730a0778ab72b1400f9c60a",
        "5730a314069b5314008321eb",
        "57286951ff5b5019007da20e",
        "571144d1a58dae1900cd6d6e",
        "5728fc9e1d04691400778f11",
        "570d2556fed7b91900d45c47",
        "572825a92ca10214002d9efe",
        "573796edc3c5551400e51f33"
      ],
      "candidate_sha256": [
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "dea705fdf75f529664435816398522b553e31b0965a1287f3375c31a630e304e",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668"
      ],
      "example_id": "56dde1d966d3e219004dad8d",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        7,
        15,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.34364011,
      "neural_order": [
        0,
        13,
        3,
        10,
        8,
        9,
        12,
        2,
        14,
        15,
        11,
        5,
        1,
        4,
        6,
        7
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "052ae51aabe2252ec7831afe749eb2ff9429891554ce96e3604730f3bf5e5d56",
      "trial_index": 4
    },
    {
      "answer_sha256": "7f0fd862f87056e16d05e4cde8a8478e7040058b770411a0d9b08308e7e38113",
      "candidate_ids": [
        "57335c20d058e614000b58f9",
        "572a11663f37b31900478693",
        "5727c55bff5b5019007d94ce",
        "5730005db2c2fd1400568703",
        "57096b66200fba1400367fa7",
        "5733e771d058e614000b6545",
        "57094b4f9928a814004714f6",
        "57309446396df919000961b8",
        "56dde27d9a695914005b9651",
        "56de4b074396321400ee2793",
        "573092088ab72b1400f9c594",
        "57286bf84b864d19001649d2",
        "5728fb002ca10214002dab6a",
        "57300ec0947a6a140053d004",
        "572a213e6aef0514001552ee",
        "572822233acd2414000df555"
      ],
      "candidate_sha256": [
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b",
        "3243e4c3f26f22351bf8c52e8438c396a19b76af187bad37a4ca97532160e33d",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "33e49ae942c39aefd29d877e8334707b666747d16641fd8392ae988a08b47f20",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758"
      ],
      "example_id": "56dde27d9a695914005b9651",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": false,
      "neural_margin": 0.04536313,
      "neural_order": [
        9,
        8,
        3,
        13,
        15,
        5,
        10,
        7,
        0,
        11,
        2,
        1,
        4,
        6,
        12,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "19714ee1d61133eecaef7c75ba973bc0a24a19267543c99328afc05d20d46c5d",
      "trial_index": 5
    },
    {
      "answer_sha256": "f56d7d974ac863f7f42e46a8a331bf383c71b18f5ab81094d43eec05cbc12604",
      "candidate_ids": [
        "57265526708984140094c2bd",
        "5725f00938643c19005aced7",
        "5730a314069b5314008321eb",
        "5728827b2ca10214002da42a",
        "57302cd004bcaa1900d772d7",
        "570d2f5bfed7b91900d45ccf",
        "5733ea04d058e614000b6594",
        "572885c44b864d1900164a78",
        "56dde2fa66d3e219004dad9b",
        "57296d571d04691400779413",
        "57287b322ca10214002da3be",
        "57274f49f1498d1400e8f61e",
        "5727c8863acd2414000dec49",
        "5727f05b4b864d1900164068",
        "5727482bf1498d1400e8f5a4",
        "5726f0e35951b619008f82f7"
      ],
      "candidate_sha256": [
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "8ec9dcc4c2fe0ea5e8f0e2e102236786bbdd27dc8fb4452718334cf4545b34bf",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "255db98fb0b6e32abd3b8317106e8eeb1841b1f3860f985c69bb911aae85aadc",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "27749618f729fa62e79b34e0666fb6ce7980b9e0e582584b2a78d0294543512a",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf"
      ],
      "example_id": "56dde2fa66d3e219004dad9b",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        3,
        9,
        1,
        0,
        2,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.27103879,
      "neural_order": [
        8,
        0,
        6,
        3,
        11,
        1,
        2,
        10,
        14,
        13,
        9,
        4,
        5,
        7,
        12,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "a0a1d4cafd76f7842f56c28a0b951eab2216173d7ef25d26355d3c2f184f6085",
      "trial_index": 6
    },
    {
      "answer_sha256": "dcd8ac9e2c01d7e19c228b97aeb767b502d3ce691afe412fc17d3d8674584bb1",
      "candidate_ids": [
        "572ffce5a23a5019007fcc15",
        "56e1a38de3433e140042305c",
        "57335ddbd058e614000b592f",
        "57302bd0b2c2fd14005689db",
        "5710f114a58dae1900cd6b60",
        "5726b929f1498d1400e8e8e9",
        "571c96095efbb31900334dbc",
        "56de0f6a4396321400ee257f",
        "57306797396df919000960ee",
        "5727ee372ca10214002d99ec",
        "571a50df4faf5e1900b8a960",
        "5727515f708984140094dc11",
        "573406d1d058e614000b6801",
        "57269e3bf1498d1400e8e516",
        "571114cfb654c5140001fb09",
        "5733a6ac4776f41900660f58"
      ],
      "candidate_sha256": [
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "1629b48b9aae07c00bf8bd9ec7d76d9982b5e645f832551c105d868a0f34f81e",
        "3ee61e6d7605bacdeb4f37ab798daa449f57668fd923e6b04e1a516237821782",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "291631d73d5dcdb57b0941017d7b1ab7a9501e869a33f4c995211b70c5487c88",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8"
      ],
      "example_id": "56de0f6a4396321400ee257f",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.40853219,
      "neural_order": [
        7,
        12,
        4,
        14,
        2,
        8,
        3,
        15,
        13,
        0,
        5,
        10,
        1,
        6,
        9,
        11
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "28eb9e07c9f80a1abb87860594e081ebd6e97e23a2addb20c158287857a0f00d",
      "trial_index": 7
    },
    {
      "answer_sha256": "2df8e96ccd085cced7a458b374f8ea59685ff4e6ef5945a1cbc4b6d5079c34cf",
      "candidate_ids": [
        "572980f9af94a219006aa4d1",
        "573796edc3c5551400e51f33",
        "56de0ffd4396321400ee258d",
        "571cc3dedd7acb1400e4c146",
        "573062662461fd1900a9cdf7",
        "5737a7351c456719005744f1",
        "5726462b708984140094c117",
        "5733fc6ed058e614000b670f",
        "5733f7b9d058e614000b66a9",
        "57281edd3acd2414000df4eb",
        "5727565ef1498d1400e8f674",
        "572647935951b619008f6ec9",
        "5737a25ac3c5551400e51f51",
        "57271f125951b619008f8635",
        "572970c11d04691400779463",
        "5734025d4776f419006616c3"
      ],
      "candidate_sha256": [
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "8d0f136ca3e21898494994833851f5c381fc610d00c5a286083f4b15d65caad4",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "5d46e9f7c620b7d8f42a0c90216d4bb7625ee95e4d9af99b663434ae1e08a483",
        "2271df8c8c1b2d712c122939c9c53f699509a751bc040f6c45e9660ceb9c2177",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662"
      ],
      "example_id": "56de0ffd4396321400ee258d",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.30976336,
      "neural_order": [
        2,
        7,
        11,
        15,
        13,
        8,
        4,
        5,
        1,
        0,
        3,
        6,
        9,
        10,
        12,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "ef4569b24b58831e2598d385e3ee7453ea32eb2d6fc392d2fc6f85b2c1e8cc29",
      "trial_index": 8
    },
    {
      "answer_sha256": "06328b61f6829b37a2f8a30d25f191519e2aea5742d7c2d2e1874b53a8406061",
      "candidate_ids": [
        "56de1645cffd8e1900b4b5d0",
        "571c8eb9dd7acb1400e4c0f6",
        "572629c6271a42140099d6a3",
        "572ff760b2c2fd1400568677",
        "5730ac782461fd1900a9cf73",
        "5725ec8289a1e219009ac0ae",
        "5733a6ac4776f41900660f58",
        "572871bd3acd2414000dfa03",
        "571a484210f8ca1400304fbe",
        "57377083c3c5551400e51edf",
        "56de10b44396321400ee2593",
        "572684365951b619008f753f",
        "571cda1bdd7acb1400e4c196",
        "572ff935b2c2fd140056869b",
        "5737a7351c456719005744f1",
        "5705ec1675f01819005e776e"
      ],
      "candidate_sha256": [
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "e1e68b90e348d860a1f3c0cfe6e1367d9c8c8a57ad577d564b36e00900222682",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1"
      ],
      "example_id": "56de10b44396321400ee2593",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        2,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.09132634,
      "neural_order": [
        10,
        0,
        4,
        11,
        3,
        5,
        14,
        1,
        12,
        2,
        9,
        13,
        7,
        6,
        8,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "8eab15799f71d4a98e565edc4418f8b5e38ddc28d403ca9a093ef416ea0701be",
      "trial_index": 9
    },
    {
      "answer_sha256": "3a5e29a8628f0b74476c00573b1b3f2302e6845a579d1c608bc1b5568b20b458",
      "candidate_ids": [
        "57267947f1498d1400e8e0ec",
        "572ff86004bcaa1900d76f65",
        "57339555d058e614000b5df3",
        "572f65e9b2c2fd14005680cb",
        "570d44abb3d812140066d5fd",
        "5726f2375951b619008f830f",
        "572853e8ff5b5019007da188",
        "572fc5a1947a6a140053cc8a",
        "5710f2e2a58dae1900cd6b72",
        "56de11154396321400ee25aa",
        "571a4ead10f8ca1400304fdd",
        "5711607f2419e314009555cd",
        "56e181d9e3433e1400422fa0",
        "57294e6b1d04691400779275",
        "5727de862ca10214002d9860",
        "5730a0778ab72b1400f9c60a"
      ],
      "candidate_sha256": [
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "3bafbc5d56c1857ae4a5346827458b7900e775e4ebe79997c59538a3013f8c57",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e"
      ],
      "example_id": "56de11154396321400ee25aa",
      "gated_correct": false,
      "gated_top": 0,
      "gold_index": 9,
      "lexical_correct": false,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.42455577,
      "neural_order": [
        9,
        10,
        15,
        8,
        3,
        2,
        11,
        1,
        0,
        4,
        5,
        6,
        7,
        12,
        13,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "05f78d4b9cb5788438b1dc1b83cce46e2d148c325a49e0f8c7f9c32f64360cd1",
      "trial_index": 10
    },
    {
      "answer_sha256": "2e2ef14d2d071e085b363318f3db5bff6446510b9156124ead4400dfc33e861a",
      "candidate_ids": [
        "572827fc3acd2414000df5bb",
        "572996c73f37b319004784b3",
        "572fd1c4947a6a140053cd02",
        "572a0d21af94a219006aa783",
        "56de148dcffd8e1900b4b5bc",
        "5727aeac3acd2414000de987",
        "572a0bafaf94a219006aa765",
        "57267640f1498d1400e8e074",
        "57299021af94a219006aa50b",
        "5728705c2ca10214002da35a",
        "5730b4282461fd1900a9cfc5",
        "56e1fc57e3433e140042322c",
        "570d2af0fed7b91900d45c93",
        "5725bad5271a42140099d0bd",
        "56e1dc62cd28a01900c67bcb",
        "5726baf2dd62a815002e8e74"
      ],
      "candidate_sha256": [
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "536e195415da4934032881cb450b11e570caec121170eb2072ecccea49d36996",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "c71b1eee7e8e8a47719b20f0d4f2e9ca9aaeec5e9d9082387579fe235a47016d",
        "b471e50d3722fdd0e8fac7b90182b37de8e917f2b6ef94945670aafaf625f2bc",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c"
      ],
      "example_id": "56de148dcffd8e1900b4b5bc",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.23291985,
      "neural_order": [
        4,
        14,
        0,
        6,
        8,
        9,
        5,
        3,
        15,
        7,
        1,
        2,
        10,
        11,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "1495d1211f28c819bac5d6c0b5c759d938a3c3e222fc4399f77ef9a5ba62f8ab",
      "trial_index": 11
    },
    {
      "answer_sha256": "70b8c33c04d94f6f504eb2e961feb5e5c201f81854ba9f21cddf356b106c8d3a",
      "candidate_ids": [
        "572a1dbb3f37b319004786f5",
        "572684365951b619008f753f",
        "56e1aff7cd28a01900c67a68",
        "57339902d058e614000b5e70",
        "572f5703a23a5019007fc573",
        "5737a0acc3c5551400e51f47",
        "5733a1854776f41900660f0a",
        "5705f09e75f01819005e77a4",
        "5711163bb654c5140001fb13",
        "5733f410d058e614000b6663",
        "5728667eff5b5019007da1fc",
        "572a12386aef051400155234",
        "5725c57a89a1e219009abe5e",
        "56de15104396321400ee25b7",
        "572facb0a23a5019007fc863",
        "5737a4511c456719005744df"
      ],
      "candidate_sha256": [
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "acd49776c5c867dd78a5c6b915a917f39479b502b183b3b4b0c8d4934fc1536a",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "65a4f29a52c31659b66baa59545297d8af56b2b1f884801f998c7d4c6e8396d8",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451"
      ],
      "example_id": "56de15104396321400ee25b7",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.29572196,
      "neural_order": [
        13,
        14,
        7,
        9,
        8,
        4,
        6,
        15,
        0,
        1,
        2,
        3,
        5,
        10,
        11,
        12
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "c46740e2d216a70437ec7041fe9ac23ba87f2f50974f746bab5a34a146dc47a2",
      "trial_index": 12
    },
    {
      "answer_sha256": "efec9aaf21433bf806e7681de337cac7dbecfbf17b22ad3bcdfe5e46e564f32f",
      "candidate_ids": [
        "57265d86f1498d1400e8dd50",
        "572fca7eb2c2fd1400568473",
        "572655e5f1498d1400e8dc5e",
        "56de3efccffd8e1900b4b6fe",
        "56de1563cffd8e1900b4b5c2",
        "572860e03acd2414000df977",
        "572f7588947a6a140053c984",
        "5726219489a1e219009ac2d0",
        "56de40da4396321400ee2708",
        "56e1ce08e3433e14004231a4",
        "57309446396df919000961b8",
        "57287c142ca10214002da3d0",
        "5728f2e26aef051400154896",
        "572fd7b8947a6a140053cd3a",
        "57297a276aef051400154f88",
        "57267d52708984140094c7d9"
      ],
      "candidate_sha256": [
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "218bcafd4ea9a6566900672f26303adc874834255c2b12646ddef2c72e9093f2",
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d",
        "ecce67f5ebd93f7f51596b468b36d1918c0f3d320dd7e6bd4fa7ac0ca537f812",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b"
      ],
      "example_id": "56de1563cffd8e1900b4b5c2",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.32796365,
      "neural_order": [
        4,
        8,
        3,
        14,
        15,
        6,
        13,
        10,
        2,
        9,
        1,
        0,
        5,
        7,
        11,
        12
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "9ea99c610aaea0914b5b7b3b2f164c39586044342cbb31e0337e3175be296e49",
      "trial_index": 13
    },
    {
      "answer_sha256": "26848e63cbd88f2214cc8909ae3918bac77ac91537a5ce3bc814a45faa52f56d",
      "candidate_ids": [
        "56e17644e3433e1400422f40",
        "57113ba6b654c5140001fc1f",
        "57296d571d04691400779413",
        "56de40da4396321400ee2708",
        "572750e8dd62a815002e9af0",
        "5737aafd1c456719005744fb",
        "5726ba2c708984140094cf59",
        "5729a03f1d04691400779593",
        "57115ff82419e314009555c3",
        "57294209af94a219006aa201",
        "56de15dbcffd8e1900b4b5c8",
        "5727e0474b864d1900163f08",
        "57263b1638643c19005ad333",
        "57264c42dd62a815002e80c6",
        "57293e983f37b3190047818b",
        "57265ceddd62a815002e82b8"
      ],
      "candidate_sha256": [
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "d404471721922f4e521c100992e50a7d7424e6de36fb7f727122c1b4b42aaf6a",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00"
      ],
      "example_id": "56de15dbcffd8e1900b4b5c8",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.28873048,
      "neural_order": [
        10,
        3,
        12,
        7,
        9,
        6,
        0,
        1,
        2,
        4,
        5,
        8,
        11,
        13,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "9a3ea85dd5776d1208adb2c12f70f0c4ac7092f92ceebb69887c3210df3be631",
      "trial_index": 14
    },
    {
      "answer_sha256": "5d2b5917e81d3f14b32985159f94d3af449ebc4a852f7b2a3099f89a00c8c92d",
      "candidate_ids": [
        "56de1645cffd8e1900b4b5d0",
        "57114f0050c2381900b54a65",
        "57263fd138643c19005ad37b",
        "56de4b074396321400ee2793",
        "572fd8efb2c2fd14005684fa",
        "57265d86f1498d1400e8dd50",
        "5726e5c95951b619008f81cb",
        "571a53d410f8ca1400304fe5",
        "5711119cb654c5140001fae3",
        "572686ac5951b619008f75a9",
        "57335fcad058e614000b5970",
        "5727dc473acd2414000dee43",
        "572a020f6aef051400155198",
        "57339a554776f41900660e74",
        "5733d2dbd058e614000b6339",
        "572a0ecb1d04691400779717"
      ],
      "candidate_sha256": [
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835"
      ],
      "example_id": "56de1645cffd8e1900b4b5d0",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.48160292,
      "neural_order": [
        0,
        3,
        8,
        9,
        14,
        15,
        7,
        5,
        2,
        1,
        4,
        6,
        10,
        11,
        12,
        13
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "f744c05dd67516aa9865f1aca5be4b46a344628a3cba24cb501824ec0c823d38",
      "trial_index": 15
    },
    {
      "answer_sha256": "253179b7231f221a65e3f1cd7a6ddf7fd59614f7a47237cce158d7ae84f6cc0b",
      "candidate_ids": [
        "5728f3724b864d1900165116",
        "5730088e947a6a140053cfac",
        "57265bdfdd62a815002e829e",
        "57265700dd62a815002e820e",
        "57264b3edd62a815002e80aa",
        "5733cf61d058e614000b62e9",
        "571cc8815efbb31900334dee",
        "5711669550c2381900b54adf",
        "573028fa04bcaa1900d77287",
        "57377862c3c5551400e51ef3",
        "572fad30a23a5019007fc86d",
        "572881704b864d1900164a50",
        "572f55e8a23a5019007fc56b",
        "5733f5f24776f419006615c1",
        "5732b6b5328d981900602021",
        "56de16ca4396321400ee25c5"
      ],
      "candidate_sha256": [
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "e26d3a98154d79e4fb93dfbc9b446801d4e38dc1c4681b39878f40025a6b4fce",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee"
      ],
      "example_id": "56de16ca4396321400ee25c5",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.22609733,
      "neural_order": [
        15,
        13,
        0,
        1,
        3,
        11,
        4,
        10,
        6,
        5,
        14,
        2,
        7,
        8,
        9,
        12
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "83272368e23b8caf3739970f61e737dbcd19324d4fe8c5a1d7bec4af872eac42",
      "trial_index": 16
    },
    {
      "answer_sha256": "086dfc983d2299d66a7d4d7ca36edb4b13422525821c82b151315e2cc59394f1",
      "candidate_ids": [
        "5706143575f01819005e7950",
        "572a2224af94a219006aa823",
        "57300bf504bcaa1900d77087",
        "5725e45689a1e219009ac048",
        "572fce12a23a5019007fca11",
        "57287d4a2ca10214002da3e4",
        "572812e74b864d19001643ce",
        "57373a9fc3c5551400e51e7b",
        "5728223cff5b5019007d9dc4",
        "57332e48d058e614000b5762",
        "57308f6b8ab72b1400f9c580",
        "5705fc3a52bb89140068976a",
        "5726f7715951b619008f838d",
        "56de1728cffd8e1900b4b5d7",
        "571135b8a58dae1900cd6d0e",
        "5727d88b4b864d1900163e5e"
      ],
      "candidate_sha256": [
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "18893f714ac102ba660ec3790204ffdf2ee86fd4cd069c50ec0ceeaed01cefbd",
        "7a54e69984565794b95d2e675d9ceb87e6f076fbf18c6fc5f9f4161c8750a672",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "e77cc2842075cf67e0f42d9c55fea1df5857504c655c9e1e596c5f52dc859507",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "4aaa6faf86477d3be34fa2e2263e88b5c583216a96351af8fd82b9f4924bd81e"
      ],
      "example_id": "56de1728cffd8e1900b4b5d7",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.52851241,
      "neural_order": [
        13,
        10,
        4,
        14,
        6,
        2,
        8,
        7,
        9,
        0,
        1,
        3,
        5,
        11,
        12,
        15
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "aed8e58d73e75b942bea2c81e7abe92460a4fa76a524f5b57a9ef2d8a3fe64df",
      "trial_index": 17
    },
    {
      "answer_sha256": "36d144ea081f24500bf72163ccde3d47487366cc47e9ea2fa199f60bbbcee648",
      "candidate_ids": [
        "5727b8df3acd2414000dea99",
        "570d2c20fed7b91900d45ca7",
        "572885c44b864d1900164a78",
        "56de17f9cffd8e1900b4b5e0",
        "570d32e4fed7b91900d45d01",
        "5711658e50c2381900b54ad5",
        "57271f125951b619008f8635",
        "572fbea404bcaa1900d76c5b",
        "56de179dcffd8e1900b4b5da",
        "5729f1fcaf94a219006aa6d3",
        "573081c2069b531400832133",
        "57264a74708984140094c18b",
        "5725cb33271a42140099d1db",
        "57308cf88ab72b1400f9c576",
        "57302faa04bcaa1900d77311",
        "57097c8fed30961900e841f2"
      ],
      "candidate_sha256": [
        "286230116fa743f77871c0721b3874ae2889664d4113752586c5f53755626f21",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "99afd74c371b051e0dd2704df0676d8589e7c060ad964a7cae71b5a5bde23c1e",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336"
      ],
      "example_id": "56de179dcffd8e1900b4b5da",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.17611584,
      "neural_order": [
        8,
        3,
        5,
        14,
        1,
        12,
        11,
        6,
        0,
        10,
        13,
        2,
        4,
        7,
        9,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "8366c1620ea1abfa5110740e3b789c874c5739e454bee57f2892787266f05631",
      "trial_index": 18
    },
    {
      "answer_sha256": "8849853b957fe153b7056d0e7d65f99fb21070daf5122ddf1d7c942d4643c33d",
      "candidate_ids": [
        "571a52cb4faf5e1900b8a969",
        "57094b4f9928a814004714f6",
        "5706143575f01819005e7950",
        "5733d68ed058e614000b637f",
        "56de17f9cffd8e1900b4b5e0",
        "57283f014b864d19001647e8",
        "5729f4e46aef051400155156",
        "572ffd9e04bcaa1900d76fc7",
        "57302a3aa23a5019007fcecf",
        "572881022ca10214002da416",
        "56de15104396321400ee25b7",
        "56e1febfe3433e1400423236",
        "5728fc2eaf94a219006a9ec7",
        "57264845f1498d1400e8db0a",
        "5727c3b02ca10214002d95ba",
        "5725b92e38643c19005acbd3"
      ],
      "candidate_sha256": [
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "0fe05bd1833002131a7e69e040cc114a9dfc4409000d5d0674152325ab91cb19",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d"
      ],
      "example_id": "56de17f9cffd8e1900b4b5e0",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.08146016,
      "neural_order": [
        4,
        10,
        13,
        3,
        14,
        5,
        15,
        9,
        7,
        0,
        1,
        2,
        6,
        8,
        11,
        12
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "33b7e3e806da10c9dbfc442caf3d5081cb9ad466d370fcc273165fdbb590e273",
      "trial_index": 19
    },
    {
      "answer_sha256": "ebf61aeacd768f8a2a1a3f4790105df8721e3fb96eaa158635dcef2d15bfd437",
      "candidate_ids": [
        "57060a6e52bb8914006897f8",
        "56de1563cffd8e1900b4b5c2",
        "5710eca0a58dae1900cd6b3a",
        "56de3cd0cffd8e1900b4b6be",
        "57377083c3c5551400e51edf",
        "57300bf504bcaa1900d77087",
        "572879574b864d1900164a14",
        "572883153acd2414000dfa6f",
        "5725f5b1271a42140099d371",
        "573020f7b2c2fd14005688f7",
        "571a52cb4faf5e1900b8a969",
        "57281edd3acd2414000df4eb",
        "5728710c3acd2414000df9ef",
        "5726385e271a42140099d797",
        "57097141200fba1400367ff9",
        "572a12386aef051400155234"
      ],
      "candidate_sha256": [
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "7a54e69984565794b95d2e675d9ceb87e6f076fbf18c6fc5f9f4161c8750a672",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "2271df8c8c1b2d712c122939c9c53f699509a751bc040f6c45e9660ceb9c2177",
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023"
      ],
      "example_id": "56de3cd0cffd8e1900b4b6be",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.16806096,
      "neural_order": [
        3,
        1,
        2,
        5,
        10,
        4,
        7,
        8,
        13,
        0,
        6,
        9,
        11,
        12,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "2daba63dd75aec76ecff45a8c69f203a1d294211de914f2148e9251e72af9537",
      "trial_index": 20
    },
    {
      "answer_sha256": "59d78a7178c4d8a4f9c7e93beb2a30d83d0ffa11b4e87f35824976cb245f6cd5",
      "candidate_ids": [
        "57283db6ff5b5019007d9fbc",
        "56de3d594396321400ee26ca",
        "5705f36452bb891400689718",
        "57332b66d058e614000b5758",
        "572810ec2ca10214002d9d06",
        "57332e48d058e614000b5762",
        "57268c01dd62a815002e8912",
        "56e1c3e1e3433e1400423148",
        "572a0e4b6aef051400155214",
        "5726f2375951b619008f830f",
        "5710e9f8a58dae1900cd6b30",
        "57339a554776f41900660e74",
        "5726446a89cfff1900a8404d",
        "56de49564396321400ee277a",
        "572fcb6da23a5019007fc9f1",
        "572826634b864d19001645be"
      ],
      "candidate_sha256": [
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2"
      ],
      "example_id": "56de3d594396321400ee26ca",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.21895205,
      "neural_order": [
        1,
        13,
        0,
        10,
        9,
        6,
        5,
        4,
        2,
        3,
        7,
        8,
        11,
        12,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "9914db49a85748913e0039f9e5395c2c341a70ec05b3d91bf426b7e74d017429",
      "trial_index": 21
    },
    {
      "answer_sha256": "6baade0e6ad44a503ff0d82b52fb878d3bfc5b61313e558fa03b7cbaeab2c311",
      "candidate_ids": [
        "572853e8ff5b5019007da188",
        "5727ff083acd2414000df1ab",
        "5730ab63396df91900096260",
        "57060a6e52bb8914006897f8",
        "572ff932a23a5019007fcbd5",
        "5725cfd0271a42140099d225",
        "57264d9edd62a815002e80fe",
        "5728f9882ca10214002dab5a",
        "5729f799af94a219006aa707",
        "572659535951b619008f703f",
        "572a0ecb1d04691400779717",
        "57111b95a58dae1900cd6c50",
        "5728fb6a1d04691400778ef5",
        "5733a45d4776f41900660f23",
        "5725e1c4271a42140099d2d7",
        "56de3dbacffd8e1900b4b6d2"
      ],
      "candidate_sha256": [
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "fbdd4577c5f9e5f4af71320383d43f13f7697f04d5670525f7d260f7ff56759d",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607"
      ],
      "example_id": "56de3dbacffd8e1900b4b6d2",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.32718124,
      "neural_order": [
        15,
        11,
        2,
        13,
        6,
        4,
        1,
        9,
        10,
        3,
        0,
        5,
        7,
        8,
        12,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "a219b232329dc36fecec9c5f96900262e2fb5ac705e65dbbb645529cae02a251",
      "trial_index": 22
    },
    {
      "answer_sha256": "9989a37538faecc495bbe774b30fb447a1582b8f1bc14ac88dfd84996a7b799a",
      "candidate_ids": [
        "57264684708984140094c123",
        "572871bc4b864d1900164a04",
        "57263cfcec44d21400f3dc8d",
        "57108073b654c5140001f925",
        "572a06866aef0514001551be",
        "56de3e414396321400ee26d8",
        "572faec7b2c2fd1400568333",
        "5725d34aec44d21400f3d639",
        "573003dd947a6a140053cf42",
        "57111b95a58dae1900cd6c50",
        "57269f3ef1498d1400e8e534",
        "5726534d708984140094c26d",
        "572885c44b864d1900164a78",
        "572fdbb004bcaa1900d76dd9",
        "56de17f9cffd8e1900b4b5e0",
        "5725daa8ec44d21400f3d6b3"
      ],
      "candidate_sha256": [
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc"
      ],
      "example_id": "56de3e414396321400ee26d8",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        10,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.11563536,
      "neural_order": [
        5,
        1,
        3,
        11,
        7,
        9,
        10,
        14,
        4,
        13,
        8,
        0,
        2,
        6,
        12,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "247c2d7875473264201a59af67cd48d1adec7ed466199852be484f22214dbaec",
      "trial_index": 23
    },
    {
      "answer_sha256": "235aa062e6372588dbae00552abf36b8ff9c315e3da56cf02786980e764630e9",
      "candidate_ids": [
        "5726a5525951b619008f78dd",
        "571a4d1a4faf5e1900b8a958",
        "57263ea0271a42140099d7c3",
        "56de4bb84396321400ee27a2",
        "5725bcb6271a42140099d0eb",
        "572885023acd2414000dfa83",
        "5727cd7dff5b5019007d958e",
        "56de3ebc4396321400ee26e6",
        "57112686b654c5140001fbd3",
        "5706094b52bb8914006897de",
        "57263dcd89a1e219009ac5a2",
        "5726a7ecf1498d1400e8e654",
        "572647935951b619008f6ec9",
        "57340111d058e614000b677d",
        "5729fb003f37b31900478627",
        "572861cc4b864d190016495e"
      ],
      "candidate_sha256": [
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "ffaee1c83ab04918603b9940620c0c8b52e675143bc31a91f1fccac00d2b1bf7",
        "3f83987ebc0c4b6d662d8cc6134fe19bdcd18d63555f50b4c6841486edd3606d"
      ],
      "example_id": "56de3ebc4396321400ee26e6",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.21965782,
      "neural_order": [
        7,
        13,
        10,
        3,
        12,
        4,
        1,
        2,
        0,
        5,
        6,
        8,
        9,
        11,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "710eceecbf7c4b6fba75bc43ac5801cfa55e24f701297d34d69596dc628bed15",
      "trial_index": 24
    },
    {
      "answer_sha256": "a090c49ff9864d2bb5e8e29aa8d6ddd115ce178626b479189758116669952839",
      "candidate_ids": [
        "5725e748ec44d21400f3d733",
        "57300ec0947a6a140053d004",
        "572fc5a1947a6a140053cc8a",
        "5726dcbddd62a815002e9320",
        "573093598ab72b1400f9c5ae",
        "5726f2375951b619008f830f",
        "5726487b5951b619008f6edd",
        "5710f114a58dae1900cd6b60",
        "572863c72ca10214002da2d4",
        "56de3efccffd8e1900b4b6fe",
        "572650325951b619008f6fa9",
        "57267ebfdd62a815002e872c",
        "56e1c9bfe3433e1400423192",
        "570603c475f01819005e7882",
        "57115f652419e314009555b9",
        "56e1fc57e3433e140042322c"
      ],
      "candidate_sha256": [
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "33e49ae942c39aefd29d877e8334707b666747d16641fd8392ae988a08b47f20",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "ccfb450b288c15536e52e7e1c54aa458c0e2c2936babc78f6cab1b418fa9e1db",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "218bcafd4ea9a6566900672f26303adc874834255c2b12646ddef2c72e9093f2",
        "b0008033f8b97b0cf458d8e505384798d796a2fc78a3fced41f39a2591ef64cd",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "752151b83536aa2b39e17ee8104540f3b94aa10671b826eb5b19cc6d1e68a8ae",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace"
      ],
      "example_id": "56de3efccffd8e1900b4b6fe",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        15,
        4,
        8,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.34971404,
      "neural_order": [
        9,
        6,
        8,
        1,
        7,
        4,
        14,
        2,
        3,
        10,
        12,
        5,
        11,
        0,
        13,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "c00699c74a620486f97aaaf7d8b2b4d21ffca8087b007482bbb23decf2290d1c",
      "trial_index": 25
    },
    {
      "answer_sha256": "671d155e4c4c29379b80f1e0cc0b5373b02c7d152137c7295262cf71e7b527a0",
      "candidate_ids": [
        "572ff626947a6a140053ce8e",
        "56e1b355e3433e14004230b0",
        "5705ec1675f01819005e776e",
        "5737a84dc3c5551400e51f59",
        "572663a9f1498d1400e8ddf2",
        "5725c95f38643c19005accf3",
        "572ff760b2c2fd1400568677",
        "572629c6271a42140099d6a3",
        "572f7588947a6a140053c984",
        "572978f91d046914007794d3",
        "56de3f784396321400ee26fa",
        "5730b541396df919000962c2",
        "5730b6592461fd1900a9cfcf",
        "572a1c943f37b319004786e1",
        "5726f48df1498d1400e8f0da",
        "5726c3da708984140094d0d9"
      ],
      "candidate_sha256": [
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "5bae1cf69d61dbf0b59d2020a2b8aaf2c59dfea5d8efc8e4b560e663cbaefcbd",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc"
      ],
      "example_id": "56de3f784396321400ee26fa",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.13906793,
      "neural_order": [
        10,
        5,
        9,
        13,
        12,
        7,
        11,
        4,
        8,
        0,
        1,
        2,
        3,
        6,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "7900fa134f87270eeddae3d47ca30095a9036d03fa6b070bde90ca730e2e59e4",
      "trial_index": 26
    },
    {
      "answer_sha256": "43a3999986187fb2558032b23a6df2853fc09e7bd58be4816655edc30de847bd",
      "candidate_ids": [
        "57114aceb654c5140001fc47",
        "57094d489928a8140047150a",
        "57268527708984140094c8bf",
        "57332a734776f41900660726",
        "5726fa525951b619008f83f7",
        "572810ec2ca10214002d9d06",
        "5727ea45ff5b5019007d986e",
        "572f567cb2c2fd140056803f",
        "572a1f086aef0514001552c0",
        "5705ec1675f01819005e776e",
        "57264d9edd62a815002e80fe",
        "56de40da4396321400ee2708",
        "5726f7715951b619008f838d",
        "572a135daf94a219006aa79f",
        "57286ab72ca10214002da31e",
        "572871bc4b864d1900164a04"
      ],
      "candidate_sha256": [
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "75bcc54c3f28b2056c91642f34ed043daecd110b921b6d80aee1698da2f1af22",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534"
      ],
      "example_id": "56de40da4396321400ee2708",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.01421141,
      "neural_order": [
        11,
        15,
        14,
        9,
        10,
        2,
        5,
        1,
        6,
        8,
        0,
        3,
        4,
        7,
        12,
        13
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "a46a231b8e5702bfec5c2d19efc66e73bc3e004175adad36cd3216f3325910e9",
      "trial_index": 27
    },
    {
      "answer_sha256": "3f5053b3ae70794bff96eb3501a85eedda4b3ee740c9207e6568df248a867c13",
      "candidate_ids": [
        "57300a9a04bcaa1900d77063",
        "57096c95200fba1400367fbb",
        "57285213ff5b5019007da182",
        "5737a25ac3c5551400e51f51",
        "572fdc34a23a5019007fca93",
        "57094b4f9928a814004714f6",
        "57108073b654c5140001f925",
        "56de49564396321400ee277a",
        "5733e771d058e614000b6545",
        "572a0ecb1d04691400779717",
        "57263c78ec44d21400f3dc7b",
        "57287ccb2ca10214002da3da",
        "572ff626947a6a140053ce8e",
        "572908c13f37b31900477fbd",
        "571cd3b55efbb31900334e03",
        "5729a3716aef05140015506a"
      ],
      "candidate_sha256": [
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "9f7d44852206b170e9506d2e1aace68791eb27a740bd612bbe067f4feddf0fd8",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea"
      ],
      "example_id": "56de49564396321400ee277a",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.22379951,
      "neural_order": [
        7,
        8,
        4,
        3,
        2,
        1,
        6,
        15,
        0,
        5,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "4d4d935b994acca22721e6ac14be2239358f51b75d3e8771f6821c04b28717ab",
      "trial_index": 28
    },
    {
      "answer_sha256": "2d2ecaf59c9bb73e94bbb8fb593bc941f07afe3bb76031bf7f1d723812d30a42",
      "candidate_ids": [
        "56de49a8cffd8e1900b4b7a7",
        "571166352419e314009555f1",
        "5728855d3acd2414000dfa8d",
        "57340111d058e614000b677d",
        "57299021af94a219006aa50b",
        "571c9348dd7acb1400e4c114",
        "57377ec7c3c5551400e51f05",
        "5705ec1675f01819005e776e",
        "56e1b00ce3433e140042309e",
        "56de16ca4396321400ee25c5",
        "5726ba2c708984140094cf59",
        "5727526cdd62a815002e9b0e",
        "57274e975951b619008f87f9",
        "5737a4511c456719005744df",
        "572ffb02b2c2fd14005686b7",
        "57302bd0b2c2fd14005689db"
      ],
      "candidate_sha256": [
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518"
      ],
      "example_id": "56de49a8cffd8e1900b4b7a7",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        2,
        9,
        5,
        1,
        3,
        4,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.47006405,
      "neural_order": [
        0,
        3,
        9,
        2,
        7,
        5,
        1,
        13,
        4,
        10,
        6,
        8,
        11,
        12,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "9ecb187c6533548c69792941b85909a196eb77e1df9716b52bdb4812c08ad31c",
      "trial_index": 29
    },
    {
      "answer_sha256": "8267ff7e88875b32ca38e3f5b09e2dd71b38c457f756451d58756f43209ed0a8",
      "candidate_ids": [
        "57115ff82419e314009555c3",
        "573092088ab72b1400f9c594",
        "5730bc308ab72b1400f9c73c",
        "57297547af94a219006aa45c",
        "57274e975951b619008f87f9",
        "5725fe63ec44d21400f3d7dd",
        "570d4030fed7b91900d45d9f",
        "5730b255396df919000962b0",
        "56de3dbacffd8e1900b4b6d2",
        "5706094b52bb8914006897de",
        "5706155352bb891400689894",
        "5705eee952bb8914006896de",
        "57332b66d058e614000b5758",
        "5729a03f1d04691400779593",
        "56de4a474396321400ee2786",
        "570602fa52bb89140068979e"
      ],
      "candidate_sha256": [
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b",
        "d28ae44a1bd659369983c5d7b048773512040e9150f20bcb6fbae1f618e3266a",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d"
      ],
      "example_id": "56de4a474396321400ee2786",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.20777449,
      "neural_order": [
        14,
        15,
        10,
        8,
        0,
        12,
        11,
        9,
        6,
        1,
        2,
        3,
        4,
        5,
        7,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "7429dbac6b43caf2d24a9b24de34082df272a9a8d440dabe0deeabd9b4f31eff",
      "trial_index": 30
    },
    {
      "answer_sha256": "b4ee12cc4279d819f04056553b5732f113e10c21ba5c69f8fa2a2a733b9f917f",
      "candidate_ids": [
        "572ff7ab04bcaa1900d76f51",
        "5725cf3238643c19005acd61",
        "571c7d55dd7acb1400e4c0c4",
        "57377862c3c5551400e51ef3",
        "572a0b101d046914007796e9",
        "573099ee8ab72b1400f9c5dc",
        "56de4a89cffd8e1900b4b7bd",
        "571caac55efbb31900334dc6",
        "571c83f3dd7acb1400e4c0d8",
        "57281d494b864d19001644be",
        "5726a34bf1498d1400e8e59e",
        "56e181d9e3433e1400422fa0",
        "572fb42904bcaa1900d76c15",
        "57287d4a2ca10214002da3e4",
        "57264a0ef1498d1400e8db40",
        "57379829c3c5551400e51f3d"
      ],
      "candidate_sha256": [
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "329e2dd3e90bd25984d9c186e564a1d7ab4e07d619284ce70de287269ef4a9a8",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "36476aaf2aea4f7927c065f38bbc7e2f8ec013b37ee0a2f1c7f545368f0dc62e",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c"
      ],
      "example_id": "56de4a89cffd8e1900b4b7bd",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.55537542,
      "neural_order": [
        6,
        14,
        2,
        10,
        11,
        9,
        7,
        8,
        1,
        3,
        0,
        15,
        4,
        5,
        12,
        13
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "bc3f6ddd631e439683b84937378866962634466f4d158604256622f57e60d5b8",
      "trial_index": 31
    },
    {
      "answer_sha256": "7f8291deb9180eed9170419a35a0ba6627a942bba83d245b454fd5dec7469568",
      "candidate_ids": [
        "57268f05dd62a815002e8990",
        "57094ca7efce8f15003a7dd6",
        "572885023acd2414000dfa83",
        "57264a0ef1498d1400e8db40",
        "57264a8cdd62a815002e808c",
        "57287ddf3acd2414000dfa3f",
        "572a10cd6aef051400155222",
        "5728759cff5b5019007da25a",
        "572a0c541d046914007796f3",
        "5726b58f5951b619008f7b55",
        "572853e8ff5b5019007da188",
        "5727213c708984140094da35",
        "5733d4c8d058e614000b6353",
        "57293f353f37b3190047819b",
        "5730876a396df9190009617a",
        "56de4b074396321400ee2793"
      ],
      "candidate_sha256": [
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "dea705fdf75f529664435816398522b553e31b0965a1287f3375c31a630e304e",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de"
      ],
      "example_id": "56de4b074396321400ee2793",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.59427546,
      "neural_order": [
        15,
        0,
        3,
        14,
        11,
        6,
        2,
        9,
        1,
        4,
        5,
        7,
        8,
        10,
        12,
        13
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "2a7f74d60a1473dea4a16420641a7fb787b5b524ffbc45f07bd889228def33e4",
      "trial_index": 32
    },
    {
      "answer_sha256": "b7342b629968dce16f56052cc86743ba4f4aeb1d46924c28acc4ae4258aae264",
      "candidate_ids": [
        "56ddde6b9a695914005b9628",
        "57332562d058e614000b5730",
        "5727f6723acd2414000df0e7",
        "56de4b5c4396321400ee2799",
        "57264e455951b619008f6f65",
        "57300c67947a6a140053cff0",
        "5729fd111d046914007796a5",
        "572885023acd2414000dfa83",
        "5725ff8238643c19005acf49",
        "573312fcd058e614000b56e6",
        "572a1a5c6aef051400155284",
        "572fb42904bcaa1900d76c15",
        "572647d0708984140094c14b",
        "5726a1e5dd62a815002e8b86",
        "572a0ecb1d04691400779717",
        "572867543acd2414000df9a1"
      ],
      "candidate_sha256": [
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "b11a179d53281b2c32154de7d72bc5f8e3c0c35320a1c9d0f9b20fb0e505678d"
      ],
      "example_id": "56de4b5c4396321400ee2799",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.19194838,
      "neural_order": [
        3,
        0,
        1,
        2,
        7,
        4,
        15,
        13,
        10,
        14,
        5,
        6,
        8,
        9,
        11,
        12
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "d22b1c9b298159692ed318b08ea6c790a2f1e96dfb545aaa16ec92c6fbb26cae",
      "trial_index": 33
    },
    {
      "answer_sha256": "0311fc66ff8b0cf80103792799fe145c595c99ad23d49179b9532f078dd5dd89",
      "candidate_ids": [
        "56e1c2eee3433e1400423134",
        "56de4bb84396321400ee27a2",
        "56e1ce08e3433e14004231a4",
        "57111b95a58dae1900cd6c50",
        "57060eaf75f01819005e7910",
        "57377aac1c45671900574479",
        "57286192ff5b5019007da1dc",
        "5726f48df1498d1400e8f0da",
        "571135b8a58dae1900cd6d0e",
        "56e1b00ce3433e140042309e",
        "572648ed5951b619008f6f03",
        "57282358ff5b5019007d9dde",
        "573408ef4776f41900661757",
        "5730040f947a6a140053cf4c",
        "57115c7450c2381900b54a9d",
        "570602fa52bb89140068979e"
      ],
      "candidate_sha256": [
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "52dde7db662fc424e04bd1ed3cd943e7d11c0868e7f691a8c6cac05cb561b057",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d"
      ],
      "example_id": "56de4bb84396321400ee27a2",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        12,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.25296228,
      "neural_order": [
        1,
        12,
        10,
        3,
        5,
        6,
        13,
        0,
        11,
        8,
        2,
        4,
        7,
        9,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "0c3a373d6636f48c879545f4281a46798f7213b48ef8a591beddd9fc7dd1073c",
      "trial_index": 34
    },
    {
      "answer_sha256": "1f276c3d7706932b737ee3063b3310549272996e77fac5aa2d0a7c457923c29c",
      "candidate_ids": [
        "56de3efccffd8e1900b4b6fe",
        "5725d34aec44d21400f3d639",
        "572699db5951b619008f7799",
        "57294e6b1d04691400779275",
        "57287ccb2ca10214002da3da",
        "5733314e4776f4190066076a",
        "5705e4fe75f01819005e7704",
        "5733dab4d058e614000b6409",
        "5710e8c8a58dae1900cd6b26",
        "572879574b864d1900164a14",
        "5726462b708984140094c117",
        "57268c01dd62a815002e8912",
        "57300e2604bcaa1900d770b7",
        "57294209af94a219006aa201",
        "56de4c324396321400ee27ab",
        "5725ce4d38643c19005acd4d"
      ],
      "candidate_sha256": [
        "218bcafd4ea9a6566900672f26303adc874834255c2b12646ddef2c72e9093f2",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "403b83bcd02b8204d4620813147424fdb54ec54909eb54e867858a16ee0028ec",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf"
      ],
      "example_id": "56de4c324396321400ee27ab",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.37116843,
      "neural_order": [
        14,
        8,
        9,
        13,
        7,
        0,
        4,
        15,
        1,
        2,
        3,
        5,
        6,
        10,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "83ded35a8591aabc2e5316dc049de67dbbe0d04c0ed5010a4b4926e377148e73",
      "trial_index": 35
    },
    {
      "answer_sha256": "b195f4372ea5f1ad9b9e37e2ae6246766406af04f81fd7b6799a248a5b6c0ef8",
      "candidate_ids": [
        "57281ab63acd2414000df493",
        "5733f5264776f419006615a3",
        "57114667a58dae1900cd6d80",
        "56de51244396321400ee27ef",
        "57273e50dd62a815002e9a02",
        "5728659f4b864d190016498a",
        "5727c94bff5b5019007d954a",
        "572a020f6aef051400155198",
        "571a53d410f8ca1400304fe5",
        "5733d249d058e614000b6331",
        "5733314e4776f4190066076a",
        "57115ff82419e314009555c3",
        "570d4606b3d812140066d619",
        "57377862c3c5551400e51ef3",
        "572f609ca23a5019007fc5af",
        "57111b95a58dae1900cd6c50"
      ],
      "candidate_sha256": [
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "9384b2ae8d06f600126ecfff9a80168fbb32d4cda2c1d70e200f7ab4d0578b47",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e"
      ],
      "example_id": "56de51244396321400ee27ef",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        1,
        0,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.51942502,
      "neural_order": [
        3,
        0,
        2,
        8,
        6,
        12,
        4,
        7,
        11,
        15,
        14,
        1,
        5,
        9,
        10,
        13
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "e869a776c43d766e17856a3cc72710e7b7f1bc7b4b10f0be95500028af9485cd",
      "trial_index": 36
    },
    {
      "answer_sha256": "00e2ba0f54e367393473c32c8d3ae41084895e8e7d60e07dcba37a3216c3abe0",
      "candidate_ids": [
        "571112ada58dae1900cd6bcc",
        "56de51c64396321400ee27f7",
        "5729727baf94a219006aa437",
        "57263ea0271a42140099d7c3",
        "56e2042ecd28a01900c67c1e",
        "572820512ca10214002d9e72",
        "56de179dcffd8e1900b4b5da",
        "5725d662ec44d21400f3d687",
        "571094b7a58dae1900cd6a66",
        "571ccfbadd7acb1400e4c164",
        "5729f12e3f37b319004785e1",
        "57282358ff5b5019007d9dde",
        "5725cfd0271a42140099d225",
        "57337ddc4776f41900660bbb",
        "572ffd9e04bcaa1900d76fc7",
        "57108ee6a58dae1900cd6a1a"
      ],
      "candidate_sha256": [
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "0771c41591ce859133397b70f7fe295ca178804c352b68cc34f3f35caa1c92ab",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6"
      ],
      "example_id": "56de51c64396321400ee27f7",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        14,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.30920501,
      "neural_order": [
        1,
        15,
        6,
        0,
        8,
        4,
        5,
        2,
        3,
        12,
        9,
        11,
        7,
        14,
        10,
        13
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "c95bae310cbaa9d9c53ebccc91ea033cadf4e9bed930d5ad85d6f1cfa47e6f1b",
      "trial_index": 37
    },
    {
      "answer_sha256": "b13b2da6444151968cd97107173fdb6abee346dcf8c57e8c81451b4de2bda9ed",
      "candidate_ids": [
        "57335c20d058e614000b58f9",
        "57336755d058e614000b5a3c",
        "572f5875947a6a140053c89a",
        "56e1b754cd28a01900c67abc",
        "57297781af94a219006aa4a3",
        "5710ed7bb654c5140001fa2b",
        "5733314e4776f4190066076a",
        "572fc80b04bcaa1900d76d11",
        "572a1a5c6aef051400155284",
        "57114dfb50c2381900b54a53",
        "5725d34089a1e219009abf50",
        "56de52614396321400ee27fb",
        "5725ff8238643c19005acf49",
        "572fbf21a23a5019007fc937",
        "57114e8d50c2381900b54a5b",
        "572a11663f37b31900478693"
      ],
      "candidate_sha256": [
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "4b2a2a7e8bce520e5e42162e14db64c90808895b36ef2edfc527bbc3df66971c",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e"
      ],
      "example_id": "56de52614396321400ee27fb",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.26685634,
      "neural_order": [
        11,
        5,
        2,
        1,
        15,
        0,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "cd0e038820541e92471c49e934d187bb37c90b155adb13322bb93a425e3c0c82",
      "trial_index": 38
    },
    {
      "answer_sha256": "b539056a5c8ba02e4481d0656075fa95ef401954a96ea9ab28a400fc81c7ea1a",
      "candidate_ids": [
        "57338255d058e614000b5c0d",
        "572a0e4b6aef051400155214",
        "5711607f2419e314009555cd",
        "57337f6ad058e614000b5bcb",
        "57275409708984140094dc35",
        "570d4a6bfed7b91900d45e13",
        "5729a3716aef05140015506a",
        "572643de5951261400b5195a",
        "56e181d9e3433e1400422fa0",
        "56e16182e3433e1400422e28",
        "57109275b654c5140001f99f",
        "57267b3c5951b619008f7423",
        "5728fb002ca10214002dab6a",
        "573028fa04bcaa1900d77287",
        "57263cfcec44d21400f3dc8d",
        "57281edd3acd2414000df4eb"
      ],
      "candidate_sha256": [
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "2271df8c8c1b2d712c122939c9c53f699509a751bc040f6c45e9660ceb9c2177"
      ],
      "example_id": "56e16182e3433e1400422e28",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.20855751,
      "neural_order": [
        9,
        8,
        14,
        11,
        4,
        2,
        12,
        6,
        7,
        1,
        5,
        0,
        3,
        10,
        13,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "c7c7f74145760fb49273069e2d2556a95829055ca566193d884d2c55dc2195e9",
      "trial_index": 39
    },
    {
      "answer_sha256": "b808d099aafc4c86df9d3bd3eb75a65ba97e9eb6d50c5748d3c792d8cf72d08b",
      "candidate_ids": [
        "56de49564396321400ee277a",
        "572864dd4b864d1900164976",
        "56e16839cd28a01900c67887",
        "57264e455951b619008f6f65",
        "5726f7715951b619008f838d",
        "57265d08708984140094c397",
        "572867543acd2414000df9a1",
        "57274f67708984140094dbf5",
        "5730040f947a6a140053cf4c",
        "5726ba2c708984140094cf59",
        "57283f014b864d19001647e8",
        "571154c72419e31400955588",
        "572a0e4b6aef051400155214",
        "5729a3716aef05140015506a",
        "572647935951b619008f6ec9",
        "57107c24a58dae1900cd69e8"
      ],
      "candidate_sha256": [
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "b11a179d53281b2c32154de7d72bc5f8e3c0c35320a1c9d0f9b20fb0e505678d",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604"
      ],
      "example_id": "56e16839cd28a01900c67887",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        7,
        0,
        1,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.36575237,
      "neural_order": [
        2,
        7,
        13,
        12,
        4,
        5,
        3,
        8,
        0,
        1,
        6,
        9,
        10,
        11,
        14,
        15
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "3b627a00d428c04f9fc021772c819d80fb9e98b01fe6112d26c5ef71c9ea3146",
      "trial_index": 40
    },
    {
      "answer_sha256": "35569104d07399f48c4553ce6ddc9e4aad89c04fc3495798e0cf12dc12f93d82",
      "candidate_ids": [
        "5705f09e75f01819005e77a4",
        "57261f9f271a42140099d4a9",
        "57373d0cc3c5551400e51e85",
        "5729f60caf94a219006aa6ef",
        "5725b5a689a1e219009abd28",
        "56e17644e3433e1400422f40",
        "5727d1c93acd2414000ded3f",
        "5725e748ec44d21400f3d733",
        "5727f2714b864d1900164072",
        "56e1a564cd28a01900c67a48",
        "5727c3b02ca10214002d95ba",
        "56dddf4066d3e219004dad5f",
        "572985011d04691400779501",
        "572658daf1498d1400e8dcac",
        "5733a2a9d058e614000b5f28",
        "5727cff1ff5b5019007d95c4"
      ],
      "candidate_sha256": [
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "ce701ae7bea9bf9e47c5972ca7624bc90227bd2c59f41f67a73bf89c12c44680",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6"
      ],
      "example_id": "56e17644e3433e1400422f40",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        9,
        15,
        2,
        0,
        1,
        3,
        4,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.28254158,
      "neural_order": [
        5,
        9,
        12,
        2,
        7,
        11,
        8,
        14,
        10,
        1,
        0,
        3,
        4,
        6,
        13,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "dee02107116c156e4cfb1b6d8823bcdee182aa6979b0cde28fc0b2acbc88c52a",
      "trial_index": 41
    },
    {
      "answer_sha256": "62f7482f73a722e2f98d89e5a4b27d1654e2892101f0d97601ea295b5096e903",
      "candidate_ids": [
        "57264a0ef1498d1400e8db40",
        "5726219489a1e219009ac2d0",
        "572970c11d04691400779463",
        "5729f39a6aef05140015514c",
        "5725f190ec44d21400f3d76f",
        "5728809f2ca10214002da40c",
        "5705f36452bb891400689718",
        "570d2af0fed7b91900d45c93",
        "5711628a2419e314009555dd",
        "5733f410d058e614000b6663",
        "57115f652419e314009555b9",
        "572643de5951261400b5195a",
        "56e17a7ccd28a01900c679a1",
        "5733a2a9d058e614000b5f28",
        "5730bd00069b5314008322d7",
        "5737a9afc3c5551400e51f61"
      ],
      "candidate_sha256": [
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "099340918f9359979a06067b9361381a993dbef386a4851ba6bb74d73cd5dc60",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "29fcf6218856cfd64395c81351e873a0769b0f291577dd5933cc1ac378f40976",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31"
      ],
      "example_id": "56e17a7ccd28a01900c679a1",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        2,
        8,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.50660408,
      "neural_order": [
        12,
        2,
        5,
        11,
        1,
        8,
        15,
        10,
        6,
        3,
        4,
        0,
        7,
        9,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "90b8726e88a5aeb649d0e5cb353d2f0f29b6f52b8cd587d4a64b111ec0763a31",
      "trial_index": 42
    },
    {
      "answer_sha256": "81a83544cf93c245178cbc1620030f1123f435af867c79d87135983c52ab39d9",
      "candidate_ids": [
        "570d28bdb3d812140066d4a3",
        "572a20816aef0514001552e4",
        "572755b7708984140094dc4d",
        "56e17e6ee3433e1400422f7f",
        "57273f27dd62a815002e9a0a",
        "57096f37200fba1400367fe5",
        "57308ddc396df919000961a4",
        "57294baaaf94a219006aa26b",
        "5737958b1c456719005744c3",
        "572a0a686aef0514001551ec",
        "56e1fc57e3433e140042322c",
        "5727d6154b864d1900163e34",
        "5726a14c708984140094cc51",
        "57113c6da58dae1900cd6d32",
        "572655e5f1498d1400e8dc5e",
        "57268f05dd62a815002e8990"
      ],
      "candidate_sha256": [
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "9bfd34dc5945f3f1cb2977bcaf00d7025ecfd1ae65eff22ed5f0ad4225e7ab2e",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "00b76f48835bfc2434242a5dea1681e9e4d9c363cc74180aa0b9b9fec6f02bef",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b"
      ],
      "example_id": "56e17e6ee3433e1400422f7f",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        9,
        5,
        0,
        14,
        10,
        1,
        2,
        4,
        6,
        7,
        8,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.33638613,
      "neural_order": [
        3,
        10,
        4,
        7,
        0,
        2,
        13,
        5,
        9,
        14,
        1,
        6,
        8,
        11,
        12,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "838b34cc2a52c53a2ac0b86ab47c3a1577fe1e25b2ee3dd5079705bcb89b5854",
      "trial_index": 43
    },
    {
      "answer_sha256": "62f7482f73a722e2f98d89e5a4b27d1654e2892101f0d97601ea295b5096e903",
      "candidate_ids": [
        "57296d571d04691400779413",
        "5729081d3f37b31900477fab",
        "572fdd03a23a5019007fca9d",
        "57111b95a58dae1900cd6c50",
        "573380e0d058e614000b5be9",
        "5730005db2c2fd1400568703",
        "5725cf3238643c19005acd61",
        "57332e48d058e614000b5762",
        "5710968aa58dae1900cd6a86",
        "571a49de4faf5e1900b8a94a",
        "56e181d9e3433e1400422fa0",
        "5726e313f1498d1400e8eeb2",
        "56e1b754cd28a01900c67abc",
        "57286192ff5b5019007da1dc",
        "572fcc43b2c2fd140056847d",
        "56e1b62ecd28a01900c67aa3"
      ],
      "candidate_sha256": [
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "db6de70b42aaae5f96618f4be9863dbd2b0618f69ea12eeb5dd9dbfb1a8bcef5",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379"
      ],
      "example_id": "56e181d9e3433e1400422fa0",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        15,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        14
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.39177629,
      "neural_order": [
        10,
        15,
        12,
        13,
        0,
        11,
        1,
        8,
        9,
        14,
        2,
        4,
        3,
        5,
        6,
        7
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "b0efdbe0d64eb6408d592a48cece5322eecdaf0bc69e48195bfe75d5f49639eb",
      "trial_index": 44
    },
    {
      "answer_sha256": "0afd47d6913d59f26fb2093b527fa85b47bfc9fab13177912db53229d4b216b2",
      "candidate_ids": [
        "5733647e4776f419006609ae",
        "5737a7351c456719005744f1",
        "572ff673b2c2fd1400568669",
        "572fc6f204bcaa1900d76cf5",
        "572fc41f947a6a140053cc80",
        "56e190bce3433e1400422fc8",
        "57265c10f1498d1400e8dd36",
        "572fbfa504bcaa1900d76c73",
        "57281940ff5b5019007d9d44",
        "571c83f3dd7acb1400e4c0d8",
        "572a13841d0469140077973b",
        "5730a951069b531400832213",
        "57296d571d04691400779413",
        "57274f67708984140094dbf5",
        "572f65e9b2c2fd14005680cb",
        "572fc5a1947a6a140053cc8a"
      ],
      "candidate_sha256": [
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "19620cc8402206f6bca616c6a305ca911d96b87a8f3423850512f4f84e4120b1",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "58de06c671d2cd29f3b8bde136326194d51c92607fc4a624d1d5a6e90dd5cf33",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "3bafbc5d56c1857ae4a5346827458b7900e775e4ebe79997c59538a3013f8c57",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4"
      ],
      "example_id": "56e190bce3433e1400422fc8",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        14,
        9,
        13,
        2,
        8,
        12,
        0,
        1,
        3,
        4,
        6,
        7,
        10,
        11,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.25063099,
      "neural_order": [
        5,
        13,
        12,
        7,
        4,
        1,
        10,
        9,
        3,
        8,
        0,
        15,
        6,
        2,
        11,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "493ddd37c5852d291e6336fbcf37502df1bd9b364ffe0a7652264fee3532bbd1",
      "trial_index": 45
    },
    {
      "answer_sha256": "bf4b51586fce552a4006a05ab24a51a0e79b2e66a9ec7d4c8bcb46d8a71658ec",
      "candidate_ids": [
        "57300200b2c2fd1400568729",
        "57264cac708984140094c1b3",
        "570d4a6bfed7b91900d45e13",
        "57265360dd62a815002e819a",
        "572fc6f204bcaa1900d76cf5",
        "571114cfb654c5140001fb09",
        "572647e2dd62a815002e805c",
        "57109180a58dae1900cd6a40",
        "5733d2dbd058e614000b6339",
        "57377083c3c5551400e51edf",
        "56e19557e3433e1400422fee",
        "57309564069b5314008321a5",
        "57286ab72ca10214002da31e",
        "5737432bc3c5551400e51e99",
        "5727ea45ff5b5019007d986e",
        "5729a3716aef05140015506a"
      ],
      "candidate_sha256": [
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "19620cc8402206f6bca616c6a305ca911d96b87a8f3423850512f4f84e4120b1",
        "291631d73d5dcdb57b0941017d7b1ab7a9501e869a33f4c995211b70c5487c88",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea"
      ],
      "example_id": "56e19557e3433e1400422fee",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        15,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.65230532,
      "neural_order": [
        10,
        15,
        14,
        1,
        6,
        9,
        3,
        11,
        8,
        13,
        12,
        4,
        0,
        2,
        5,
        7
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "f4ba4af56e34c77aa012425c7efb9feda81fe569f031034bd9c9b68a6faeabd2",
      "trial_index": 46
    },
    {
      "answer_sha256": "4cd13b779b95df9b3090095f13ca87b3b9689fc076402d2cfa1c4cfd51097293",
      "candidate_ids": [
        "57267b3c5951b619008f7423",
        "57269f3ef1498d1400e8e534",
        "572996c73f37b319004784b3",
        "5727ec062ca10214002d99b6",
        "572647e2dd62a815002e805c",
        "56e19724cd28a01900c679f6",
        "5726e08e5951b619008f810f",
        "572a0f073f37b3190047867f",
        "57299021af94a219006aa50b",
        "5726e5c95951b619008f81cb",
        "5733ef47d058e614000b662b",
        "573792ee1c456719005744b9",
        "572a0bf96aef051400155204",
        "572754dd708984140094dc3c",
        "5725c0f289a1e219009abdf2",
        "5733314e4776f4190066076a"
      ],
      "candidate_sha256": [
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b"
      ],
      "example_id": "56e19724cd28a01900c679f6",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        2,
        3,
        8,
        7,
        0,
        1,
        4,
        6,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.46790768,
      "neural_order": [
        5,
        8,
        3,
        2,
        1,
        9,
        11,
        7,
        0,
        10,
        6,
        13,
        4,
        12,
        14,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "8f01a10a8d9e39b2cc0c0dbafac10474759d0f03c5ec6bb9eae57a093c0abd04",
      "trial_index": 47
    },
    {
      "answer_sha256": "0afd47d6913d59f26fb2093b527fa85b47bfc9fab13177912db53229d4b216b2",
      "candidate_ids": [
        "572648d1708984140094c15d",
        "5725d183271a42140099d23d",
        "5709686c200fba1400367f77",
        "56e1d9fee3433e14004231cb",
        "571cb010dd7acb1400e4c128",
        "572a09abaf94a219006aa75b",
        "56de4b5c4396321400ee2799",
        "57267947f1498d1400e8e0ec",
        "57264d58f1498d1400e8db7a",
        "572ff673b2c2fd1400568669",
        "572940246aef051400154bec",
        "5729f39a6aef05140015514c",
        "56e1a0dccd28a01900c67a2e",
        "572ffd9e04bcaa1900d76fc7",
        "5733834ed058e614000b5c26",
        "57287d4a2ca10214002da3e4"
      ],
      "candidate_sha256": [
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548"
      ],
      "example_id": "56e1a0dccd28a01900c67a2e",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        3,
        10,
        6,
        0,
        1,
        2,
        4,
        5,
        7,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.39626804,
      "neural_order": [
        12,
        3,
        10,
        5,
        15,
        1,
        7,
        2,
        11,
        0,
        4,
        6,
        8,
        9,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "a295c49f876432f794d04b76d06fe19d1b76f07b206a44638cddc33527c2b046",
      "trial_index": 48
    },
    {
      "answer_sha256": "66cb38b5a0290791fb06f2fa47eab2f78faa53fdd68a0c9e2636618bfc4572dd",
      "candidate_ids": [
        "5733dab4d058e614000b6409",
        "5710eca0a58dae1900cd6b3a",
        "572883a33acd2414000dfa79",
        "5733d7cbd058e614000b63ab",
        "57060a1175f01819005e78d2",
        "57286fa83acd2414000df9e5",
        "57286010ff5b5019007da1ca",
        "5728683b3acd2414000df9af",
        "570d4329fed7b91900d45dd7",
        "570d32e4fed7b91900d45d01",
        "572fb059947a6a140053cb80",
        "5726249538643c19005ad07f",
        "57340111d058e614000b677d",
        "573407d7d058e614000b6813",
        "56e1a38de3433e140042305c",
        "5729f9953f37b3190047861f"
      ],
      "candidate_sha256": [
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "55bbfa9052d82fc2074807e0795fc8957a358974ef289c3f0e19e4e8dbb35f85",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "c4fd59d733b1633198ab8d307fb4d275d55c73f2c3fa5e12deb57198789613e4",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "a4008fb587348767c254824b7e5bb46aaec5eaa762404c37c3e9e2f52fc93eac",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "d03773b18a1be50107069edcf0e896834515c0cc39a5f2a1177372718099eaff",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad"
      ],
      "example_id": "56e1a38de3433e140042305c",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.64155341,
      "neural_order": [
        14,
        6,
        8,
        11,
        7,
        4,
        0,
        1,
        2,
        3,
        5,
        9,
        10,
        12,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "d80b206f46b9b37a76b10167d5dfb05b3ec2ac545a320e762c6dea5b8bb1a463",
      "trial_index": 49
    },
    {
      "answer_sha256": "c37b590f9a4c94f2d44d379715d838dc0e904411bd86d89edaa1a408f2460d00",
      "candidate_ids": [
        "5726dcbddd62a815002e9320",
        "5726887e708984140094c917",
        "572885023acd2414000dfa83",
        "572812e74b864d19001643ce",
        "57293f8a6aef051400154bde",
        "5726a1e5dd62a815002e8b86",
        "56e1a564cd28a01900c67a48",
        "572900f73f37b31900477f69",
        "572684365951b619008f753f",
        "5727d9c43acd2414000dee17",
        "57287c142ca10214002da3d0",
        "57094d489928a8140047150a",
        "5725e45689a1e219009ac048",
        "572885c44b864d1900164a78",
        "5729a03f1d04691400779593",
        "573092088ab72b1400f9c594"
      ],
      "candidate_sha256": [
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "4e041656788e4036188e81c0a35175217b17488e8ea0585f48b4719b1fac1708",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b"
      ],
      "example_id": "56e1a564cd28a01900c67a48",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        5,
        14,
        3,
        9,
        1,
        0,
        2,
        4,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.28088623,
      "neural_order": [
        6,
        14,
        3,
        5,
        10,
        4,
        9,
        0,
        11,
        1,
        2,
        7,
        8,
        12,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "8407f25ec34fce2c397262e5b3bd24269605999c611d19e86f7039f954964d26",
      "trial_index": 50
    },
    {
      "answer_sha256": "43afae10a414d00cf3ad46e7b613594ea3cf45b6258712324ecb6168e7f1bac7",
      "candidate_ids": [
        "57300a9a04bcaa1900d77063",
        "5727c8863acd2414000dec49",
        "56e1aba0e3433e1400423094",
        "572843304b864d1900164848",
        "572fcb6da23a5019007fc9f1",
        "5706155352bb891400689894",
        "5727500f708984140094dbfd",
        "5728fd3c3f37b31900477f2b",
        "572991943f37b319004784a1",
        "5725f00938643c19005aced7",
        "5726c5a9f1498d1400e8eac4",
        "572945b11d0469140077922f",
        "572a1fe16aef0514001552d0",
        "57115bf350c2381900b54a93",
        "5727cb4b2ca10214002d9676",
        "57273d8bf1498d1400e8f4fe"
      ],
      "candidate_sha256": [
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "55e3030ac51ce0d76d4fcad6c72bf5b57a9192da5256c5e338b03fe44bf2bb59",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "0303b992326c30ab7517bcf7737bdc345dbbbc7eae232baca3af64f05f6565a4"
      ],
      "example_id": "56e1aba0e3433e1400423094",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        15,
        0,
        10,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.50251155,
      "neural_order": [
        2,
        15,
        12,
        5,
        8,
        0,
        7,
        3,
        13,
        6,
        9,
        1,
        4,
        10,
        11,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "ad1f36ac61709b326f978207955835f14756cf344ce9bb5325236f90bf93c3dd",
      "trial_index": 51
    },
    {
      "answer_sha256": "c408f17fc8d716e09658eaa94c2e3f46e5cc171872efd1217945664fec1855cf",
      "candidate_ids": [
        "5730b1022461fd1900a9cfa3",
        "56e1aff7cd28a01900c67a68",
        "56e1c7e2cd28a01900c67b74",
        "572822233acd2414000df555",
        "5725b7f389a1e219009abd5d",
        "5730bc308ab72b1400f9c73c",
        "5709686c200fba1400367f77",
        "572872822ca10214002da374",
        "56de17f9cffd8e1900b4b5e0",
        "5730b4282461fd1900a9cfc5",
        "5733647e4776f419006609ae",
        "571112ada58dae1900cd6bcc",
        "5729a26d6aef05140015505a",
        "57287ccb2ca10214002da3da",
        "572ff890a23a5019007fcbcd",
        "573755afc3c5551400e51eb3"
      ],
      "candidate_sha256": [
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "d28ae44a1bd659369983c5d7b048773512040e9150f20bcb6fbae1f618e3266a",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "b471e50d3722fdd0e8fac7b90182b37de8e917f2b6ef94945670aafaf625f2bc",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "02473abeaeaf0a7ac1221ede2323c119c62182f573df438534fcb86c2c6943d4",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41",
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0"
      ],
      "example_id": "56e1aff7cd28a01900c67a68",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        13,
        9,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.26570451,
      "neural_order": [
        1,
        2,
        12,
        5,
        13,
        4,
        9,
        14,
        10,
        15,
        0,
        7,
        3,
        6,
        8,
        11
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "19156ad7256f1e77263d63a98f5073ca459477883d721eb2f50f3fc5c44d0d5c",
      "trial_index": 52
    },
    {
      "answer_sha256": "61c2f47614599052971e0ef05ce8692004e365045c556208527e35a47cbda253",
      "candidate_ids": [
        "56e1b00ce3433e140042309e",
        "572f609ca23a5019007fc5af",
        "571cd5b1dd7acb1400e4c178",
        "5727c8863acd2414000dec49",
        "572669a9dd62a815002e8416",
        "57302ad804bcaa1900d772af",
        "5727526cdd62a815002e9b0e",
        "5727565ef1498d1400e8f674",
        "57264845f1498d1400e8db0a",
        "572facb0a23a5019007fc863",
        "5706149552bb891400689880",
        "57302a3aa23a5019007fcecf",
        "5725c57a89a1e219009abe5e",
        "5727d1c93acd2414000ded3f",
        "57286c8cff5b5019007da218",
        "572ffa79a23a5019007fcbe9"
      ],
      "candidate_sha256": [
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "65a4f29a52c31659b66baa59545297d8af56b2b1f884801f998c7d4c6e8396d8",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862"
      ],
      "example_id": "56e1b00ce3433e140042309e",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        3,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.46309907,
      "neural_order": [
        0,
        4,
        9,
        14,
        12,
        6,
        15,
        2,
        11,
        7,
        1,
        3,
        5,
        8,
        10,
        13
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "13a53a787235eabfc519d0d5c62ad205e3b5457b619acd58aac5aa0a9364da86",
      "trial_index": 53
    },
    {
      "answer_sha256": "cefe1a2e5aae61884bc3b34be33eb69991b0df9ca2f5be33bfee7761d9f598b5",
      "candidate_ids": [
        "57264f18f1498d1400e8dbae",
        "5727e9523acd2414000def95",
        "572667e2f1498d1400e8de90",
        "572a0ecb1d04691400779717",
        "572f5533a23a5019007fc55c",
        "571cc3dedd7acb1400e4c146",
        "572a0f073f37b3190047867f",
        "56e1b169cd28a01900c67a72",
        "572864542ca10214002da2de",
        "56e1ba41cd28a01900c67ae0",
        "5729fd56af94a219006aa72f",
        "5710eca0a58dae1900cd6b3a",
        "57300761947a6a140053cf9a",
        "57300200b2c2fd1400568729",
        "57286f373acd2414000df9db",
        "56de40da4396321400ee2708"
      ],
      "candidate_sha256": [
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "7c28011f4adb5bc9518e20aa534136a5ab7d9e920f0263cb0d482a6ce7859ede",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "a21efd80668b227ef68dd022f77a31b092d02d626d2e188520af3bf6059d57dc",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d"
      ],
      "example_id": "56e1b169cd28a01900c67a72",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        9,
        2,
        6,
        0,
        1,
        3,
        4,
        5,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.58667854,
      "neural_order": [
        7,
        9,
        14,
        3,
        5,
        8,
        2,
        1,
        13,
        11,
        4,
        0,
        6,
        10,
        12,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "e82d18b7675670f8c2ef91cef9d9db0dd4919e75d5fad45182c9144718b4ad4e",
      "trial_index": 54
    },
    {
      "answer_sha256": "0f044946d7a9e781398a5b968fee73bb4524ce78d907aed5b8aa2057d28a5f7e",
      "candidate_ids": [
        "5725edfe38643c19005ace9f",
        "57287fec4b864d1900164a3c",
        "5726701add62a815002e84c8",
        "5730ab63396df91900096260",
        "5728fb002ca10214002dab6a",
        "572750df5951b619008f882f",
        "57274beff1498d1400e8f5e4",
        "572a1f086aef0514001552c0",
        "56e1b355e3433e14004230b0",
        "57376828c3c5551400e51eb9",
        "56e1e9dfe3433e14004231fc",
        "57282dfb4b864d1900164668",
        "5726472bdd62a815002e8042",
        "57274e975951b619008f87f9",
        "5725fb8138643c19005acf3f",
        "57284e9fff5b5019007da150"
      ],
      "candidate_sha256": [
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "8075afe5649850130d6277b4d42a263b335970200c99b9786462feb745bd73d5",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2"
      ],
      "example_id": "56e1b355e3433e14004230b0",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        10,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.15745742,
      "neural_order": [
        8,
        10,
        9,
        2,
        11,
        6,
        4,
        13,
        1,
        0,
        3,
        5,
        7,
        12,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "ca782502956faed4144b2f823e4cd17196ef3c788e0c5a73f2ddbf225f1d6003",
      "trial_index": 55
    },
    {
      "answer_sha256": "3fd86d98d82d5208a615feaef4d269957aacf00e22809e8253c923c56ada3436",
      "candidate_ids": [
        "5729fb003f37b31900478627",
        "57287c2bff5b5019007da26c",
        "570d4606b3d812140066d619",
        "57111428b654c5140001faff",
        "572fd7b8947a6a140053cd3a",
        "572757bef1498d1400e8f690",
        "5727ff083acd2414000df1ab",
        "572fc41f947a6a140053cc80",
        "56e1b62ecd28a01900c67aa3",
        "56de3cd0cffd8e1900b4b6be",
        "57269bb8708984140094cb95",
        "572872dd2ca10214002da37e",
        "57284e9fff5b5019007da150",
        "5706155352bb891400689894",
        "57287b4a4b864d1900164a28",
        "57293bc91d0469140077919b"
      ],
      "candidate_sha256": [
        "ffaee1c83ab04918603b9940620c0c8b52e675143bc31a91f1fccac00d2b1bf7",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89"
      ],
      "example_id": "56e1b62ecd28a01900c67aa3",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        4,
        7,
        1,
        9,
        0,
        2,
        3,
        5,
        6,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.34758305,
      "neural_order": [
        8,
        4,
        7,
        11,
        15,
        1,
        10,
        2,
        13,
        0,
        14,
        3,
        5,
        6,
        9,
        12
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "070290cfeb8b384376e7b8ea8e1e9b8cad6e89e876ff99b26f195e0f8f7a20bd",
      "trial_index": 56
    },
    {
      "answer_sha256": "c4578666e6aa23c23a4596d1c4d7110f7acd119190ca02043647879fab55d7ad",
      "candidate_ids": [
        "57264d9edd62a815002e80fe",
        "56e1b754cd28a01900c67abc",
        "573796edc3c5551400e51f33",
        "572878942ca10214002da3a2",
        "57264e455951b619008f6f65",
        "572671a5f1498d1400e8dfe4",
        "571154c72419e31400955588",
        "570d2417fed7b91900d45c3d",
        "57297a276aef051400154f88",
        "5728fb6a1d04691400778ef5",
        "573312fcd058e614000b56e6",
        "571cd3b55efbb31900334e03",
        "57111380a58dae1900cd6bd6",
        "572864542ca10214002da2de",
        "572863c72ca10214002da2d4",
        "56de179dcffd8e1900b4b5da"
      ],
      "candidate_sha256": [
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2"
      ],
      "example_id": "56e1b754cd28a01900c67abc",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        15,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.2118853,
      "neural_order": [
        1,
        0,
        6,
        5,
        2,
        3,
        8,
        10,
        7,
        11,
        4,
        9,
        12,
        13,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "d1e973b6be4833f3afe3f2551a4f135beb6b4cbc20f6a508734b2ee5f60f0991",
      "trial_index": 57
    },
    {
      "answer_sha256": "9a3342696978880af79104c61e8248afd96cba39d352305337fa7f0f414cc54a",
      "candidate_ids": [
        "5729f4e46aef051400155156",
        "5733d13e4776f419006612c4",
        "5725b76389a1e219009abd4a",
        "571cd11add7acb1400e4c16e",
        "5726f36cdd62a815002e95fe",
        "57294baaaf94a219006aa26b",
        "57282358ff5b5019007d9dde",
        "57286bb84b864d19001649c8",
        "56e1b8f3e3433e14004230e6",
        "5726bc1add62a815002e8ea6",
        "57309ef18ab72b1400f9c600",
        "56de51244396321400ee27ef",
        "5725be0f271a42140099d117",
        "572863c72ca10214002da2d4",
        "570953a7efce8f15003a7dfe",
        "5734025d4776f419006616c3"
      ],
      "candidate_sha256": [
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "4cb08df8d00c330db23afb5af129580ac1c4437b39cc7557de63a8b111d7abe4",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "8ffda8441c55fe91c1d7c352b984785d42dee97084adfcaec49f89486beb85ef",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662"
      ],
      "example_id": "56e1b8f3e3433e14004230e6",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        4,
        15,
        9,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.45389714,
      "neural_order": [
        8,
        9,
        5,
        4,
        10,
        0,
        7,
        14,
        15,
        3,
        1,
        2,
        6,
        11,
        12,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "b2b0aacd91ea9f6bdb787f3d8f004de21b78bf5f380af0c6f2fa9ca59f13ffe0",
      "trial_index": 58
    },
    {
      "answer_sha256": "ae477519c631b7338113d4bf37405774b192990a7a2775a1795c326e90a1e973",
      "candidate_ids": [
        "5733638fd058e614000b59e8",
        "5730ab63396df91900096260",
        "5726f7715951b619008f838d",
        "572871bd3acd2414000dfa03",
        "5729fefbaf94a219006aa739",
        "571c8198dd7acb1400e4c0ce",
        "5727cc15ff5b5019007d9576",
        "5725b5a689a1e219009abd28",
        "56e1ba41cd28a01900c67ae0",
        "5725db98ec44d21400f3d6c5",
        "57300200b2c2fd1400568729",
        "57096b66200fba1400367fa7",
        "570d529fb3d812140066d6bb",
        "57273cca708984140094db33",
        "57308f6b8ab72b1400f9c580",
        "5725ff8238643c19005acf49"
      ],
      "candidate_sha256": [
        "05e6b70a4c996f49687d72f7a373609343967f6fed5be1f3bda6965dcfad141e",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "81ba9c103184caabf51fa7a6f0ba2856254a93dd30897f55cc1cd699bd203a42",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "a21efd80668b227ef68dd022f77a31b092d02d626d2e188520af3bf6059d57dc",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308"
      ],
      "example_id": "56e1ba41cd28a01900c67ae0",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.34771746,
      "neural_order": [
        8,
        13,
        15,
        12,
        4,
        2,
        7,
        10,
        0,
        1,
        3,
        5,
        6,
        9,
        11,
        14
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "789152dc40953165f97faeab3058f89a94acae8e42038a50cf98308e882ca195",
      "trial_index": 59
    },
    {
      "answer_sha256": "786ab4ef6862d1a2901a9a24135ddca9399c672cb217dea4fabe02d6f5cb9d34",
      "candidate_ids": [
        "572885023acd2414000dfa83",
        "573060b48ab72b1400f9c4c6",
        "570966e0200fba1400367f4f",
        "573784fa1c45671900574483",
        "56de40da4396321400ee2708",
        "572883a33acd2414000dfa79",
        "56e1fc57e3433e140042322c",
        "56e1bc3ae3433e1400423104",
        "57107c24a58dae1900cd69e8",
        "56de15104396321400ee25b7",
        "57286192ff5b5019007da1dc",
        "5726701add62a815002e84c8",
        "57300c67947a6a140053cff0",
        "57268a37f1498d1400e8e33c",
        "5710e9f8a58dae1900cd6b30",
        "5726e985dd62a815002e94da"
      ],
      "candidate_sha256": [
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "b7d78d9fd858b52f42115f7144f2b2a6844dffc86f3a087c6833c6f78182e39e",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "55bbfa9052d82fc2074807e0795fc8957a358974ef289c3f0e19e4e8dbb35f85",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9"
      ],
      "example_id": "56e1bc3ae3433e1400423104",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        3,
        6,
        9,
        4,
        5,
        1,
        2,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.25661927,
      "neural_order": [
        7,
        6,
        0,
        1,
        11,
        12,
        13,
        15,
        3,
        5,
        2,
        4,
        8,
        9,
        10,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "78d3bb366fe842f03bec79780cecbab2437ad4af2b48cb182a82c75f24c76af1",
      "trial_index": 60
    },
    {
      "answer_sha256": "a47f09d9af3ed8f71efad0de8faac95f1022c9bc07f1f2b78d543683943dce64",
      "candidate_ids": [
        "572663a9f1498d1400e8ddf2",
        "5730a4d02461fd1900a9cf29",
        "5705e99452bb891400689688",
        "57263b1638643c19005ad333",
        "56e1bd4acd28a01900c67afc",
        "5728661e2ca10214002da2e8",
        "5737898f1c45671900574495",
        "572683f95951b619008f7525",
        "572818f54b864d190016446c",
        "5711619950c2381900b54aaf",
        "57284d484b864d1900164900",
        "5728f6446aef0514001548e4",
        "572fc41f947a6a140053cc80",
        "5727de862ca10214002d9860",
        "57300ec0947a6a140053d004",
        "570d529fb3d812140066d6bb"
      ],
      "candidate_sha256": [
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "3344ef93052b85063f75bebf68306da4193c68170ffc88ba5cf12a5bf36faa69",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "25160eaaf4cedb4feaad143ed4ed40d1a5aeba443aaa0327bbecb05f058934a7",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "40206469ec21c4f98d7c72cb3c313a8bdfc89251b79140b6c63a48b1afe123d0",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "33e49ae942c39aefd29d877e8334707b666747d16641fd8392ae988a08b47f20",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683"
      ],
      "example_id": "56e1bd4acd28a01900c67afc",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        3,
        9,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.46232294,
      "neural_order": [
        4,
        12,
        0,
        9,
        8,
        3,
        11,
        15,
        1,
        2,
        5,
        6,
        7,
        10,
        13,
        14
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "4bc8bfb9830451e04c9fac52b2d2a6363a0d049f0acee8bd97fa4d0b2257f376",
      "trial_index": 61
    },
    {
      "answer_sha256": "61c2f47614599052971e0ef05ce8692004e365045c556208527e35a47cbda253",
      "candidate_ids": [
        "5705eee952bb8914006896de",
        "57269e3bf1498d1400e8e516",
        "56e1c0f6cd28a01900c67b2c",
        "5733d4c8d058e614000b6353",
        "5725ce4d38643c19005acd4d",
        "57267b3c5951b619008f7423",
        "572885c44b864d1900164a78",
        "571161092419e314009555d7",
        "57287fec4b864d1900164a3c",
        "572ff7ab04bcaa1900d76f51",
        "57332a734776f41900660726",
        "57286ec63acd2414000df9d1",
        "5733f5f24776f419006615c1",
        "5730040f947a6a140053cf4c",
        "5727da564b864d1900163e8f",
        "5728ed94ff5b5019007da97c"
      ],
      "candidate_sha256": [
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "e26d3a98154d79e4fb93dfbc9b446801d4e38dc1c4681b39878f40025a6b4fce",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2"
      ],
      "example_id": "56e1c0f6cd28a01900c67b2c",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        1,
        0,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.28139041,
      "neural_order": [
        2,
        1,
        15,
        5,
        6,
        0,
        11,
        3,
        7,
        4,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "d8f391d7e4b29a6fd09b8091e95514ca8e67378d116499b36219ea4e563cb1cc",
      "trial_index": 62
    },
    {
      "answer_sha256": "a970dfa2d5665e9bd6543adfe65d033959f302f10f5d50ab5084c9b3ce9ad115",
      "candidate_ids": [
        "5725e45689a1e219009ac048",
        "5733e5a14776f4190066145b",
        "572a1a5c6aef051400155284",
        "57111713a58dae1900cd6c00",
        "572fc78d04bcaa1900d76d07",
        "5726a299dd62a815002e8b9e",
        "570d4a6bfed7b91900d45e13",
        "571cd703dd7acb1400e4c182",
        "5725f190ec44d21400f3d76f",
        "5726431aec44d21400f3dd13",
        "57286ec63acd2414000df9d1",
        "5726472bdd62a815002e8042",
        "5725ce4d38643c19005acd4d",
        "57294279af94a219006aa209",
        "56e1c2eee3433e1400423134",
        "57306797396df919000960ee"
      ],
      "candidate_sha256": [
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "8657b03977310d90fc72a13d570f1c07e80c6100a3b2eeb92d2acdff9adb8b16",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a"
      ],
      "example_id": "56e1c2eee3433e1400423134",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.56663153,
      "neural_order": [
        14,
        10,
        2,
        7,
        4,
        13,
        15,
        5,
        8,
        9,
        11,
        12,
        0,
        3,
        1,
        6
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "d4c1256ee0e652c82ff3c1b48d16eeddce36e8f164ec867bd233f7c7ec954edd",
      "trial_index": 63
    },
    {
      "answer_sha256": "395886de87511e10937b93d7b6104f4208d1e7a750b3ee0d7194aee8eae7aa3c",
      "candidate_ids": [
        "5711488ab654c5140001fc3d",
        "57097c8fed30961900e841f2",
        "572f64ccb2c2fd14005680b7",
        "57265a58dd62a815002e8270",
        "57332c1e4776f4190066073a",
        "57280e1aff5b5019007d9bec",
        "572754dd708984140094dc3c",
        "56dde1d966d3e219004dad8d",
        "56e1c3e1e3433e1400423148",
        "57094a79efce8f15003a7dc4",
        "57263ea0271a42140099d7c3",
        "5726701add62a815002e84c8",
        "572fca7eb2c2fd1400568473",
        "5726e179dd62a815002e93ae",
        "572a03a06aef0514001551aa",
        "5710f4b8b654c5140001fa45"
      ],
      "candidate_sha256": [
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6"
      ],
      "example_id": "56e1c3e1e3433e1400423148",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        11,
        1,
        15,
        13,
        0,
        10,
        9,
        2,
        3,
        4,
        5,
        6,
        7,
        12,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.60558924,
      "neural_order": [
        8,
        12,
        0,
        10,
        13,
        11,
        6,
        7,
        5,
        1,
        2,
        3,
        4,
        9,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "f3bea06da922b3e99669cf33c327f3d6f053a3ac960286831141d7ef1b71f00b",
      "trial_index": 64
    },
    {
      "answer_sha256": "145ab1303c7474fa79554e4f72710ae40e8fc970acf5b83385907a0ef9aa04f1",
      "candidate_ids": [
        "56e1c4fce3433e140042314e",
        "57111b95a58dae1900cd6c50",
        "5733a45d4776f41900660f23",
        "5734025d4776f419006616c3",
        "573796edc3c5551400e51f33",
        "57115ac550c2381900b54a77",
        "571caac55efbb31900334dc6",
        "57286010ff5b5019007da1ca",
        "57299c2c6aef051400155020",
        "57339c16d058e614000b5ec5",
        "570d3468b3d812140066d543",
        "57268a37f1498d1400e8e33c",
        "573314e3d058e614000b56f0",
        "572881022ca10214002da416",
        "572fd47fa23a5019007fca55",
        "572989846aef051400154fc0"
      ],
      "candidate_sha256": [
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "36476aaf2aea4f7927c065f38bbc7e2f8ec013b37ee0a2f1c7f545368f0dc62e",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f"
      ],
      "example_id": "56e1c4fce3433e140042314e",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        12,
        10,
        3,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.50878437,
      "neural_order": [
        0,
        15,
        8,
        5,
        7,
        14,
        6,
        3,
        2,
        4,
        12,
        1,
        13,
        9,
        11,
        10
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "b4f1354338eea7fb12f98cb7a8dd65986881d825e261a1e3fc120917e2c2b719",
      "trial_index": 65
    },
    {
      "answer_sha256": "1ffad39ea8cc6c8d35eb3cd6c6e97177087b6eb313b9f4610167ddf973512cab",
      "candidate_ids": [
        "572812e74b864d19001643ce",
        "57111ab8a58dae1900cd6c3c",
        "56e1c720e3433e140042316a",
        "5727dc473acd2414000dee43",
        "570d418dfed7b91900d45dbf",
        "57286c8cff5b5019007da218",
        "56de51c64396321400ee27f7",
        "57283c464b864d19001647c8",
        "5729f1fcaf94a219006aa6d3",
        "5726542ff1498d1400e8dc28",
        "56e1c9bfe3433e1400423192",
        "5711648850c2381900b54ac3",
        "572fc41f947a6a140053cc80",
        "56de3d594396321400ee26ca",
        "571155ae2419e31400955591",
        "572691bedd62a815002e89dc"
      ],
      "candidate_sha256": [
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "631ae85a58426197f43bce08d34df2914057034648942e41a9d822902b61b3d3",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "99afd74c371b051e0dd2704df0676d8589e7c060ad964a7cae71b5a5bde23c1e",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62"
      ],
      "example_id": "56e1c720e3433e140042316a",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        8,
        10,
        13,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": false,
      "neural_margin": 0.04122707,
      "neural_order": [
        10,
        2,
        0,
        11,
        12,
        7,
        6,
        4,
        15,
        1,
        13,
        3,
        5,
        8,
        9,
        14
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "210f8f2d94f31cbe6fa9043a9ae89fac5b3f2719f1068fd490db273a14336cc3",
      "trial_index": 66
    },
    {
      "answer_sha256": "e5409e106e69a1a472ede194252d570c3e73aced7001e12877e4d882523c6619",
      "candidate_ids": [
        "57286bf84b864d19001649d2",
        "5730b1022461fd1900a9cfa3",
        "5727f6723acd2414000df0e7",
        "5727da564b864d1900163e8f",
        "56e1c7e2cd28a01900c67b74",
        "570d30fdfed7b91900d45ce3",
        "5725ce4d38643c19005acd4d",
        "5726e179dd62a815002e93ae",
        "5737958b1c456719005744c3",
        "57377862c3c5551400e51ef3",
        "5727ef664b864d1900164060",
        "5726559edd62a815002e81c8",
        "572a06866aef0514001551be",
        "572fd1c4947a6a140053cd02",
        "57115e532419e314009555af",
        "572ff5fcb2c2fd1400568657"
      ],
      "candidate_sha256": [
        "3243e4c3f26f22351bf8c52e8438c396a19b76af187bad37a4ca97532160e33d",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875"
      ],
      "example_id": "56e1c7e2cd28a01900c67b74",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        9,
        10,
        7,
        15,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.52904475,
      "neural_order": [
        4,
        8,
        13,
        14,
        2,
        12,
        10,
        7,
        3,
        1,
        9,
        0,
        5,
        6,
        11,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "7c0df9e7e32b1c7c5a132d8b4c121a5563ae3280a8e276a6922a804224ba0a15",
      "trial_index": 67
    },
    {
      "answer_sha256": "46401dc068e724efd350dc439b592ec536156dc51ebbd9179ed93985315e553c",
      "candidate_ids": [
        "572905ce1d04691400778f83",
        "571a50df4faf5e1900b8a960",
        "57280e1aff5b5019007d9bec",
        "5728fc9e1d04691400778f11",
        "5729f5a03f37b31900478605",
        "5711119cb654c5140001fae3",
        "5727aa413acd2414000de921",
        "56dddf4066d3e219004dad5f",
        "5728fb6a1d04691400778ef5",
        "571cbe35dd7acb1400e4c13c",
        "573792ee1c456719005744b9",
        "56e1c0f6cd28a01900c67b2c",
        "56e1c9bfe3433e1400423192",
        "57060a6e52bb8914006897f8",
        "571ce5055efbb31900334e2a",
        "57264b3edd62a815002e80aa"
      ],
      "candidate_sha256": [
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038"
      ],
      "example_id": "56e1c9bfe3433e1400423192",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        11,
        4,
        3,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": false,
      "neural_margin": 0.23109308,
      "neural_order": [
        11,
        12,
        1,
        0,
        6,
        3,
        15,
        7,
        10,
        14,
        8,
        2,
        4,
        9,
        13,
        5
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "3edc7bc307259f98a5d3a87d43dca26a52370448d425bff4f48f05426de9b36b",
      "trial_index": 68
    },
    {
      "answer_sha256": "8d71a0728b0ff340bb2f4078ea5fa7a825ebf62d49de1656bee005306b915888",
      "candidate_ids": [
        "57284d484b864d1900164900",
        "5725d42a89a1e219009abf58",
        "571163172419e314009555e7",
        "571c8198dd7acb1400e4c0ce",
        "5705e99452bb891400689688",
        "571ce3745efbb31900334e20",
        "56e1cbe2cd28a01900c67bac",
        "5733d5704776f4190066130e",
        "572941273f37b319004781ad",
        "5725bcb6271a42140099d0eb",
        "572863363acd2414000df985",
        "571ccfbadd7acb1400e4c164",
        "5733266d4776f41900660712",
        "5729fe5c3f37b3190047862f",
        "57286fa83acd2414000df9e5",
        "57263b1638643c19005ad333"
      ],
      "candidate_sha256": [
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "18017326aa99d6478493c6fd9c81f3144c242ec0a8dc61e8a64d3d1ab8e0507e",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "c4fd59d733b1633198ab8d307fb4d275d55c73f2c3fa5e12deb57198789613e4",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24"
      ],
      "example_id": "56e1cbe2cd28a01900c67bac",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.27855105,
      "neural_order": [
        6,
        11,
        2,
        3,
        8,
        9,
        10,
        13,
        15,
        5,
        4,
        7,
        0,
        1,
        12,
        14
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "bd8cab67b4e2f3ab30bde627d4aebbdf3f1ee0bf0a2a9aa350a58b53e97eaf68",
      "trial_index": 69
    },
    {
      "answer_sha256": "169927019cc4343ade8b7f94042923cf65f1f739fbdc10aec91f3df0bbd9fe5c",
      "candidate_ids": [
        "57281940ff5b5019007d9d44",
        "57281c594b864d19001644a2",
        "56e20a27cd28a01900c67c24",
        "56e190bce3433e1400422fc8",
        "5733f8dc4776f419006615f7",
        "57111b95a58dae1900cd6c50",
        "573406d1d058e614000b6801",
        "56e1ce08e3433e14004231a4",
        "5725d34aec44d21400f3d639",
        "56de1645cffd8e1900b4b5d0",
        "571144d1a58dae1900cd6d6e",
        "5730005db2c2fd1400568703",
        "5733e8ae4776f419006614a6",
        "5728f50baf94a219006a9e55",
        "5730042804bcaa1900d77011",
        "572671a5f1498d1400e8dfe4"
      ],
      "candidate_sha256": [
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "c2c1868add9d6aa35780c0c8d4492524306055ae928ea56b315e6e9e6969d494",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667"
      ],
      "example_id": "56e1ce08e3433e14004231a4",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        3,
        15,
        0,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.16332626,
      "neural_order": [
        7,
        3,
        2,
        13,
        1,
        15,
        9,
        11,
        6,
        10,
        0,
        12,
        4,
        5,
        8,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "32c5d083752c4ef46df8e28a7383ab9209eb43ac1d6bf78a7221db38545bdbef",
      "trial_index": 70
    },
    {
      "answer_sha256": "52d50a9cca5e6e4eef98a5a8dad76db401a92d76cbf15710ecaee4654a8d1653",
      "candidate_ids": [
        "57115f652419e314009555b9",
        "570967c4ed30961900e840ba",
        "572756265951b619008f886e",
        "5727502f708984140094dc07",
        "5730131c947a6a140053d052",
        "57287e512ca10214002da3f8",
        "56de4a474396321400ee2786",
        "56de3dbacffd8e1900b4b6d2",
        "56e1d9fee3433e14004231cb",
        "5737432bc3c5551400e51e99",
        "57107d73b654c5140001f91d",
        "572647e2dd62a815002e805c",
        "5733fd66d058e614000b6735",
        "572ffd9e04bcaa1900d76fc7",
        "57284d484b864d1900164900",
        "571c7d55dd7acb1400e4c0c4"
      ],
      "candidate_sha256": [
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "ce72fed7b084941e90079823ae33cc19adb21829e9f5aa99491ba4d96d782570",
        "e32237eb13f46727755bd53347a189c0620b3ffff778cecbcd627a94fbebc022",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "a828c58f3d12ed3b04a42280e3053c818961fb2b99589a63f4145c2a9ab650b5",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75"
      ],
      "example_id": "56e1d9fee3433e14004231cb",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.48273018,
      "neural_order": [
        8,
        3,
        11,
        15,
        2,
        9,
        5,
        14,
        4,
        0,
        1,
        6,
        7,
        10,
        12,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "a5299b7cbc5bf33b9e5ea4088c9d12073ac841f5720903b1f92b881895e40254",
      "trial_index": 71
    },
    {
      "answer_sha256": "b0bced3d7bebd6f16d5f85546062e3702c9309c41a83c5ce35e3138dda49b642",
      "candidate_ids": [
        "57108d69b654c5140001f983",
        "5725fe63ec44d21400f3d7dd",
        "57283f014b864d19001647e8",
        "57302bd0b2c2fd14005689db",
        "572fb42904bcaa1900d76c15",
        "573776eec3c5551400e51ee9",
        "57337ea24776f41900660bd0",
        "56e1dc62cd28a01900c67bcb",
        "572fd1c4947a6a140053cd02",
        "57264cc6dd62a815002e80e4",
        "572a2224af94a219006aa823",
        "571a4d1a4faf5e1900b8a958",
        "572a10cd6aef051400155222",
        "56de4a89cffd8e1900b4b7bd",
        "573062662461fd1900a9cdf7",
        "5711163bb654c5140001fb13"
      ],
      "candidate_sha256": [
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "18893f714ac102ba660ec3790204ffdf2ee86fd4cd069c50ec0ceeaed01cefbd",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0"
      ],
      "example_id": "56e1dc62cd28a01900c67bcb",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.59268305,
      "neural_order": [
        7,
        11,
        5,
        12,
        2,
        14,
        13,
        9,
        3,
        10,
        4,
        8,
        1,
        15,
        0,
        6
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "7d49705d4b9392355c1ded33b2d82487740272d5da24fc7470516e13be260d8b",
      "trial_index": 72
    },
    {
      "answer_sha256": "f0bf0d21e8669f517aad02eb149226d1c130d2c388899b0bcd8db8dc00c08f27",
      "candidate_ids": [
        "57339eb9d058e614000b5ef6",
        "572a04d51d046914007796cd",
        "57378c9b1c456719005744a7",
        "57309446396df919000961b8",
        "572fc80b04bcaa1900d76d11",
        "56e1ddfce3433e14004231d5",
        "572a02483f37b3190047864b",
        "57373a9fc3c5551400e51e7b",
        "5730bd00069b5314008322d7",
        "572673f5708984140094c69b",
        "570d4030fed7b91900d45d9f",
        "57264efddd62a815002e8134",
        "5727d3843acd2414000ded69",
        "572ff932a23a5019007fcbd5",
        "5727565ef1498d1400e8f674",
        "5730b255396df919000962b0"
      ],
      "candidate_sha256": [
        "de308e41aa84b64f8c0380e4021ce99dfb8b520d42b8a543f77ef7f6f3c88a00",
        "215da96f975d61a3dc7af5d1e2b75129453d3b8e5a3b4af20a46de0b25ce3030",
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "eb6de5520e4ff5dd74226484694777930247235a992261581ce74e58cdb39246",
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0"
      ],
      "example_id": "56e1ddfce3433e14004231d5",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.62267139,
      "neural_order": [
        5,
        6,
        4,
        12,
        14,
        2,
        11,
        10,
        1,
        13,
        0,
        3,
        7,
        8,
        9,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "3972aaaf8435c56d22c7f522ef3ed6b1c00c7383317779da5736243c0207f63c",
      "trial_index": 73
    },
    {
      "answer_sha256": "9d9f53cb5e795612c251a3d891e94a89b7e134fc71be22353fcc38e060b04b4c",
      "candidate_ids": [
        "572970c11d04691400779463",
        "57114b1a2419e31400955575",
        "57308cf88ab72b1400f9c576",
        "572ff7ab04bcaa1900d76f51",
        "57097c8fed30961900e841f2",
        "56e1ded7cd28a01900c67bd4",
        "5733e8ae4776f419006614a6",
        "572fc78d04bcaa1900d76d07",
        "572669a9dd62a815002e8416",
        "571cd11add7acb1400e4c16e",
        "572a1ba46aef05140015528e",
        "5725f46289a1e219009ac0fa",
        "5711648850c2381900b54ac3",
        "5725f5b1271a42140099d371",
        "5705eb3375f01819005e7764",
        "572ff83ab2c2fd1400568681"
      ],
      "candidate_sha256": [
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "aa95ca2f8cb345761597dedb47b8dc46be2773412d98ec4f473e747ae7e16b2a",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc"
      ],
      "example_id": "56e1ded7cd28a01900c67bd4",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.36375185,
      "neural_order": [
        5,
        0,
        2,
        8,
        4,
        7,
        11,
        12,
        15,
        6,
        10,
        1,
        3,
        9,
        13,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "88124718012693aee6b1078dff9795b2d36d9f867eda09432ec87182b57156c2",
      "trial_index": 74
    },
    {
      "answer_sha256": "573044d1dd9b3dd35a501fc98edf9770f573555ef316f925f0b155c57a033932",
      "candidate_ids": [
        "57373d0cc3c5551400e51e85",
        "5726f868dd62a815002e9684",
        "572991943f37b319004784a1",
        "57115dbe2419e314009555a5",
        "57332442d058e614000b5720",
        "5725ce4d38643c19005acd4d",
        "57060eaf75f01819005e7910",
        "56e1e9dfe3433e14004231fc",
        "5726ba2c708984140094cf59",
        "570d391fb3d812140066d575",
        "571cc3dedd7acb1400e4c146",
        "5725f46289a1e219009ac0fa",
        "572756fe708984140094dc71",
        "5728710c3acd2414000df9ef",
        "572759dbdd62a815002e9b94",
        "5728759cff5b5019007da25a"
      ],
      "candidate_sha256": [
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "fcb9ab642fd54564ba8561d66afa42c7a340302550f69419acc9e102acd011a5",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5"
      ],
      "example_id": "56e1e9dfe3433e14004231fc",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        8,
        1,
        2,
        3,
        4,
        5,
        6,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.6231974,
      "neural_order": [
        7,
        2,
        0,
        5,
        1,
        3,
        12,
        11,
        8,
        10,
        4,
        13,
        15,
        6,
        9,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "ee78088680fe73af4b0cc1754f79e6a733b912a54be30f4903e7ea3440287309",
      "trial_index": 75
    },
    {
      "answer_sha256": "e8d48e332bdb8620ac08c3fa0b47b7d12eb9c6375cd366e29dfbe64585690fc9",
      "candidate_ids": [
        "57280f974b864d1900164370",
        "57264684708984140094c123",
        "57273cca708984140094db33",
        "56de15104396321400ee25b7",
        "56e17a7ccd28a01900c679a1",
        "5732b6b5328d981900602021",
        "5733a560d058e614000b5f77",
        "56e1ec83cd28a01900c67c0a",
        "572843304b864d1900164848",
        "571c9348dd7acb1400e4c114",
        "5727d0f73acd2414000ded13",
        "5727de862ca10214002d9860",
        "5726f868dd62a815002e9684",
        "572a03a06aef0514001551aa",
        "5737a0acc3c5551400e51f47",
        "572825a92ca10214002d9efe"
      ],
      "candidate_sha256": [
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "29fcf6218856cfd64395c81351e873a0769b0f291577dd5933cc1ac378f40976",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415"
      ],
      "example_id": "56e1ec83cd28a01900c67c0a",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        4,
        11,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.28236045,
      "neural_order": [
        7,
        4,
        2,
        9,
        11,
        14,
        12,
        13,
        5,
        1,
        8,
        0,
        3,
        6,
        10,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "7b63c7eb9b15f917a96ddfeb38650ba2d35a337fb7bcb9a37f7ddeb5db127bb3",
      "trial_index": 76
    },
    {
      "answer_sha256": "7c19d60a7a29ce5c861b976d9830fd66c63174b3b0f76dfe4d4ff6a7ba06d45b",
      "candidate_ids": [
        "572a1046af94a219006aa78d",
        "5726545f708984140094c2a5",
        "56e1ee4de3433e1400423210",
        "572fd6aa04bcaa1900d76d9f",
        "572871bc4b864d1900164a04",
        "5725fe63ec44d21400f3d7dd",
        "5730bb522461fd1900a9d011",
        "5725d42a89a1e219009abf58",
        "57285213ff5b5019007da182",
        "572872822ca10214002da374",
        "57281d494b864d19001644be",
        "57111992b654c5140001fb43",
        "56e1ec83cd28a01900c67c0a",
        "57264a8cdd62a815002e808c",
        "5733faaf4776f4190066162f",
        "572ff83ab2c2fd1400568681"
      ],
      "candidate_sha256": [
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "d23507599fde3bea64581b405ee4729f02882f17e7602e4f065a2b2a98228ea6",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc"
      ],
      "example_id": "56e1ee4de3433e1400423210",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        12,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.42992714,
      "neural_order": [
        2,
        12,
        0,
        8,
        6,
        1,
        10,
        4,
        3,
        5,
        9,
        11,
        15,
        7,
        13,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "6b84aa2c1d62e82cf66449cff9cf84bf1dd551aa9f5124e40e44b7816d3b9f6e",
      "trial_index": 77
    },
    {
      "answer_sha256": "467b5f096070b67d15d2190b0933105afa629f6fb1a4621cc643fe08ae0f5782",
      "candidate_ids": [
        "572643de5951261400b5195a",
        "572885023acd2414000dfa83",
        "57115f0a50c2381900b54aa7",
        "571156152419e3140095559b",
        "57264cac708984140094c1b3",
        "5728759cff5b5019007da25a",
        "57299ec43f37b3190047850d",
        "5730126ba23a5019007fcd09",
        "57286bf84b864d19001649d2",
        "57299021af94a219006aa50b",
        "56e1efa0e3433e140042321a",
        "5725cfd0271a42140099d225",
        "5727cb4b2ca10214002d9676",
        "572943ab1d04691400779219",
        "572f64ccb2c2fd14005680b7",
        "5710968aa58dae1900cd6a86"
      ],
      "candidate_sha256": [
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "3243e4c3f26f22351bf8c52e8438c396a19b76af187bad37a4ca97532160e33d",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca"
      ],
      "example_id": "56e1efa0e3433e140042321a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        1,
        9,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.53264872,
      "neural_order": [
        10,
        9,
        6,
        0,
        13,
        1,
        8,
        15,
        3,
        5,
        2,
        4,
        7,
        11,
        12,
        14
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "d8fcdf6485d914cbbe2ab4085e559a9a49b0f929656b3bfe58f9a946eaaa9d74",
      "trial_index": 78
    },
    {
      "answer_sha256": "ec97645e9bfd81bb1c640d4eae92d9732d8fe377dc1350260dd88e49cfba905b",
      "candidate_ids": [
        "5726eb4b5951b619008f826b",
        "5705eee952bb8914006896de",
        "5725d34aec44d21400f3d639",
        "57335c20d058e614000b58f9",
        "57274eca5951b619008f8800",
        "5726414e271a42140099d7e5",
        "56e1f10ee3433e1400423223",
        "57265200708984140094c237",
        "57294baaaf94a219006aa26b",
        "5728facd4b864d1900165170",
        "572673f5708984140094c69b",
        "572870b2ff5b5019007da222",
        "57264c42dd62a815002e80c6",
        "572a0b101d046914007796e9",
        "5733834ed058e614000b5c26",
        "57287ee3ff5b5019007da274"
      ],
      "candidate_sha256": [
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "46ceed087a89c8c5146679d04d65800e1d5b68fadcd272ac5f73357c18ad83be",
        "8680c2d1511073dfa6129eaab02ad788c74a59c1cb87f2123d2a4768527f9002",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "75ac617b40cb38bf492969cedae24066684167be82eea9788bf5bc732da485ac",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "329e2dd3e90bd25984d9c186e564a1d7ab4e07d619284ce70de287269ef4a9a8",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a"
      ],
      "example_id": "56e1f10ee3433e1400423223",
      "gated_correct": false,
      "gated_top": 1,
      "gold_index": 6,
      "lexical_correct": false,
      "lexical_order": [
        1,
        12,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.33469061,
      "neural_order": [
        6,
        11,
        8,
        0,
        9,
        4,
        7,
        10,
        1,
        2,
        3,
        5,
        12,
        13,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "c58395a832136c32b9d835937eec8b5166a9f109555bc78eddee66b3c1dde9c0",
      "trial_index": 79
    },
    {
      "answer_sha256": "ff7bb87ed28678f29fbcc1094f4ee6d17e2079c2e46ce4905c3ab3601db1d4bf",
      "candidate_ids": [
        "5725dd7d89a1e219009abfea",
        "56de4c324396321400ee27ab",
        "57332e48d058e614000b5762",
        "57282358ff5b5019007d9dde",
        "571c83f3dd7acb1400e4c0d8",
        "572985011d04691400779501",
        "56e1e9dfe3433e14004231fc",
        "56e1fc57e3433e140042322c",
        "5727c0402ca10214002d9564",
        "572745c6708984140094db99",
        "56e1c9bfe3433e1400423192",
        "56de11154396321400ee25aa",
        "57092322efce8f15003a7db0",
        "5726e179dd62a815002e93ae",
        "570d35b7b3d812140066d54d",
        "57309564069b5314008321a5"
      ],
      "candidate_sha256": [
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "ce701ae7bea9bf9e47c5972ca7624bc90227bd2c59f41f67a73bf89c12c44680",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "bc5f1c2743b9de3bbf84078611d839665374c71f9f5ce23ea41ca3d84c8793ba",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "ca0f7b7018127b2736fe74e8ea343545834b32796fc74beb48cde30324c80067",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2"
      ],
      "example_id": "56e1fc57e3433e140042322c",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        6,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.19195805,
      "neural_order": [
        7,
        10,
        6,
        9,
        8,
        5,
        13,
        15,
        0,
        1,
        2,
        3,
        4,
        11,
        12,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "419f7d8b59296db312061187688be80ffa2423d776e2d6b437bccbeace3c49c5",
      "trial_index": 80
    },
    {
      "answer_sha256": "1d4f79ebcbf98fd2c989f9b610afe39c08350b3e471557c92bdbf85aee5ac983",
      "candidate_ids": [
        "57274e975951b619008f87f9",
        "572a020f6aef051400155198",
        "57268882f1498d1400e8e306",
        "57264228ec44d21400f3dcf5",
        "571c91c8dd7acb1400e4c10a",
        "56e1febfe3433e1400423236",
        "572989846aef051400154fc0",
        "57332a734776f41900660726",
        "572fca7eb2c2fd1400568473",
        "57287ee3ff5b5019007da274",
        "5730a951069b531400832213",
        "572a0bf96aef051400155204",
        "56de51c64396321400ee27f7",
        "57297781af94a219006aa4a3",
        "57108073b654c5140001f925",
        "572fc6f204bcaa1900d76cf5"
      ],
      "candidate_sha256": [
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "19620cc8402206f6bca616c6a305ca911d96b87a8f3423850512f4f84e4120b1"
      ],
      "example_id": "56e1febfe3433e1400423236",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        6,
        13,
        0,
        1,
        2,
        3,
        4,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.10989904,
      "neural_order": [
        5,
        13,
        6,
        2,
        12,
        8,
        9,
        1,
        10,
        11,
        0,
        3,
        4,
        7,
        14,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "6d8d802185c315624c49276e23f87058f17181a44863fdddb7308c02bc4af1e2",
      "trial_index": 81
    },
    {
      "answer_sha256": "002cc5e2eb7bab6c6a8735a23c076e980cf7571cfb3826c8df9c683584e809f0",
      "candidate_ids": [
        "570d2c20fed7b91900d45ca7",
        "570602fa52bb89140068979e",
        "57265e11708984140094c3bb",
        "5728f50baf94a219006a9e55",
        "5727d3843acd2414000ded69",
        "5725ca4389a1e219009abeb4",
        "57332c1e4776f4190066073a",
        "5726559edd62a815002e81c8",
        "570d3468b3d812140066d543",
        "56e1a0dccd28a01900c67a2e",
        "5730ac782461fd1900a9cf73",
        "572881d34b864d1900164a5a",
        "56e200e4cd28a01900c67c14",
        "5727c8863acd2414000dec49",
        "572945b11d0469140077922f",
        "572881704b864d1900164a50"
      ],
      "candidate_sha256": [
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "eb6de5520e4ff5dd74226484694777930247235a992261581ce74e58cdb39246",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "f7140e68e270abe4b07098bebe1f59cce949b799b8237e1c88451a74c65cd4a0",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa"
      ],
      "example_id": "56e200e4cd28a01900c67c14",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        9,
        10,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        11,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.22174919,
      "neural_order": [
        12,
        9,
        4,
        7,
        15,
        14,
        3,
        2,
        0,
        11,
        10,
        8,
        6,
        1,
        5,
        13
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "6e51abab0548685828aac6f15e1831585fb21213fba5118584d28cc52bfc2dbc",
      "trial_index": 82
    },
    {
      "answer_sha256": "8177d0e8a5968412d18872d573561d698622665626682c99b80b39d39aef8bd3",
      "candidate_ids": [
        "5705e99452bb891400689688",
        "572a135daf94a219006aa79f",
        "57268a37f1498d1400e8e33c",
        "5737a9afc3c5551400e51f61",
        "56ddde6b9a695914005b9628",
        "572673f5708984140094c69b",
        "572fadcbb2c2fd1400568329",
        "5729f8516aef05140015516c",
        "57339a554776f41900660e74",
        "572869b84b864d19001649ae",
        "572fc659b2c2fd1400568447",
        "57265642f1498d1400e8dc68",
        "572820512ca10214002d9e72",
        "572812e74b864d19001643ce",
        "5725f2c838643c19005aceed",
        "56e202e9e3433e1400423240"
      ],
      "candidate_sha256": [
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "03eb6accc47f8f37613f49787d90a7cde1808cd11e3eca574b283b3e63a9aa56"
      ],
      "example_id": "56e202e9e3433e1400423240",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.57471153,
      "neural_order": [
        15,
        8,
        3,
        11,
        12,
        13,
        9,
        1,
        5,
        10,
        14,
        0,
        4,
        2,
        6,
        7
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "c20e95cea6c561f886bb60a8b1638baee0f3e3607728f34a4f4264e647b5cd84",
      "trial_index": 83
    },
    {
      "answer_sha256": "f8801b70574906901c91de873d820f814f552c31ed43c20ab2f14b08f4d14749",
      "candidate_ids": [
        "56de17f9cffd8e1900b4b5e0",
        "56e2042ecd28a01900c67c1e",
        "5733fb7bd058e614000b66ff",
        "572a2224af94a219006aa823",
        "5726deed5951b619008f80c5",
        "57097b50200fba140036804d",
        "57336755d058e614000b5a3c",
        "57287b4a4b864d1900164a28",
        "5729f799af94a219006aa707",
        "572fbf21a23a5019007fc937",
        "57302ad804bcaa1900d772af",
        "570602fa52bb89140068979e",
        "5705f36452bb891400689718",
        "5729f69caf94a219006aa6f9",
        "572a0c541d046914007796f3",
        "57114b1a2419e31400955575"
      ],
      "candidate_sha256": [
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "0771c41591ce859133397b70f7fe295ca178804c352b68cc34f3f35caa1c92ab",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "18893f714ac102ba660ec3790204ffdf2ee86fd4cd069c50ec0ceeaed01cefbd",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "d5f4c4227c07c52cff10ca527f73efedf7e6c5aa9c01a3d9bf8a9468a6ae6939",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707"
      ],
      "example_id": "56e2042ecd28a01900c67c1e",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        12,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.47134374,
      "neural_order": [
        1,
        15,
        8,
        12,
        14,
        3,
        13,
        7,
        0,
        10,
        5,
        9,
        2,
        4,
        6,
        11
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "1e9005e59123ec51075e3aca4056b35b6ec976ee1859590928c1e97d7a8cb9d7",
      "trial_index": 84
    },
    {
      "answer_sha256": "c10e5a4db60d48b2b3dcb7e524aa5d55619586e05bee2fff5ac342d87950a28e",
      "candidate_ids": [
        "57303048947a6a140053d254",
        "57338007d058e614000b5bda",
        "5733e771d058e614000b6545",
        "5725bad5271a42140099d0bd",
        "572a0c541d046914007796f3",
        "5725f190ec44d21400f3d76f",
        "56e20a27cd28a01900c67c24",
        "573003dd947a6a140053cf42",
        "56de16ca4396321400ee25c5",
        "573011de04bcaa1900d770f9",
        "5725f00938643c19005aced7",
        "57108073b654c5140001f925",
        "570d3468b3d812140066d543",
        "573009a004bcaa1900d7704f",
        "571a4ead10f8ca1400304fdd",
        "56de52614396321400ee27fb"
      ],
      "candidate_sha256": [
        "13ad58ca96a59b05a605a8e8fd1cadd65b62ca392d3e04f125096c9b8294e267",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066"
      ],
      "example_id": "56e20a27cd28a01900c67c24",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.44608752,
      "neural_order": [
        6,
        4,
        14,
        10,
        5,
        0,
        9,
        1,
        2,
        3,
        7,
        8,
        11,
        12,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "ba4d8179c51815382a2af06a3910f3fd57f36de41a8d31b11d93009e5c5c8281",
      "trial_index": 85
    },
    {
      "answer_sha256": "05b886ef6437c33895b7a272f07c965fcbd7a20d2e69ec2d648f39854389fd6c",
      "candidate_ids": [
        "5726a14c708984140094cc51",
        "572940973f37b319004781a5",
        "571077ecb654c5140001f909",
        "570d50a5fed7b91900d45e7d",
        "5725edfe38643c19005ace9f",
        "57106644b654c5140001f8e5",
        "572a0bebaf94a219006aa76f",
        "56de4b074396321400ee2793",
        "57113c6da58dae1900cd6d32",
        "5710e9f8a58dae1900cd6b30",
        "57297547af94a219006aa45c",
        "57300c67947a6a140053cff0",
        "57264c42dd62a815002e80c6",
        "570d4c3bfed7b91900d45e31",
        "56e20a3ae3433e140042324a",
        "571c8198dd7acb1400e4c0ce"
      ],
      "candidate_sha256": [
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "db91f2599d75c551b0b759a111af3526cadd1aaf2de507a25b546a5c61b6d412",
        "395b1cfae86dbeb34c8873339b090ededf002969f69503f5b785bd284dc9d2ca",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119"
      ],
      "example_id": "56e20a3ae3433e140042324a",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        12,
        15,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.32003883,
      "neural_order": [
        14,
        10,
        0,
        15,
        8,
        1,
        11,
        7,
        5,
        12,
        2,
        3,
        4,
        6,
        9,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "da8eb69e9381d58f06db9e1c5a5142b4dd34a1376942fd943ee5c618d476277f",
      "trial_index": 86
    },
    {
      "answer_sha256": "00c1c9a927cc3ccb964c52b50edddc3f38b60eb6975476c472260e56d265e570",
      "candidate_ids": [
        "57263c78ec44d21400f3dc7b",
        "572a0bebaf94a219006aa76f",
        "5730b1022461fd1900a9cfa3",
        "5727db85ff5b5019007d96fc",
        "573383d0d058e614000b5c35",
        "572a0a391d046914007796df",
        "5711541350c2381900b54a6f",
        "5705e26d75f01819005e76d4",
        "57284142ff5b5019007da00a",
        "572655e5f1498d1400e8dc5e",
        "5733ef47d058e614000b662b",
        "5725c0f289a1e219009abdf2",
        "5728eb1a3acd2414000e01c5",
        "5728510f4b864d1900164936",
        "57339ad74776f41900660e86",
        "572970c11d04691400779463"
      ],
      "candidate_sha256": [
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f"
      ],
      "example_id": "5705e26d75f01819005e76d4",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        9,
        6,
        3,
        0,
        1,
        2,
        4,
        5,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.55955391,
      "neural_order": [
        7,
        1,
        5,
        4,
        10,
        2,
        0,
        12,
        3,
        6,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "a7e6e63900248e27ab4b71d3f67152e6850fca376222a9735bdadc80b7de3f56",
      "trial_index": 87
    },
    {
      "answer_sha256": "d4dea9d2600070f9e2b51a3857c39f6bfde8d799146da6b9bbcc7345ecef1f15",
      "candidate_ids": [
        "57269016708984140094ca41",
        "571090abb654c5140001f995",
        "57273f27dd62a815002e9a0a",
        "572fb059947a6a140053cb80",
        "572683f95951b619008f7525",
        "57108d69b654c5140001f983",
        "5705ffde52bb891400689784",
        "572ffa79a23a5019007fcbe9",
        "5705e33f52bb89140068964c",
        "5726a14c708984140094cc51",
        "56de4c324396321400ee27ab",
        "5706074552bb8914006897d4",
        "57378e311c456719005744af",
        "572a20816aef0514001552e4",
        "57107932a58dae1900cd69ca",
        "5729f60caf94a219006aa6ef"
      ],
      "candidate_sha256": [
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "0c60fccfa0a5891c548480d13173fbf66ab99f403f068010ba84b6402ed6daff",
        "9bfd34dc5945f3f1cb2977bcaf00d7025ecfd1ae65eff22ed5f0ad4225e7ab2e",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a"
      ],
      "example_id": "5705e33f52bb89140068964c",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        11,
        3,
        1,
        5,
        0,
        2,
        4,
        6,
        7,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.21941089,
      "neural_order": [
        8,
        11,
        7,
        2,
        6,
        1,
        14,
        5,
        4,
        3,
        15,
        13,
        9,
        0,
        10,
        12
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "25f7cc10a5d40030bf6fae23e0d34dcff1e665cce4d822250a61b338ab92196f",
      "trial_index": 88
    },
    {
      "answer_sha256": "7b989fe5889d6645056f3124f6992096225bb949894c59e50fba3ef42a8b8fde",
      "candidate_ids": [
        "57293f8a6aef051400154bde",
        "5727cebc2ca10214002d96f0",
        "5705e3f252bb89140068966a",
        "5728f7774b864d190016512e",
        "5727d0f73acd2414000ded13",
        "57297547af94a219006aa45c",
        "57376df3c3c5551400e51ed7",
        "572a0ecb1d04691400779717",
        "571c83f3dd7acb1400e4c0d8",
        "572fc659b2c2fd1400568447",
        "572a09abaf94a219006aa75b",
        "57332b66d058e614000b5758",
        "5705f36452bb891400689718",
        "5733f410d058e614000b6663",
        "571cb27fdd7acb1400e4c132",
        "56de1563cffd8e1900b4b5c2"
      ],
      "candidate_sha256": [
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "511ddcc5ddbeeaf06bef1b76bc323fc7607c5658b94f72c360e5919a5c54656e",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d"
      ],
      "example_id": "5705e3f252bb89140068966a",
      "gated_correct": false,
      "gated_top": 12,
      "gold_index": 2,
      "lexical_correct": false,
      "lexical_order": [
        12,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.02773711,
      "neural_order": [
        2,
        12,
        10,
        15,
        6,
        13,
        14,
        7,
        4,
        0,
        1,
        3,
        5,
        8,
        9,
        11
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "be2a14f2e4c8a58a592729a75be3524a0884fc42cbaf5749a065ae65124b4b24",
      "trial_index": 89
    },
    {
      "answer_sha256": "d119738b3320b131520ab5085430d5cdb898aa00f028881a75c4316c201a1edc",
      "candidate_ids": [
        "5705fc3a52bb89140068976a",
        "56e1fc57e3433e140042322c",
        "572684365951b619008f753f",
        "57265bdfdd62a815002e829e",
        "5705e4fe75f01819005e7704",
        "5725b76389a1e219009abd4a",
        "572824f13acd2414000df58f",
        "5726385e271a42140099d797",
        "57340b1bd058e614000b6869",
        "572881d34b864d1900164a5a",
        "57280f974b864d1900164370",
        "572f57c704bcaa1900d7686d",
        "572905ce1d04691400778f83",
        "5711541350c2381900b54a6f",
        "572fda6fb2c2fd140056850b",
        "5730ab63396df91900096260"
      ],
      "candidate_sha256": [
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "5b1d7779ff634f8b4049e70e99b5eaadf538e5a3d3a2aa5069b096c68cb79e5d",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d"
      ],
      "example_id": "5705e4fe75f01819005e7704",
      "gated_correct": false,
      "gated_top": 11,
      "gold_index": 4,
      "lexical_correct": false,
      "lexical_order": [
        11,
        4,
        2,
        12,
        6,
        0,
        1,
        3,
        5,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.04238963,
      "neural_order": [
        4,
        11,
        2,
        0,
        15,
        9,
        12,
        5,
        6,
        7,
        1,
        3,
        8,
        10,
        13,
        14
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "e60ab89c31ba14a4a98c72b18e847e8cd4e4c82913e6945fd6f640250d604f8a",
      "trial_index": 90
    },
    {
      "answer_sha256": "7e8eea5cc60980270c9ceb75ce8c087d48d726110fd3d17921f774eefd8e18d8",
      "candidate_ids": [
        "57266e72f1498d1400e8df8c",
        "56de1728cffd8e1900b4b5d7",
        "571a50df4faf5e1900b8a960",
        "572fbea404bcaa1900d76c5b",
        "572a20816aef0514001552e4",
        "571cdcb85efbb31900334e0c",
        "56de51244396321400ee27ef",
        "56e1e9dfe3433e14004231fc",
        "57273cca708984140094db33",
        "57265d08708984140094c397",
        "571127a5a58dae1900cd6cc4",
        "57268da7f1498d1400e8e39c",
        "5705e63175f01819005e7720",
        "572a020f6aef051400155198",
        "572ffd9e04bcaa1900d76fc7",
        "56e1dc62cd28a01900c67bcb"
      ],
      "candidate_sha256": [
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "e77cc2842075cf67e0f42d9c55fea1df5857504c655c9e1e596c5f52dc859507",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "9bfd34dc5945f3f1cb2977bcaf00d7025ecfd1ae65eff22ed5f0ad4225e7ab2e",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "8394917ed3c6abd9f899d73ce839433e519acb0219643e9bfa9a48f92940892c",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d"
      ],
      "example_id": "5705e63175f01819005e7720",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        5,
        4,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.50322405,
      "neural_order": [
        12,
        8,
        9,
        1,
        6,
        0,
        2,
        3,
        4,
        5,
        7,
        10,
        11,
        13,
        14,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "6d77fa2672dae890a2f6038f96b22215c049f632af4bd04d08756075d29328bb",
      "trial_index": 91
    },
    {
      "answer_sha256": "9f2608067816e38c85edfb0c3985feff32def8b5dc17bb522ffc2e877e9b386b",
      "candidate_ids": [
        "572822233acd2414000df555",
        "5726a299dd62a815002e8b9e",
        "57280f974b864d1900164370",
        "5733a5f54776f41900660f44",
        "5705fec152bb89140068977a",
        "57286d4f2ca10214002da328",
        "573776eec3c5551400e51ee9",
        "5725fabc89a1e219009ac128",
        "5737a4511c456719005744df",
        "570d2af0fed7b91900d45c93",
        "5705e99452bb891400689688",
        "570953a7efce8f15003a7dfe",
        "572fd7b8947a6a140053cd3a",
        "57293d6d1d046914007791b7",
        "57300761947a6a140053cf9a",
        "57115f652419e314009555b9"
      ],
      "candidate_sha256": [
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01"
      ],
      "example_id": "5705e99452bb891400689688",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        5,
        12,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.6354389,
      "neural_order": [
        10,
        4,
        9,
        7,
        5,
        14,
        13,
        2,
        0,
        1,
        3,
        6,
        8,
        11,
        12,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "4216aecb6b1164126baec4ca8b50d05b0c1a7755c7b3ed38a0429320a6ff5e48",
      "trial_index": 92
    },
    {
      "answer_sha256": "c630cb195f52c883a377b3c1d5e403df5743bd33088e3812aa507d12909dbd10",
      "candidate_ids": [
        "5726a3c6f1498d1400e8e5ae",
        "571093aba58dae1900cd6a5c",
        "572fd8efb2c2fd14005684fa",
        "5706111a52bb89140068984c",
        "5705e26d75f01819005e76d4",
        "572872dd2ca10214002da37e",
        "5730208fa23a5019007fcded",
        "5728fc9e1d04691400778f11",
        "5726f96ddd62a815002e9698",
        "5705eb3375f01819005e7764",
        "5705f7c875f01819005e77dc",
        "572867212ca10214002da2f2",
        "57274a8ff1498d1400e8f5bc",
        "5737a7351c456719005744f1",
        "571161092419e314009555d7",
        "5725f2c838643c19005aceed"
      ],
      "candidate_sha256": [
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b"
      ],
      "example_id": "5705eb3375f01819005e7764",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        13,
        2,
        11,
        7,
        0,
        1,
        3,
        4,
        5,
        6,
        8,
        10,
        12,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.43910683,
      "neural_order": [
        9,
        10,
        3,
        4,
        15,
        13,
        8,
        6,
        7,
        5,
        0,
        1,
        2,
        11,
        12,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "9716ef917792adc12ec90d2de4ec57397306fa03d6f7605c96cec93c7910889f",
      "trial_index": 93
    },
    {
      "answer_sha256": "2406f46e506c14b5a93679beacf9a89f6878521a444455866726d7a4ea9f6fe9",
      "candidate_ids": [
        "572fc49d04bcaa1900d76ccb",
        "572fd264b2c2fd14005684a9",
        "571c9348dd7acb1400e4c114",
        "57264b1ddd62a815002e80a0",
        "5705fc3a52bb89140068976a",
        "572759dbdd62a815002e9b94",
        "57281c594b864d19001644a2",
        "572647e2dd62a815002e805c",
        "5725b5a689a1e219009abd28",
        "571156152419e3140095559b",
        "572f609ca23a5019007fc5af",
        "5705ec1675f01819005e776e",
        "572a0d21af94a219006aa783",
        "570d4329fed7b91900d45dd7",
        "5733dab4d058e614000b6409",
        "572ff890a23a5019007fcbcd"
      ],
      "candidate_sha256": [
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41"
      ],
      "example_id": "5705ec1675f01819005e776e",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": false,
      "neural_margin": 0.01461301,
      "neural_order": [
        4,
        11,
        15,
        14,
        10,
        5,
        3,
        6,
        7,
        8,
        12,
        13,
        0,
        1,
        2,
        9
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "02d68473dbf571db207ba432a07e0ccf9ba1ad6d9179cc8e925a61e8b7ffff5a",
      "trial_index": 94
    },
    {
      "answer_sha256": "42f13cb242dfc68373a46bebfc681d5b86ae35ef9887f406d7a5724f9283faa7",
      "candidate_ids": [
        "5729f39a6aef05140015514c",
        "56de0f6a4396321400ee257f",
        "572a0bebaf94a219006aa76f",
        "57280fd3ff5b5019007d9c26",
        "572980f9af94a219006aa4d1",
        "5725bbec271a42140099d0d1",
        "57097c8fed30961900e841f2",
        "572fff45947a6a140053cf26",
        "57299326af94a219006aa515",
        "5728827b2ca10214002da42a",
        "57300a9a04bcaa1900d77063",
        "571126dfa58dae1900cd6cb2",
        "572a12386aef051400155234",
        "5705eccb52bb8914006896b8",
        "5727b0892ca10214002d93e8",
        "572faf74b2c2fd1400568347"
      ],
      "candidate_sha256": [
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "f54da4edb14741599b317e246ffe56becbfd450806ace7773701841b9f5b06c4",
        "8ec9dcc4c2fe0ea5e8f0e2e102236786bbdd27dc8fb4452718334cf4545b34bf",
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4"
      ],
      "example_id": "5705eccb52bb8914006896b8",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.25427338,
      "neural_order": [
        13,
        3,
        2,
        7,
        1,
        4,
        10,
        9,
        15,
        12,
        0,
        5,
        6,
        8,
        11,
        14
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "d5f3e5e19986e7822de60f4f49195d2c50c2b51774aec8136b03f0b7e2acfa23",
      "trial_index": 95
    },
    {
      "answer_sha256": "89fb288c085ec17c9c394e3e8536c55a4bb52d4323c4aa2c5924281700967223",
      "candidate_ids": [
        "5705edcd52bb8914006896ca",
        "5706094b52bb8914006897de",
        "5728223cff5b5019007d9dc4",
        "56e1c720e3433e140042316a",
        "57300f8504bcaa1900d770d1",
        "571098f8a58dae1900cd6ab2",
        "5725ff8238643c19005acf49",
        "5725fcbe271a42140099d3ad",
        "5733140a4776f419006606e0",
        "5727b8df3acd2414000dea99",
        "571cda1bdd7acb1400e4c196",
        "57113639a58dae1900cd6d18",
        "572fae4b04bcaa1900d76bdf",
        "5711623e50c2381900b54ab9",
        "5733a45d4776f41900660f23",
        "57269016708984140094ca41"
      ],
      "candidate_sha256": [
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "196983e68e85513a7fe6690715cedcdf5521e1e6f3f26798c48cc4618989f754",
        "286230116fa743f77871c0721b3874ae2889664d4113752586c5f53755626f21",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f"
      ],
      "example_id": "5705edcd52bb8914006896ca",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        7,
        2,
        6,
        1,
        5,
        3,
        4,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.41098198,
      "neural_order": [
        0,
        7,
        1,
        6,
        2,
        15,
        14,
        13,
        9,
        3,
        4,
        5,
        8,
        10,
        11,
        12
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "ee447855213b39ee078fde10de6bc7a4fc598fabfed27a0ebc833cf1a39d31f8",
      "trial_index": 96
    },
    {
      "answer_sha256": "8be2978e343799a3747a31957706009d207d6a3de7978e63f00b2a41cc99355f",
      "candidate_ids": [
        "5725f46289a1e219009ac0fa",
        "57108d69b654c5140001f983",
        "57377862c3c5551400e51ef3",
        "56dde0379a695914005b9636",
        "57268f05dd62a815002e8990",
        "572ff7ab04bcaa1900d76f51",
        "5705eee952bb8914006896de",
        "572ffbaa947a6a140053cee6",
        "57300c67947a6a140053cff0",
        "572ffc99947a6a140053cef6",
        "5737a25ac3c5551400e51f51",
        "572649d8f1498d1400e8db37",
        "57111428b654c5140001faff",
        "57115f652419e314009555b9",
        "56e1a0dccd28a01900c67a2e",
        "571127a5a58dae1900cd6cc4"
      ],
      "candidate_sha256": [
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "72457fa0c9150fb2a7fed021650358b9295b523f55397d61363a047683297908",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "9a0ea788d4b66d4412369821a33e96287b8b91a59dbbe258cb494dfacfb20ad4",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "8394917ed3c6abd9f899d73ce839433e519acb0219643e9bfa9a48f92940892c"
      ],
      "example_id": "5705eee952bb8914006896de",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        13,
        12,
        11,
        4,
        0,
        1,
        2,
        3,
        5,
        7,
        8,
        9,
        10,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.39261067,
      "neural_order": [
        6,
        1,
        12,
        11,
        0,
        2,
        13,
        5,
        3,
        10,
        4,
        9,
        7,
        8,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "1e96eaac91936be67c7b1e2bfb4244e39c765e23a2d055aa9491c43d86110dec",
      "trial_index": 97
    },
    {
      "answer_sha256": "1709f87fa47a848f790d0d0a105b772e585623b660de8f920bf4e7a5f38f56fc",
      "candidate_ids": [
        "572663a9f1498d1400e8ddf2",
        "572647d0708984140094c14b",
        "57268a37f1498d1400e8e33c",
        "572ffd75b2c2fd14005686e5",
        "57109180a58dae1900cd6a40",
        "572a005f1d046914007796b7",
        "5733f9fa4776f4190066161f",
        "5705f09e75f01819005e77a4",
        "572681c1dd62a815002e8796",
        "5733eb34d058e614000b65cc",
        "572fadcbb2c2fd1400568329",
        "5730035e04bcaa1900d77001",
        "5725c69738643c19005accb9",
        "5729fd111d046914007796a5",
        "57284142ff5b5019007da00a",
        "5727565ef1498d1400e8f674"
      ],
      "candidate_sha256": [
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "14e1246d6c57cac98cd47a35fe5c0801832d2bfe7bc8d39af8efc222d99387f9",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875"
      ],
      "example_id": "5705f09e75f01819005e77a4",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        3,
        14,
        0,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.36610095,
      "neural_order": [
        7,
        6,
        10,
        5,
        9,
        14,
        4,
        11,
        0,
        13,
        12,
        3,
        2,
        8,
        1,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "048788e40e4e0145f1f7b815b1b1dce566e73f112f802e77258ab741742f46e1",
      "trial_index": 98
    },
    {
      "answer_sha256": "02ea91dd4f9fcc1e94373c0f7a75c2b0256cb0940bc14addd247d987320f28a6",
      "candidate_ids": [
        "57281ab63acd2414000df493",
        "572654e2708984140094c2b7",
        "57299021af94a219006aa50b",
        "5733f9fa4776f4190066161f",
        "5725edfe38643c19005ace9f",
        "572751b4708984140094dc1b",
        "57288428ff5b5019007da28e",
        "5705f13d52bb8914006896f0",
        "57269aa65951b619008f77ab",
        "5726701add62a815002e84c8",
        "572fc49d04bcaa1900d76ccb",
        "56de4a474396321400ee2786",
        "57115f0a50c2381900b54aa7",
        "57281bb84b864d190016449a",
        "5711475ca58dae1900cd6d8b",
        "5733ffa7d058e614000b674f"
      ],
      "candidate_sha256": [
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "9d0852e7a492edfb020138aeed2e89901eaf6cbc0450e6da752320d2c7c024e0",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "34b3cf8b2ee4c5e03d253cbd14d037704ed4a2f0fbffba0798b1eb00604b8c7e",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32"
      ],
      "example_id": "5705f13d52bb8914006896f0",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.34203868,
      "neural_order": [
        7,
        4,
        0,
        13,
        10,
        15,
        3,
        11,
        6,
        1,
        2,
        5,
        8,
        9,
        12,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "3f60260c084e9917e33a4ac4be9cf28e110a67002b1ca7adc98358c6a8db5cc8",
      "trial_index": 99
    },
    {
      "answer_sha256": "f45f9b4924358737d3783dfc209ed8243202ba5672cf05cd527d89d2b53ef3ef",
      "candidate_ids": [
        "5710f4b8b654c5140001fa45",
        "573368044776f41900660a29",
        "57114f0050c2381900b54a65",
        "57273a465951b619008f86ff",
        "57286192ff5b5019007da1dc",
        "572fff45947a6a140053cf26",
        "5728f50baf94a219006a9e55",
        "572a0d21af94a219006aa783",
        "57309446396df919000961b8",
        "5705f36452bb891400689718",
        "5705ffde52bb891400689784",
        "57274046dd62a815002e9a12",
        "573009a004bcaa1900d7704f",
        "56e1efa0e3433e140042321a",
        "573314e3d058e614000b56f0",
        "572f60f4947a6a140053c8ea"
      ],
      "candidate_sha256": [
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08"
      ],
      "example_id": "5705f36452bb891400689718",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.24672869,
      "neural_order": [
        9,
        8,
        0,
        1,
        3,
        7,
        2,
        6,
        10,
        14,
        4,
        15,
        11,
        12,
        5,
        13
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "b6e8a1fad4d11122404bee3876041f74632f2a6aae362e49311241a788bd643c",
      "trial_index": 100
    },
    {
      "answer_sha256": "b1e99324505bd32da0e1f85dcf5e19a09db0481e8a15f62c41eb320304a8e927",
      "candidate_ids": [
        "5726c002708984140094d073",
        "572867d72ca10214002da2fc",
        "57300911947a6a140053cfb6",
        "57115dbe2419e314009555a5",
        "572f6a0ba23a5019007fc5eb",
        "572656e4dd62a815002e81fa",
        "57115ac550c2381900b54a77",
        "57306797396df919000960ee",
        "572827fc3acd2414000df5bb",
        "57113639a58dae1900cd6d18",
        "5706111a52bb89140068984c",
        "5705f7c875f01819005e77dc",
        "5705e4fe75f01819005e7704",
        "572ffce5a23a5019007fcc15",
        "57373f80c3c5551400e51e8f",
        "57261f9f271a42140099d4a9"
      ],
      "candidate_sha256": [
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "3c1fdff7bdb3cfd3ccc8ba02719a8a942b0f26957f044cbeb9e587d6f71ae0cd",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670"
      ],
      "example_id": "5705f7c875f01819005e77dc",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.30286611,
      "neural_order": [
        11,
        8,
        10,
        12,
        3,
        7,
        0,
        15,
        4,
        13,
        9,
        5,
        1,
        2,
        6,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "cf6d0d3413955a7ac8584f155c7a8b24c0a0cdd953f4a736cb07dcc68d4fb8a5",
      "trial_index": 101
    },
    {
      "answer_sha256": "4c6ba7b1e67a4c7e33bbc00966118d828ca25b74684b97024ea6a833f6c47f22",
      "candidate_ids": [
        "5705fb7f52bb891400689750",
        "5706094b52bb8914006897de",
        "573099ee8ab72b1400f9c5dc",
        "56de10b44396321400ee2593",
        "57115e532419e314009555af",
        "5725c57a89a1e219009abe5e",
        "572a019f3f37b31900478643",
        "5726446a89cfff1900a8404d",
        "56de51c64396321400ee27f7",
        "5727e6cbff5b5019007d97ee",
        "5725e1c4271a42140099d2d7",
        "5725be0f271a42140099d117",
        "5733cf61d058e614000b62e9",
        "573792ee1c456719005744b9",
        "571cd11add7acb1400e4c16e",
        "5727d1c93acd2414000ded3f"
      ],
      "candidate_sha256": [
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "09dae1e2b2708f999627efe257489cfaad54e793df884b8877c2d638640ffa20",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6"
      ],
      "example_id": "5705fb7f52bb891400689750",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        2,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": false,
      "neural_margin": 0.04910248,
      "neural_order": [
        1,
        4,
        0,
        11,
        2,
        12,
        9,
        7,
        15,
        8,
        5,
        10,
        6,
        3,
        13,
        14
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "837c8698f454fb0d054503748600cbf665e41b90858d4d962888bf6cf6700e79",
      "trial_index": 102
    },
    {
      "answer_sha256": "92fbde133a4cba333888a6d70782107eca1d8b87eb09643fa1b880d247db81b6",
      "candidate_ids": [
        "5728facd4b864d1900165170",
        "57267ebfdd62a815002e872c",
        "571112ada58dae1900cd6bcc",
        "57300580b2c2fd140056874d",
        "57267d52708984140094c7d9",
        "5728855d3acd2414000dfa8d",
        "5728501aff5b5019007da166",
        "571117d4a58dae1900cd6c0a",
        "5726f48df1498d1400e8f0da",
        "5705fc3a52bb89140068976a",
        "5710f2e2a58dae1900cd6b72",
        "572683075951b619008f7513",
        "5729fb003f37b31900478627",
        "5730b541396df919000962c2",
        "572951f16aef051400154cce",
        "57267b3c5951b619008f7423"
      ],
      "candidate_sha256": [
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "8e983a2167b912359aa8e1110455d27a279e9b3baaac76337616f57b19396e19",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "ffaee1c83ab04918603b9940620c0c8b52e675143bc31a91f1fccac00d2b1bf7",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915"
      ],
      "example_id": "5705fc3a52bb89140068976a",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.49309014,
      "neural_order": [
        9,
        14,
        1,
        3,
        12,
        11,
        15,
        4,
        0,
        13,
        10,
        2,
        5,
        6,
        7,
        8
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "a72f87711a8b9330e961bd20223ec95b77e9371c2bcce4eec0fbe4fd8e80390f",
      "trial_index": 103
    },
    {
      "answer_sha256": "81f1787b530b558d6718ef248531518ed449e2c5125e86144a78d5feda9722a8",
      "candidate_ids": [
        "57060a6e52bb8914006897f8",
        "5729f69caf94a219006aa6f9",
        "57269e3bf1498d1400e8e516",
        "56e1febfe3433e1400423236",
        "572810ec2ca10214002d9d06",
        "572a005f1d046914007796b7",
        "5728742cff5b5019007da246",
        "570d4e6cb3d812140066d66d",
        "5705fd8475f01819005e7840",
        "572881022ca10214002da416",
        "573380e0d058e614000b5be9",
        "57302faa04bcaa1900d77311",
        "5725ec8289a1e219009ac0ae",
        "572f60f4947a6a140053c8ea",
        "56e20a3ae3433e140042324a",
        "57264f18f1498d1400e8dbae"
      ],
      "candidate_sha256": [
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "d5f4c4227c07c52cff10ca527f73efedf7e6c5aa9c01a3d9bf8a9468a6ae6939",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "b850237f43814691bedb635f5b07cea17e5bd3a2be801e564b61359a1746c8c6",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94"
      ],
      "example_id": "5705fd8475f01819005e7840",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        11,
        15,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.31647662,
      "neural_order": [
        8,
        0,
        12,
        5,
        13,
        4,
        10,
        7,
        14,
        3,
        6,
        1,
        2,
        9,
        11,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "4f3f2a9aed1166ab461816e162816bff258bd7a405127587dc01be34c10d64f6",
      "trial_index": 104
    },
    {
      "answer_sha256": "28ec039832f5bc96c2be0eaee016dafeb7547ed76b510ed5a24d25b180493271",
      "candidate_ids": [
        "57335ddbd058e614000b592f",
        "571097baa58dae1900cd6a98",
        "5730b255396df919000962b0",
        "5726581fdd62a815002e823a",
        "57097b50200fba140036804d",
        "5705edcd52bb8914006896ca",
        "5725c95f38643c19005accf3",
        "5730131c947a6a140053d052",
        "571154c72419e31400955588",
        "573407d7d058e614000b6813",
        "5705fec152bb89140068977a",
        "5725b81b271a42140099d097",
        "5737a5931c456719005744e7",
        "57106644b654c5140001f8e5",
        "572742bd5951b619008f8785",
        "572978f91d046914007794d3"
      ],
      "candidate_sha256": [
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "d03773b18a1be50107069edcf0e896834515c0cc39a5f2a1177372718099eaff",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "ce8d2359f23498cb006a1bf2f91d5494c463bc986a83749d5f9208b28fa63c51",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e"
      ],
      "example_id": "5705fec152bb89140068977a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.23061667,
      "neural_order": [
        10,
        5,
        6,
        1,
        14,
        11,
        0,
        4,
        15,
        13,
        2,
        3,
        7,
        8,
        9,
        12
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "eaced30cda6483a9905830d9cd1c722e6eacd927cacf075a8c85223115d8e22c",
      "trial_index": 105
    },
    {
      "answer_sha256": "8d3515626c4b80c92acfde8518e149fa87e306db7eefb6ab1d55622948186de4",
      "candidate_ids": [
        "570d2417fed7b91900d45c3d",
        "572a0ecb1d04691400779717",
        "573368e54776f41900660a53",
        "572726c9708984140094da7b",
        "57335c20d058e614000b58f9",
        "572a09abaf94a219006aa75b",
        "57265e455951b619008f70bb",
        "57114e8d50c2381900b54a5b",
        "572ffc0f947a6a140053cef0",
        "5705ffde52bb891400689784",
        "5726deed5951b619008f80c5",
        "572810ec2ca10214002d9d06",
        "57264586f1498d1400e8dac6",
        "572744aff1498d1400e8f588",
        "57267ebfdd62a815002e872c",
        "56e19557e3433e1400422fee"
      ],
      "candidate_sha256": [
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "da7e3caa2252604ebed016e003b315efd2a4a47f3b063950ffca6d27e6dd03f2",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086"
      ],
      "example_id": "5705ffde52bb891400689784",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.36229534,
      "neural_order": [
        9,
        14,
        4,
        1,
        5,
        7,
        6,
        2,
        13,
        8,
        3,
        0,
        10,
        11,
        12,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "38a3d29739d5da4b286a440040179c765d0b5440ae9301e992b6fea497ce8e02",
      "trial_index": 106
    },
    {
      "answer_sha256": "6662a9fea29c613c1793d926b7d4ee267b4e8ac5ac5bffe7ca0c30f3bfd80cc9",
      "candidate_ids": [
        "5733cff84776f419006612b4",
        "57293d116aef051400154bc8",
        "570602fa52bb89140068979e",
        "572ff5fcb2c2fd1400568657",
        "5730a314069b5314008321eb",
        "57265aaf5951b619008f706b",
        "57264e66dd62a815002e811a",
        "5726dcbddd62a815002e9320",
        "5725fcbe271a42140099d3ad",
        "57114aceb654c5140001fc47",
        "5726a14c708984140094cc51",
        "56de3e414396321400ee26d8",
        "5729f5a03f37b31900478605",
        "572949306aef051400154c68",
        "572756fe708984140094dc71",
        "5730ab63396df91900096260"
      ],
      "candidate_sha256": [
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d"
      ],
      "example_id": "570602fa52bb89140068979e",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        15,
        8,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.49929175,
      "neural_order": [
        2,
        4,
        8,
        15,
        5,
        0,
        13,
        6,
        12,
        7,
        1,
        11,
        14,
        3,
        9,
        10
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "c9e27b264c5214376b2f23ebc70ec89157c33bfc5f338a3a4fc48ed854490049",
      "trial_index": 107
    },
    {
      "answer_sha256": "7d12ba56e9f8b3dc64f77c87318c4f37bc12cfbf1a37573cdf3e4fa683f20155",
      "candidate_ids": [
        "572fdb17b2c2fd140056851d",
        "5729a26d6aef05140015505a",
        "56dde0ba66d3e219004dad75",
        "57265aaf5951b619008f706b",
        "56e16839cd28a01900c67887",
        "57060eaf75f01819005e7910",
        "57271f125951b619008f8635",
        "5733f5f24776f419006615c1",
        "570603c475f01819005e7882",
        "571126dfa58dae1900cd6cb2",
        "57332f81d058e614000b5776",
        "571155ae2419e31400955591",
        "5733638fd058e614000b59e8",
        "5726545f708984140094c2a5",
        "5727c0402ca10214002d9564",
        "572754dd708984140094dc3c"
      ],
      "candidate_sha256": [
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "02473abeaeaf0a7ac1221ede2323c119c62182f573df438534fcb86c2c6943d4",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "e26d3a98154d79e4fb93dfbc9b446801d4e38dc1c4681b39878f40025a6b4fce",
        "752151b83536aa2b39e17ee8104540f3b94aa10671b826eb5b19cc6d1e68a8ae",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "25632077ec283d4cc31c80588ea6810e247141a57355bd6b1a28a5238b8acb10",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "05e6b70a4c996f49687d72f7a373609343967f6fed5be1f3bda6965dcfad141e",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a"
      ],
      "example_id": "570603c475f01819005e7882",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        5,
        13,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.71807477,
      "neural_order": [
        8,
        5,
        7,
        2,
        0,
        4,
        15,
        11,
        9,
        1,
        3,
        6,
        10,
        12,
        13,
        14
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "41ab438cb26d3f2afd318fde64cf32355cf9176f1f53afd58076e2619cea3185",
      "trial_index": 108
    },
    {
      "answer_sha256": "e6d49b901b65d3336837067f5849474c5d5a8a6ba761a4c79acbc20d0b644c76",
      "candidate_ids": [
        "572864542ca10214002da2de",
        "56e1b62ecd28a01900c67aa3",
        "572973f76aef051400154f0b",
        "5730bd00069b5314008322d7",
        "5706074552bb8914006897d4",
        "572980f9af94a219006aa4d1",
        "56de3f784396321400ee26fa",
        "572ff626947a6a140053ce8e",
        "5726a7ecf1498d1400e8e654",
        "57269aa65951b619008f77ab",
        "572fd7b8947a6a140053cd3a",
        "5725fcbe271a42140099d3ad",
        "571cda1bdd7acb1400e4c196",
        "57377083c3c5551400e51edf",
        "57274beff1498d1400e8f5e4",
        "56e1c7e2cd28a01900c67b74"
      ],
      "candidate_sha256": [
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6"
      ],
      "example_id": "5706074552bb8914006897d4",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        11,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.25833089,
      "neural_order": [
        4,
        11,
        14,
        15,
        7,
        3,
        1,
        8,
        0,
        2,
        5,
        6,
        9,
        10,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "221bf51403f332d5ce4444cd1d3fc31ccc4709a61ebb7a3aecb4ad0cdb0dd32c",
      "trial_index": 109
    },
    {
      "answer_sha256": "9f2608067816e38c85edfb0c3985feff32def8b5dc17bb522ffc2e877e9b386b",
      "candidate_ids": [
        "57297a276aef051400154f88",
        "56de17f9cffd8e1900b4b5e0",
        "56e1b62ecd28a01900c67aa3",
        "57298ef11d0469140077952d",
        "570607f575f01819005e78b4",
        "5727f05b4b864d1900164068",
        "5733e8ae4776f419006614a6",
        "572a2224af94a219006aa823",
        "573060b48ab72b1400f9c4c6",
        "571c83f3dd7acb1400e4c0d8",
        "57113be3a58dae1900cd6d28",
        "57265aaf5951b619008f706b",
        "57287c2bff5b5019007da26c",
        "572671a5f1498d1400e8dfe4",
        "5728659f4b864d190016498a",
        "57294baaaf94a219006aa26b"
      ],
      "candidate_sha256": [
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "18893f714ac102ba660ec3790204ffdf2ee86fd4cd069c50ec0ceeaed01cefbd",
        "b7d78d9fd858b52f42115f7144f2b2a6844dffc86f3a087c6833c6f78182e39e",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1"
      ],
      "example_id": "570607f575f01819005e78b4",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.6228571,
      "neural_order": [
        4,
        14,
        6,
        11,
        12,
        2,
        5,
        13,
        3,
        0,
        1,
        7,
        8,
        9,
        10,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "8b13bfa7c2543c94a9333bccc78e32ee4841c0890e70613783e8132065d4aa23",
      "trial_index": 110
    },
    {
      "answer_sha256": "b4212b193c22b64d7c6ca4d747621420f4b8b5d5450bc9a658ce299c49985d8b",
      "candidate_ids": [
        "57302a3aa23a5019007fcecf",
        "572ff932a23a5019007fcbd5",
        "5725db98ec44d21400f3d6c5",
        "57300761947a6a140053cf9a",
        "57274971708984140094dbbb",
        "5706149552bb891400689880",
        "571cca4add7acb1400e4c150",
        "5706094b52bb8914006897de",
        "5733647e4776f419006609ae",
        "573060b48ab72b1400f9c4c6",
        "572a02483f37b3190047864b",
        "5737477bc3c5551400e51ea3",
        "57096e1ced30961900e84102",
        "56e17a7ccd28a01900c679a1",
        "572fc41f947a6a140053cc80",
        "572843304b864d1900164848"
      ],
      "candidate_sha256": [
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "b7d78d9fd858b52f42115f7144f2b2a6844dffc86f3a087c6833c6f78182e39e",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "5b8656a663456a61c1d9692e2b9d3858aa19a6b21b538a75b82941eb5206617c",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "29fcf6218856cfd64395c81351e873a0769b0f291577dd5933cc1ac378f40976",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7"
      ],
      "example_id": "5706094b52bb8914006897de",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.40571875,
      "neural_order": [
        7,
        2,
        5,
        15,
        9,
        13,
        4,
        3,
        12,
        0,
        1,
        6,
        8,
        10,
        11,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "4ce5bd50296649d65aaa27babadf15827091f5c7982fa84cacfacc5b92e033f9",
      "trial_index": 111
    },
    {
      "answer_sha256": "a40d6a5eb98165725d806378b92d9924969a699d2249adeb40005c35d852a174",
      "candidate_ids": [
        "571a49de4faf5e1900b8a94a",
        "57060a1175f01819005e78d2",
        "572978f91d046914007794d3",
        "5728f50baf94a219006a9e55",
        "572872dd2ca10214002da37e",
        "5726bc1add62a815002e8ea6",
        "57267d52708984140094c7d9",
        "572a11663f37b31900478693",
        "572ffe6fb2c2fd14005686ef",
        "5726545f708984140094c2a5",
        "5726a09f708984140094cc39",
        "5711541350c2381900b54a6f",
        "572fdbb004bcaa1900d76dd9",
        "572ff760b2c2fd1400568677",
        "5727ee372ca10214002d99ec",
        "572a0f073f37b3190047867f"
      ],
      "candidate_sha256": [
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "808648d31e3c0a580e378e5e38b6e262d80eae03ea2f5f3343f713667cc29b1d",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2"
      ],
      "example_id": "57060a1175f01819005e78d2",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        10,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.55490241,
      "neural_order": [
        1,
        14,
        11,
        7,
        2,
        0,
        4,
        3,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "7ccab42dc83ce0d39c913c52f4699722ef95f04946a37982071518fc7e83d445",
      "trial_index": 112
    },
    {
      "answer_sha256": "11d40417959631d3d2420e8cd8709893c11cd7a4db737af63e8d56cfa7866f85",
      "candidate_ids": [
        "57264684708984140094c123",
        "5730876a396df9190009617a",
        "57340a094776f4190066177d",
        "5726f635dd62a815002e9656",
        "572fb42904bcaa1900d76c15",
        "57339dd94776f41900660ecc",
        "573750f51c45671900574467",
        "5728f9882ca10214002dab5a",
        "572a06866aef0514001551be",
        "572869b84b864d19001649ae",
        "572a07c11d046914007796d5",
        "573093598ab72b1400f9c5ae",
        "5727cb4b2ca10214002d9676",
        "5734025d4776f419006616c3",
        "57060a6e52bb8914006897f8",
        "57114667a58dae1900cd6d80"
      ],
      "candidate_sha256": [
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd",
        "fbdd4577c5f9e5f4af71320383d43f13f7697f04d5670525f7d260f7ff56759d",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27",
        "ccfb450b288c15536e52e7e1c54aa458c0e2c2936babc78f6cab1b418fa9e1db",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea"
      ],
      "example_id": "57060a6e52bb8914006897f8",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        3,
        10,
        2,
        7,
        1,
        0,
        4,
        5,
        6,
        8,
        9,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.45895558,
      "neural_order": [
        14,
        12,
        8,
        2,
        10,
        4,
        3,
        0,
        1,
        5,
        6,
        7,
        9,
        11,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "81dcf84f4aeb445c5b3131522240ffccdf474d0bd5b4a3dbabb97a25c3545ad9",
      "trial_index": 113
    },
    {
      "answer_sha256": "11d40417959631d3d2420e8cd8709893c11cd7a4db737af63e8d56cfa7866f85",
      "candidate_ids": [
        "5733e771d058e614000b6545",
        "57060cc352bb89140068980e",
        "5726722bdd62a815002e8528",
        "56de40da4396321400ee2708",
        "572ff5fcb2c2fd1400568657",
        "56e1fc57e3433e140042322c",
        "571136b8a58dae1900cd6d22",
        "57097d63ed30961900e841fc",
        "573083dc2461fd1900a9ce6d",
        "57339dd94776f41900660ecc",
        "5729a3716aef05140015506a",
        "5725fb8138643c19005acf3f",
        "5730131c947a6a140053d052",
        "571a4b0f10f8ca1400304fd5",
        "56dde2fa66d3e219004dad9b",
        "572648ed5951b619008f6f03"
      ],
      "candidate_sha256": [
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "27749618f729fa62e79b34e0666fb6ce7980b9e0e582584b2a78d0294543512a",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee"
      ],
      "example_id": "57060cc352bb89140068980e",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        5,
        0,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.58724195,
      "neural_order": [
        1,
        11,
        12,
        9,
        8,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        10,
        13,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "b8437c99ec51c9db65d323c76bb0861e5ee04f011439afd3aa30aa1c6066b14e",
      "trial_index": 114
    },
    {
      "answer_sha256": "11d40417959631d3d2420e8cd8709893c11cd7a4db737af63e8d56cfa7866f85",
      "candidate_ids": [
        "571142b3a58dae1900cd6d5a",
        "57335ddbd058e614000b592f",
        "5726a00cf1498d1400e8e550",
        "5727cb4b2ca10214002d9676",
        "573383d0d058e614000b5c35",
        "570d2556fed7b91900d45c47",
        "572a135daf94a219006aa79f",
        "571c8539dd7acb1400e4c0e2",
        "572881d34b864d1900164a5a",
        "5726e5c95951b619008f81cb",
        "572750df5951b619008f882f",
        "57060df252bb891400689820",
        "573003dd947a6a140053cf42",
        "57308ddc396df919000961a4",
        "573062662461fd1900a9cdf7",
        "57293c246aef051400154bb8"
      ],
      "candidate_sha256": [
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31"
      ],
      "example_id": "57060df252bb891400689820",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        1,
        3,
        0,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.59479778,
      "neural_order": [
        11,
        3,
        15,
        1,
        9,
        5,
        8,
        12,
        13,
        0,
        2,
        4,
        6,
        7,
        10,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "211e64e5335d917f73e1e6cc7b2854bc032e4c917a673af6b1f0c3a38636482b",
      "trial_index": 115
    },
    {
      "answer_sha256": "1b4c9133da73a711322404314402765ab0d23fd362a167d6f0c65bb215113d94",
      "candidate_ids": [
        "57097141200fba1400367ff9",
        "57284b904b864d19001648e2",
        "57286b003acd2414000df9c1",
        "5733f7b9d058e614000b66a9",
        "57294279af94a219006aa209",
        "57337ea24776f41900660bd0",
        "57096b66200fba1400367fa7",
        "572667e2f1498d1400e8de90",
        "57060eaf75f01819005e7910",
        "571cd88ddd7acb1400e4c18d",
        "5727f2714b864d1900164072",
        "572a0ce11d046914007796fd",
        "57294209af94a219006aa201",
        "57265e455951b619008f70bb",
        "5737958b1c456719005744c3",
        "57265c10f1498d1400e8dd36"
      ],
      "candidate_sha256": [
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "5d46e9f7c620b7d8f42a0c90216d4bb7625ee95e4d9af99b663434ae1e08a483",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377"
      ],
      "example_id": "57060eaf75f01819005e7910",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.39982801,
      "neural_order": [
        8,
        1,
        2,
        6,
        5,
        0,
        3,
        4,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "8f690634633ceb46970677b3f07d383c31c9a997eb08d268ca73d3f36d321943",
      "trial_index": 116
    },
    {
      "answer_sha256": "340a3211ddb62608932085d446e114b2ba35aefc749d26ec114a85473c06edfd",
      "candidate_ids": [
        "571095a8a58dae1900cd6a76",
        "57060f3e75f01819005e7922",
        "57264b1ddd62a815002e80a0",
        "57107a3ea58dae1900cd69de",
        "5726545f708984140094c2a5",
        "5729f9953f37b3190047861f",
        "5733d3cb4776f419006612ea",
        "56de4a89cffd8e1900b4b7bd",
        "5733a45d4776f41900660f23",
        "5733d13e4776f419006612c4",
        "56de3cd0cffd8e1900b4b6be",
        "572864dd4b864d1900164976",
        "57302e45947a6a140053d22e",
        "5727502f708984140094dc07",
        "5733d4c8d058e614000b6353",
        "572a09abaf94a219006aa75b"
      ],
      "candidate_sha256": [
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "2d5af9a4d02fc579e6e15f098bcffbcc6459cdee51fd51b38ea0075eee7a728a",
        "e32237eb13f46727755bd53347a189c0620b3ffff778cecbcd627a94fbebc022",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027"
      ],
      "example_id": "57060f3e75f01819005e7922",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.69811561,
      "neural_order": [
        1,
        2,
        13,
        7,
        15,
        4,
        3,
        11,
        0,
        5,
        6,
        8,
        9,
        10,
        12,
        14
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "6c6e99e069abb4867ba6ff51caecb758b23cbba97f04692bb07ca3535db30889",
      "trial_index": 117
    },
    {
      "answer_sha256": "e7ef80b6ff4a3c1c4978a3f74349a6c6ae15bff998aee247982f00feb4c70e4f",
      "candidate_ids": [
        "570610b275f01819005e792a",
        "57287fec4b864d1900164a3c",
        "5728855d3acd2414000dfa8d",
        "5711163bb654c5140001fb13",
        "571ce6655efbb31900334e34",
        "5729a26d6aef05140015505a",
        "56de15104396321400ee25b7",
        "572853e8ff5b5019007da188",
        "571cd11add7acb1400e4c16e",
        "570d4c3bfed7b91900d45e31",
        "571cbe35dd7acb1400e4c13c",
        "5726c5a9f1498d1400e8eac4",
        "57286ec63acd2414000df9d1",
        "57265bdfdd62a815002e829e",
        "57111380a58dae1900cd6bd6",
        "57263ea0271a42140099d7c3"
      ],
      "candidate_sha256": [
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "02473abeaeaf0a7ac1221ede2323c119c62182f573df438534fcb86c2c6943d4",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30"
      ],
      "example_id": "570610b275f01819005e792a",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        3,
        11,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.54329664,
      "neural_order": [
        0,
        8,
        4,
        13,
        7,
        3,
        15,
        10,
        5,
        6,
        1,
        2,
        9,
        11,
        12,
        14
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "470a58df88a7f815825de6c1a234b873bab36b570f2142231e5f47f5b06ddc50",
      "trial_index": 118
    },
    {
      "answer_sha256": "91c8358fc3513fac050a40c2c5a0821b0cdb767b6784853d263d37d9f9fa5bb4",
      "candidate_ids": [
        "57286951ff5b5019007da20e",
        "5726446a89cfff1900a8404d",
        "57308ddc396df919000961a4",
        "56dddf4066d3e219004dad5f",
        "57377aac1c45671900574479",
        "5726431aec44d21400f3dd13",
        "57113ba6b654c5140001fc1f",
        "57264991f1498d1400e8db2e",
        "572fdc34a23a5019007fca93",
        "571135b8a58dae1900cd6d0e",
        "57373d0cc3c5551400e51e85",
        "56e20a27cd28a01900c67c24",
        "5706111a52bb89140068984c",
        "57300761947a6a140053cf9a",
        "572ff7ab04bcaa1900d76f51",
        "5705ffde52bb891400689784"
      ],
      "candidate_sha256": [
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228"
      ],
      "example_id": "5706111a52bb89140068984c",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.33322798,
      "neural_order": [
        12,
        1,
        5,
        9,
        8,
        6,
        15,
        10,
        11,
        4,
        2,
        0,
        3,
        7,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "5dc6af2baa7eec3780968d1d9df9d0305ebd94456d76206dc2425e6559d2f6ab",
      "trial_index": 119
    },
    {
      "answer_sha256": "d045a80935bc0afbfa05ce6319c10d0c853b38c6e32b1d27b4cce3bf87829e39",
      "candidate_ids": [
        "5727dd2e4b864d1900163eba",
        "5705fc3a52bb89140068976a",
        "5711669550c2381900b54adf",
        "5727d1c93acd2414000ded3f",
        "572fdd03a23a5019007fca9d",
        "570611c475f01819005e793c",
        "573020f7b2c2fd14005688f7",
        "5727e21e4b864d1900163f34",
        "57294baaaf94a219006aa26b",
        "5727213c708984140094da35",
        "571090abb654c5140001f995",
        "571c96095efbb31900334dbc",
        "57339902d058e614000b5e70",
        "5711163bb654c5140001fb13",
        "572667e2f1498d1400e8de90",
        "5727aec03acd2414000de991"
      ],
      "candidate_sha256": [
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "db91c1023a94550f1ffe6e9f87e0b45aaad95f3bef9ada0d0c3872b88286bf7c",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "874533677a269c5d5fb8bd3a3213307ea1632c2236ea48b8da8182d07905fcc8",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "3ee61e6d7605bacdeb4f37ab798daa449f57668fd923e6b04e1a516237821782",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "afa714a72ebd441495497201ca8e5b34e7410ce35809dcecc075c5b3d84568f3"
      ],
      "example_id": "570611c475f01819005e793c",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.4359902,
      "neural_order": [
        5,
        15,
        0,
        13,
        3,
        1,
        11,
        12,
        2,
        6,
        4,
        7,
        8,
        9,
        10,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "0894f30f4c912ceacadaeca2371ec52e6c91ff04350ac4ef4d72d4a8e682d0b0",
      "trial_index": 120
    },
    {
      "answer_sha256": "6acae2f43df22db866f30fe29ff02b1d1e25d8abb384bcff4b17b182a71a2dd7",
      "candidate_ids": [
        "57264a8cdd62a815002e808c",
        "572810ec2ca10214002d9d06",
        "57263b1638643c19005ad333",
        "5726446a89cfff1900a8404d",
        "5706139252bb891400689864",
        "5729f4e46aef051400155156",
        "572871bc4b864d1900164a04",
        "57264fe65951b619008f6fa1",
        "572989846aef051400154fc0",
        "5728170d3acd2414000df443",
        "5733ef47d058e614000b662b",
        "5705edcd52bb8914006896ca",
        "57268220f1498d1400e8e216",
        "5705fb7f52bb891400689750",
        "5726e179dd62a815002e93ae",
        "572826634b864d19001645be"
      ],
      "candidate_sha256": [
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "3a95085253039f4bf191f1037c44a2641c44c877382c059105507c459d53eaba",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2"
      ],
      "example_id": "5706139252bb891400689864",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        2,
        6,
        0,
        1,
        3,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.18948466,
      "neural_order": [
        4,
        3,
        5,
        12,
        13,
        11,
        14,
        15,
        6,
        2,
        8,
        1,
        7,
        0,
        9,
        10
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "a0975837c697608d194f62f2c6aa1e9feaebc21650295203bba729d8175e0526",
      "trial_index": 121
    },
    {
      "answer_sha256": "196e2aa52f81dffeeaad4e6ee89acbf57bc6a98e31f2e286fdf57bc7f3258190",
      "candidate_ids": [
        "571112ada58dae1900cd6bcc",
        "57265285708984140094c25b",
        "57286dfa2ca10214002da332",
        "57267ebfdd62a815002e872c",
        "57264e66dd62a815002e811a",
        "5733a2a9d058e614000b5f28",
        "5733fd66d058e614000b6735",
        "57302faa04bcaa1900d77311",
        "5706143575f01819005e7950",
        "56dde27d9a695914005b9651",
        "572ff5fcb2c2fd1400568657",
        "572a058aaf94a219006aa751",
        "573755afc3c5551400e51eb3",
        "5728659f4b864d190016498a",
        "572a11663f37b31900478693",
        "5730b541396df919000962c2"
      ],
      "candidate_sha256": [
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936",
        "87c3a5e21ea40ce9396fd4bd04485fc5c291473b22cdd4b5feb623f3ece0e993",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701",
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7"
      ],
      "example_id": "5706143575f01819005e7950",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.34043027,
      "neural_order": [
        8,
        13,
        11,
        4,
        9,
        0,
        12,
        5,
        1,
        2,
        3,
        6,
        7,
        10,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "b3ac3e9e8728cdb016d510b973368595d2477f925ca928fec8aaa4bb245b029c",
      "trial_index": 122
    },
    {
      "answer_sha256": "cb2574fe8d77d066295067fd7fc402b5da5bf55601033bb0bde3620da9217d18",
      "candidate_ids": [
        "5725cc38ec44d21400f3d5bb",
        "57285213ff5b5019007da182",
        "56de3ebc4396321400ee26e6",
        "5726462b708984140094c117",
        "5726da89dd62a815002e92b2",
        "572f609ca23a5019007fc5af",
        "57298ef11d0469140077952d",
        "56e1dc62cd28a01900c67bcb",
        "572744aff1498d1400e8f588",
        "56e19724cd28a01900c679f6",
        "57308cf88ab72b1400f9c576",
        "572fd264b2c2fd14005684a9",
        "56de179dcffd8e1900b4b5da",
        "57274f67708984140094dbf5",
        "5706149552bb891400689880",
        "57309bfb8ab72b1400f9c5e6"
      ],
      "candidate_sha256": [
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51",
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee"
      ],
      "example_id": "5706149552bb891400689880",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.59727323,
      "neural_order": [
        14,
        0,
        3,
        1,
        6,
        10,
        2,
        4,
        5,
        7,
        8,
        9,
        11,
        12,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "f5b9d2067903f328115f08da561ec64d2c4297da06d33ee35878a0a80ef95520",
      "trial_index": 123
    },
    {
      "answer_sha256": "776af9401679287d8d332e26e99062919d58a135f4ef5ba85d4ec981b50f58ae",
      "candidate_ids": [
        "57274beff1498d1400e8f5e4",
        "57338007d058e614000b5bda",
        "573382d24776f41900660c37",
        "5730bd00069b5314008322d7",
        "56e1bc3ae3433e1400423104",
        "57264cac708984140094c1b3",
        "572f5533a23a5019007fc55c",
        "57109275b654c5140001f99f",
        "5705e4fe75f01819005e7704",
        "57332c1e4776f4190066073a",
        "570614ff52bb89140068988a",
        "57115dbe2419e314009555a5",
        "572a1c943f37b319004786e1",
        "5730042804bcaa1900d77011",
        "57308ddc396df919000961a4",
        "57287c142ca10214002da3d0"
      ],
      "candidate_sha256": [
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "7c28011f4adb5bc9518e20aa534136a5ab7d9e920f0263cb0d482a6ce7859ede",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5"
      ],
      "example_id": "570614ff52bb89140068988a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.41700009,
      "neural_order": [
        10,
        8,
        11,
        14,
        13,
        7,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        9,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "74bdd7f27d024aa88e2291f6e2c023ac82ff393b4e623c534d9579e15fd6cb73",
      "trial_index": 124
    },
    {
      "answer_sha256": "3e9df2cb288efe7a21cc992d70d29fc1de1b04306d5ad3ab4378713381657b7c",
      "candidate_ids": [
        "57060cc352bb89140068980e",
        "5728fd206aef05140015494c",
        "56de11154396321400ee25aa",
        "57306797396df919000960ee",
        "5726581fdd62a815002e823a",
        "5706155352bb891400689894",
        "57264c42dd62a815002e80c6",
        "573380e0d058e614000b5be9",
        "5727cd0f4b864d1900163d72",
        "5733ffa7d058e614000b674f",
        "56e1bc3ae3433e1400423104",
        "572ff5fcb2c2fd1400568657",
        "5733314e4776f4190066076a",
        "57265ceddd62a815002e82b8",
        "57265285708984140094c25b",
        "56dddf4066d3e219004dad5f"
      ],
      "candidate_sha256": [
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760"
      ],
      "example_id": "5706155352bb891400689894",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        6,
        15,
        0,
        1,
        2,
        3,
        4,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.32850294,
      "neural_order": [
        5,
        0,
        8,
        13,
        9,
        4,
        7,
        3,
        15,
        1,
        2,
        6,
        10,
        11,
        12,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "18fe28222845b69619bdcedf1a82c4d3600439213999b55da12e01254cda5179",
      "trial_index": 125
    },
    {
      "answer_sha256": "45ead65f6948112309927802e02f91ebe0108f2c5c02d7b97bbf3d69975fbfb4",
      "candidate_ids": [
        "572fcb6da23a5019007fc9f1",
        "57299a6f6aef051400155016",
        "57286b003acd2414000df9c1",
        "5705f13d52bb8914006896f0",
        "57092322efce8f15003a7db0",
        "5730a951069b531400832213",
        "572a0e4b6aef051400155214",
        "5728661e2ca10214002da2e8",
        "56de4a474396321400ee2786",
        "571c8eb9dd7acb1400e4c0f6",
        "57113639a58dae1900cd6d18",
        "572a0c541d046914007796f3",
        "5726c5a9f1498d1400e8eac4",
        "5727515f708984140094dc11",
        "572a02483f37b3190047864b",
        "57265746dd62a815002e8218"
      ],
      "candidate_sha256": [
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "c426efe1359054866c8d322fd02f908d61dd09289424401de4416d000f338ca6",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "bc5f1c2743b9de3bbf84078611d839665374c71f9f5ce23ea41ca3d84c8793ba",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7"
      ],
      "example_id": "57092322efce8f15003a7db0",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        12,
        6,
        0,
        1,
        2,
        3,
        5,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.64522627,
      "neural_order": [
        4,
        12,
        9,
        3,
        11,
        13,
        10,
        14,
        5,
        0,
        1,
        2,
        6,
        7,
        8,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "439e8d828058e21425f95fbfe2867c97c913ca07dcb6e08288647483cf71c13e",
      "trial_index": 126
    },
    {
      "answer_sha256": "6f6a4e56098cfd9af29e3ae549503b370211a4e94421457fe4cfd39a38a1fa08",
      "candidate_ids": [
        "57264e455951b619008f6f65",
        "5726eb4b5951b619008f826b",
        "572754fff1498d1400e8f660",
        "5730131c947a6a140053d052",
        "5725f6e138643c19005acf19",
        "57332562d058e614000b5730",
        "57308ddc396df919000961a4",
        "5705eb3375f01819005e7764",
        "571a52cb4faf5e1900b8a969",
        "572647e2dd62a815002e805c",
        "5737432bc3c5551400e51e99",
        "5725b92e38643c19005acbd3",
        "571c879bdd7acb1400e4c0ec",
        "5737898f1c45671900574495",
        "573403394776f419006616dd",
        "57094a79efce8f15003a7dc4"
      ],
      "candidate_sha256": [
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60"
      ],
      "example_id": "57094a79efce8f15003a7dc4",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        1,
        4,
        0,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.34702119,
      "neural_order": [
        15,
        7,
        9,
        0,
        6,
        13,
        11,
        4,
        1,
        2,
        3,
        5,
        8,
        10,
        12,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "76fa8c38a840cc03cfa3b2f9cbac09f5c1348591f7d9bf30e3bcbf26786f96f4",
      "trial_index": 127
    },
    {
      "answer_sha256": "7ac9b8dd44d3db4e095ff473094ba7e37f9db84d4aa1c491a3a45974d1b9c245",
      "candidate_ids": [
        "5726a34bf1498d1400e8e59e",
        "57287c2bff5b5019007da26c",
        "571a484210f8ca1400304fbe",
        "57280e1aff5b5019007d9bec",
        "56de179dcffd8e1900b4b5da",
        "572f5875947a6a140053c89a",
        "5727f6723acd2414000df0e7",
        "5728e8212ca10214002daa6c",
        "5705e3f252bb89140068966a",
        "573406d1d058e614000b6801",
        "5733dab4d058e614000b6409",
        "57280fd3ff5b5019007d9c26",
        "57287d4a2ca10214002da3e4",
        "5729081d3f37b31900477fab",
        "57094b4f9928a814004714f6",
        "5705f36452bb891400689718"
      ],
      "candidate_sha256": [
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "e1e68b90e348d860a1f3c0cfe6e1367d9c8c8a57ad577d564b36e00900222682",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "db6de70b42aaae5f96618f4be9863dbd2b0618f69ea12eeb5dd9dbfb1a8bcef5",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba"
      ],
      "example_id": "57094b4f9928a814004714f6",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.33837308,
      "neural_order": [
        14,
        6,
        12,
        1,
        10,
        15,
        4,
        3,
        0,
        2,
        5,
        7,
        8,
        9,
        11,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "85cdfe6645a5668c0697603f1ebe6d9ca9900636cfa7e140d9b1ea9ca9a66a8a",
      "trial_index": 128
    },
    {
      "answer_sha256": "c89afdbfd951cb7da687e597d615cdbab5dfe267bdc4318cb8e9134ac7b99dac",
      "candidate_ids": [
        "5710eb6fb654c5140001fa17",
        "572fbf21a23a5019007fc937",
        "572a019f3f37b31900478643",
        "5706074552bb8914006897d4",
        "572fd47fa23a5019007fca55",
        "5726509bdd62a815002e815a",
        "57096c95200fba1400367fbb",
        "5733f1784776f41900661575",
        "5729fd56af94a219006aa72f",
        "57115c7450c2381900b54a9d",
        "5727d6faff5b5019007d969a",
        "571127a5a58dae1900cd6cc4",
        "5705ec1675f01819005e776e",
        "57094ca7efce8f15003a7dd6",
        "57264cac708984140094c1b3",
        "57106d2fb654c5140001f8ef"
      ],
      "candidate_sha256": [
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "8394917ed3c6abd9f899d73ce839433e519acb0219643e9bfa9a48f92940892c",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109"
      ],
      "example_id": "57094ca7efce8f15003a7dd6",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        6,
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.2855597,
      "neural_order": [
        13,
        6,
        5,
        10,
        14,
        12,
        0,
        1,
        2,
        3,
        4,
        7,
        8,
        9,
        11,
        15
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "00c355ba02cd524e2fd70cae64033b723739a8299d79b7e9935dcd0df0f684e3",
      "trial_index": 129
    },
    {
      "answer_sha256": "9b258f648f00aee39f2cf41e7418d529c74674d0426207621b7467da16584552",
      "candidate_ids": [
        "5710e8c8a58dae1900cd6b26",
        "5725d662ec44d21400f3d687",
        "5729feaf6aef051400155188",
        "572fbea404bcaa1900d76c5b",
        "57096f37200fba1400367fe5",
        "57115ac550c2381900b54a77",
        "570963a5200fba1400367f33",
        "57284b904b864d19001648e2",
        "57377083c3c5551400e51edf",
        "572fbb04a23a5019007fc8f7",
        "57094d489928a8140047150a",
        "5725cbb289a1e219009abed2",
        "5725b76389a1e219009abd4a",
        "572943ab1d04691400779219",
        "5727fc37ff5b5019007d9a18",
        "56e1d9fee3433e14004231cb"
      ],
      "candidate_sha256": [
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "3895e66f384dc9afdd15ffe1c33b4044506755bd22d33e8c94583b00ed06854d",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "40f970a92f69db6815bd67e7934fc725f910e6a6e08c5ff4e55067dcc721b938",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "06dabbfdce88f733bf8b7c056c89d87a5ec555147201d86e7703ff7355c1b756",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "040a5627ef7bb5d949592bbcd7381d5c49b141f464054a8ba8c03c8253480ddb",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42"
      ],
      "example_id": "57094d489928a8140047150a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        4,
        1,
        6,
        0,
        2,
        3,
        5,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.21919768,
      "neural_order": [
        10,
        4,
        6,
        3,
        9,
        13,
        7,
        11,
        1,
        8,
        0,
        2,
        5,
        12,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "5f5fd676e3851b3ef3efe115178db8c405a3d30c8aa10046ec3be3ab485a081b",
      "trial_index": 130
    },
    {
      "answer_sha256": "f1cfa5ebb149e8099d561aae57beed6c68f990f45a910ea9d7b460dbcc5350be",
      "candidate_ids": [
        "571cd703dd7acb1400e4c182",
        "5729f799af94a219006aa707",
        "5727c55bff5b5019007d94ce",
        "5733e5a14776f4190066145b",
        "5706149552bb891400689880",
        "5733834ed058e614000b5c26",
        "57263cfcec44d21400f3dc8d",
        "572ffe6fb2c2fd14005686ef",
        "571077ecb654c5140001f909",
        "573083dc2461fd1900a9ce6d",
        "573383d0d058e614000b5c35",
        "57299021af94a219006aa50b",
        "572f6a0ba23a5019007fc5eb",
        "5710f2e2a58dae1900cd6b72",
        "570953a7efce8f15003a7dfe",
        "57337ddc4776f41900660bbb"
      ],
      "candidate_sha256": [
        "8657b03977310d90fc72a13d570f1c07e80c6100a3b2eeb92d2acdff9adb8b16",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "808648d31e3c0a580e378e5e38b6e262d80eae03ea2f5f3343f713667cc29b1d",
        "395b1cfae86dbeb34c8873339b090ededf002969f69503f5b785bd284dc9d2ca",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a"
      ],
      "example_id": "570953a7efce8f15003a7dfe",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.65440427,
      "neural_order": [
        14,
        4,
        6,
        5,
        10,
        2,
        3,
        11,
        0,
        1,
        7,
        8,
        9,
        12,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "fb02b3211dda0d9deae1ad4096c3510198e522bad82334e03891a0fed25f8dd3",
      "trial_index": 131
    },
    {
      "answer_sha256": "9cf50fc5da34dc81c16bba0fb846cb1e7ba240913bfd72d08cd0a80f4d8b3452",
      "candidate_ids": [
        "572996c73f37b319004784b3",
        "5729f69caf94a219006aa6f9",
        "5726f635dd62a815002e9656",
        "5727da564b864d1900163e8f",
        "572fd264b2c2fd14005684a9",
        "573081c2069b531400832133",
        "572908c13f37b31900477fbd",
        "570d2417fed7b91900d45c3d",
        "570960cf200fba1400367f01",
        "5727cb4b2ca10214002d9676",
        "5733a2a9d058e614000b5f28",
        "573009a004bcaa1900d7704f",
        "570d4030fed7b91900d45d9f",
        "572fdc34a23a5019007fca93",
        "57109180a58dae1900cd6a40",
        "571ce3745efbb31900334e20"
      ],
      "candidate_sha256": [
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "d5f4c4227c07c52cff10ca527f73efedf7e6c5aa9c01a3d9bf8a9468a6ae6939",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "9f7d44852206b170e9506d2e1aace68791eb27a740bd612bbe067f4feddf0fd8",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c"
      ],
      "example_id": "570960cf200fba1400367f01",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        14,
        5,
        4,
        1,
        0,
        2,
        3,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.5826524,
      "neural_order": [
        8,
        4,
        2,
        15,
        3,
        12,
        5,
        14,
        0,
        1,
        6,
        7,
        9,
        10,
        11,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "caa54ef4a9581c1618f04261a64fa88ec58d7f42a85045ba61942f06fb7472f3",
      "trial_index": 132
    },
    {
      "answer_sha256": "b196f3ea0f89b0dc15a9f5e21be95902f17a55da93454b34b217c1debd4bc31f",
      "candidate_ids": [
        "5710f4b8b654c5140001fa45",
        "57266193dd62a815002e832a",
        "57111ab8a58dae1900cd6c3c",
        "57332c1e4776f4190066073a",
        "572a07c11d046914007796d5",
        "57299021af94a219006aa50b",
        "57096c95200fba1400367fbb",
        "572863c72ca10214002da2d4",
        "5730088e947a6a140053cfac",
        "57273cca708984140094db33",
        "5705ffde52bb891400689784",
        "57373f80c3c5551400e51e8f",
        "5728705c2ca10214002da35a",
        "570961aa200fba1400367f15",
        "570953a7efce8f15003a7dfe",
        "57296f293f37b319004783a3"
      ],
      "candidate_sha256": [
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "c71b1eee7e8e8a47719b20f0d4f2e9ca9aaeec5e9d9082387579fe235a47016d",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6"
      ],
      "example_id": "570961aa200fba1400367f15",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        6,
        3,
        0,
        1,
        2,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.05294839,
      "neural_order": [
        13,
        3,
        14,
        0,
        12,
        7,
        6,
        9,
        1,
        2,
        4,
        5,
        8,
        10,
        11,
        15
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "acc99fe69ff6e5b169c469162f42fdb71cef01d4a8d18795ad546dd935f79530",
      "trial_index": 133
    },
    {
      "answer_sha256": "45ead65f6948112309927802e02f91ebe0108f2c5c02d7b97bbf3d69975fbfb4",
      "candidate_ids": [
        "5726219489a1e219009ac2d0",
        "571a49de4faf5e1900b8a94a",
        "572754fff1498d1400e8f660",
        "57283f014b864d19001647e8",
        "570963a5200fba1400367f33",
        "572ffd9e04bcaa1900d76fc7",
        "5725b92e38643c19005acbd3",
        "57288428ff5b5019007da28e",
        "56e1b355e3433e14004230b0",
        "5728284e3acd2414000df5cd",
        "572a213e6aef0514001552ee",
        "5726472bdd62a815002e8042",
        "5727d0f73acd2414000ded13",
        "57274e0d708984140094dbe5",
        "5727cebc2ca10214002d96f0",
        "57308ddc396df919000961a4"
      ],
      "candidate_sha256": [
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "40f970a92f69db6815bd67e7934fc725f910e6a6e08c5ff4e55067dcc721b938",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893"
      ],
      "example_id": "570963a5200fba1400367f33",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        14,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.50905227,
      "neural_order": [
        4,
        11,
        13,
        10,
        15,
        5,
        12,
        1,
        2,
        6,
        0,
        3,
        7,
        8,
        9,
        14
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "f6daaeb449f176df9a6d0bcc749cf2ac8980befcd9eed8f7d5487d27efd300a7",
      "trial_index": 134
    },
    {
      "answer_sha256": "a4c09ae019f5d62ce4a3972e4e2602e74ef4163bdf228ad8051df9f88f93b01b",
      "candidate_ids": [
        "572a020f6aef051400155198",
        "5705fb7f52bb891400689750",
        "572940246aef051400154bec",
        "5726f635dd62a815002e9656",
        "57273a465951b619008f86ff",
        "5733d249d058e614000b6331",
        "572885c44b864d1900164a78",
        "572ffd75b2c2fd14005686e5",
        "572fb42904bcaa1900d76c15",
        "56de40da4396321400ee2708",
        "572758e0f1498d1400e8f6ae",
        "5729fb003f37b31900478627",
        "5727ee372ca10214002d99ec",
        "56de15dbcffd8e1900b4b5c8",
        "57300c67947a6a140053cff0",
        "57096505ed30961900e84082"
      ],
      "candidate_sha256": [
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "ffaee1c83ab04918603b9940620c0c8b52e675143bc31a91f1fccac00d2b1bf7",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "2892ce1ee90d7e758e8bd4b12fd7bebc4276caad2f0d0af29d0444e47b6a4a05"
      ],
      "example_id": "57096505ed30961900e84082",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.56568837,
      "neural_order": [
        15,
        2,
        6,
        8,
        4,
        0,
        7,
        1,
        3,
        5,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "dac05f52961d17e3cf31e2c1e671979bddcfacd4f8e646e546214920e954be48",
      "trial_index": 135
    },
    {
      "answer_sha256": "2836088e259a5ec855cffc93f44b14511a375bf6c50b10aca10dda459535a9e6",
      "candidate_ids": [
        "57378b141c4567190057449f",
        "57336755d058e614000b5a3c",
        "5711623e50c2381900b54ab9",
        "5705e26d75f01819005e76d4",
        "56e1ce08e3433e14004231a4",
        "57265360dd62a815002e819a",
        "57265700dd62a815002e820e",
        "572a142e3f37b319004786b9",
        "56e1c7e2cd28a01900c67b74",
        "57096f37200fba1400367fe5",
        "57268527708984140094c8bf",
        "570966e0200fba1400367f4f",
        "572750df5951b619008f882f",
        "5726e08e5951b619008f810f",
        "571094b7a58dae1900cd6a66",
        "572a019f3f37b31900478643"
      ],
      "candidate_sha256": [
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9",
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b"
      ],
      "example_id": "570966e0200fba1400367f4f",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        9,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.13282902,
      "neural_order": [
        11,
        9,
        13,
        12,
        3,
        8,
        4,
        2,
        0,
        1,
        5,
        6,
        7,
        10,
        14,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "4a5e39f6f56565bbeebc04c5a7f0dd301d0504e5c3c497c20f32513b449b54b7",
      "trial_index": 136
    },
    {
      "answer_sha256": "d54123de468bd42ea00dafbd777f85fe5fa1ff6404d9838c007953c25c92a1c5",
      "candidate_ids": [
        "572758e0f1498d1400e8f6ae",
        "57280f974b864d1900164370",
        "572855973acd2414000df925",
        "572807802ca10214002d9bf8",
        "5726e985dd62a815002e94da",
        "56e1febfe3433e1400423236",
        "5726bc1add62a815002e8ea6",
        "5725be0f271a42140099d117",
        "57268c01dd62a815002e8912",
        "57297ed93f37b3190047845f",
        "57297781af94a219006aa4a3",
        "57267ebfdd62a815002e872c",
        "5706155352bb891400689894",
        "571090abb654c5140001f995",
        "570967c4ed30961900e840ba",
        "57378b141c4567190057449f"
      ],
      "candidate_sha256": [
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef"
      ],
      "example_id": "570967c4ed30961900e840ba",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.58410465,
      "neural_order": [
        14,
        2,
        15,
        7,
        0,
        12,
        5,
        10,
        1,
        3,
        4,
        6,
        8,
        9,
        11,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "16c81678c6d60b5918d9ecc2cfb122fe58003cc7da1f2f7f396ad2374d2dcdcd",
      "trial_index": 137
    },
    {
      "answer_sha256": "e2a04eeda0421c96437c21b55270c9405305c56a26646044421456e4d1736b1f",
      "candidate_ids": [
        "5726ba2c708984140094cf59",
        "572824f13acd2414000df58f",
        "57339eb9d058e614000b5ef6",
        "57265e455951b619008f70bb",
        "57300580b2c2fd140056874d",
        "57115f0a50c2381900b54aa7",
        "57339c16d058e614000b5ec5",
        "572a064a3f37b3190047865d",
        "5727dd2e4b864d1900163eba",
        "571cd5b1dd7acb1400e4c178",
        "57286bb84b864d19001649c8",
        "5727c0402ca10214002d9564",
        "5728661e2ca10214002da2e8",
        "5709686c200fba1400367f77",
        "572fc78d04bcaa1900d76d07",
        "5727ee372ca10214002d99ec"
      ],
      "candidate_sha256": [
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "de308e41aa84b64f8c0380e4021ce99dfb8b520d42b8a543f77ef7f6f3c88a00",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "8e983a2167b912359aa8e1110455d27a279e9b3baaac76337616f57b19396e19",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076"
      ],
      "example_id": "5709686c200fba1400367f77",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.65159629,
      "neural_order": [
        13,
        6,
        0,
        11,
        15,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        14
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "27cd39b7cc3c3dddb8e74f928d85697aca62f718ca2b8acbe16ac68573ba0823",
      "trial_index": 138
    },
    {
      "answer_sha256": "730435c99dc505914c89f4dfade2116d4ed77146040c983ab94b31600d8964b4",
      "candidate_ids": [
        "572658daf1498d1400e8dcac",
        "57297a276aef051400154f88",
        "5711541350c2381900b54a6f",
        "572864542ca10214002da2de",
        "5726c5a9f1498d1400e8eac4",
        "5727da564b864d1900163e8f",
        "57264b1ddd62a815002e80a0",
        "572825a92ca10214002d9efe",
        "572fd8efb2c2fd14005684fa",
        "5730069004bcaa1900d7702d",
        "57096b66200fba1400367fa7",
        "56de3e414396321400ee26d8",
        "5725ca4389a1e219009abeb4",
        "57285ed5ff5b5019007da1b6",
        "572a213e6aef0514001552ee",
        "57287f6a3acd2414000dfa51"
      ],
      "candidate_sha256": [
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3"
      ],
      "example_id": "57096b66200fba1400367fa7",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.71849053,
      "neural_order": [
        10,
        3,
        7,
        1,
        14,
        4,
        13,
        0,
        2,
        5,
        6,
        8,
        9,
        11,
        12,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "d966ee131ca934c75c791282aed3e5e8b5eed4b453f344814270f5e2ec22c4fb",
      "trial_index": 139
    },
    {
      "answer_sha256": "3b08028ef27b017166442ebd73a405ef4e287af357bc1fb7b59acd3f8b1c429e",
      "candidate_ids": [
        "572a058aaf94a219006aa751",
        "56de15dbcffd8e1900b4b5c8",
        "56de0f6a4396321400ee257f",
        "570610b275f01819005e792a",
        "573010fab2c2fd14005687d7",
        "572ffc0f947a6a140053cef0",
        "572f60f4947a6a140053c8ea",
        "570d32e4fed7b91900d45d01",
        "57287f6a3acd2414000dfa51",
        "572ffd75b2c2fd14005686e5",
        "5733dab4d058e614000b6409",
        "5725cb33271a42140099d1db",
        "57096c95200fba1400367fbb",
        "5725b5a689a1e219009abd28",
        "572a02483f37b3190047864b",
        "5737a84dc3c5551400e51f59"
      ],
      "candidate_sha256": [
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "da7e3caa2252604ebed016e003b315efd2a4a47f3b063950ffca6d27e6dd03f2",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "5bae1cf69d61dbf0b59d2020a2b8aaf2c59dfea5d8efc8e4b560e663cbaefcbd"
      ],
      "example_id": "57096c95200fba1400367fbb",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        7,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.46548986,
      "neural_order": [
        12,
        3,
        7,
        0,
        9,
        1,
        2,
        4,
        5,
        6,
        8,
        10,
        11,
        13,
        14,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "338b17309394abb254545f7e362868214c6a7a179a473888a449cb1918d3d3c2",
      "trial_index": 140
    },
    {
      "answer_sha256": "8e9b669109df89620b94f2387dc53206a82ddc71d658f8f7a2b3a9b417370d3e",
      "candidate_ids": [
        "5737804dc3c5551400e51f0f",
        "5711119cb654c5140001fae3",
        "5730bb522461fd1900a9d011",
        "57287338ff5b5019007da232",
        "5726f48df1498d1400e8f0da",
        "5711669550c2381900b54adf",
        "571a52cb4faf5e1900b8a969",
        "5737432bc3c5551400e51e99",
        "57115c7450c2381900b54a9d",
        "57335fcad058e614000b5970",
        "57113be3a58dae1900cd6d28",
        "5726241189a1e219009ac2de",
        "57096e1ced30961900e84102",
        "5727d1c93acd2414000ded3f",
        "5727f05b4b864d1900164068",
        "56e1ded7cd28a01900c67bd4"
      ],
      "candidate_sha256": [
        "f940260eaf284a909912a1db1a7d78a8a63d57532e47769c4005b748d582708e",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "bb7672c097755bb254dcda07123304b126ddc10e9fc8c157410b39caff2b9954",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "61af1803b0730548c2562a214669d95520c7f20b1ca36c0cd3e8cbc90c590f5b",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5"
      ],
      "example_id": "57096e1ced30961900e84102",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        11,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.26439007,
      "neural_order": [
        12,
        1,
        9,
        5,
        3,
        2,
        15,
        11,
        0,
        4,
        6,
        7,
        8,
        10,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "1cc1d6802b1d113b665aca9a440122e35c1c48256cc4e103609df27df52d7384",
      "trial_index": 141
    },
    {
      "answer_sha256": "f8812df5b31cb4355f91e551377020e7457a6c3cc440306b22147336697679de",
      "candidate_ids": [
        "56de10b44396321400ee2593",
        "572824f13acd2414000df58f",
        "573796edc3c5551400e51f33",
        "5725b92e38643c19005acbd3",
        "572a0bafaf94a219006aa765",
        "57264b3edd62a815002e80aa",
        "57293f8a6aef051400154bde",
        "5705e99452bb891400689688",
        "57264b1ddd62a815002e80a0",
        "5727500f708984140094dbfd",
        "57096f37200fba1400367fe5",
        "5733e8ae4776f419006614a6",
        "5729fe5c3f37b3190047862f",
        "572a0d21af94a219006aa783",
        "57060f3e75f01819005e7922",
        "5728fb6a1d04691400778ef5"
      ],
      "candidate_sha256": [
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260"
      ],
      "example_id": "57096f37200fba1400367fe5",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.45066721,
      "neural_order": [
        10,
        9,
        6,
        15,
        13,
        7,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        8,
        11,
        12
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "c25a5cd65fb49c40b9d6a7720ce34c54941a53ddd269388e42dbf7c18c0a7cd8",
      "trial_index": 142
    },
    {
      "answer_sha256": "883c55079a39e34219da344753ec0e9bee9fc6f3542a7fcec96269c89438add3",
      "candidate_ids": [
        "5727cc15ff5b5019007d9576",
        "5730bc308ab72b1400f9c73c",
        "57302ad804bcaa1900d772af",
        "57284b904b864d19001648e2",
        "5733ef47d058e614000b662b",
        "5733dab4d058e614000b6409",
        "57269bb8708984140094cb95",
        "57094d489928a8140047150a",
        "56de15dbcffd8e1900b4b5c8",
        "5737a84dc3c5551400e51f59",
        "56e1e9dfe3433e14004231fc",
        "5726bc1add62a815002e8ea6",
        "572a12381d0469140077972b",
        "572945b11d0469140077922f",
        "57097051ed30961900e84132",
        "571142b3a58dae1900cd6d5a"
      ],
      "candidate_sha256": [
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "d28ae44a1bd659369983c5d7b048773512040e9150f20bcb6fbae1f618e3266a",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "5bae1cf69d61dbf0b59d2020a2b8aaf2c59dfea5d8efc8e4b560e663cbaefcbd",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a"
      ],
      "example_id": "57097051ed30961900e84132",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        7,
        15,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        12,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.2051597,
      "neural_order": [
        14,
        7,
        10,
        6,
        11,
        13,
        4,
        3,
        12,
        0,
        1,
        2,
        5,
        8,
        9,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "dc579e274cb65177cb0e792cc74660a345464d1c8aa6babcead408ab51e183d5",
      "trial_index": 143
    },
    {
      "answer_sha256": "f1cfa5ebb149e8099d561aae57beed6c68f990f45a910ea9d7b460dbcc5350be",
      "candidate_ids": [
        "5730285a04bcaa1900d7726d",
        "5725c57a89a1e219009abe5e",
        "572a0b0b6aef0514001551f6",
        "5730ac782461fd1900a9cf73",
        "57115f0a50c2381900b54aa7",
        "57286192ff5b5019007da1dc",
        "572fffb404bcaa1900d76fef",
        "572754dd708984140094dc3c",
        "57107932a58dae1900cd69ca",
        "57293d6d1d046914007791b7",
        "56dde1d966d3e219004dad8d",
        "5725c69738643c19005accb9",
        "5730bc308ab72b1400f9c73c",
        "5726a299dd62a815002e8b9e",
        "57097141200fba1400367ff9",
        "56e1f10ee3433e1400423223"
      ],
      "candidate_sha256": [
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "d28ae44a1bd659369983c5d7b048773512040e9150f20bcb6fbae1f618e3266a",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2"
      ],
      "example_id": "57097141200fba1400367ff9",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.61557146,
      "neural_order": [
        14,
        7,
        13,
        8,
        0,
        15,
        1,
        2,
        3,
        4,
        5,
        6,
        9,
        10,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "4430f8baefde318b2a7b8413726e56b98702eaad52d68d302a3655b16319f1e6",
      "trial_index": 144
    },
    {
      "answer_sha256": "8a7fc0e8e2f45e33173d1499a1d2fa0834732068b6019560db00acf0501110ff",
      "candidate_ids": [
        "57265aaf5951b619008f706b",
        "57274971708984140094dbbb",
        "5726a46cdd62a815002e8bd0",
        "57300c67947a6a140053cff0",
        "57297781af94a219006aa4a3",
        "57340b1bd058e614000b6869",
        "57294baaaf94a219006aa26b",
        "572673f5708984140094c69b",
        "57097b50200fba140036804d",
        "5727ea45ff5b5019007d986e",
        "572a020f6aef051400155198",
        "5730040f947a6a140053cf4c",
        "5727526cdd62a815002e9b0e",
        "573368e54776f41900660a53",
        "572751b4708984140094dc1b",
        "572881022ca10214002da416"
      ],
      "candidate_sha256": [
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6"
      ],
      "example_id": "57097b50200fba140036804d",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        3,
        12,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.70552653,
      "neural_order": [
        8,
        10,
        3,
        12,
        13,
        6,
        5,
        11,
        0,
        1,
        2,
        4,
        7,
        9,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "bbd72c65008f862254beac8350077bc41bbd7e0e470578a15d3d9d0ca37f9be1",
      "trial_index": 145
    },
    {
      "answer_sha256": "1cd6c45dc63ae91621e0767f34e1989d6abc47c8169ee572e179da42c8b3ac5e",
      "candidate_ids": [
        "573784fa1c45671900574483",
        "571a4b0f10f8ca1400304fd5",
        "56dde0379a695914005b9636",
        "57293f8a6aef051400154bde",
        "571a4d1a4faf5e1900b8a958",
        "5725b5a689a1e219009abd28",
        "5705e3f252bb89140068966a",
        "57097c8fed30961900e841f2",
        "57265d86f1498d1400e8dd50",
        "572fadcbb2c2fd1400568329",
        "572681c1dd62a815002e8796",
        "5706111a52bb89140068984c",
        "572f64ccb2c2fd14005680b7",
        "5730876a396df9190009617a",
        "5725e748ec44d21400f3d733",
        "5725b76389a1e219009abd4a"
      ],
      "candidate_sha256": [
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "72457fa0c9150fb2a7fed021650358b9295b523f55397d61363a047683297908",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585"
      ],
      "example_id": "57097c8fed30961900e841f2",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.31310379,
      "neural_order": [
        7,
        5,
        15,
        12,
        4,
        3,
        9,
        13,
        1,
        11,
        0,
        2,
        6,
        8,
        10,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "0da32f234a56dec8bf055af3e7b17d1a3f5aa7b7b87e8d421ff1e60db4b831da",
      "trial_index": 146
    },
    {
      "answer_sha256": "ac937d0243e9c180e811cee5015c94a19f0cfae0286ea201e89f7c7622cebaff",
      "candidate_ids": [
        "57274046dd62a815002e9a12",
        "5705eee952bb8914006896de",
        "57284e9fff5b5019007da150",
        "572fc5a1947a6a140053cc8a",
        "571cde695efbb31900334e16",
        "57097d63ed30961900e841fc",
        "571cc3dedd7acb1400e4c146",
        "5726516a708984140094c223",
        "572faf74b2c2fd1400568347",
        "5726a299dd62a815002e8b9e",
        "572758e0f1498d1400e8f6ae",
        "572ffbaa947a6a140053cee6",
        "5710f114a58dae1900cd6b60",
        "572815d9ff5b5019007d9cbc",
        "5733f1784776f41900661575",
        "57376a1bc3c5551400e51ec3"
      ],
      "candidate_sha256": [
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a"
      ],
      "example_id": "57097d63ed30961900e841fc",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.44847327,
      "neural_order": [
        5,
        7,
        10,
        3,
        1,
        13,
        14,
        0,
        2,
        4,
        6,
        8,
        9,
        11,
        12,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "0d3c0108ad04d7504b1b38fc84964cdc0ee089334df409fa5e11acf06205779f",
      "trial_index": 147
    },
    {
      "answer_sha256": "3faa58dacca8a596e1e09c855dbc912b4aa7c4a82442392a0e15084239fa418d",
      "candidate_ids": [
        "56de3dbacffd8e1900b4b6d2",
        "572987e46aef051400154fa2",
        "5705f13d52bb8914006896f0",
        "570d2417fed7b91900d45c3d",
        "5728683b3acd2414000df9af",
        "572ffbaa947a6a140053cee6",
        "5737898f1c45671900574495",
        "572872822ca10214002da374",
        "5706111a52bb89140068984c",
        "5727da564b864d1900163e8f",
        "5725c57a89a1e219009abe5e",
        "572a03086aef0514001551a2",
        "572fc49d04bcaa1900d76ccb",
        "572667e2f1498d1400e8de90",
        "572867d72ca10214002da2fc",
        "57308ddc396df919000961a4"
      ],
      "candidate_sha256": [
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607",
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893"
      ],
      "example_id": "570d2417fed7b91900d45c3d",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.24541127,
      "neural_order": [
        3,
        15,
        8,
        2,
        9,
        6,
        4,
        5,
        7,
        12,
        0,
        1,
        10,
        11,
        13,
        14
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "1d03fb088e3e459148afc2771d7dcc0c17398601d521e2a89c19ef6b73a346ca",
      "trial_index": 148
    },
    {
      "answer_sha256": "dadc466c22a1bdc4adba2c016c9c821a26d88503052ae523132027fb7c559493",
      "candidate_ids": [
        "5725c57a89a1e219009abe5e",
        "5727cb4b2ca10214002d9676",
        "570d2556fed7b91900d45c47",
        "572a12386aef051400155234",
        "572ffee1947a6a140053cf14",
        "5726446a89cfff1900a8404d",
        "5729727baf94a219006aa437",
        "572885023acd2414000dfa83",
        "5730b1022461fd1900a9cfa3",
        "5726db5add62a815002e92d4",
        "572a020f6aef051400155198",
        "56e1cbe2cd28a01900c67bac",
        "5733f8dc4776f419006615f7",
        "56de4bb84396321400ee27a2",
        "56e1efa0e3433e140042321a",
        "5730bd00069b5314008322d7"
      ],
      "candidate_sha256": [
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "804a1a2d7b5d5f57198945102d763975f6edcf55d1fa34d39bc21af2a7e82d7e",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "c2c1868add9d6aa35780c0c8d4492524306055ae928ea56b315e6e9e6969d494",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485"
      ],
      "example_id": "570d2556fed7b91900d45c47",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": false,
      "neural_margin": 0.07602265,
      "neural_order": [
        7,
        2,
        8,
        5,
        15,
        10,
        4,
        13,
        12,
        1,
        6,
        11,
        0,
        3,
        9,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "b2e5f0e164ffe90e3ac6f4bf27594cbffc757c1bc0ee893f022ff83322f56ff1",
      "trial_index": 149
    },
    {
      "answer_sha256": "d003acc4fa11ecbd52d5df3744bea7019afe4b7aa082f25ed79a8840fd8eb9c3",
      "candidate_ids": [
        "5729f39a6aef05140015514c",
        "572a12381d0469140077972b",
        "573408ef4776f41900661757",
        "572726c9708984140094da7b",
        "572a135daf94a219006aa79f",
        "5737a7351c456719005744f1",
        "5727c69d3acd2414000dec13",
        "570d26efb3d812140066d48f",
        "57286fa83acd2414000df9e5",
        "573361404776f4190066093c",
        "571ce5055efbb31900334e2a",
        "56de3d594396321400ee26ca",
        "572fb42904bcaa1900d76c15",
        "56e1c4fce3433e140042314e",
        "571a50df4faf5e1900b8a960",
        "57268882f1498d1400e8e306"
      ],
      "candidate_sha256": [
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "52dde7db662fc424e04bd1ed3cd943e7d11c0868e7f691a8c6cac05cb561b057",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "63083aa84294da2de8401c0b772521bbfe426e2587fa80448b3830f78d5bb8ae",
        "2ef6bc8b598c3ae95dea517f30b72d286a3c1ea1c56e1f73f6bd8f2ab9cb63ef",
        "c4fd59d733b1633198ab8d307fb4d275d55c73f2c3fa5e12deb57198789613e4",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe"
      ],
      "example_id": "570d26efb3d812140066d48f",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        5,
        4,
        0,
        1,
        2,
        3,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.32784141,
      "neural_order": [
        7,
        12,
        13,
        9,
        3,
        5,
        15,
        4,
        11,
        0,
        6,
        1,
        2,
        8,
        10,
        14
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "f36e36275c37dcb80572fbd814eb33750ee869fd19398a979620318d06392e89",
      "trial_index": 150
    },
    {
      "answer_sha256": "43545285ba3708a37b17c2adb9100eb997c3d64ec3963010cb6043075ccf1c7d",
      "candidate_ids": [
        "572a0a686aef0514001551ec",
        "570d28bdb3d812140066d4a3",
        "572864dd4b864d1900164976",
        "5725cf3238643c19005acd61",
        "5711669550c2381900b54adf",
        "571cd3b55efbb31900334e03",
        "571a50df4faf5e1900b8a960",
        "5726e985dd62a815002e94da",
        "57332e48d058e614000b5762",
        "572a0bebaf94a219006aa76f",
        "57108198b654c5140001f937",
        "57293d6d1d046914007791b7",
        "572fcb6da23a5019007fc9f1",
        "57269e3bf1498d1400e8e516",
        "5726a09f708984140094cc39",
        "57308ddc396df919000961a4"
      ],
      "candidate_sha256": [
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893"
      ],
      "example_id": "570d28bdb3d812140066d4a3",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        4,
        11,
        0,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.41896652,
      "neural_order": [
        1,
        10,
        12,
        15,
        0,
        6,
        5,
        14,
        3,
        2,
        4,
        7,
        8,
        9,
        11,
        13
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "bfd036ab4d157a29c981fd892305d12749403cf2ef31dd099b3d4e9105dfabc6",
      "trial_index": 151
    },
    {
      "answer_sha256": "cf57059bc4d8423eeeebc2b5287e20be84a2dcaeb5cb2f12e1e271b10068d856",
      "candidate_ids": [
        "5737898f1c45671900574495",
        "5726431aec44d21400f3dd13",
        "5725cfd0271a42140099d225",
        "5727213c708984140094da35",
        "572fc80b04bcaa1900d76d11",
        "5727526cdd62a815002e9b0e",
        "5725b92e38643c19005acbd3",
        "5726a00cf1498d1400e8e550",
        "572691bedd62a815002e89dc",
        "57274b35f1498d1400e8f5d4",
        "572a135daf94a219006aa79f",
        "5706074552bb8914006897d4",
        "570d29c2b3d812140066d4bd",
        "57267d52708984140094c7d9",
        "5730069004bcaa1900d7702d",
        "5728ec6a4b864d19001650aa"
      ],
      "candidate_sha256": [
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "e83edd2f6eb1fa0981acc0119542b4a54c226b0e0249686e43082425f6d974b0"
      ],
      "example_id": "570d29c2b3d812140066d4bd",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.61789415,
      "neural_order": [
        12,
        6,
        4,
        2,
        9,
        11,
        0,
        14,
        3,
        1,
        5,
        7,
        8,
        10,
        13,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "b78ca8bae4537e4afa2bb7bdc036c5f1bf71daf72728820bf6c02e8f3a0c8924",
      "trial_index": 152
    },
    {
      "answer_sha256": "ec47424feecf45e5cc2afdaa1cd5b77aa06ba85ef504c61e26f5b160ebc48380",
      "candidate_ids": [
        "5726bcde708984140094cfbf",
        "5711541350c2381900b54a6f",
        "570d2af0fed7b91900d45c93",
        "572a1f086aef0514001552c0",
        "571135b8a58dae1900cd6d0e",
        "56dde27d9a695914005b9651",
        "5730b8ca8ab72b1400f9c704",
        "573330444776f41900660758",
        "570d2f5bfed7b91900d45ccf",
        "5727cc15ff5b5019007d9576",
        "572989846aef051400154fc0",
        "57274e975951b619008f87f9",
        "5726701add62a815002e84c8",
        "57274a1edd62a815002e9a9b",
        "57263c78ec44d21400f3dc7b",
        "56e1bc3ae3433e1400423104"
      ],
      "candidate_sha256": [
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "dba710cd82bb65136087ed545607820a13e196f28ad71ddb50f8213f409b789b",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8"
      ],
      "example_id": "570d2af0fed7b91900d45c93",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        9,
        8,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.34585648,
      "neural_order": [
        2,
        8,
        0,
        9,
        6,
        14,
        1,
        3,
        4,
        5,
        7,
        10,
        11,
        12,
        13,
        15
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "cf1840cf335a2462816c75e3917723c5d3caffdb047f828d9e8261fd0e1b287b",
      "trial_index": 153
    },
    {
      "answer_sha256": "c92e05833f5a2488a26d30bfe82ae6303be0c5ec645e5ab1ed4f1b3699168f07",
      "candidate_ids": [
        "57060a6e52bb8914006897f8",
        "5727d6154b864d1900163e34",
        "57287b4a4b864d1900164a28",
        "56de49a8cffd8e1900b4b7a7",
        "57094a79efce8f15003a7dc4",
        "5725bbec271a42140099d0d1",
        "5725b7f389a1e219009abd5d",
        "5733faaf4776f4190066162f",
        "570d2c20fed7b91900d45ca7",
        "5727ea45ff5b5019007d986e",
        "5726a638dd62a815002e8bf6",
        "572864dd4b864d1900164976",
        "572fd7b8947a6a140053cd3a",
        "573380e0d058e614000b5be9",
        "573399b54776f41900660e64",
        "572849b4ff5b5019007da0f2"
      ],
      "candidate_sha256": [
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "184d8e2bfbbe465d571014652c9207a041ec5654986198ec457160d7a42c77dc",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de"
      ],
      "example_id": "570d2c20fed7b91900d45ca7",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.40306114,
      "neural_order": [
        8,
        7,
        12,
        2,
        3,
        4,
        0,
        1,
        5,
        6,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "46e369fc1f35ccec793e89582d16de33a7bfa3e3c9a78ed0fe150f8c11202ec8",
      "trial_index": 154
    },
    {
      "answer_sha256": "fa4ba7afb5175b94be767bd2c3a3544b1c0ad6dcaeaadd896ee0909d0ba3f3e2",
      "candidate_ids": [
        "5726a14c708984140094cc51",
        "57274712708984140094dbad",
        "57266c015951b619008f7237",
        "57108d69b654c5140001f983",
        "572fb059947a6a140053cb80",
        "57302cd004bcaa1900d772d7",
        "570d2d68fed7b91900d45cbb",
        "57115ac550c2381900b54a77",
        "572744aff1498d1400e8f588",
        "5728fc9e1d04691400778f11",
        "572f60f4947a6a140053c8ea",
        "5705fb7f52bb891400689750",
        "5733f309d058e614000b6648",
        "5710ed7bb654c5140001fa2b",
        "57377ec7c3c5551400e51f05",
        "5737534ec3c5551400e51eab"
      ],
      "candidate_sha256": [
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "5b469d64ced1175805ddbb53037730159f46f770f6697c84c8b9e8738d9bbb19",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "d2c301f3e303e0529f1b5862d9c77adabe2cb026b4683982583601544f992a66"
      ],
      "example_id": "570d2d68fed7b91900d45cbb",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.57246812,
      "neural_order": [
        6,
        11,
        1,
        7,
        3,
        4,
        14,
        2,
        8,
        0,
        5,
        9,
        10,
        12,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "b6652029f789fcee86dc2a9f87304ecff104786b82e5b2ef59dc517617031bc2",
      "trial_index": 155
    },
    {
      "answer_sha256": "20ee235b5de5b36244da6f9aa1cbdd032a90867ba92276ccc8c38c0d0d57fcec",
      "candidate_ids": [
        "572a06866aef0514001551be",
        "570d2f5bfed7b91900d45ccf",
        "571097baa58dae1900cd6a98",
        "57332e48d058e614000b5762",
        "56de4b5c4396321400ee2799",
        "57287b4a4b864d1900164a28",
        "572f57c704bcaa1900d7686d",
        "572883153acd2414000dfa6f",
        "57287c142ca10214002da3d0",
        "5733266d4776f41900660712",
        "5706111a52bb89140068984c",
        "5725c63438643c19005acc9f",
        "571095a8a58dae1900cd6a76",
        "57335fcad058e614000b5970",
        "572a064a3f37b3190047865d",
        "57107e6ca58dae1900cd69f2"
      ],
      "candidate_sha256": [
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565"
      ],
      "example_id": "570d2f5bfed7b91900d45ccf",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.547535,
      "neural_order": [
        1,
        12,
        9,
        8,
        7,
        4,
        2,
        14,
        13,
        0,
        15,
        6,
        5,
        10,
        3,
        11
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "15dcb47b5e334f7ffc02d798891d78c7c77bac2dc67c3906bd30384ba3845d4c",
      "trial_index": 156
    },
    {
      "answer_sha256": "d14420a53218d85b3517903d1770a9dcd5c5f0b2c7b7bada6b172c5c1ff515a3",
      "candidate_ids": [
        "572970c11d04691400779463",
        "570d391fb3d812140066d575",
        "5725b33f6a3fe71400b8952d",
        "572a12386aef051400155234",
        "57287d4a2ca10214002da3e4",
        "5737432bc3c5551400e51e99",
        "57060df252bb891400689820",
        "571cc6f85efbb31900334de4",
        "571ccc00dd7acb1400e4c15a",
        "57265f605951b619008f70db",
        "572684365951b619008f753f",
        "570d30fdfed7b91900d45ce3",
        "572f7b33947a6a140053c9a2",
        "5730b541396df919000962c2",
        "572671a5f1498d1400e8dfe4",
        "573007fab2c2fd140056876b"
      ],
      "candidate_sha256": [
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "c456b31b38f7b3d656d571ec7289b0630f5f2d8ce31a3965fddb4291053d73ef",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "d206c39c98d46bd01f841c82284343c93a112882c53094883d72423bafd6ee01"
      ],
      "example_id": "570d30fdfed7b91900d45ce3",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.27392989,
      "neural_order": [
        11,
        1,
        12,
        15,
        7,
        14,
        10,
        6,
        9,
        8,
        3,
        0,
        2,
        4,
        5,
        13
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "1b795668c4fd971386890c0fe1952d7035d9f99a4d66e57f47600cc4cda23b80",
      "trial_index": 157
    },
    {
      "answer_sha256": "54653ea833a78b4e706047690f63bea9d08968253ac9c024065982fe69f6a5dc",
      "candidate_ids": [
        "57266e72f1498d1400e8df8c",
        "56e1fc57e3433e140042322c",
        "56de15104396321400ee25b7",
        "57284e9fff5b5019007da150",
        "570d32e4fed7b91900d45d01",
        "57286f373acd2414000df9db",
        "571ce5055efbb31900334e2a",
        "572ffa79a23a5019007fcbe9",
        "57336755d058e614000b5a3c",
        "570953a7efce8f15003a7dfe",
        "57108073b654c5140001f925",
        "572649d8f1498d1400e8db37",
        "570d2417fed7b91900d45c3d",
        "56de15dbcffd8e1900b4b5c8",
        "570614ff52bb89140068988a",
        "573092088ab72b1400f9c594"
      ],
      "candidate_sha256": [
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b"
      ],
      "example_id": "570d32e4fed7b91900d45d01",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        12,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.34422237,
      "neural_order": [
        4,
        12,
        3,
        14,
        5,
        1,
        0,
        2,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "6774b51f40b5eed3c9a981a21450e33ca685b95481d08cb4a9637ed721f2edbe",
      "trial_index": 158
    },
    {
      "answer_sha256": "17aee05c96bca801fe52f16ae99fe903c24e35fcd47cd2942af70ed793d33ec5",
      "candidate_ids": [
        "5725c743ec44d21400f3d549",
        "56e20a3ae3433e140042324a",
        "570d3468b3d812140066d543",
        "572a0e4b6aef051400155214",
        "5730ac782461fd1900a9cf73",
        "56dde0ba66d3e219004dad75",
        "57377ec7c3c5551400e51f05",
        "573083dc2461fd1900a9ce6d",
        "571095a8a58dae1900cd6a76",
        "56e1c720e3433e140042316a",
        "57293ca73f37b3190047815b",
        "5726545f708984140094c2a5",
        "5705e33f52bb89140068964c",
        "5705e26d75f01819005e76d4",
        "572991943f37b319004784a1",
        "57293f8a6aef051400154bde"
      ],
      "candidate_sha256": [
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7"
      ],
      "example_id": "570d3468b3d812140066d543",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        14,
        3,
        0,
        1,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.53688436,
      "neural_order": [
        2,
        15,
        4,
        8,
        10,
        13,
        3,
        0,
        1,
        5,
        6,
        7,
        9,
        11,
        12,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "44bb6fd85e87fc693b8672dcafc848864d6a80eb7d7e55ac9c4cc6a0a758ccc0",
      "trial_index": 159
    },
    {
      "answer_sha256": "1765706d579fc5d05a80a5f96747c368eddc39fd3b8c2ccf85df2e72c392f94e",
      "candidate_ids": [
        "57115e532419e314009555af",
        "572843304b864d1900164848",
        "57264684708984140094c123",
        "572fbea404bcaa1900d76c5b",
        "571166352419e314009555f1",
        "5726a3c6f1498d1400e8e5ae",
        "5733d4c8d058e614000b6353",
        "572648e8dd62a815002e8076",
        "5733266d4776f41900660712",
        "572a0ce11d046914007796fd",
        "57287f6a3acd2414000dfa51",
        "572818f54b864d190016446c",
        "570d35b7b3d812140066d54d",
        "571c96095efbb31900334dbc",
        "571099b2b654c5140001f9b3",
        "5711119cb654c5140001fae3"
      ],
      "candidate_sha256": [
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "ca0f7b7018127b2736fe74e8ea343545834b32796fc74beb48cde30324c80067",
        "3ee61e6d7605bacdeb4f37ab798daa449f57668fd923e6b04e1a516237821782",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438"
      ],
      "example_id": "570d35b7b3d812140066d54d",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.36300304,
      "neural_order": [
        12,
        15,
        3,
        14,
        13,
        7,
        2,
        6,
        5,
        4,
        0,
        1,
        8,
        9,
        10,
        11
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "6c27e0cb63409cb7b40ad45393ad452ac6de4751c2f4eec0f05598bfbb66ee5b",
      "trial_index": 160
    },
    {
      "answer_sha256": "f7a869561117fce513dad89503cca4dc6a26a99e6b2d3ee991efd24a96cb95d4",
      "candidate_ids": [
        "57274d9bf1498d1400e8f5f6",
        "570d2417fed7b91900d45c3d",
        "572a1046af94a219006aa78d",
        "572677e7708984140094c723",
        "5727482bf1498d1400e8f5a4",
        "56e1efa0e3433e140042321a",
        "570d30fdfed7b91900d45ce3",
        "572754dd708984140094dc3c",
        "5729f3831d0469140077967b",
        "5705e3f252bb89140068966a",
        "57264684708984140094c123",
        "570d391fb3d812140066d575",
        "57300a25a23a5019007fcc97",
        "5733638fd058e614000b59e8",
        "572691bedd62a815002e89dc",
        "572746addd62a815002e9a5e"
      ],
      "candidate_sha256": [
        "d3224d2a7c9b1fb0c025ab44b397ab073a37fb28162d6b3730660c29e767b600",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "05e6b70a4c996f49687d72f7a373609343967f6fed5be1f3bda6965dcfad141e",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d"
      ],
      "example_id": "570d391fb3d812140066d575",
      "gated_correct": false,
      "gated_top": 1,
      "gold_index": 11,
      "lexical_correct": false,
      "lexical_order": [
        1,
        6,
        11,
        0,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.06101341,
      "neural_order": [
        11,
        6,
        1,
        9,
        8,
        3,
        10,
        12,
        7,
        0,
        2,
        4,
        5,
        13,
        14,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "2ddf8a4cdc88f96ed83dab3a3969be22b601ea560c38899731f6cff5d2d35679",
      "trial_index": 161
    },
    {
      "answer_sha256": "4b05ae4c1d7f67b22d112218bd53d20bab31c37bf585b08321e595393ae31b8b",
      "candidate_ids": [
        "573098f38ab72b1400f9c5d2",
        "5733ef47d058e614000b662b",
        "570960cf200fba1400367f01",
        "57264cac708984140094c1b3",
        "5726472bdd62a815002e8042",
        "572a0ce11d046914007796fd",
        "572a142e3f37b319004786b9",
        "570d4030fed7b91900d45d9f",
        "57108c95b654c5140001f979",
        "571156152419e3140095559b",
        "5725b33f6a3fe71400b8952d",
        "572fc5a1947a6a140053cc8a",
        "57267b3c5951b619008f7423",
        "57300c67947a6a140053cff0",
        "572684365951b619008f753f",
        "57280f974b864d1900164370"
      ],
      "candidate_sha256": [
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a"
      ],
      "example_id": "570d4030fed7b91900d45d9f",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.48163865,
      "neural_order": [
        7,
        11,
        4,
        3,
        1,
        9,
        2,
        0,
        6,
        10,
        8,
        5,
        12,
        13,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "27e6d0b04ae606782cc7b6ba238bb588f577c99dbedee86e70242bdf1f50f923",
      "trial_index": 162
    },
    {
      "answer_sha256": "439c6f56990f597fbe5133af65ea3cebc318abc18f3917680a28ffd319558b9c",
      "candidate_ids": [
        "573004bf947a6a140053cf56",
        "57302a3aa23a5019007fcecf",
        "571136b8a58dae1900cd6d22",
        "57266e72f1498d1400e8df8c",
        "570d418dfed7b91900d45dbf",
        "572f609ca23a5019007fc5af",
        "572815d9ff5b5019007d9cbc",
        "571c97e2dd7acb1400e4c11e",
        "57337ddc4776f41900660bbb",
        "5729727baf94a219006aa437",
        "5726e179dd62a815002e93ae",
        "5727213c708984140094da35",
        "56de3ebc4396321400ee26e6",
        "572650325951b619008f6fa9",
        "5726edecdd62a815002e957f",
        "572a1dbb3f37b319004786f5"
      ],
      "candidate_sha256": [
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "631ae85a58426197f43bce08d34df2914057034648942e41a9d822902b61b3d3",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "b0008033f8b97b0cf458d8e505384798d796a2fc78a3fced41f39a2591ef64cd",
        "c59e7593c862a0c68cee17ab74270592bac6538a668da2cdfed71c7db8087fec",
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943"
      ],
      "example_id": "570d418dfed7b91900d45dbf",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        15,
        7,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.35629474,
      "neural_order": [
        4,
        15,
        0,
        2,
        11,
        14,
        13,
        1,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "b7e48eed273e423613032609fc70a307baea634ab092a79d00d94a19d34fb152",
      "trial_index": 163
    },
    {
      "answer_sha256": "23779c9d6c459f5385ede2469d65b0912c70f0e3f67b54615e737cad33976bb3",
      "candidate_ids": [
        "5711488ab654c5140001fc3d",
        "5733fe73d058e614000b673d",
        "572fff45947a6a140053cf26",
        "57115f0a50c2381900b54aa7",
        "57300e2604bcaa1900d770b7",
        "56e1cbe2cd28a01900c67bac",
        "57339a554776f41900660e74",
        "57293e983f37b3190047818b",
        "57107932a58dae1900cd69ca",
        "57267b3c5951b619008f7423",
        "5733d2dbd058e614000b6339",
        "570d4329fed7b91900d45dd7",
        "572fdbb004bcaa1900d76dd9",
        "571c879bdd7acb1400e4c0ec",
        "5729a26d6aef05140015505a",
        "57274eca5951b619008f8800"
      ],
      "candidate_sha256": [
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "403b83bcd02b8204d4620813147424fdb54ec54909eb54e867858a16ee0028ec",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "02473abeaeaf0a7ac1221ede2323c119c62182f573df438534fcb86c2c6943d4",
        "46ceed087a89c8c5146679d04d65800e1d5b68fadcd272ac5f73357c18ad83be"
      ],
      "example_id": "570d4329fed7b91900d45dd7",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        8,
        1,
        15,
        10,
        7,
        0,
        2,
        3,
        4,
        5,
        6,
        9,
        12,
        13,
        14
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.26130405,
      "neural_order": [
        11,
        12,
        15,
        8,
        10,
        1,
        7,
        9,
        2,
        13,
        3,
        4,
        0,
        5,
        6,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "f8aadd47ad368537b562825842d9be148a41d8f35b5ea0c5e8ae573c39ac5941",
      "trial_index": 164
    },
    {
      "answer_sha256": "d2907764880b07555381ac97b7a1839ac9f1e1ac6c01f325dd2303ac8db1f5e9",
      "candidate_ids": [
        "5733cff84776f419006612b4",
        "5727ed2e3acd2414000df001",
        "5725c63438643c19005acc9f",
        "5728683b3acd2414000df9af",
        "572944e03f37b319004781e1",
        "573776eec3c5551400e51ee9",
        "57300bf504bcaa1900d77087",
        "57287d4a2ca10214002da3e4",
        "5727d6faff5b5019007d969a",
        "573786b51c4567190057448d",
        "57097141200fba1400367ff9",
        "57097d63ed30961900e841fc",
        "570d44abb3d812140066d5fd",
        "5726a34bf1498d1400e8e59e",
        "5727f6723acd2414000df0e7",
        "5711651050c2381900b54acb"
      ],
      "candidate_sha256": [
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667",
        "b38835b0da15953f81e7fa39f6745a69d2e09f953b85ebc4995b8b8252bb8940",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "7a54e69984565794b95d2e675d9ceb87e6f076fbf18c6fc5f9f4161c8750a672",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07"
      ],
      "example_id": "570d44abb3d812140066d5fd",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.59045358,
      "neural_order": [
        12,
        3,
        10,
        8,
        14,
        2,
        4,
        9,
        13,
        11,
        0,
        1,
        5,
        6,
        7,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "c923b5ae77aef4bf9f7e51257fb67c57650f93f636cb2a1c087b95d3835d4176",
      "trial_index": 165
    },
    {
      "answer_sha256": "ab1cb712f2dca756105160805501f4d6d8657d93d40b16eee4ecb5fd048d26eb",
      "candidate_ids": [
        "570d4606b3d812140066d619",
        "5733d2dbd058e614000b6339",
        "5726542ff1498d1400e8dc28",
        "5729f3831d0469140077967b",
        "57265bdfdd62a815002e829e",
        "572686fc708984140094c8e5",
        "573399b54776f41900660e64",
        "5726f7715951b619008f838d",
        "57308ddc396df919000961a4",
        "571126dfa58dae1900cd6cb2",
        "570961aa200fba1400367f15",
        "56de11154396321400ee25aa",
        "57264a8cdd62a815002e808c",
        "5729fe5c3f37b3190047862f",
        "5727ea45ff5b5019007d986e",
        "57293bc91d0469140077919b"
      ],
      "candidate_sha256": [
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "184d8e2bfbbe465d571014652c9207a041ec5654986198ec457160d7a42c77dc",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89"
      ],
      "example_id": "570d4606b3d812140066d619",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        8,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.31625135,
      "neural_order": [
        0,
        8,
        14,
        1,
        9,
        3,
        11,
        7,
        2,
        4,
        5,
        6,
        10,
        12,
        13,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "0ce2168a988f46cf2cdf9f1fc06696a5d599cea1df7aa9642b5204d5547d3198",
      "trial_index": 166
    },
    {
      "answer_sha256": "3258ffd26326c80002fc8762ca53ba0abc98e49e75b346272702170d53747ac0",
      "candidate_ids": [
        "5725f46289a1e219009ac0fa",
        "5727526cdd62a815002e9b0e",
        "5729f12e3f37b319004785e1",
        "56e1a0dccd28a01900c67a2e",
        "572faec7b2c2fd1400568333",
        "572970c11d04691400779463",
        "57105da9a58dae1900cd699e",
        "5730bb522461fd1900a9d011",
        "5705f09e75f01819005e77a4",
        "5725d7e438643c19005acdf9",
        "5728facd4b864d1900165170",
        "572f6c85947a6a140053c940",
        "5705edcd52bb8914006896ca",
        "56de51244396321400ee27ef",
        "570d47b8b3d812140066d62d",
        "572f60f4947a6a140053c8ea"
      ],
      "candidate_sha256": [
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "bb3727c243a6cb744686ba6886c54791cd7745eca592eef8baf9c5b06c996b3e",
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08"
      ],
      "example_id": "570d47b8b3d812140066d62d",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.57496866,
      "neural_order": [
        14,
        15,
        1,
        4,
        8,
        11,
        9,
        12,
        7,
        13,
        0,
        2,
        3,
        5,
        6,
        10
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "41cbe21f8936415b7436ba8fb9a5870618a3daeaa4620b5ff1f2f273e2b86139",
      "trial_index": 167
    },
    {
      "answer_sha256": "ad7a0849a3ea6e33c1e2b80f50222f76eda56d2f4c154039a99aa710f7800c7b",
      "candidate_ids": [
        "5733a560d058e614000b5f77",
        "5728809f2ca10214002da40c",
        "5727cebc2ca10214002d96f0",
        "57265e11708984140094c3bb",
        "56e1c0f6cd28a01900c67b2c",
        "57296f293f37b319004783a3",
        "5730b255396df919000962b0",
        "57376df3c3c5551400e51ed7",
        "570d4a6bfed7b91900d45e13",
        "57264865dd62a815002e8062",
        "57302bd0b2c2fd14005689db",
        "57266193dd62a815002e832a",
        "57287ccb2ca10214002da3da",
        "57265ceddd62a815002e82b8",
        "5728eef92ca10214002daab0",
        "571cca4add7acb1400e4c150"
      ],
      "candidate_sha256": [
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f"
      ],
      "example_id": "570d4a6bfed7b91900d45e13",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        10,
        13,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        9,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.52123432,
      "neural_order": [
        8,
        15,
        6,
        0,
        4,
        10,
        14,
        9,
        1,
        2,
        3,
        5,
        7,
        11,
        12,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "80f2ace290c4660e1823cabbbd6431502f85ed5c27374515635fd1f8e5e8dc70",
      "trial_index": 168
    },
    {
      "answer_sha256": "f7ec2d2600c8998dd7ebb42ea2367dc30615d60a801383b4bcb319a5b376e6aa",
      "candidate_ids": [
        "56de52614396321400ee27fb",
        "5725b7f389a1e219009abd5d",
        "57267d52708984140094c7d9",
        "56de51c64396321400ee27f7",
        "5730a951069b531400832213",
        "572872dd2ca10214002da37e",
        "57339c16d058e614000b5ec5",
        "572f6ec7a23a5019007fc621",
        "570d4c3bfed7b91900d45e31",
        "571ccd9b5efbb31900334df8",
        "5733db8dd058e614000b6428",
        "572a07a86aef0514001551d2",
        "5729f39a6aef05140015514c",
        "5725d42a89a1e219009abf58",
        "5728804b4b864d1900164a46",
        "57115f652419e314009555b9"
      ],
      "candidate_sha256": [
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "9e5f19ae8c4bcea47d2097bb803a6f512f53874779686d5848e1aec6bdadef81",
        "7544f3f342f82852b6fd8a74d46285e0772381cc4cd0b61c47fea00590e8938e",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "f9d0ba77dd8dff4446935bfec391a5986633d68d8bdf2319ad27e1fd5890d87c",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01"
      ],
      "example_id": "570d4c3bfed7b91900d45e31",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.40106592,
      "neural_order": [
        8,
        13,
        10,
        2,
        4,
        6,
        15,
        5,
        14,
        7,
        11,
        1,
        0,
        3,
        9,
        12
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "3dead0535cfb59dc1bc73e8d07e383804158ac70f9983112e1a837240943d6fb",
      "trial_index": 169
    },
    {
      "answer_sha256": "8f1beb4e7d9255300d69e3bf230a4bfd98aa7815944f89affd1fac4f83088d8a",
      "candidate_ids": [
        "5733cff84776f419006612b4",
        "5728eb1a3acd2414000e01c5",
        "57264cac708984140094c1b3",
        "5725b76389a1e219009abd4a",
        "5730035e04bcaa1900d77001",
        "571cc3dedd7acb1400e4c146",
        "56de15dbcffd8e1900b4b5c8",
        "570d4e6cb3d812140066d66d",
        "5733e5a14776f4190066145b",
        "5728facd4b864d1900165170",
        "5729727baf94a219006aa437",
        "56e16182e3433e1400422e28",
        "5727c8863acd2414000dec49",
        "572fd1c4947a6a140053cd02",
        "573005b9947a6a140053cf6a",
        "5730a951069b531400832213"
      ],
      "candidate_sha256": [
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35"
      ],
      "example_id": "570d4e6cb3d812140066d66d",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        9,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.26548665,
      "neural_order": [
        7,
        13,
        4,
        1,
        8,
        9,
        15,
        10,
        6,
        14,
        0,
        2,
        3,
        5,
        11,
        12
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "1ade619ad17be9cba731067e28e7d8f9b98d51ab71036f632fb7a1e6b37199b5",
      "trial_index": 170
    },
    {
      "answer_sha256": "1fefe13afb37635edf3ceb61a59607faebd890b5d966f0b6a1e013e6567a9a26",
      "candidate_ids": [
        "5710f2e2a58dae1900cd6b72",
        "572a11663f37b31900478693",
        "5729fa40af94a219006aa70f",
        "570960cf200fba1400367f01",
        "5725fabc89a1e219009ac128",
        "56e190bce3433e1400422fc8",
        "572815d9ff5b5019007d9cbc",
        "572ff760b2c2fd1400568677",
        "57269016708984140094ca41",
        "570d50a5fed7b91900d45e7d",
        "5727f7523acd2414000df10d",
        "56de179dcffd8e1900b4b5da",
        "5725c41eec44d21400f3d50b",
        "57284d484b864d1900164900",
        "5706149552bb891400689880",
        "57275409708984140094dc35"
      ],
      "candidate_sha256": [
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "9819559c80db18970983cbfdf2bcb3f75443deb410b562890bbe13cea74a03dc",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720"
      ],
      "example_id": "570d50a5fed7b91900d45e7d",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        2,
        15,
        13,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        14
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.31160451,
      "neural_order": [
        9,
        1,
        10,
        12,
        4,
        6,
        0,
        15,
        3,
        2,
        5,
        7,
        8,
        11,
        13,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "1bcaebd68aa6dc1ff287fd66abb1958343a5c45ddf1a27e9f5930e1d2b084b46",
      "trial_index": 171
    },
    {
      "answer_sha256": "8bd0f4bb7d222c5a808c14ccee156dc0e0ec82ceb3811d628572de3783718223",
      "candidate_ids": [
        "56de148dcffd8e1900b4b5bc",
        "572fd47fa23a5019007fca55",
        "56e1ba41cd28a01900c67ae0",
        "57274712708984140094dbad",
        "57094b4f9928a814004714f6",
        "57265d08708984140094c397",
        "5726eb4b5951b619008f826b",
        "572a05eb3f37b31900478653",
        "5706155352bb891400689894",
        "56de4a474396321400ee2786",
        "570d529fb3d812140066d6bb",
        "573362b94776f41900660974",
        "572fadcbb2c2fd1400568329",
        "5725c337271a42140099d163",
        "57109275b654c5140001f99f",
        "57274e975951b619008f87f9"
      ],
      "candidate_sha256": [
        "536e195415da4934032881cb450b11e570caec121170eb2072ecccea49d36996",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "a21efd80668b227ef68dd022f77a31b092d02d626d2e188520af3bf6059d57dc",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "ee73f593c26ad38e58d299361ba118fcf077591f71e9d4ed9515efb8b2fc75f2",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac"
      ],
      "example_id": "570d529fb3d812140066d6bb",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.37088894,
      "neural_order": [
        10,
        7,
        8,
        1,
        15,
        14,
        11,
        5,
        3,
        12,
        4,
        2,
        0,
        6,
        9,
        13
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "621f39f5fe977aa2ddd4dba8ef6a812880bd34fd0812715183962bebf57d4e27",
      "trial_index": 172
    },
    {
      "answer_sha256": "3d6f5bda88308214de2476806a5aca5588d1f6de5a0453bbd082108f94317f8e",
      "candidate_ids": [
        "5710f4b8b654c5140001fa45",
        "571cb010dd7acb1400e4c128",
        "57115f652419e314009555b9",
        "572f567cb2c2fd140056803f",
        "570d32e4fed7b91900d45d01",
        "572665ff708984140094c4c3",
        "56e1d9fee3433e14004231cb",
        "5711119cb654c5140001fae3",
        "56de10b44396321400ee2593",
        "5726487b5951b619008f6edd",
        "572807802ca10214002d9bf8",
        "571161092419e314009555d7",
        "57105da9a58dae1900cd699e",
        "572fae4b04bcaa1900d76bdf",
        "57106185b654c5140001f8db",
        "5711619950c2381900b54aaf"
      ],
      "candidate_sha256": [
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "75bcc54c3f28b2056c91642f34ed043daecd110b921b6d80aee1698da2f1af22",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "e71eab6fb81f391a6f9f6f7ddf6a435221f7159360e4d7a0549e3b969d010395",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b"
      ],
      "example_id": "57105da9a58dae1900cd699e",
      "gated_correct": false,
      "gated_top": 14,
      "gold_index": 12,
      "lexical_correct": false,
      "lexical_order": [
        14,
        12,
        0,
        9,
        7,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.02922466,
      "neural_order": [
        12,
        7,
        0,
        14,
        8,
        9,
        3,
        13,
        6,
        10,
        15,
        2,
        1,
        4,
        5,
        11
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "d68f251d21505b6136d3b26927b51ed7889a5bb5e3dfa88d46725470bb0cee6e",
      "trial_index": 173
    },
    {
      "answer_sha256": "df1f442dff44a97b5091433d7b82f1d4ac617ff8887a6e737bd2c69265cbeab4",
      "candidate_ids": [
        "5705f13d52bb8914006896f0",
        "57265e97708984140094c3c3",
        "5726a09f708984140094cc39",
        "572a1dbb3f37b319004786f5",
        "5728759cff5b5019007da25a",
        "56e1b62ecd28a01900c67aa3",
        "5725fabc89a1e219009ac128",
        "5727f7523acd2414000df10d",
        "5725bad5271a42140099d0bd",
        "5726a638dd62a815002e8bf6",
        "5727e9523acd2414000def95",
        "57115dbe2419e314009555a5",
        "571cb27fdd7acb1400e4c132",
        "57265642f1498d1400e8dc68",
        "572fc80b04bcaa1900d76d11",
        "57106185b654c5140001f8db"
      ],
      "candidate_sha256": [
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "9819559c80db18970983cbfdf2bcb3f75443deb410b562890bbe13cea74a03dc",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b"
      ],
      "example_id": "57106185b654c5140001f8db",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.52014024,
      "neural_order": [
        15,
        7,
        0,
        13,
        4,
        11,
        10,
        3,
        1,
        9,
        12,
        2,
        6,
        5,
        8,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "0f2ef6ef9b5e2cf15b4b2a8f84ccefd8499f7080b3b235ca3e78d43e28aa08e4",
      "trial_index": 174
    },
    {
      "answer_sha256": "4967cbc85834f83170eb6c23ef34de52aa8dbc5c6ab683c1273f432a7f4f59f3",
      "candidate_ids": [
        "5737a5931c456719005744e7",
        "5733f7b9d058e614000b66a9",
        "56de4c324396321400ee27ab",
        "5725bbec271a42140099d0d1",
        "570603c475f01819005e7882",
        "572843304b864d1900164848",
        "572667e2f1498d1400e8de90",
        "572a0ce11d046914007796fd",
        "5727490bdd62a815002e9a82",
        "5729517d6aef051400154cc8",
        "5730bb522461fd1900a9d011",
        "571cb27fdd7acb1400e4c132",
        "5728667eff5b5019007da1fc",
        "57281c594b864d19001644a2",
        "57274eca5951b619008f8800",
        "57106644b654c5140001f8e5"
      ],
      "candidate_sha256": [
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "5d46e9f7c620b7d8f42a0c90216d4bb7625ee95e4d9af99b663434ae1e08a483",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "752151b83536aa2b39e17ee8104540f3b94aa10671b826eb5b19cc6d1e68a8ae",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "46ceed087a89c8c5146679d04d65800e1d5b68fadcd272ac5f73357c18ad83be",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc"
      ],
      "example_id": "57106644b654c5140001f8e5",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.34303775,
      "neural_order": [
        15,
        2,
        1,
        14,
        10,
        13,
        8,
        3,
        5,
        4,
        0,
        6,
        7,
        9,
        11,
        12
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "acf38f7f259b015f07c90d35a2b569f0f20155b679448d174fa9d91368b4748d",
      "trial_index": 175
    },
    {
      "answer_sha256": "69d2c5c0d93110bf1afe581c7ba6fcffeeca431a2a716dcbb81757b3be1d49e8",
      "candidate_ids": [
        "57265285708984140094c25b",
        "57286c8cff5b5019007da218",
        "5726385e271a42140099d797",
        "57060eaf75f01819005e7910",
        "57115c7450c2381900b54a9d",
        "56e1b00ce3433e140042309e",
        "571cca4add7acb1400e4c150",
        "56e1ddfce3433e14004231d5",
        "572861cc4b864d190016495e",
        "57332b66d058e614000b5758",
        "57115f0a50c2381900b54aa7",
        "572a10cd6aef051400155222",
        "5726431aec44d21400f3dd13",
        "56e1fc57e3433e140042322c",
        "572fbf21a23a5019007fc937",
        "57106d2fb654c5140001f8ef"
      ],
      "candidate_sha256": [
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "3f83987ebc0c4b6d662d8cc6134fe19bdcd18d63555f50b4c6841486edd3606d",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109"
      ],
      "example_id": "57106d2fb654c5140001f8ef",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.32671106,
      "neural_order": [
        15,
        0,
        1,
        12,
        3,
        9,
        5,
        11,
        2,
        4,
        6,
        7,
        8,
        10,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "9f3c96eb38235cc8f5965ee641179bfcbaecca36ea7b1bd60674b068526203a5",
      "trial_index": 176
    },
    {
      "answer_sha256": "7a72b4bd751db159994328e207466704d2b256e8acf0b64563dc70f440324025",
      "candidate_ids": [
        "5728fd206aef05140015494c",
        "57338007d058e614000b5bda",
        "572659ea5951b619008f7051",
        "572ff890a23a5019007fcbcd",
        "57267ebfdd62a815002e872c",
        "57284142ff5b5019007da00a",
        "572985011d04691400779501",
        "56de1645cffd8e1900b4b5d0",
        "5729fe5c3f37b3190047862f",
        "56e1a38de3433e140042305c",
        "57378862c3c5551400e51f21",
        "572745c6708984140094db99",
        "5725ec8289a1e219009ac0ae",
        "571077ecb654c5140001f909",
        "5725f8f5ec44d21400f3d7b1",
        "572629c6271a42140099d6a3"
      ],
      "candidate_sha256": [
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "ce701ae7bea9bf9e47c5972ca7624bc90227bd2c59f41f67a73bf89c12c44680",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "06a36c034cb1681bb677178da6125ae4b9b63cd95527bdc90f5a7af7da958523",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "395b1cfae86dbeb34c8873339b090ededf002969f69503f5b785bd284dc9d2ca",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0"
      ],
      "example_id": "571077ecb654c5140001f909",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        12,
        6,
        10,
        5,
        0,
        1,
        2,
        3,
        4,
        7,
        8,
        9,
        11,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.42364165,
      "neural_order": [
        13,
        7,
        14,
        1,
        12,
        2,
        15,
        10,
        3,
        5,
        4,
        9,
        0,
        6,
        8,
        11
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "44aca4a135a41ce7d6294c66a477cde23477e53e921539cdd28ea7d224424171",
      "trial_index": 177
    },
    {
      "answer_sha256": "3ea77fc339263dd319e3dab93c288d0afcbd8ba8f96c20cfd6b42b5e8f3cf458",
      "candidate_ids": [
        "572ffc0f947a6a140053cef0",
        "5727502f708984140094dc07",
        "5725edfe38643c19005ace9f",
        "572fc659b2c2fd1400568447",
        "5730b7ce069b5314008322c3",
        "5710e9f8a58dae1900cd6b30",
        "5732b6b5328d981900602021",
        "5737a25ac3c5551400e51f51",
        "57283db6ff5b5019007d9fbc",
        "56de0f6a4396321400ee257f",
        "572944e03f37b319004781e1",
        "57300761947a6a140053cf9a",
        "57107932a58dae1900cd69ca",
        "57094ca7efce8f15003a7dd6",
        "57287b4a4b864d1900164a28",
        "57111428b654c5140001faff"
      ],
      "candidate_sha256": [
        "da7e3caa2252604ebed016e003b315efd2a4a47f3b063950ffca6d27e6dd03f2",
        "e32237eb13f46727755bd53347a189c0620b3ffff778cecbcd627a94fbebc022",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "66cbf5ef7ba18a058b7e88c4c66ba9e11f4ef0efae05c7495c2ee6e756648699",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884"
      ],
      "example_id": "57107932a58dae1900cd69ca",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        4,
        5,
        15,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.04998391,
      "neural_order": [
        12,
        5,
        4,
        9,
        15,
        11,
        10,
        14,
        0,
        2,
        7,
        1,
        3,
        6,
        8,
        13
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "4ab6aa77ebae0869830876f0c6dc50c5e55a96d50dfafc23e7dd9bd22fa0b619",
      "trial_index": 178
    },
    {
      "answer_sha256": "b2ca310c103217fd85f60f25f54535a7ec3b424f14e81993695553df821c988e",
      "candidate_ids": [
        "570d28bdb3d812140066d4a3",
        "572fc8a904bcaa1900d76d1f",
        "571166352419e314009555f1",
        "56e1c9bfe3433e1400423192",
        "57264a8cdd62a815002e808c",
        "57377ec7c3c5551400e51f05",
        "57264a74708984140094c18b",
        "57274d9bf1498d1400e8f5f6",
        "5726dcbddd62a815002e9320",
        "5725d42a89a1e219009abf58",
        "5726f868dd62a815002e9684",
        "57107a3ea58dae1900cd69de",
        "5730b8ca8ab72b1400f9c704",
        "5727ef664b864d1900164060",
        "572a0bf96aef051400155204",
        "572810ec2ca10214002d9d06"
      ],
      "candidate_sha256": [
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "d3224d2a7c9b1fb0c025ab44b397ab073a37fb28162d6b3730660c29e767b600",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87"
      ],
      "example_id": "57107a3ea58dae1900cd69de",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.37785914,
      "neural_order": [
        11,
        15,
        6,
        9,
        0,
        8,
        10,
        12,
        1,
        2,
        3,
        4,
        5,
        7,
        13,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "89b1b5472013d10c8f19bd492f3c70ef9f6a881e3db21539939ffe31db43ca21",
      "trial_index": 179
    },
    {
      "answer_sha256": "da515cfe5df51d912fbf6d610e2507075048373afc61d058d1a17189bdf8bd60",
      "candidate_ids": [
        "572683075951b619008f7513",
        "572856beff5b5019007da190",
        "572867d72ca10214002da2fc",
        "57339a554776f41900660e74",
        "5705ec1675f01819005e776e",
        "57107c24a58dae1900cd69e8",
        "57113ba6b654c5140001fc1f",
        "57263dcd89a1e219009ac5a2",
        "57096c95200fba1400367fbb",
        "5733834ed058e614000b5c26",
        "57282dfb4b864d1900164668",
        "5725d7e438643c19005acdf9",
        "570d529fb3d812140066d6bb",
        "570d391fb3d812140066d575",
        "5727aa413acd2414000de921",
        "572818f54b864d190016446c"
      ],
      "candidate_sha256": [
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660"
      ],
      "example_id": "57107c24a58dae1900cd69e8",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        7,
        1,
        0,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.33417426,
      "neural_order": [
        5,
        2,
        11,
        7,
        14,
        12,
        9,
        15,
        10,
        0,
        1,
        3,
        4,
        6,
        8,
        13
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "88912d3b2bfd3254ce853da85f9efe8bc634bd254052ae342810f8315fb6321e",
      "trial_index": 180
    },
    {
      "answer_sha256": "eefcace3be30bda37134513aa6a654ece88ec3ec135a1282de01355ab50a5bec",
      "candidate_ids": [
        "57285213ff5b5019007da182",
        "57339c16d058e614000b5ec5",
        "57097b50200fba140036804d",
        "57107d73b654c5140001f91d",
        "570610b275f01819005e792a",
        "5727f2714b864d1900164072",
        "572822233acd2414000df555",
        "56dde1d966d3e219004dad8d",
        "5726e313f1498d1400e8eeb2",
        "572ff760b2c2fd1400568677",
        "56e1ec83cd28a01900c67c0a",
        "572815d9ff5b5019007d9cbc",
        "57282358ff5b5019007d9dde",
        "5711623e50c2381900b54ab9",
        "571112ada58dae1900cd6bcc",
        "5729f799af94a219006aa707"
      ],
      "candidate_sha256": [
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e"
      ],
      "example_id": "57107d73b654c5140001f91d",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        14,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.31624196,
      "neural_order": [
        3,
        14,
        6,
        5,
        7,
        11,
        9,
        12,
        13,
        15,
        1,
        0,
        2,
        4,
        8,
        10
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "8c4f24e966c2a5e4b38d6130b4cc0471f25755e85647b9f7e873485e680c2680",
      "trial_index": 181
    },
    {
      "answer_sha256": "3ab31331e63b99bd9d32e51fb237f201c2e5aa1e7cfb5bba5758f96cd8da9266",
      "candidate_ids": [
        "572a0a686aef0514001551ec",
        "57293e221d046914007791d5",
        "5733e771d058e614000b6545",
        "572745c6708984140094db99",
        "5729fd56af94a219006aa72f",
        "572991943f37b319004784a1",
        "571a52cb4faf5e1900b8a969",
        "5726581fdd62a815002e823a",
        "57107e6ca58dae1900cd69f2",
        "5710f2e2a58dae1900cd6b72",
        "57378e311c456719005744af",
        "573098f38ab72b1400f9c5d2",
        "5727e9523acd2414000def95",
        "5725d183271a42140099d23d",
        "57286951ff5b5019007da20e",
        "5725d7e438643c19005acdf9"
      ],
      "candidate_sha256": [
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "0c60fccfa0a5891c548480d13173fbf66ab99f403f068010ba84b6402ed6daff",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52"
      ],
      "example_id": "57107e6ca58dae1900cd69f2",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.0933128,
      "neural_order": [
        8,
        9,
        2,
        11,
        7,
        10,
        0,
        5,
        13,
        14,
        1,
        3,
        4,
        6,
        12,
        15
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "ae24014d443d7465bfe59770661820d77d52bd76b7c17c482c098bcdd2a676de",
      "trial_index": 182
    },
    {
      "answer_sha256": "e741bdad56c495cc7b9ec5424a4588f7f666482183de099d3194f35f476aaef6",
      "candidate_ids": [
        "5733a5f54776f41900660f44",
        "5733dab4d058e614000b6409",
        "57114f0050c2381900b54a65",
        "57108073b654c5140001f925",
        "57302efe04bcaa1900d772f5",
        "57300bf504bcaa1900d77087",
        "57294e6b1d04691400779275",
        "57274e0d708984140094dbe5",
        "572fc659b2c2fd1400568447",
        "57268a37f1498d1400e8e33c",
        "57264fe65951b619008f6fa1",
        "57264c42dd62a815002e80c6",
        "5737a5931c456719005744e7",
        "57264d58f1498d1400e8db7a",
        "57265746dd62a815002e8218",
        "572647e2dd62a815002e805c"
      ],
      "candidate_sha256": [
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "7a54e69984565794b95d2e675d9ceb87e6f076fbf18c6fc5f9f4161c8750a672",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f"
      ],
      "example_id": "57108073b654c5140001f925",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        10,
        11,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.33606547,
      "neural_order": [
        3,
        1,
        2,
        5,
        13,
        11,
        10,
        14,
        4,
        0,
        12,
        6,
        7,
        8,
        9,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "8a1a8f3f6167ce423f5a87fae5f42679cf7bdde6769e6c36db9359cb7ed1b791",
      "trial_index": 183
    },
    {
      "answer_sha256": "65065e4a2fe533b13e5be30cfd3580fc7a477b06fdaf7a900cc448092f7020cd",
      "candidate_ids": [
        "5727c94bff5b5019007d954a",
        "573011de04bcaa1900d770f9",
        "572faf74b2c2fd1400568347",
        "572ff935b2c2fd140056869b",
        "5726887e708984140094c917",
        "572f609ca23a5019007fc5af",
        "572684365951b619008f753f",
        "572fca7eb2c2fd1400568473",
        "5727ee372ca10214002d99ec",
        "57287b322ca10214002da3be",
        "570966e0200fba1400367f4f",
        "57108198b654c5140001f937",
        "5728e8212ca10214002daa6c",
        "57297547af94a219006aa45c",
        "571c97e2dd7acb1400e4c11e",
        "572a03a06aef0514001551aa"
      ],
      "candidate_sha256": [
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247"
      ],
      "example_id": "57108198b654c5140001f937",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        3,
        6,
        13,
        15,
        12,
        0,
        1,
        2,
        4,
        5,
        7,
        8,
        9,
        10,
        14
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.30482821,
      "neural_order": [
        11,
        0,
        5,
        2,
        1,
        13,
        8,
        3,
        14,
        4,
        6,
        7,
        9,
        10,
        12,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "5e1f2dc60b19789fa3f3d8d4c0991a88e00528d81e6e8ef39f0738ab40d74309",
      "trial_index": 184
    },
    {
      "answer_sha256": "0aeef45fae64678e813c7368f9ff77d05bfb57f23bd41d383ce96dbb392a2262",
      "candidate_ids": [
        "57108c95b654c5140001f979",
        "5705eccb52bb8914006896b8",
        "57264f18f1498d1400e8dbae",
        "57268066708984140094c821",
        "571a52cb4faf5e1900b8a969",
        "57297781af94a219006aa4a3",
        "56de16ca4396321400ee25c5",
        "572fac17947a6a140053cb54",
        "572f7588947a6a140053c984",
        "57265a58dd62a815002e8270",
        "573099ee8ab72b1400f9c5dc",
        "57302efe04bcaa1900d772f5",
        "572a096e6aef0514001551e4",
        "57096505ed30961900e84082",
        "57269cc3dd62a815002e8b12",
        "57111ab8a58dae1900cd6c3c"
      ],
      "candidate_sha256": [
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee",
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "2892ce1ee90d7e758e8bd4b12fd7bebc4276caad2f0d0af29d0444e47b6a4a05",
        "6d2627b1297d1087a3e3c836fbeab911593b2c58f4b2ae45d6c39dd4ba695bea",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2"
      ],
      "example_id": "57108c95b654c5140001f979",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        15,
        1,
        8,
        13,
        7,
        2,
        3,
        4,
        5,
        6,
        9,
        10,
        11,
        12,
        14
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.26605754,
      "neural_order": [
        0,
        15,
        2,
        8,
        3,
        6,
        1,
        7,
        4,
        5,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "a92f5d5b7e564d51f16e38c2fc216bc548be0e0af94989e7c9c0fba5c26ec571",
      "trial_index": 185
    },
    {
      "answer_sha256": "2b1a417eaa0d0dcff645e9b113b326733deac014859bf52bd5cdf905800e28d1",
      "candidate_ids": [
        "57264e2f708984140094c1e1",
        "57115e532419e314009555af",
        "573005b9947a6a140053cf6a",
        "56de0f6a4396321400ee257f",
        "572882242ca10214002da420",
        "571cd5b1dd7acb1400e4c178",
        "571cc8815efbb31900334dee",
        "57287c2bff5b5019007da26c",
        "5726a7ecf1498d1400e8e654",
        "571cc3dedd7acb1400e4c146",
        "56e1bc3ae3433e1400423104",
        "57108d69b654c5140001f983",
        "5728eff82ca10214002daadc",
        "5733faaf4776f4190066162f",
        "5725c0f289a1e219009abdf2",
        "57294279af94a219006aa209"
      ],
      "candidate_sha256": [
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c"
      ],
      "example_id": "57108d69b654c5140001f983",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        8,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.44393235,
      "neural_order": [
        11,
        3,
        2,
        0,
        13,
        7,
        4,
        5,
        8,
        12,
        14,
        1,
        6,
        9,
        10,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "16662d8c57a0db43ed3498cf566e161dcea9b6fcbc141bca4d54ba43c589c7ec",
      "trial_index": 186
    },
    {
      "answer_sha256": "69b60601ca00d2b5a1f3dec125570a37db10b0a25a5657f42bee853265b745d9",
      "candidate_ids": [
        "5705e4fe75f01819005e7704",
        "572755b7708984140094dc4d",
        "57281c594b864d19001644a2",
        "5727ff083acd2414000df1ab",
        "57309bfb8ab72b1400f9c5e6",
        "5725b92e38643c19005acbd3",
        "572683f95951b619008f7525",
        "57115bf350c2381900b54a93",
        "57266c015951b619008f7237",
        "573368e54776f41900660a53",
        "57108ee6a58dae1900cd6a1a",
        "5725d42a89a1e219009abf58",
        "573792ee1c456719005744b9",
        "570d2f5bfed7b91900d45ccf",
        "572659535951b619008f703f",
        "570d4606b3d812140066d619"
      ],
      "candidate_sha256": [
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757"
      ],
      "example_id": "57108ee6a58dae1900cd6a1a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.33048112,
      "neural_order": [
        10,
        0,
        9,
        4,
        6,
        2,
        3,
        11,
        13,
        5,
        1,
        7,
        8,
        12,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "e05c88a0acb762a54269ff82a7c148ad0cdfbf51ed059a2bc08dfba088f424e7",
      "trial_index": 187
    },
    {
      "answer_sha256": "b0942022159de90c288e007306f4984b0882a32c55bcd98739acfe9de0e78099",
      "candidate_ids": [
        "57284b904b864d19001648e2",
        "571090abb654c5140001f995",
        "5733f5f24776f419006615c1",
        "5727aa413acd2414000de921",
        "570961aa200fba1400367f15",
        "57300888b2c2fd1400568775",
        "57302e45947a6a140053d22e",
        "5725e748ec44d21400f3d733",
        "573098f38ab72b1400f9c5d2",
        "5727cd0f4b864d1900163d72",
        "571a49de4faf5e1900b8a94a",
        "572658435951b619008f7025",
        "5727d6154b864d1900163e34",
        "5733dc95d058e614000b644a",
        "570602fa52bb89140068979e",
        "572746addd62a815002e9a5e"
      ],
      "candidate_sha256": [
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "e26d3a98154d79e4fb93dfbc9b446801d4e38dc1c4681b39878f40025a6b4fce",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "2d5af9a4d02fc579e6e15f098bcffbcc6459cdee51fd51b38ea0075eee7a728a",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d",
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d"
      ],
      "example_id": "571090abb654c5140001f995",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.18240794,
      "neural_order": [
        1,
        13,
        8,
        2,
        5,
        9,
        14,
        0,
        7,
        12,
        3,
        4,
        6,
        10,
        11,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "e97d908c14bdb603fe335d39678dd2d17841ffa709c2f369ac88fa12e2851ed8",
      "trial_index": 188
    },
    {
      "answer_sha256": "fc5fc0c3433b24a138b5f7446f09639622b8121ba60c2bb4d46428b5a99021be",
      "candidate_ids": [
        "57109180a58dae1900cd6a40",
        "5726f2375951b619008f830f",
        "573027d6a23a5019007fce9d",
        "5727e21e4b864d1900163f34",
        "57274e145951b619008f87e7",
        "57094a79efce8f15003a7dc4",
        "572a11663f37b31900478693",
        "5730a0778ab72b1400f9c60a",
        "5727fc37ff5b5019007d9a18",
        "56e1ddfce3433e14004231d5",
        "572750e8dd62a815002e9af0",
        "572871bd3acd2414000dfa03",
        "572f609ca23a5019007fc5af",
        "57269016708984140094ca41",
        "57293e221d046914007791d5",
        "57300888b2c2fd1400568775"
      ],
      "candidate_sha256": [
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "874533677a269c5d5fb8bd3a3213307ea1632c2236ea48b8da8182d07905fcc8",
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21"
      ],
      "example_id": "57109180a58dae1900cd6a40",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        5,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.5497895,
      "neural_order": [
        0,
        15,
        11,
        6,
        12,
        7,
        4,
        10,
        5,
        8,
        1,
        2,
        3,
        9,
        13,
        14
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "e0a84648a9926b51c0907ac15a046f1ade34d05517f9724f4fd3a99fd85642d7",
      "trial_index": 189
    },
    {
      "answer_sha256": "574ae59e58f11e58ada5817cf997621a8cdd5e9d1594c994536e1aafe9de1cc4",
      "candidate_ids": [
        "572818f54b864d190016446c",
        "57094d489928a8140047150a",
        "571cca4add7acb1400e4c150",
        "5725f00938643c19005aced7",
        "573010fab2c2fd14005687d7",
        "5727d9c43acd2414000dee17",
        "5728f50baf94a219006a9e55",
        "57115dbe2419e314009555a5",
        "57109275b654c5140001f99f",
        "5733a5f54776f41900660f44",
        "5725fe63ec44d21400f3d7dd",
        "571a4d1a4faf5e1900b8a958",
        "572677e7708984140094c723",
        "5729a3716aef05140015506a",
        "57296d571d04691400779413",
        "571098f8a58dae1900cd6ab2"
      ],
      "candidate_sha256": [
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554"
      ],
      "example_id": "57109275b654c5140001f99f",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        15,
        6,
        4,
        13,
        11,
        0,
        1,
        2,
        3,
        5,
        7,
        9,
        10,
        12,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.18786166,
      "neural_order": [
        8,
        15,
        13,
        0,
        4,
        6,
        12,
        3,
        9,
        7,
        1,
        2,
        5,
        10,
        11,
        14
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "805be635fa7f6b61d1079802df5f67938e1f97a55952c0838ca7cb7a53b1593b",
      "trial_index": 190
    },
    {
      "answer_sha256": "cff91d95b4eed96ddbfb33b9ed41cc0d642dfbff5a6721feff0c87c82b3f4992",
      "candidate_ids": [
        "5733e5a14776f4190066145b",
        "5737821cc3c5551400e51f19",
        "57274d1cdd62a815002e9aae",
        "571136b8a58dae1900cd6d22",
        "57339a554776f41900660e74",
        "5728759cff5b5019007da25a",
        "56de0f6a4396321400ee257f",
        "57283f014b864d19001647e8",
        "56e190bce3433e1400422fc8",
        "572ff83ab2c2fd1400568681",
        "571093aba58dae1900cd6a5c",
        "56de51244396321400ee27ef",
        "572885c44b864d1900164a78",
        "5733d2dbd058e614000b6339",
        "572fdb17b2c2fd140056851d",
        "5725fabc89a1e219009ac128"
      ],
      "candidate_sha256": [
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "2a88a6e435244f1025c628b748e860ab699fad659a4359c6fb5c85d3ee365576",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4"
      ],
      "example_id": "571093aba58dae1900cd6a5c",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.16782111,
      "neural_order": [
        10,
        11,
        6,
        9,
        13,
        0,
        14,
        3,
        2,
        12,
        1,
        4,
        5,
        7,
        8,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "b0be0fc69ed0de0498d074848052389a4a50f597139653f982c1aaf99d012d1b",
      "trial_index": 191
    },
    {
      "answer_sha256": "bc3be8783c2ce53e710b6d0d6f0414ffe0f6c48aba3e0d9c50aae5e1446fc97a",
      "candidate_ids": [
        "573368044776f41900660a29",
        "572659ea5951b619008f7051",
        "5730a40f396df91900096234",
        "571094b7a58dae1900cd6a66",
        "572751b4708984140094dc1b",
        "5733d5704776f4190066130e",
        "57286bb84b864d19001649c8",
        "56de3cd0cffd8e1900b4b6be",
        "571c8198dd7acb1400e4c0ce",
        "572878942ca10214002da3a2",
        "5730005db2c2fd1400568703",
        "5726d9935951b619008f7fed",
        "57115ac550c2381900b54a77",
        "57263c78ec44d21400f3dc7b",
        "5733d13e4776f419006612c4",
        "5727e6cbff5b5019007d97ee"
      ],
      "candidate_sha256": [
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "18017326aa99d6478493c6fd9c81f3144c242ec0a8dc61e8a64d3d1ab8e0507e",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "09dae1e2b2708f999627efe257489cfaad54e793df884b8877c2d638640ffa20"
      ],
      "example_id": "571094b7a58dae1900cd6a66",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.29454728,
      "neural_order": [
        3,
        14,
        7,
        10,
        5,
        9,
        2,
        13,
        1,
        12,
        0,
        4,
        6,
        8,
        11,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "a0460ae0e810b0de568f34efcefddf346aa01e7764947d22623edc2cccb32b78",
      "trial_index": 192
    },
    {
      "answer_sha256": "bfe9ad907e252b40e407caea5c334f1c1371491d6be1092e64527bb41e32728c",
      "candidate_ids": [
        "5730b255396df919000962b0",
        "573766251c45671900574471",
        "56de0ffd4396321400ee258d",
        "56de40da4396321400ee2708",
        "571156152419e3140095559b",
        "57283db6ff5b5019007d9fbc",
        "57115dbe2419e314009555a5",
        "573749741c4567190057445d",
        "5728e8212ca10214002daa6c",
        "57378b141c4567190057449f",
        "572883153acd2414000dfa6f",
        "5733a5f54776f41900660f44",
        "57273cca708984140094db33",
        "571095a8a58dae1900cd6a76",
        "572f57c704bcaa1900d7686d",
        "57284456ff5b5019007da05c"
      ],
      "candidate_sha256": [
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "f4a941b8fb7b8c4a1f8010d912eaa080c3e3ddf68b132d4a96fc1da34d4ea0d2",
        "8d0f136ca3e21898494994833851f5c381fc610d00c5a286083f4b15d65caad4",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "0427123719c1fd7ac7026780bf08a83e759f63ccba9a9425627df9633f21559d"
      ],
      "example_id": "571095a8a58dae1900cd6a76",
      "gated_correct": false,
      "gated_top": 14,
      "gold_index": 13,
      "lexical_correct": false,
      "lexical_order": [
        14,
        9,
        3,
        13,
        2,
        11,
        4,
        15,
        5,
        0,
        1,
        6,
        7,
        8,
        10,
        12
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.19208768,
      "neural_order": [
        13,
        14,
        2,
        3,
        11,
        12,
        0,
        10,
        6,
        15,
        1,
        4,
        5,
        7,
        8,
        9
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "31ce1f1e2d32fb6f3644f17913bd7cc29babc0e0daf265eddabfe8297eceedad",
      "trial_index": 193
    },
    {
      "answer_sha256": "83ba553cdbd31bb7a9f869b2f6ccfb24057cb917dd97855fc346b5e1c7316cd0",
      "candidate_ids": [
        "570966e0200fba1400367f4f",
        "57274eca5951b619008f8800",
        "5729f1fcaf94a219006aa6d3",
        "5710968aa58dae1900cd6a86",
        "571117d4a58dae1900cd6c0a",
        "57293e983f37b3190047818b",
        "571a49de4faf5e1900b8a94a",
        "5730208fa23a5019007fcded",
        "5727526cdd62a815002e9b0e",
        "573403394776f419006616dd",
        "5711669550c2381900b54adf",
        "5726a46cdd62a815002e8bd0",
        "5737aafd1c456719005744fb",
        "5725c95f38643c19005accf3",
        "5727aeac3acd2414000de987",
        "57281c594b864d19001644a2"
      ],
      "candidate_sha256": [
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "46ceed087a89c8c5146679d04d65800e1d5b68fadcd272ac5f73357c18ad83be",
        "99afd74c371b051e0dd2704df0676d8589e7c060ad964a7cae71b5a5bde23c1e",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d"
      ],
      "example_id": "5710968aa58dae1900cd6a86",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        4,
        9,
        5,
        0,
        1,
        2,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.40706488,
      "neural_order": [
        3,
        4,
        9,
        13,
        10,
        1,
        5,
        12,
        11,
        0,
        2,
        6,
        7,
        8,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "006adc544c780de53590aa5f9676ba685d84ec6150900598d199cf1fb1286774",
      "trial_index": 194
    },
    {
      "answer_sha256": "57b7c17f2f10dee9687b205eb5e82f5bab90300ad9452b53938fc83a73bb0738",
      "candidate_ids": [
        "57378862c3c5551400e51f21",
        "57115b8b50c2381900b54a89",
        "573099ee8ab72b1400f9c5dc",
        "573368044776f41900660a29",
        "5728510f4b864d1900164936",
        "5726a34bf1498d1400e8e59e",
        "5726baf2dd62a815002e8e74",
        "57263fd138643c19005ad37b",
        "57115b2850c2381900b54a7f",
        "5705f36452bb891400689718",
        "5730a40f396df91900096234",
        "5710e9f8a58dae1900cd6b30",
        "570960cf200fba1400367f01",
        "571097baa58dae1900cd6a98",
        "572744aff1498d1400e8f588",
        "56e1e9dfe3433e14004231fc"
      ],
      "candidate_sha256": [
        "06a36c034cb1681bb677178da6125ae4b9b63cd95527bdc90f5a7af7da958523",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4"
      ],
      "example_id": "571097baa58dae1900cd6a98",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.09345018,
      "neural_order": [
        13,
        11,
        2,
        9,
        10,
        1,
        6,
        8,
        0,
        15,
        3,
        4,
        5,
        7,
        12,
        14
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "82e61fa2d2dd7fc92cf7687b29b2d3cf3140d7c24b34bd11ef6749f749bdebab",
      "trial_index": 195
    },
    {
      "answer_sha256": "97eb1d6128536a2628d9df27a1d218258bf483b889d106d6c0b254895919c4cc",
      "candidate_ids": [
        "57261dab38643c19005ad037",
        "572750df5951b619008f882f",
        "56e16182e3433e1400422e28",
        "572681c1dd62a815002e8796",
        "5733d7cbd058e614000b63ab",
        "5725c57a89a1e219009abe5e",
        "57377862c3c5551400e51ef3",
        "57300911947a6a140053cfb6",
        "571098f8a58dae1900cd6ab2",
        "5729517d6aef051400154cc8",
        "57107c24a58dae1900cd69e8",
        "57275409708984140094dc35",
        "572648e8dd62a815002e8076",
        "56e1c720e3433e140042316a",
        "572849b4ff5b5019007da0f2",
        "5705eccb52bb8914006896b8"
      ],
      "candidate_sha256": [
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1"
      ],
      "example_id": "571098f8a58dae1900cd6ab2",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.28849784,
      "neural_order": [
        8,
        14,
        4,
        1,
        15,
        7,
        10,
        12,
        0,
        6,
        3,
        5,
        13,
        2,
        9,
        11
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "bee879557a98400d19312dd88aeae528a67ecc1534ebfb0f41946b54ed50e4e7",
      "trial_index": 196
    },
    {
      "answer_sha256": "e08a885cc63242cd47191675df217e195e3685a5460c93cd23e376184f950d7a",
      "candidate_ids": [
        "571099b2b654c5140001f9b3",
        "57286010ff5b5019007da1ca",
        "572f6c85947a6a140053c940",
        "56e1a38de3433e140042305c",
        "56de3e414396321400ee26d8",
        "5733fe73d058e614000b673d",
        "572820512ca10214002d9e72",
        "572853e8ff5b5019007da188",
        "57294baaaf94a219006aa26b",
        "573314e3d058e614000b56f0",
        "572826634b864d19001645be",
        "5725cc38ec44d21400f3d5bb",
        "5727ec062ca10214002d99b6",
        "56e1bc3ae3433e1400423104",
        "571c97e2dd7acb1400e4c11e",
        "5726d9935951b619008f7fed"
      ],
      "candidate_sha256": [
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "bb3727c243a6cb744686ba6886c54791cd7745eca592eef8baf9c5b06c996b3e",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8"
      ],
      "example_id": "571099b2b654c5140001f9b3",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        9,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.39591823,
      "neural_order": [
        0,
        2,
        9,
        1,
        4,
        5,
        6,
        11,
        8,
        10,
        7,
        3,
        12,
        13,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "46e4380690baae5034c82d6798d1339db8d77d7c22a0d88a2f32fc8591590a26",
      "trial_index": 197
    },
    {
      "answer_sha256": "f3c7b97f38a6b79850568c285b575e6e08c485a84f84b82949138e763d5aa238",
      "candidate_ids": [
        "5726e313f1498d1400e8eeb2",
        "57265360dd62a815002e819a",
        "572ff890a23a5019007fcbcd",
        "57282358ff5b5019007d9dde",
        "5726bcde708984140094cfbf",
        "5705fb7f52bb891400689750",
        "5730b8ca8ab72b1400f9c704",
        "5737432bc3c5551400e51e99",
        "570d2d68fed7b91900d45cbb",
        "56e20a27cd28a01900c67c24",
        "5725e45689a1e219009ac048",
        "5710e8c8a58dae1900cd6b26",
        "5725b92e38643c19005acbd3",
        "5728848cff5b5019007da298",
        "573796edc3c5551400e51f33",
        "5710f2e2a58dae1900cd6b72"
      ],
      "candidate_sha256": [
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed"
      ],
      "example_id": "5710e8c8a58dae1900cd6b26",
      "gated_correct": false,
      "gated_top": 1,
      "gold_index": 11,
      "lexical_correct": false,
      "lexical_order": [
        1,
        11,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.15031108,
      "neural_order": [
        11,
        15,
        13,
        4,
        1,
        6,
        2,
        12,
        8,
        10,
        0,
        3,
        5,
        7,
        9,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "e50aba78a3e9e551dc5f40059f302c3103a4977cd6752f1e0556a0191fc4f742",
      "trial_index": 198
    },
    {
      "answer_sha256": "0355d63cdb3c0c538c28dd3f52c920a3ef18b86efeabd0a14334dd765ba4cd39",
      "candidate_ids": [
        "571cb010dd7acb1400e4c128",
        "572863363acd2414000df985",
        "57274beff1498d1400e8f5e4",
        "5726ddf6f1498d1400e8ee04",
        "5725d34089a1e219009abf50",
        "57274ca8dd62a815002e9aa4",
        "57309564069b5314008321a5",
        "5710e9f8a58dae1900cd6b30",
        "57267d52708984140094c7d9",
        "5728fd206aef05140015494c",
        "570d2556fed7b91900d45c47",
        "56de17f9cffd8e1900b4b5e0",
        "572ffbaa947a6a140053cee6",
        "5725d662ec44d21400f3d687",
        "5728705c2ca10214002da35a",
        "5726e08e5951b619008f810f"
      ],
      "candidate_sha256": [
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "d0c3193100a2c9cefa5efa046ca711a3a7943ddbb731e1aa29b247fe5eb27c7c",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "c71b1eee7e8e8a47719b20f0d4f2e9ca9aaeec5e9d9082387579fe235a47016d",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9"
      ],
      "example_id": "5710e9f8a58dae1900cd6b30",
      "gated_correct": false,
      "gated_top": 13,
      "gold_index": 7,
      "lexical_correct": false,
      "lexical_order": [
        13,
        7,
        4,
        10,
        6,
        3,
        0,
        1,
        2,
        5,
        8,
        9,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.46248787,
      "neural_order": [
        7,
        6,
        10,
        11,
        4,
        14,
        8,
        3,
        12,
        15,
        0,
        1,
        2,
        5,
        9,
        13
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "efdfb9f44868aa596962bb225f5fb65d0d38ac056f5dd7edd9b7ac9791f94c88",
      "trial_index": 199
    },
    {
      "answer_sha256": "b7729e3a6303504542061ba975fffe5a41e35ed867b92ec4d5719f6cfdc2eb05",
      "candidate_ids": [
        "5725cfd0271a42140099d225",
        "57273a465951b619008f86ff",
        "573786b51c4567190057448d",
        "572a096e6aef0514001551e4",
        "5710eb6fb654c5140001fa17",
        "5733fd66d058e614000b6735",
        "572fd7b8947a6a140053cd3a",
        "57115f0a50c2381900b54aa7",
        "5725bad5271a42140099d0bd",
        "572821274b864d1900164510",
        "5727ff083acd2414000df1ab",
        "572820512ca10214002d9e72",
        "56e17644e3433e1400422f40",
        "572826634b864d19001645be",
        "5727dc473acd2414000dee43",
        "5725c91e38643c19005acceb"
      ],
      "candidate_sha256": [
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "b67161cb70767d25408d47a9b2cd15ed23cc66a97c33ff46706b10e7a4772023"
      ],
      "example_id": "5710eb6fb654c5140001fa17",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        6,
        11,
        3,
        0,
        1,
        2,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.44825707,
      "neural_order": [
        4,
        5,
        0,
        9,
        15,
        11,
        13,
        3,
        1,
        2,
        6,
        7,
        8,
        10,
        12,
        14
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "3147b74130827faf036bfe320d19647e8cb10d56de3ec65cd33517bef909aefc",
      "trial_index": 200
    },
    {
      "answer_sha256": "0a4992ea442b53e3dca861deac09a8d4987004a8483079b12861080ea4aa1b52",
      "candidate_ids": [
        "572a0f073f37b3190047867f",
        "573003dd947a6a140053cf42",
        "570d4c3bfed7b91900d45e31",
        "57287ddf3acd2414000dfa3f",
        "57293f353f37b3190047819b",
        "5710eca0a58dae1900cd6b3a",
        "57340549d058e614000b67dd",
        "571126dfa58dae1900cd6cb2",
        "572872dd2ca10214002da37e",
        "57309446396df919000961b8",
        "5730126ba23a5019007fcd09",
        "571156152419e3140095559b",
        "5705eccb52bb8914006896b8",
        "57378b141c4567190057449f",
        "5733dd4f4776f419006613aa",
        "572ff626947a6a140053ce8e"
      ],
      "candidate_sha256": [
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91",
        "dea705fdf75f529664435816398522b553e31b0965a1287f3375c31a630e304e",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513"
      ],
      "example_id": "5710eca0a58dae1900cd6b3a",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        2,
        0,
        1,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.20843877,
      "neural_order": [
        5,
        14,
        6,
        2,
        9,
        12,
        1,
        7,
        3,
        0,
        4,
        8,
        10,
        11,
        13,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "e0913d2d7783925d79e361e211aa7074a4b0678ce35f9aab7bd20e40aede8cb0",
      "trial_index": 201
    },
    {
      "answer_sha256": "2ced72f34819bac23b9986d94316f8791c9ae20f6ca3dd0b0eb0fb65818c11e2",
      "candidate_ids": [
        "56dde0ba66d3e219004dad75",
        "5737432bc3c5551400e51e99",
        "572a0ce11d046914007796fd",
        "573796edc3c5551400e51f33",
        "56e19557e3433e1400422fee",
        "572980f9af94a219006aa4d1",
        "5725c0f289a1e219009abdf2",
        "57265aaf5951b619008f706b",
        "5727f05b4b864d1900164068",
        "57296d571d04691400779413",
        "572671a5f1498d1400e8dfe4",
        "5725d34aec44d21400f3d639",
        "5727d9c43acd2414000dee17",
        "572658daf1498d1400e8dcac",
        "5710ed7bb654c5140001fa2b",
        "57339902d058e614000b5e70"
      ],
      "candidate_sha256": [
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8"
      ],
      "example_id": "5710ed7bb654c5140001fa2b",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.26599742,
      "neural_order": [
        14,
        0,
        8,
        13,
        3,
        7,
        5,
        1,
        2,
        4,
        6,
        9,
        10,
        11,
        12,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "7be36dcf27407f060f00da1222aea7cf20221357226276f0b80b2240b1b53bd1",
      "trial_index": 202
    },
    {
      "answer_sha256": "a805d212e15b4e9b614b53b2d1e6791f22327093e790706d198d87497010bc52",
      "candidate_ids": [
        "57286dfa2ca10214002da332",
        "570d47b8b3d812140066d62d",
        "57114e8d50c2381900b54a5b",
        "5706149552bb891400689880",
        "571cca4add7acb1400e4c150",
        "5710f114a58dae1900cd6b60",
        "57060cc352bb89140068980e",
        "572fae4b04bcaa1900d76bdf",
        "5733a2a9d058e614000b5f28",
        "56dde27d9a695914005b9651",
        "5726487b5951b619008f6edd",
        "572a07fc6aef0514001551dc",
        "572905ce1d04691400778f83",
        "5727f16c3acd2414000df05b",
        "5727490bdd62a815002e9a82",
        "572ff760b2c2fd1400568677"
      ],
      "candidate_sha256": [
        "87c3a5e21ea40ce9396fd4bd04485fc5c291473b22cdd4b5feb623f3ece0e993",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "a8497768ed7455269c8c0e43100892fa8dc1ffff0a532613071c4cb9e3f05a18",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1"
      ],
      "example_id": "5710f114a58dae1900cd6b60",
      "gated_correct": false,
      "gated_top": 13,
      "gold_index": 5,
      "lexical_correct": false,
      "lexical_order": [
        13,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.21353782,
      "neural_order": [
        5,
        10,
        9,
        7,
        14,
        15,
        13,
        0,
        3,
        8,
        1,
        2,
        4,
        6,
        11,
        12
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "1d64794edbea5217b5e43910e8de933cec25c5feac3ee52817ecc09ddc7b491f",
      "trial_index": 203
    },
    {
      "answer_sha256": "fd5b1ed3f700a5c811fda4ba8427a48273232fede0668bdc565ace7bb4e6cd9f",
      "candidate_ids": [
        "57300200b2c2fd1400568729",
        "5725b81b271a42140099d097",
        "5710f2e2a58dae1900cd6b72",
        "57097c8fed30961900e841f2",
        "5728facd4b864d1900165170",
        "573010fab2c2fd14005687d7",
        "571cb010dd7acb1400e4c128",
        "57274a1edd62a815002e9a9b",
        "5725ff8238643c19005acf49",
        "572667e2f1498d1400e8de90",
        "5728151b4b864d1900164428",
        "57280fd3ff5b5019007d9c26",
        "5705fd8475f01819005e7840",
        "5725f2c838643c19005aceed",
        "571154c72419e31400955588",
        "572f64ccb2c2fd14005680b7"
      ],
      "candidate_sha256": [
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "b850237f43814691bedb635f5b07cea17e5bd3a2be801e564b61359a1746c8c6",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb"
      ],
      "example_id": "5710f2e2a58dae1900cd6b72",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        11,
        5,
        4,
        0,
        1,
        3,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.33661981,
      "neural_order": [
        2,
        11,
        15,
        14,
        3,
        4,
        5,
        6,
        9,
        12,
        0,
        7,
        1,
        8,
        10,
        13
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "ebe3d480a61c6bf84207d6d05363716620d679cf1211ed419170b6d7874b895d",
      "trial_index": 204
    },
    {
      "answer_sha256": "8739402913336603a75f57bc92d84f7649791395815dd018d3e60f37bedc96a5",
      "candidate_ids": [
        "572905ce1d04691400778f83",
        "5710f4b8b654c5140001fa45",
        "572647e2dd62a815002e805c",
        "572951f16aef051400154cce",
        "570d4e6cb3d812140066d66d",
        "573380e0d058e614000b5be9",
        "572a1a5c6aef051400155284",
        "570d418dfed7b91900d45dbf",
        "5725c41eec44d21400f3d50b",
        "572a020f6aef051400155198",
        "572699db5951b619008f7799",
        "572fd73e947a6a140053cd32",
        "572ff86004bcaa1900d76f65",
        "57373f80c3c5551400e51e8f",
        "57113c6da58dae1900cd6d32",
        "572754fff1498d1400e8f660"
      ],
      "candidate_sha256": [
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "631ae85a58426197f43bce08d34df2914057034648942e41a9d822902b61b3d3",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921"
      ],
      "example_id": "5710f4b8b654c5140001fa45",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        14,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.39960419,
      "neural_order": [
        1,
        5,
        14,
        6,
        4,
        8,
        10,
        13,
        12,
        7,
        11,
        0,
        2,
        3,
        9,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "5fa497e6b1940dced15ec80ffbc8da89b92c9db889d8259b4b1311fd057196ed",
      "trial_index": 205
    },
    {
      "answer_sha256": "290b1d79f6bebca4aed85638fe786b9a41dbfcf356d587b92dad70c622fb8129",
      "candidate_ids": [
        "5730208fa23a5019007fcded",
        "5726c002708984140094d073",
        "5733eb34d058e614000b65cc",
        "5726baf2dd62a815002e8e74",
        "571cbe35dd7acb1400e4c13c",
        "572659535951b619008f703f",
        "5705f36452bb891400689718",
        "57286d4f2ca10214002da328",
        "571144d1a58dae1900cd6d6e",
        "5727ee372ca10214002d99ec",
        "572a10cd6aef051400155222",
        "572a070c6aef0514001551c8",
        "571136b8a58dae1900cd6d22",
        "5728fd206aef05140015494c",
        "57286dfa2ca10214002da332",
        "5711119cb654c5140001fae3"
      ],
      "candidate_sha256": [
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "14e1246d6c57cac98cd47a35fe5c0801832d2bfe7bc8d39af8efc222d99387f9",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "87c3a5e21ea40ce9396fd4bd04485fc5c291473b22cdd4b5feb623f3ece0e993",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438"
      ],
      "example_id": "5711119cb654c5140001fae3",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        4,
        6,
        2,
        1,
        3,
        0,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.37996607,
      "neural_order": [
        15,
        3,
        0,
        1,
        14,
        5,
        7,
        2,
        13,
        6,
        11,
        10,
        4,
        8,
        9,
        12
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "64a6916e275138c323bf25138fc79b99d0edd660bebff9a7f724db97293db3b5",
      "trial_index": 206
    },
    {
      "answer_sha256": "4bda0ee9b9b79d8302ad4f02c9ecf63ef0cd45869b34c7c60a2f3efd19d60f82",
      "candidate_ids": [
        "5729506d6aef051400154cac",
        "57265d08708984140094c397",
        "57281940ff5b5019007d9d44",
        "572a10cd6aef051400155222",
        "572fd73e947a6a140053cd32",
        "572673f5708984140094c69b",
        "5733f1784776f41900661575",
        "57378e311c456719005744af",
        "571cd3b55efbb31900334e03",
        "571112ada58dae1900cd6bcc",
        "5728223cff5b5019007d9dc4",
        "5728fb6a1d04691400778ef5",
        "572fda6fb2c2fd140056850b",
        "573792ee1c456719005744b9",
        "573380e0d058e614000b5be9",
        "572a1a5c6aef051400155284"
      ],
      "candidate_sha256": [
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "0c60fccfa0a5891c548480d13173fbf66ab99f403f068010ba84b6402ed6daff",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "5b1d7779ff634f8b4049e70e99b5eaadf538e5a3d3a2aa5069b096c68cb79e5d",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470"
      ],
      "example_id": "571112ada58dae1900cd6bcc",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.48098153,
      "neural_order": [
        9,
        6,
        1,
        7,
        14,
        10,
        3,
        2,
        12,
        5,
        0,
        4,
        8,
        11,
        13,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "a23cd4d2bee21932d962d6ed6c83e5b77ba9e0c52224e3883318e831e2c7639c",
      "trial_index": 207
    },
    {
      "answer_sha256": "dcccf6aa990f7a107380834d327b7e6813a48ef071a47232d6e52963f5ed443b",
      "candidate_ids": [
        "571156152419e3140095559b",
        "56dddf4066d3e219004dad5f",
        "57269cc3dd62a815002e8b12",
        "5705eb3375f01819005e7764",
        "57274a1edd62a815002e9a9b",
        "5734025d4776f419006616c3",
        "57060eaf75f01819005e7910",
        "57280e1aff5b5019007d9bec",
        "57337ea24776f41900660bd0",
        "573407d7d058e614000b6813",
        "57377ec7c3c5551400e51f05",
        "57300e2604bcaa1900d770b7",
        "56e2042ecd28a01900c67c1e",
        "57111380a58dae1900cd6bd6",
        "57115ac550c2381900b54a77",
        "57282358ff5b5019007d9dde"
      ],
      "candidate_sha256": [
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "6d2627b1297d1087a3e3c836fbeab911593b2c58f4b2ae45d6c39dd4ba695bea",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "d03773b18a1be50107069edcf0e896834515c0cc39a5f2a1177372718099eaff",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "403b83bcd02b8204d4620813147424fdb54ec54909eb54e867858a16ee0028ec",
        "0771c41591ce859133397b70f7fe295ca178804c352b68cc34f3f35caa1c92ab",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d"
      ],
      "example_id": "57111380a58dae1900cd6bd6",
      "gated_correct": false,
      "gated_top": 9,
      "gold_index": 13,
      "lexical_correct": false,
      "lexical_order": [
        9,
        13,
        7,
        1,
        0,
        3,
        14,
        2,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.28615272,
      "neural_order": [
        13,
        9,
        1,
        5,
        7,
        14,
        12,
        4,
        0,
        3,
        11,
        2,
        8,
        6,
        10,
        15
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "0ea9d63bbe21e393cf88643ce8d333071e153877422ae22e47e980739757425a",
      "trial_index": 208
    },
    {
      "answer_sha256": "c24740142a6b7ccab4742066fc2aedccbac82265ba74a367cc066e833cd59408",
      "candidate_ids": [
        "57111428b654c5140001faff",
        "56e1bc3ae3433e1400423104",
        "56e16182e3433e1400422e28",
        "5711658e50c2381900b54ad5",
        "56e181d9e3433e1400422fa0",
        "5726542ff1498d1400e8dc28",
        "57265bdfdd62a815002e829e",
        "5726d8bd708984140094d35b",
        "572807802ca10214002d9bf8",
        "5725d34aec44d21400f3d639",
        "5725fb8138643c19005acf3f",
        "5726e5c95951b619008f81cb",
        "572f64ccb2c2fd14005680b7",
        "572867d72ca10214002da2fc",
        "572fbea404bcaa1900d76c5b",
        "5727dd2e4b864d1900163eba"
      ],
      "candidate_sha256": [
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a"
      ],
      "example_id": "57111428b654c5140001faff",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.50319574,
      "neural_order": [
        0,
        5,
        12,
        8,
        13,
        2,
        10,
        1,
        3,
        4,
        6,
        7,
        9,
        11,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "6036611b526bb5961bf6634f26647699cd86928a511a94c7b6079e99807c6abc",
      "trial_index": 209
    },
    {
      "answer_sha256": "dcdffe8795e2bda6bb587cc75f8a0400113b1d3e271b805de0ce02385941e920",
      "candidate_ids": [
        "57287c2bff5b5019007da26c",
        "5725c0f289a1e219009abdf2",
        "573403394776f419006616dd",
        "56de51244396321400ee27ef",
        "572a096e6aef0514001551e4",
        "5727f6723acd2414000df0e7",
        "572991943f37b319004784a1",
        "571114cfb654c5140001fb09",
        "572fd47fa23a5019007fca55",
        "57377083c3c5551400e51edf",
        "572fffb1b2c2fd14005686f9",
        "57115ff82419e314009555c3",
        "5727f05b4b864d1900164068",
        "571154c72419e31400955588",
        "5733a45d4776f41900660f23",
        "571c8539dd7acb1400e4c0e2"
      ],
      "candidate_sha256": [
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "291631d73d5dcdb57b0941017d7b1ab7a9501e869a33f4c995211b70c5487c88",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "809ab002069708b05d7022073e5692a5d269e21b863617ea0d1573256c8af15f",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d"
      ],
      "example_id": "571114cfb654c5140001fb09",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.30595895,
      "neural_order": [
        7,
        3,
        14,
        2,
        9,
        0,
        8,
        15,
        1,
        6,
        12,
        4,
        5,
        10,
        11,
        13
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "039f5365d6c7ce8f0eb2e556a94ab38cfe08756af507f4dc20008219f7864eb8",
      "trial_index": 210
    },
    {
      "answer_sha256": "3cb9c0283860785f4ba6e0719308b1ca50df2fb943e6e0d34e4ab7c1212da1fb",
      "candidate_ids": [
        "5711658e50c2381900b54ad5",
        "5711163bb654c5140001fb13",
        "572a07a86aef0514001551d2",
        "57280e1aff5b5019007d9bec",
        "5733db8dd058e614000b6428",
        "5725c071271a42140099d127",
        "57376a1bc3c5551400e51ec3",
        "5725c743ec44d21400f3d549",
        "5737a5931c456719005744e7",
        "572fc043a23a5019007fc95d",
        "5725cc38ec44d21400f3d5bb",
        "56e1d9fee3433e14004231cb",
        "5726b718dd62a815002e8dbe",
        "57274a1edd62a815002e9a9b",
        "5726f635dd62a815002e9656",
        "572757bef1498d1400e8f690"
      ],
      "candidate_sha256": [
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "7544f3f342f82852b6fd8a74d46285e0772381cc4cd0b61c47fea00590e8938e",
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a",
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "f3e994c58b1e4906937c035c80fb9519764a0f4f42ff12a50c7aeac3190ebd39",
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32"
      ],
      "example_id": "5711163bb654c5140001fb13",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        4,
        0,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.45764152,
      "neural_order": [
        1,
        14,
        3,
        4,
        12,
        10,
        13,
        2,
        0,
        5,
        6,
        7,
        8,
        9,
        11,
        15
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "66413baec79e68bf6464328571090b3c14a675d00f6215c93d4b94705512f337",
      "trial_index": 211
    },
    {
      "answer_sha256": "37f452ce99da940696706a268f6863c042db07a0361586492feffdda20881df7",
      "candidate_ids": [
        "56e1ec83cd28a01900c67c0a",
        "572fdb17b2c2fd140056851d",
        "572fad30a23a5019007fc86d",
        "572683075951b619008f7513",
        "5725d16aec44d21400f3d61b",
        "57111713a58dae1900cd6c00",
        "5730069004bcaa1900d7702d",
        "572872822ca10214002da374",
        "56e1bd4acd28a01900c67afc",
        "5727cff1ff5b5019007d95c4",
        "56e1a564cd28a01900c67a48",
        "5729f4e46aef051400155156",
        "57297547af94a219006aa45c",
        "57300911947a6a140053cfb6",
        "571155ae2419e31400955591",
        "571c8539dd7acb1400e4c0e2"
      ],
      "candidate_sha256": [
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "25160eaaf4cedb4feaad143ed4ed40d1a5aeba443aaa0327bbecb05f058934a7",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d"
      ],
      "example_id": "57111713a58dae1900cd6c00",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        4,
        14,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.36113959,
      "neural_order": [
        5,
        13,
        7,
        11,
        2,
        15,
        12,
        4,
        8,
        6,
        3,
        0,
        1,
        9,
        10,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "5de72816861da8116a2eb4562f47af160fec98368aed0afcad0f38d49e240d38",
      "trial_index": 212
    },
    {
      "answer_sha256": "7794355c26976b6a6084e2a5fec53749eeb28724f5a4957460b559c3dfd5b394",
      "candidate_ids": [
        "570d44abb3d812140066d5fd",
        "572949306aef051400154c68",
        "572905ce1d04691400778f83",
        "5733d5704776f4190066130e",
        "573028fa04bcaa1900d77287",
        "56de0f6a4396321400ee257f",
        "572864542ca10214002da2de",
        "57274ca8dd62a815002e9aa4",
        "572686ac5951b619008f75a9",
        "572978f91d046914007794d3",
        "57300911947a6a140053cfb6",
        "5705fec152bb89140068977a",
        "57274b35f1498d1400e8f5d4",
        "57114aceb654c5140001fc47",
        "5726581fdd62a815002e823a",
        "571117d4a58dae1900cd6c0a"
      ],
      "candidate_sha256": [
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "18017326aa99d6478493c6fd9c81f3144c242ec0a8dc61e8a64d3d1ab8e0507e",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032"
      ],
      "example_id": "571117d4a58dae1900cd6c0a",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.44831261,
      "neural_order": [
        15,
        10,
        3,
        5,
        14,
        12,
        13,
        8,
        9,
        0,
        1,
        2,
        4,
        6,
        7,
        11
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "734a429728cd866a62284f4e30432d57a3f85c0fda7862ef2bc00b321cd2508d",
      "trial_index": 213
    },
    {
      "answer_sha256": "4092065b691336ed78ccc893c1f961fd691b152d5786bf83b9d76360c82aee87",
      "candidate_ids": [
        "57287ccb2ca10214002da3da",
        "57111992b654c5140001fb43",
        "5728fd3c3f37b31900477f2b",
        "57286f373acd2414000df9db",
        "572755b7708984140094dc4d",
        "56e1aba0e3433e1400423094",
        "572944e03f37b319004781e1",
        "57309564069b5314008321a5",
        "572fce12a23a5019007fca11",
        "56de4b074396321400ee2793",
        "5730040f947a6a140053cf4c",
        "5726446a89cfff1900a8404d",
        "5727ea45ff5b5019007d986e",
        "5711607f2419e314009555cd",
        "57111ab8a58dae1900cd6c3c",
        "572a18a4af94a219006aa7df"
      ],
      "candidate_sha256": [
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "e72c427b33209e555b5fc09b718f5d3a1abc94dbdf4f37900c15e3a035c2a954"
      ],
      "example_id": "57111992b654c5140001fb43",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        9,
        14,
        11,
        5,
        15,
        0,
        2,
        3,
        4,
        6,
        7,
        8,
        10,
        12,
        13
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.02600144,
      "neural_order": [
        1,
        14,
        9,
        6,
        7,
        12,
        0,
        8,
        11,
        15,
        5,
        2,
        3,
        4,
        10,
        13
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "7c0d5981ad30cbf0e52b4530f507efe021573055a7d1b603abec3c5b7dbd9d6d",
      "trial_index": 214
    },
    {
      "answer_sha256": "a785bd15970f08609110cf2e9e187793b93005f0ef61ad8c7d9d51aaf75eef5e",
      "candidate_ids": [
        "5737821cc3c5551400e51f19",
        "572a1f086aef0514001552c0",
        "572647d0708984140094c14b",
        "5728855d3acd2414000dfa8d",
        "57060f3e75f01819005e7922",
        "5727cebc2ca10214002d96f0",
        "5729a26d6aef05140015505a",
        "572756265951b619008f886e",
        "57293f8a6aef051400154bde",
        "570d2d68fed7b91900d45cbb",
        "57111ab8a58dae1900cd6c3c",
        "57265aaf5951b619008f706b",
        "573776eec3c5551400e51ee9",
        "5730876a396df9190009617a",
        "5706143575f01819005e7950",
        "5725edfe38643c19005ace9f"
      ],
      "candidate_sha256": [
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "02473abeaeaf0a7ac1221ede2323c119c62182f573df438534fcb86c2c6943d4",
        "ce72fed7b084941e90079823ae33cc19adb21829e9f5aa99491ba4d96d782570",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275"
      ],
      "example_id": "57111ab8a58dae1900cd6c3c",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.3092285,
      "neural_order": [
        10,
        13,
        7,
        15,
        3,
        14,
        9,
        4,
        1,
        2,
        12,
        0,
        5,
        6,
        8,
        11
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "4bf3a5523161ed63a66f733a0f9ddb60321cd3725d4440d4c25a15c1c3b94655",
      "trial_index": 215
    },
    {
      "answer_sha256": "ddbf86b97884b84ec8388f9d3b2155fa7a6fe551ef039ecafe17d4728b68a298",
      "candidate_ids": [
        "56dde1d966d3e219004dad8d",
        "572867212ca10214002da2f2",
        "573005b9947a6a140053cf6a",
        "5725c071271a42140099d127",
        "571099b2b654c5140001f9b3",
        "571112ada58dae1900cd6bcc",
        "572991943f37b319004784a1",
        "57263ea0271a42140099d7c3",
        "5725f46289a1e219009ac0fa",
        "572f6a0ba23a5019007fc5eb",
        "57111b95a58dae1900cd6c50",
        "571a4b0f10f8ca1400304fd5",
        "572677e7708984140094c723",
        "57268f05dd62a815002e8990",
        "572a03086aef0514001551a2",
        "5725e1c4271a42140099d2d7"
      ],
      "candidate_sha256": [
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012"
      ],
      "example_id": "57111b95a58dae1900cd6c50",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.06024908,
      "neural_order": [
        10,
        4,
        5,
        0,
        9,
        2,
        1,
        11,
        13,
        3,
        6,
        7,
        8,
        12,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "27ea6245bef0a64edcb898ed8d83c6f648a7c371d880444771a7a2a07636fba6",
      "trial_index": 216
    },
    {
      "answer_sha256": "2bfbc118abcafe8c53ee6a46b79bed6638ae9eefda8dfd2d7948d02f40481739",
      "candidate_ids": [
        "5726a299dd62a815002e8b9e",
        "5711623e50c2381900b54ab9",
        "571090abb654c5140001f995",
        "572885023acd2414000dfa83",
        "56de3e414396321400ee26d8",
        "5706074552bb8914006897d4",
        "57112686b654c5140001fbd3",
        "57264865dd62a815002e8062",
        "5727c94bff5b5019007d954a",
        "572843304b864d1900164848",
        "5727b0892ca10214002d93e8",
        "57266c015951b619008f7237",
        "57269aa65951b619008f77ab",
        "5726bcde708984140094cfbf",
        "572ff626947a6a140053ce8e",
        "5737898f1c45671900574495"
      ],
      "candidate_sha256": [
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753"
      ],
      "example_id": "57112686b654c5140001fbd3",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.21600971,
      "neural_order": [
        6,
        1,
        11,
        9,
        12,
        0,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        13,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "495879efc580b069ad502ba63808c2c593c7858242c5a6c99bf0b04e72fb80e7",
      "trial_index": 217
    },
    {
      "answer_sha256": "ac863524b490b70f6188d256930fcb27bb9ba0dddaee66e7d9c5f0922c4c6c64",
      "candidate_ids": [
        "5725d183271a42140099d23d",
        "572a135daf94a219006aa79f",
        "57261dab38643c19005ad037",
        "571cda1bdd7acb1400e4c196",
        "56e1cbe2cd28a01900c67bac",
        "56e1c2eee3433e1400423134",
        "571126dfa58dae1900cd6cb2",
        "5726db5add62a815002e92d4",
        "572807802ca10214002d9bf8",
        "5733a45d4776f41900660f23",
        "5725bad5271a42140099d0bd",
        "572750df5951b619008f882f",
        "5727490bdd62a815002e9a82",
        "5705f09e75f01819005e77a4",
        "570d44abb3d812140066d5fd",
        "5705e4fe75f01819005e7704"
      ],
      "candidate_sha256": [
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440"
      ],
      "example_id": "571126dfa58dae1900cd6cb2",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.50631132,
      "neural_order": [
        6,
        10,
        9,
        7,
        13,
        3,
        4,
        5,
        12,
        0,
        1,
        2,
        8,
        11,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "4e94f702bac5b221c0ee3c484f05d1ceceac804222e8ad5123dc015716c4f5cf",
      "trial_index": 218
    },
    {
      "answer_sha256": "0e98cc0b1589b955ff17cd30084aafa0b0d2248fa64b1335bac484f39857ec04",
      "candidate_ids": [
        "571cc5c45efbb31900334dda",
        "573750f51c45671900574467",
        "57108198b654c5140001f937",
        "57108c95b654c5140001f979",
        "57283db6ff5b5019007d9fbc",
        "5729f69caf94a219006aa6f9",
        "57337ea24776f41900660bd0",
        "5728804b4b864d1900164a46",
        "57115f652419e314009555b9",
        "5727c94bff5b5019007d954a",
        "57111428b654c5140001faff",
        "57287ccb2ca10214002da3da",
        "571ce5055efbb31900334e2a",
        "572a1a5c6aef051400155284",
        "572a07c11d046914007796d5",
        "571127a5a58dae1900cd6cc4"
      ],
      "candidate_sha256": [
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6",
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "d5f4c4227c07c52cff10ca527f73efedf7e6c5aa9c01a3d9bf8a9468a6ae6939",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "f9d0ba77dd8dff4446935bfec391a5986633d68d8bdf2319ad27e1fd5890d87c",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27",
        "8394917ed3c6abd9f899d73ce839433e519acb0219643e9bfa9a48f92940892c"
      ],
      "example_id": "571127a5a58dae1900cd6cc4",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        8,
        13,
        3,
        10,
        0,
        4,
        11,
        1,
        2,
        5,
        6,
        7,
        9,
        12,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.20058698,
      "neural_order": [
        15,
        8,
        11,
        1,
        7,
        12,
        13,
        10,
        0,
        2,
        3,
        4,
        5,
        6,
        9,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "94e9622aacdd47c0a78eadc5cbd238b89c11f2266c3c9d296f082f39d9e6249c",
      "trial_index": 219
    },
    {
      "answer_sha256": "73701220d8e961c67499ad617bc0cbdde42ecb103316642d36a1f49015e26db7",
      "candidate_ids": [
        "572a1c943f37b319004786e1",
        "57377083c3c5551400e51edf",
        "5737a25ac3c5551400e51f51",
        "5728661e2ca10214002da2e8",
        "570607f575f01819005e78b4",
        "57269016708984140094ca41",
        "5726a1e5dd62a815002e8b86",
        "572658daf1498d1400e8dcac",
        "57337f6ad058e614000b5bcb",
        "57113ba6b654c5140001fc1f",
        "5728fd206aef05140015494c",
        "57263cfcec44d21400f3dc8d",
        "571135b8a58dae1900cd6d0e",
        "5729727baf94a219006aa437",
        "5705e26d75f01819005e76d4",
        "570961aa200fba1400367f15"
      ],
      "candidate_sha256": [
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c"
      ],
      "example_id": "571135b8a58dae1900cd6d0e",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        9,
        11,
        14,
        13,
        8,
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        10,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.2501937,
      "neural_order": [
        12,
        9,
        1,
        13,
        2,
        8,
        7,
        3,
        0,
        4,
        5,
        6,
        10,
        11,
        14,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "4472225269be1c9d3b5accfcf0896e77642f55fb4899023f18bc9f8648a28058",
      "trial_index": 220
    },
    {
      "answer_sha256": "8b5b9db0c13db24256c829aa364aa90c6d2eba318b9232a4ab9313b954d3555f",
      "candidate_ids": [
        "5706149552bb891400689880",
        "572f6ec7a23a5019007fc621",
        "57293e983f37b3190047818b",
        "5729f12e3f37b319004785e1",
        "57113639a58dae1900cd6d18",
        "57299ec43f37b3190047850d",
        "57335ddbd058e614000b592f",
        "5726462b708984140094c117",
        "5710eb6fb654c5140001fa17",
        "57294baaaf94a219006aa26b",
        "5726bc1add62a815002e8ea6",
        "572a0c541d046914007796f3",
        "5732b6b5328d981900602021",
        "57106644b654c5140001f8e5",
        "572a12386aef051400155234",
        "56e1b62ecd28a01900c67aa3"
      ],
      "candidate_sha256": [
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379"
      ],
      "example_id": "57113639a58dae1900cd6d18",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        3,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.57733581,
      "neural_order": [
        4,
        9,
        12,
        6,
        1,
        13,
        3,
        0,
        2,
        7,
        15,
        5,
        8,
        10,
        11,
        14
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "0fb4b37392e8f2c1464d8b440396ead83152c9cf4b8494487d6dc4b6123d8d8f",
      "trial_index": 221
    },
    {
      "answer_sha256": "1c9a0dbae92c05a957a04032ffb6973694863d20716547f7a02629d18dcd1b8d",
      "candidate_ids": [
        "5728742cff5b5019007da246",
        "5727e9523acd2414000def95",
        "5705ec1675f01819005e776e",
        "57268f05dd62a815002e8990",
        "5726722bdd62a815002e8528",
        "572867212ca10214002da2f2",
        "57340111d058e614000b677d",
        "56e1a564cd28a01900c67a48",
        "57261f9f271a42140099d4a9",
        "56e1b00ce3433e140042309e",
        "573314e3d058e614000b56f0",
        "5727c94bff5b5019007d954a",
        "5737898f1c45671900574495",
        "571136b8a58dae1900cd6d22",
        "572a13841d0469140077973b",
        "5727cff1ff5b5019007d95c4"
      ],
      "candidate_sha256": [
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6"
      ],
      "example_id": "571136b8a58dae1900cd6d22",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.56321175,
      "neural_order": [
        13,
        9,
        3,
        7,
        6,
        2,
        5,
        0,
        1,
        4,
        8,
        10,
        11,
        12,
        14,
        15
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "96e399cb77c1800eb8fd2c643c5eec9a1eb38e1f9e8bbf923d4978d3888ad33b",
      "trial_index": 222
    },
    {
      "answer_sha256": "344c8d52078007f8e88de7e01e3e4b2fd3bc9a156808875c2a11c8bd5dc0c25e",
      "candidate_ids": [
        "5737534ec3c5551400e51eab",
        "5728eef92ca10214002daab0",
        "57113ba6b654c5140001fc1f",
        "5725c95f38643c19005accf3",
        "5737a7351c456719005744f1",
        "5725d52f89a1e219009abf78",
        "571ce6655efbb31900334e34",
        "573081c2069b531400832133",
        "5730b7ce069b5314008322c3",
        "572989846aef051400154fc0",
        "572863363acd2414000df985",
        "5730a40f396df91900096234",
        "5726a5525951b619008f78dd",
        "5726516a708984140094c223",
        "57264845f1498d1400e8db0a",
        "5737a9afc3c5551400e51f61"
      ],
      "candidate_sha256": [
        "d2c301f3e303e0529f1b5862d9c77adabe2cb026b4683982583601544f992a66",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "66cbf5ef7ba18a058b7e88c4c66ba9e11f4ef0efae05c7495c2ee6e756648699",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31"
      ],
      "example_id": "57113ba6b654c5140001fc1f",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.49805715,
      "neural_order": [
        2,
        6,
        9,
        15,
        4,
        12,
        0,
        11,
        5,
        7,
        1,
        3,
        8,
        10,
        13,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "27aa03bae4933da7a1d2757b8dd7f0cf9a692ce75589a212b04857fd7a347721",
      "trial_index": 223
    },
    {
      "answer_sha256": "0fdab7eba705292fa998fc508e601740075b3756c8b524cd224845244229738c",
      "candidate_ids": [
        "5726ed6cf1498d1400e8f00c",
        "572faec7b2c2fd1400568333",
        "572a02483f37b3190047864b",
        "5730b255396df919000962b0",
        "57115bf350c2381900b54a93",
        "572867d72ca10214002da2fc",
        "5727c94bff5b5019007d954a",
        "57274e975951b619008f87f9",
        "57113be3a58dae1900cd6d28",
        "5730bb522461fd1900a9d011",
        "57115ac550c2381900b54a77",
        "573380e0d058e614000b5be9",
        "57373f80c3c5551400e51e8f",
        "572825a92ca10214002d9efe",
        "56e1c4fce3433e140042314e",
        "56e1cbe2cd28a01900c67bac"
      ],
      "candidate_sha256": [
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95"
      ],
      "example_id": "57113be3a58dae1900cd6d28",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        4,
        10,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.26373926,
      "neural_order": [
        8,
        10,
        4,
        12,
        3,
        5,
        11,
        2,
        1,
        14,
        7,
        15,
        0,
        6,
        9,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "9f1385d7cdf9f0c074d13886582bfe2d4a57bacec415f3571a087b95030fd181",
      "trial_index": 224
    },
    {
      "answer_sha256": "a7937b64b8caa58f03721bb6bacf5c78cb235febe0e70b1b84cd99541461a08e",
      "candidate_ids": [
        "56e1a38de3433e140042305c",
        "5733ef47d058e614000b662b",
        "57113c6da58dae1900cd6d32",
        "57263b1638643c19005ad333",
        "5705ffde52bb891400689784",
        "572943ab1d04691400779219",
        "56e1c9bfe3433e1400423192",
        "5725d183271a42140099d23d",
        "5733f8dc4776f419006615f7",
        "57287fec4b864d1900164a3c",
        "57274a1edd62a815002e9a9b",
        "572759dbdd62a815002e9b94",
        "57308ddc396df919000961a4",
        "57293f353f37b3190047819b",
        "572a0b0b6aef0514001551f6",
        "572ffa79a23a5019007fcbe9"
      ],
      "candidate_sha256": [
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "c2c1868add9d6aa35780c0c8d4492524306055ae928ea56b315e6e9e6969d494",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "dea705fdf75f529664435816398522b553e31b0965a1287f3375c31a630e304e",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862"
      ],
      "example_id": "57113c6da58dae1900cd6d32",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        12,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.45142337,
      "neural_order": [
        2,
        15,
        1,
        8,
        9,
        6,
        12,
        0,
        5,
        3,
        10,
        13,
        4,
        7,
        11,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "22476c35d8aac3e195b55d44a4b8499a192ae69321a46cf99f57c286c793df67",
      "trial_index": 225
    },
    {
      "answer_sha256": "ee0df97cece494b174c6844cfd00255723152d5555215073611b8ad673c4e749",
      "candidate_ids": [
        "57284e9fff5b5019007da150",
        "57094a79efce8f15003a7dc4",
        "572ffce5a23a5019007fcc15",
        "57281940ff5b5019007d9d44",
        "57113f83b654c5140001fc29",
        "56e190bce3433e1400422fc8",
        "572f6ec7a23a5019007fc621",
        "572820512ca10214002d9e72",
        "572745c6708984140094db99",
        "573361404776f4190066093c",
        "56de3dbacffd8e1900b4b6d2",
        "5727d6faff5b5019007d969a",
        "57282dfb4b864d1900164668",
        "5729fd111d046914007796a5",
        "57274e0d708984140094dbe5",
        "570d4606b3d812140066d619"
      ],
      "candidate_sha256": [
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "a6b7ec7938b966c86b4cc72fa2059dcdbd60bc13208210cff0105baccb082b8b",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757"
      ],
      "example_id": "57113f83b654c5140001fc29",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        2,
        9,
        6,
        5,
        13,
        1,
        0,
        3,
        7,
        8,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.41262587,
      "neural_order": [
        4,
        9,
        2,
        10,
        6,
        3,
        15,
        7,
        14,
        8,
        0,
        1,
        5,
        11,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "db5c38470f8c89b6f7dfc2916e7bf3be472dc3be82d0574212072e5aada5a01f",
      "trial_index": 226
    },
    {
      "answer_sha256": "792dc3f0dc1dc2d0a58fbb65ec31c33f483e4941bd8e7de669b0a8a8628653c7",
      "candidate_ids": [
        "572a1dbb3f37b319004786f5",
        "572a0a391d046914007796df",
        "57113639a58dae1900cd6d18",
        "571142b3a58dae1900cd6d5a",
        "570614ff52bb89140068988a",
        "56de51c64396321400ee27f7",
        "57097c8fed30961900e841f2",
        "5705fec152bb89140068977a",
        "572a02483f37b3190047864b",
        "572a07fc6aef0514001551dc",
        "5727aeac3acd2414000de987",
        "57287fec4b864d1900164a3c",
        "57096e1ced30961900e84102",
        "57273a465951b619008f86ff",
        "5730131c947a6a140053d052",
        "5728128cff5b5019007d9c76"
      ],
      "candidate_sha256": [
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "928990f37f000a25d64f03019258af004a5c8124a88add2049f7910fea98667f"
      ],
      "example_id": "571142b3a58dae1900cd6d5a",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        2,
        14,
        9,
        0,
        1,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.20468923,
      "neural_order": [
        3,
        2,
        5,
        8,
        9,
        11,
        0,
        13,
        4,
        15,
        1,
        6,
        7,
        10,
        12,
        14
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "e69c3c37d094e88d2422af36340708636ce483b2fcea59d6c5bea7cf9afc9413",
      "trial_index": 227
    },
    {
      "answer_sha256": "5a409129019155e294f44f3dfa3ae89fa03a43b3f55adfebc99beaf6b8b8b4ec",
      "candidate_ids": [
        "571c83f3dd7acb1400e4c0d8",
        "57107a3ea58dae1900cd69de",
        "573009a004bcaa1900d7704f",
        "573382d24776f41900660c37",
        "57286ec63acd2414000df9d1",
        "572ff83ab2c2fd1400568681",
        "571cd5b1dd7acb1400e4c178",
        "5727d0f73acd2414000ded13",
        "5726da89dd62a815002e92b2",
        "57264e455951b619008f6f65",
        "57092322efce8f15003a7db0",
        "572ff890a23a5019007fcbcd",
        "5725c0f289a1e219009abdf2",
        "5705eccb52bb8914006896b8",
        "571144d1a58dae1900cd6d6e",
        "5726edecdd62a815002e957f"
      ],
      "candidate_sha256": [
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "bc5f1c2743b9de3bbf84078611d839665374c71f9f5ce23ea41ca3d84c8793ba",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "c59e7593c862a0c68cee17ab74270592bac6538a668da2cdfed71c7db8087fec"
      ],
      "example_id": "571144d1a58dae1900cd6d6e",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        11,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.61524287,
      "neural_order": [
        14,
        11,
        0,
        5,
        2,
        3,
        1,
        6,
        12,
        4,
        7,
        8,
        9,
        10,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "16a38e0bcc2683462d5f1e545867665eb1d431e16f1d09349f4abde55cd60b0a",
      "trial_index": 228
    },
    {
      "answer_sha256": "8ca1ad21404d09c57fafdf02381bf37f24345e1ed5a0eeaadc38efae5b6cecef",
      "candidate_ids": [
        "5706149552bb891400689880",
        "572991943f37b319004784a1",
        "572987e46aef051400154fa2",
        "5725f6e138643c19005acf19",
        "572fce12a23a5019007fca11",
        "57302bd0b2c2fd14005689db",
        "5725d34089a1e219009abf50",
        "56e1a564cd28a01900c67a48",
        "5728742cff5b5019007da246",
        "5727ec062ca10214002d99b6",
        "571166352419e314009555f1",
        "571cda1bdd7acb1400e4c196",
        "570614ff52bb89140068988a",
        "5733f1784776f41900661575",
        "572fdbb004bcaa1900d76dd9",
        "57114667a58dae1900cd6d80"
      ],
      "candidate_sha256": [
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea"
      ],
      "example_id": "57114667a58dae1900cd6d80",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        10,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.42027145,
      "neural_order": [
        15,
        10,
        11,
        7,
        3,
        4,
        6,
        8,
        9,
        0,
        1,
        2,
        5,
        12,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "09c77c0b38dee548b94fd42ea632212facb1f139225b2de1c89959049ab4deec",
      "trial_index": 229
    },
    {
      "answer_sha256": "7b69759630f869f2723875f873935fed29d2d12b10ef763c1c33b8e0004cb405",
      "candidate_ids": [
        "5727f2714b864d1900164072",
        "5730a314069b5314008321eb",
        "5727e9523acd2414000def95",
        "5728f6446aef0514001548e4",
        "57265f605951b619008f70db",
        "57340b1bd058e614000b6869",
        "57309bfb8ab72b1400f9c5e6",
        "5711475ca58dae1900cd6d8b",
        "57060eaf75f01819005e7910",
        "572905ce1d04691400778f83",
        "572699db5951b619008f7799",
        "57284e9fff5b5019007da150",
        "573382d24776f41900660c37",
        "57096b66200fba1400367fa7",
        "57115dbe2419e314009555a5",
        "570966e0200fba1400367f4f"
      ],
      "candidate_sha256": [
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "40206469ec21c4f98d7c72cb3c313a8bdfc89251b79140b6c63a48b1afe123d0",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7"
      ],
      "example_id": "5711475ca58dae1900cd6d8b",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.48930766,
      "neural_order": [
        7,
        14,
        10,
        4,
        5,
        12,
        8,
        2,
        0,
        1,
        3,
        6,
        9,
        11,
        13,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "c49d132f857d119bd8a33a60a4990ccd1e3c1b08d397f47cf0e50ba7cb73f6b8",
      "trial_index": 230
    },
    {
      "answer_sha256": "2594a577866dacafccfa2e80967a39de585e34a921076127bb0f1ded93c7bf62",
      "candidate_ids": [
        "5727cff1ff5b5019007d95c4",
        "5726da89dd62a815002e92b2",
        "5726a1e5dd62a815002e8b86",
        "571c83f3dd7acb1400e4c0d8",
        "572a064a3f37b3190047865d",
        "571077ecb654c5140001f909",
        "57097051ed30961900e84132",
        "5726487b5951b619008f6edd",
        "571097baa58dae1900cd6a98",
        "5730b8ca8ab72b1400f9c704",
        "57283db6ff5b5019007d9fbc",
        "57302faa04bcaa1900d77311",
        "571c8198dd7acb1400e4c0ce",
        "56de4a89cffd8e1900b4b7bd",
        "57299ec43f37b3190047850d",
        "5711488ab654c5140001fc3d"
      ],
      "candidate_sha256": [
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "395b1cfae86dbeb34c8873339b090ededf002969f69503f5b785bd284dc9d2ca",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16"
      ],
      "example_id": "5711488ab654c5140001fc3d",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.61967778,
      "neural_order": [
        15,
        13,
        12,
        3,
        1,
        7,
        0,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "9f2c505dda72de02a05df7998150db6c8913f47c2688879ec86607a5bc14c120",
      "trial_index": 231
    },
    {
      "answer_sha256": "9f35ead4ea3d503db15339a94adffd0218270086175e86bf6ae98872ea42aec0",
      "candidate_ids": [
        "56e202e9e3433e1400423240",
        "5706149552bb891400689880",
        "571ce6655efbb31900334e34",
        "5726219489a1e219009ac2d0",
        "57097c8fed30961900e841f2",
        "572872822ca10214002da374",
        "572ff932a23a5019007fcbd5",
        "57302efe04bcaa1900d772f5",
        "57269aa65951b619008f77ab",
        "5725d34aec44d21400f3d639",
        "57114aceb654c5140001fc47",
        "570960cf200fba1400367f01",
        "57286d4f2ca10214002da328",
        "57060cc352bb89140068980e",
        "5725b5a689a1e219009abd28",
        "572881704b864d1900164a50"
      ],
      "candidate_sha256": [
        "03eb6accc47f8f37613f49787d90a7cde1808cd11e3eca574b283b3e63a9aa56",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa"
      ],
      "example_id": "57114aceb654c5140001fc47",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.60305323,
      "neural_order": [
        10,
        3,
        7,
        0,
        8,
        11,
        6,
        2,
        12,
        1,
        4,
        5,
        9,
        13,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "31368fe8bd909e4522dc6cb24b595a63e5ffba93e2ffca5b0520a2499d20c713",
      "trial_index": 232
    },
    {
      "answer_sha256": "ccbb598ae6da5326a9d56cc540865a2b47b958c84a9ca7de670cffcaf5537ae5",
      "candidate_ids": [
        "56e17644e3433e1400422f40",
        "5728284e3acd2414000df5cd",
        "572869b84b864d19001649ae",
        "573362b94776f41900660974",
        "572fdd03a23a5019007fca9d",
        "572faf74b2c2fd1400568347",
        "5725c071271a42140099d127",
        "5726f868dd62a815002e9684",
        "572825a92ca10214002d9efe",
        "57114b1a2419e31400955575",
        "5726e179dd62a815002e93ae",
        "572a1c943f37b319004786e1",
        "57274a1edd62a815002e9a9b",
        "5733d249d058e614000b6331",
        "5728855d3acd2414000dfa8d",
        "5725bbec271a42140099d0d1"
      ],
      "candidate_sha256": [
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac"
      ],
      "example_id": "57114b1a2419e31400955575",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.51543133,
      "neural_order": [
        9,
        15,
        8,
        6,
        0,
        3,
        13,
        12,
        1,
        2,
        4,
        5,
        7,
        10,
        11,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "82e7994fd5d6ad65c652330f6cae8ea6d41f73ae1f1a83335bbfded27ee30edf",
      "trial_index": 233
    },
    {
      "answer_sha256": "73d41dde3a7f1321da6dfa98cbb509abf006d88ce02d5af0cfc67006303a0778",
      "candidate_ids": [
        "57285841ff5b5019007da19b",
        "5730088e947a6a140053cfac",
        "572a058aaf94a219006aa751",
        "5728742cff5b5019007da246",
        "5726a1e5dd62a815002e8b86",
        "57114dfb50c2381900b54a53",
        "5727cb4b2ca10214002d9676",
        "57287d4a2ca10214002da3e4",
        "57265700dd62a815002e820e",
        "573088da069b53140083216b",
        "57300a25a23a5019007fcc97",
        "5730285a04bcaa1900d7726d",
        "56de4a89cffd8e1900b4b7bd",
        "56de1645cffd8e1900b4b5d0",
        "5727db85ff5b5019007d96fc",
        "571097baa58dae1900cd6a98"
      ],
      "candidate_sha256": [
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "4b2a2a7e8bce520e5e42162e14db64c90808895b36ef2edfc527bbc3df66971c",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "773a455641daa5132327b1bb66496206257ea78ed57c4ba20dd7f45d1811f08a",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2"
      ],
      "example_id": "57114dfb50c2381900b54a53",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        2,
        11,
        0,
        7,
        1,
        3,
        4,
        6,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.55173922,
      "neural_order": [
        5,
        15,
        11,
        2,
        10,
        6,
        7,
        13,
        0,
        1,
        3,
        4,
        8,
        9,
        12,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "3fa3812d50fdb50a66fe66a01e1c4e55d3afc9c2d93b5834809fe82f3800650e",
      "trial_index": 234
    },
    {
      "answer_sha256": "2205bfae1ca81e96184ebee333804db30226973d5cbd5d0d6c31b29c363ba033",
      "candidate_ids": [
        "5729f12e3f37b319004785e1",
        "5730208fa23a5019007fcded",
        "56e1a38de3433e140042305c",
        "5733fc6ed058e614000b670f",
        "57332562d058e614000b5730",
        "572a05eb3f37b31900478653",
        "572a13841d0469140077973b",
        "57106d2fb654c5140001f8ef",
        "57096e1ced30961900e84102",
        "57377862c3c5551400e51ef3",
        "57268c01dd62a815002e8912",
        "57113c6da58dae1900cd6d32",
        "57114e8d50c2381900b54a5b",
        "57263dcd89a1e219009ac5a2",
        "56e1b355e3433e14004230b0",
        "572864542ca10214002da2de"
      ],
      "candidate_sha256": [
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741"
      ],
      "example_id": "57114e8d50c2381900b54a5b",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.51858252,
      "neural_order": [
        12,
        11,
        14,
        2,
        5,
        3,
        4,
        1,
        6,
        10,
        7,
        0,
        8,
        9,
        13,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "f0390ac2ff041ef0ee3c839460653e6b947b03c2ff4485d0cfdd7841c90ec785",
      "trial_index": 235
    },
    {
      "answer_sha256": "ebc59256fb0e406a9e3dc93735f35a6a12175d528ab4f24492c23a64bd41f661",
      "candidate_ids": [
        "572fac17947a6a140053cb54",
        "57274a8ff1498d1400e8f5bc",
        "56de15dbcffd8e1900b4b5c8",
        "572fc659b2c2fd1400568447",
        "57284618ff5b5019007da0a8",
        "5711607f2419e314009555cd",
        "571cd11add7acb1400e4c16e",
        "5728fd3c3f37b31900477f2b",
        "570961aa200fba1400367f15",
        "57274f49f1498d1400e8f61e",
        "57285841ff5b5019007da19b",
        "57376828c3c5551400e51eb9",
        "573382d24776f41900660c37",
        "57294279af94a219006aa209",
        "57268882f1498d1400e8e306",
        "57114f0050c2381900b54a65"
      ],
      "candidate_sha256": [
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "8075afe5649850130d6277b4d42a263b335970200c99b9786462feb745bd73d5",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78"
      ],
      "example_id": "57114f0050c2381900b54a65",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.25004838,
      "neural_order": [
        15,
        5,
        14,
        13,
        11,
        2,
        0,
        4,
        7,
        3,
        12,
        8,
        10,
        1,
        6,
        9
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "8facc0330fa79cf9a8624c04562cd2a3c44067911881ca4a42173090cc587f6f",
      "trial_index": 236
    },
    {
      "answer_sha256": "0850cd408214bd3bc659d423073efa3281a47e3b29a3d1234a5c23b4e4c0b128",
      "candidate_ids": [
        "56e17e6ee3433e1400422f7f",
        "5727cebc2ca10214002d96f0",
        "5710e9f8a58dae1900cd6b30",
        "57261dab38643c19005ad037",
        "5730b6592461fd1900a9cfcf",
        "572fad30a23a5019007fc86d",
        "570d26efb3d812140066d48f",
        "571153422419e3140095557d",
        "5711119cb654c5140001fae3",
        "5728667eff5b5019007da1fc",
        "57094ca7efce8f15003a7dd6",
        "56de0f6a4396321400ee257f",
        "57373f80c3c5551400e51e8f",
        "572821274b864d1900164510",
        "570d4606b3d812140066d619",
        "5728e8212ca10214002daa6c"
      ],
      "candidate_sha256": [
        "00b76f48835bfc2434242a5dea1681e9e4d9c363cc74180aa0b9b9fec6f02bef",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "2ef6bc8b598c3ae95dea517f30b72d286a3c1ea1c56e1f73f6bd8f2ab9cb63ef",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea"
      ],
      "example_id": "571153422419e3140095557d",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.44630268,
      "neural_order": [
        7,
        13,
        1,
        2,
        0,
        11,
        12,
        4,
        14,
        8,
        5,
        3,
        6,
        9,
        10,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "2d46dc156c01dc88c1edaaa173e81ebefe4e843bfcaa6b62939f402fc77c871c",
      "trial_index": 237
    },
    {
      "answer_sha256": "d39385995c1ca219739d65c28266ecf94b26c6c73ad9f818125d860c58ad1675",
      "candidate_ids": [
        "5733f062d058e614000b6633",
        "572a1046af94a219006aa78d",
        "572fd6aa04bcaa1900d76d9f",
        "57265360dd62a815002e819a",
        "57111380a58dae1900cd6bd6",
        "57107932a58dae1900cd69ca",
        "5711541350c2381900b54a6f",
        "572f60f4947a6a140053c8ea",
        "572869b84b864d19001649ae",
        "57339902d058e614000b5e70",
        "57265642f1498d1400e8dc68",
        "571ce3745efbb31900334e20",
        "5725bcb6271a42140099d0eb",
        "571c9074dd7acb1400e4c100",
        "57287ccb2ca10214002da3da",
        "57267b3c5951b619008f7423"
      ],
      "candidate_sha256": [
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "069f593a47b1c7d532212a62aba1e27bdd08d0259cb3e0a96bc73f362f4036e4",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915"
      ],
      "example_id": "5711541350c2381900b54a6f",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.37577198,
      "neural_order": [
        6,
        13,
        15,
        12,
        1,
        0,
        11,
        10,
        4,
        5,
        3,
        8,
        7,
        9,
        2,
        14
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "7a2b47ce6fa004b9b8a3cfcee1f713d7d92d418edab2d9f4ebec152fb5d39170",
      "trial_index": 238
    },
    {
      "answer_sha256": "14031f5f9531b1557c3dbc93c380c7727f0f6229dfbc1d2032f247b730694b4d",
      "candidate_ids": [
        "5728151b4b864d1900164428",
        "57300761947a6a140053cf9a",
        "5733f062d058e614000b6633",
        "56e1b355e3433e14004230b0",
        "56e1d9fee3433e14004231cb",
        "5733a560d058e614000b5f77",
        "572744aff1498d1400e8f588",
        "571154c72419e31400955588",
        "573796edc3c5551400e51f33",
        "571166352419e314009555f1",
        "57097141200fba1400367ff9",
        "57264fe65951b619008f6fa1",
        "570953a7efce8f15003a7dfe",
        "5729517d6aef051400154cc8",
        "5727d88b4b864d1900163e5e",
        "572fd73e947a6a140053cd32"
      ],
      "candidate_sha256": [
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "4aaa6faf86477d3be34fa2e2263e88b5c583216a96351af8fd82b9f4924bd81e",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379"
      ],
      "example_id": "571154c72419e31400955588",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.1646896,
      "neural_order": [
        7,
        9,
        3,
        5,
        6,
        1,
        2,
        4,
        11,
        14,
        0,
        8,
        10,
        12,
        13,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "1be94c29c56319914f4e29eaefc6908ee19c55297c0f2bdde3e5bcbb6db48b29",
      "trial_index": 239
    },
    {
      "answer_sha256": "31d3aeb706c1c046b0c1551222e2444e227a46aa925e388bad364d9a970e7f09",
      "candidate_ids": [
        "57379ed81c456719005744d5",
        "5727f6723acd2414000df0e7",
        "5730aa52069b53140083221d",
        "5729727baf94a219006aa437",
        "5730909d8ab72b1400f9c58a",
        "56de3cd0cffd8e1900b4b6be",
        "571155ae2419e31400955591",
        "572f6a0ba23a5019007fc5eb",
        "57273a465951b619008f86ff",
        "573368044776f41900660a29",
        "572812e74b864d19001643ce",
        "57332a734776f41900660726",
        "5726581fdd62a815002e823a",
        "572825a92ca10214002d9efe",
        "571c9348dd7acb1400e4c114",
        "572fd8efb2c2fd14005684fa"
      ],
      "candidate_sha256": [
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "6638daf3357f97afba40505b33af08c76b24c6d5399a55772351804230526ef8",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011"
      ],
      "example_id": "571155ae2419e31400955591",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.32635431,
      "neural_order": [
        6,
        14,
        12,
        0,
        7,
        11,
        4,
        5,
        1,
        8,
        15,
        10,
        2,
        3,
        9,
        13
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "668c3925b8764f186d77ef03e2bfea61c97a1218fe9f3c5f9e768a0d767a94f1",
      "trial_index": 240
    },
    {
      "answer_sha256": "9bb09914b117aaa92837348557e3deab4ccb38d7e94e0a29189091aad0a364f2",
      "candidate_ids": [
        "5728fb002ca10214002dab6a",
        "57302bd0b2c2fd14005689db",
        "572fc659b2c2fd1400568447",
        "571156152419e3140095559b",
        "5737a7351c456719005744f1",
        "5710e9f8a58dae1900cd6b30",
        "5725bad5271a42140099d0bd",
        "572a064a3f37b3190047865d",
        "572fc8a904bcaa1900d76d1f",
        "5727b8df3acd2414000dea99",
        "57286ab72ca10214002da31e",
        "57267b3c5951b619008f7423",
        "57264e66dd62a815002e811a",
        "5727f6723acd2414000df0e7",
        "573749741c4567190057445d",
        "5727d88b4b864d1900163e5e"
      ],
      "candidate_sha256": [
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "286230116fa743f77871c0721b3874ae2889664d4113752586c5f53755626f21",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "4aaa6faf86477d3be34fa2e2263e88b5c583216a96351af8fd82b9f4924bd81e"
      ],
      "example_id": "571156152419e3140095559b",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.50200659,
      "neural_order": [
        3,
        13,
        6,
        2,
        10,
        9,
        11,
        4,
        0,
        1,
        5,
        7,
        8,
        12,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "ac5b76eeb2067fe404c19f5e947496638d123527e846f8c7f9c12d6a566b0163",
      "trial_index": 241
    },
    {
      "answer_sha256": "792dc3f0dc1dc2d0a58fbb65ec31c33f483e4941bd8e7de669b0a8a8628653c7",
      "candidate_ids": [
        "571c879bdd7acb1400e4c0ec",
        "5726c19add62a815002e8f8a",
        "572869b84b864d19001649ae",
        "572ff5fcb2c2fd1400568657",
        "572f6a0ba23a5019007fc5eb",
        "57263ea0271a42140099d7c3",
        "56de4c324396321400ee27ab",
        "56de10b44396321400ee2593",
        "56e1bd4acd28a01900c67afc",
        "5733ffa7d058e614000b674f",
        "573408ef4776f41900661757",
        "57115ac550c2381900b54a77",
        "5728fb6a1d04691400778ef5",
        "572a213e6aef0514001552ee",
        "5726d8bd708984140094d35b",
        "5725fe63ec44d21400f3d7dd"
      ],
      "candidate_sha256": [
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "25160eaaf4cedb4feaad143ed4ed40d1a5aeba443aaa0327bbecb05f058934a7",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "52dde7db662fc424e04bd1ed3cd943e7d11c0868e7f691a8c6cac05cb561b057",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd"
      ],
      "example_id": "57115ac550c2381900b54a77",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.69076432,
      "neural_order": [
        11,
        3,
        4,
        0,
        6,
        2,
        7,
        10,
        13,
        9,
        15,
        5,
        1,
        8,
        12,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "fe3ffadb6fe4f7518b28bc2fb8baabbce158ed2f92b98a7cbf7e06fdb923bd15",
      "trial_index": 242
    },
    {
      "answer_sha256": "d740156a97f113b0a04160efef28910c00d4e54a6b067fee776cd8b903f625c1",
      "candidate_ids": [
        "5725ec8289a1e219009ac0ae",
        "5726f48df1498d1400e8f0da",
        "57115b2850c2381900b54a7f",
        "57264cac708984140094c1b3",
        "572864542ca10214002da2de",
        "5727d6faff5b5019007d969a",
        "5726a46cdd62a815002e8bd0",
        "57293c246aef051400154bb8",
        "5726577f708984140094c301",
        "572871bc4b864d1900164a04",
        "57265ceddd62a815002e82b8",
        "56e1aff7cd28a01900c67a68",
        "57264d58f1498d1400e8db7a",
        "572849b4ff5b5019007da0f2",
        "56de3e414396321400ee26d8",
        "572677e7708984140094c723"
      ],
      "candidate_sha256": [
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db"
      ],
      "example_id": "57115b2850c2381900b54a7f",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        14,
        4,
        1,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.38408069,
      "neural_order": [
        2,
        13,
        3,
        9,
        0,
        10,
        4,
        11,
        6,
        14,
        1,
        5,
        7,
        8,
        12,
        15
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "c7cff089e107a288df7ccffb975cefb1c9f16587326fe41da6b62b196fceabc2",
      "trial_index": 243
    },
    {
      "answer_sha256": "73aeea990667008cfe3f454ec468bac7dccf6e512461b630998b1d7935c30157",
      "candidate_ids": [
        "57264845f1498d1400e8db0a",
        "57281d494b864d19001644be",
        "57268527708984140094c8bf",
        "572fce12a23a5019007fca11",
        "57297d421d046914007794e5",
        "573312fcd058e614000b56e6",
        "56e20a27cd28a01900c67c24",
        "570d4e6cb3d812140066d66d",
        "5726bcde708984140094cfbf",
        "5725cb33271a42140099d1db",
        "5725f7cd38643c19005acf23",
        "57115b8b50c2381900b54a89",
        "5706149552bb891400689880",
        "57294279af94a219006aa209",
        "57113ba6b654c5140001fc1f",
        "5737534ec3c5551400e51eab"
      ],
      "candidate_sha256": [
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "576db96228f229ebd5188bfb187b0285ce8e7a6eae31bd74cad56712e9e871a5",
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "d2c301f3e303e0529f1b5862d9c77adabe2cb026b4683982583601544f992a66"
      ],
      "example_id": "57115b8b50c2381900b54a89",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.48663611,
      "neural_order": [
        11,
        14,
        2,
        13,
        1,
        0,
        6,
        5,
        3,
        4,
        7,
        8,
        9,
        10,
        12,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "1836a4fd3fab6614bd8b3df3b7735dcc264da2a32a15799101fdd0c578cead96",
      "trial_index": 244
    },
    {
      "answer_sha256": "8aa56cb291253c086f06872989f6f6bd4debd6731092b62c14822b7acca9249a",
      "candidate_ids": [
        "57266193dd62a815002e832a",
        "57377083c3c5551400e51edf",
        "57265700dd62a815002e820e",
        "5710f2e2a58dae1900cd6b72",
        "5725e1c4271a42140099d2d7",
        "57284618ff5b5019007da0a8",
        "572f5533a23a5019007fc55c",
        "572826634b864d19001645be",
        "571cd5b1dd7acb1400e4c178",
        "56e1b754cd28a01900c67abc",
        "56de49564396321400ee277a",
        "5733f9fa4776f4190066161f",
        "57115bf350c2381900b54a93",
        "5725d52f89a1e219009abf78",
        "572f58d9a23a5019007fc57d",
        "572f7588947a6a140053c984"
      ],
      "candidate_sha256": [
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "7c28011f4adb5bc9518e20aa534136a5ab7d9e920f0263cb0d482a6ce7859ede",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "46a60c1fb7a3dc1eb8d8008909c9080e1b010b9c75f648c33a07f6ce1e997157",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a"
      ],
      "example_id": "57115bf350c2381900b54a93",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        1,
        4,
        0,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.38851189,
      "neural_order": [
        12,
        3,
        14,
        5,
        8,
        4,
        9,
        1,
        6,
        10,
        0,
        2,
        7,
        11,
        13,
        15
      ],
      "neural_top": 12,
      "partition": "calibration",
      "question_sha256": "031487d129d8e1238dc5816446a2953a36783722c271bc76f2d42a6ce0edea0d",
      "trial_index": 245
    },
    {
      "answer_sha256": "e67be447ad6e8744c004fbaf59905f813f218ed33508a5f6a46c12a28e175128",
      "candidate_ids": [
        "57302e45947a6a140053d22e",
        "57115c7450c2381900b54a9d",
        "5733314e4776f4190066076a",
        "57265526708984140094c2bd",
        "57287d4a2ca10214002da3e4",
        "57097c8fed30961900e841f2",
        "571135b8a58dae1900cd6d0e",
        "572ffb02b2c2fd14005686b7",
        "57297ed93f37b3190047845f",
        "5726c5a9f1498d1400e8eac4",
        "5729f8516aef05140015516c",
        "5729fa40af94a219006aa70f",
        "57302faa04bcaa1900d77311",
        "573749741c4567190057445d",
        "5733a45d4776f41900660f23",
        "5726887e708984140094c917"
      ],
      "candidate_sha256": [
        "2d5af9a4d02fc579e6e15f098bcffbcc6459cdee51fd51b38ea0075eee7a728a",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd"
      ],
      "example_id": "57115c7450c2381900b54a9d",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        6,
        14,
        0,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.12780085,
      "neural_order": [
        1,
        6,
        7,
        15,
        2,
        8,
        9,
        5,
        10,
        14,
        11,
        4,
        0,
        3,
        12,
        13
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "0d9d42ea1bf487963395d49fe4c2a27e686aefa503011d52ba310ee7d8dd7716",
      "trial_index": 246
    },
    {
      "answer_sha256": "00b2578b85e1356408648586fd6b0e6894aa3957437fe82fdc9d26b978b92714",
      "candidate_ids": [
        "57094a79efce8f15003a7dc4",
        "5728804b4b864d1900164a46",
        "5725b5a689a1e219009abd28",
        "571c879bdd7acb1400e4c0ec",
        "57284618ff5b5019007da0a8",
        "5727c69d3acd2414000dec13",
        "573099ee8ab72b1400f9c5dc",
        "57271c235951b619008f860b",
        "572fd8efb2c2fd14005684fa",
        "57115dbe2419e314009555a5",
        "5725d16aec44d21400f3d61b",
        "57265360dd62a815002e819a",
        "57293c246aef051400154bb8",
        "572fc78d04bcaa1900d76d07",
        "571166352419e314009555f1",
        "57284d484b864d1900164900"
      ],
      "candidate_sha256": [
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "f9d0ba77dd8dff4446935bfec391a5986633d68d8bdf2319ad27e1fd5890d87c",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "63083aa84294da2de8401c0b772521bbfe426e2587fa80448b3830f78d5bb8ae",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf"
      ],
      "example_id": "57115dbe2419e314009555a5",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.11505039,
      "neural_order": [
        9,
        14,
        6,
        1,
        7,
        4,
        3,
        15,
        0,
        11,
        2,
        5,
        8,
        10,
        12,
        13
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "2038c36d9ec771dc7e8a08f77ff889128c23352a07704e097bb95f8f0a844946",
      "trial_index": 247
    },
    {
      "answer_sha256": "cdc8d01a6ab341278ee6ff93d8c004b649197d74da145b63c10f93d442553239",
      "candidate_ids": [
        "571a4ead10f8ca1400304fdd",
        "572667e2f1498d1400e8de90",
        "571166352419e314009555f1",
        "57115e532419e314009555af",
        "5728223cff5b5019007d9dc4",
        "57094b4f9928a814004714f6",
        "572754fff1498d1400e8f660",
        "5729f8516aef05140015516c",
        "56de51244396321400ee27ef",
        "5726241189a1e219009ac2de",
        "573735e8c3c5551400e51e71",
        "5725d16aec44d21400f3d61b",
        "57264b3edd62a815002e80aa",
        "57287c142ca10214002da3d0",
        "5733fe73d058e614000b673d",
        "571142b3a58dae1900cd6d5a"
      ],
      "candidate_sha256": [
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "61af1803b0730548c2562a214669d95520c7f20b1ca36c0cd3e8cbc90c590f5b",
        "80e800db40294efc728c4dcbf06c36437000cef741a99c5f9ed99d6d0756badc",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a"
      ],
      "example_id": "57115e532419e314009555af",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        15,
        10,
        13,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        14
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.31027525,
      "neural_order": [
        3,
        15,
        0,
        9,
        2,
        1,
        8,
        10,
        12,
        4,
        13,
        14,
        11,
        5,
        6,
        7
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "be5923a926b8d39330a3e040d10c778b22504077f84393d9d8e07ecbaeab0c82",
      "trial_index": 248
    },
    {
      "answer_sha256": "66d1ffc26dad0d16481859a02570739c4d21c845eeeed2dfac5c7f3906cc931a",
      "candidate_ids": [
        "5705e63175f01819005e7720",
        "57113be3a58dae1900cd6d28",
        "571c8539dd7acb1400e4c0e2",
        "5705f09e75f01819005e77a4",
        "56e1b169cd28a01900c67a72",
        "5728fb6a1d04691400778ef5",
        "572a03086aef0514001551a2",
        "5729f799af94a219006aa707",
        "572f609ca23a5019007fc5af",
        "57115f0a50c2381900b54aa7",
        "572a06866aef0514001551be",
        "57286010ff5b5019007da1ca",
        "57265526708984140094c2bd",
        "5733cf61d058e614000b62e9",
        "5729a3716aef05140015506a",
        "5727d1c93acd2414000ded3f"
      ],
      "candidate_sha256": [
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6"
      ],
      "example_id": "57115f0a50c2381900b54aa7",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        4,
        15,
        12,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        13,
        14
      ],
      "lexical_top": 9,
      "neural_correct": false,
      "neural_margin": 0.04647003,
      "neural_order": [
        15,
        9,
        4,
        8,
        5,
        7,
        11,
        0,
        1,
        2,
        3,
        6,
        10,
        12,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "0c996164b09a37710e3e09326d1de127a4a0084aac87d95cab5e213c2a6b35b0",
      "trial_index": 249
    },
    {
      "answer_sha256": "b1c7f1f6d6bf4fc7abcdfa7b475ac6a8950854e2e129deb10e25ff446ffa251c",
      "candidate_ids": [
        "570610b275f01819005e792a",
        "5729f5a03f37b31900478605",
        "5728facd4b864d1900165170",
        "573004bf947a6a140053cf56",
        "571136b8a58dae1900cd6d22",
        "56de15104396321400ee25b7",
        "573406d1d058e614000b6801",
        "57115f652419e314009555b9",
        "5729fa40af94a219006aa70f",
        "572ff83ab2c2fd1400568681",
        "5733d7cbd058e614000b63ab",
        "57332b66d058e614000b5758",
        "572812e74b864d19001643ce",
        "57300580b2c2fd140056874d",
        "57286d4f2ca10214002da328",
        "5727c94bff5b5019007d954a"
      ],
      "candidate_sha256": [
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "78549308d98b83d2e71231ce03fc2e9764fe1d8c295bef8561a4ce2a7802cb01",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "8e983a2167b912359aa8e1110455d27a279e9b3baaac76337616f57b19396e19",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062"
      ],
      "example_id": "57115f652419e314009555b9",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        4,
        3,
        5,
        14,
        0,
        1,
        2,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.29784131,
      "neural_order": [
        7,
        4,
        13,
        14,
        6,
        2,
        9,
        10,
        0,
        1,
        3,
        5,
        8,
        11,
        12,
        15
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "660df7b62096f605a592b17251953dbe82836a89cb3d692c1fd7a36a1e090254",
      "trial_index": 250
    },
    {
      "answer_sha256": "8a63beb2ae644b7b197e0187e25adbcb56a4ec11bd1d010a8f3bf4284737e89f",
      "candidate_ids": [
        "57261f9f271a42140099d4a9",
        "57286d7d4b864d19001649e0",
        "5725f8f5ec44d21400f3d7b1",
        "572a142e3f37b319004786b9",
        "57340549d058e614000b67dd",
        "57286ec63acd2414000df9d1",
        "57266193dd62a815002e832a",
        "57060df252bb891400689820",
        "572fc80b04bcaa1900d76d11",
        "572742bd5951b619008f8785",
        "572f76d1b2c2fd1400568153",
        "5725bcb6271a42140099d0eb",
        "5725fe63ec44d21400f3d7dd",
        "57115ff82419e314009555c3",
        "571c8539dd7acb1400e4c0e2",
        "56e1b754cd28a01900c67abc"
      ],
      "candidate_sha256": [
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "ce8d2359f23498cb006a1bf2f91d5494c463bc986a83749d5f9208b28fa63c51",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3"
      ],
      "example_id": "57115ff82419e314009555c3",
      "gated_correct": false,
      "gated_top": 0,
      "gold_index": 13,
      "lexical_correct": false,
      "lexical_order": [
        0,
        13,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.30775591,
      "neural_order": [
        13,
        8,
        10,
        12,
        6,
        9,
        3,
        14,
        15,
        7,
        0,
        1,
        2,
        4,
        5,
        11
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "75b7d4f2f7f989088bb651e8db6ff44efa0831cdbbc9abc57159ad28b6ff0e68",
      "trial_index": 251
    },
    {
      "answer_sha256": "8c6fb1e9e37a1aea1d308c785192e1a17d71cef08c7f50a68d2e0ab292b2e7f4",
      "candidate_ids": [
        "572883153acd2414000dfa6f",
        "573004bf947a6a140053cf56",
        "57108d69b654c5140001f983",
        "57105da9a58dae1900cd699e",
        "5711607f2419e314009555cd",
        "57114b1a2419e31400955575",
        "57338160d058e614000b5bf9",
        "571090abb654c5140001f995",
        "5737a9afc3c5551400e51f61",
        "572750e8dd62a815002e9af0",
        "56de49564396321400ee277a",
        "57283f014b864d19001647e8",
        "5726bc1add62a815002e8ea6",
        "57337f6ad058e614000b5bcb",
        "572864dd4b864d1900164976",
        "5733fd66d058e614000b6735"
      ],
      "candidate_sha256": [
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7"
      ],
      "example_id": "5711607f2419e314009555cd",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        5,
        8,
        0,
        12,
        1,
        2,
        3,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.33723057,
      "neural_order": [
        4,
        5,
        8,
        13,
        2,
        3,
        12,
        11,
        9,
        6,
        0,
        1,
        7,
        10,
        14,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "02cfff6af7f935d78d1575b776ec6bd1aa83a80e9d1e2bc6db2c26ecfb3fa8c1",
      "trial_index": 252
    },
    {
      "answer_sha256": "41a3c63123a7cea108b2fcf51faa06ada65f2754068710564b71fcc95fc4f4e9",
      "candidate_ids": [
        "571cbe35dd7acb1400e4c13c",
        "572ff5fcb2c2fd1400568657",
        "572882242ca10214002da420",
        "570d28bdb3d812140066d4a3",
        "572a0b0b6aef0514001551f6",
        "5710f2e2a58dae1900cd6b72",
        "5728710c3acd2414000df9ef",
        "56de15dbcffd8e1900b4b5c8",
        "571161092419e314009555d7",
        "572a12381d0469140077972b",
        "5705e4fe75f01819005e7704",
        "57115ac550c2381900b54a77",
        "572fcc43b2c2fd140056847d",
        "57373f80c3c5551400e51e8f",
        "5706094b52bb8914006897de",
        "57274e0d708984140094dbe5"
      ],
      "candidate_sha256": [
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657"
      ],
      "example_id": "571161092419e314009555d7",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        4,
        13,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.17723451,
      "neural_order": [
        8,
        11,
        13,
        7,
        4,
        1,
        15,
        14,
        9,
        12,
        3,
        2,
        0,
        5,
        6,
        10
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "468f81743a86f65f582bff7bf50000718fad68c62ab5f6ee353581df287550b0",
      "trial_index": 253
    },
    {
      "answer_sha256": "8fe1029e7f1d04f6911b35ca2c666d172ed8605550e8c55397bedd823b4e6686",
      "candidate_ids": [
        "5726542ff1498d1400e8dc28",
        "5725f8f5ec44d21400f3d7b1",
        "570d4329fed7b91900d45dd7",
        "5725ca4389a1e219009abeb4",
        "572a1c943f37b319004786e1",
        "57106644b654c5140001f8e5",
        "57335c20d058e614000b58f9",
        "5725cc38ec44d21400f3d5bb",
        "57265e97708984140094c3c3",
        "5706111a52bb89140068984c",
        "57377862c3c5551400e51ef3",
        "5728128cff5b5019007d9c76",
        "57113be3a58dae1900cd6d28",
        "572a07c11d046914007796d5",
        "5711619950c2381900b54aaf",
        "5730b8ca8ab72b1400f9c704"
      ],
      "candidate_sha256": [
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "928990f37f000a25d64f03019258af004a5c8124a88add2049f7910fea98667f",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936"
      ],
      "example_id": "5711619950c2381900b54aaf",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        12,
        5,
        7,
        15,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.12784587,
      "neural_order": [
        14,
        12,
        4,
        15,
        5,
        2,
        7,
        8,
        9,
        0,
        1,
        3,
        6,
        10,
        11,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "23a955768c0c1bfa6e74e8a40b7325ee487ee40e384cc9fa0b07c8ff31a260c0",
      "trial_index": 254
    },
    {
      "answer_sha256": "3fc4ccfe745870e2c0d99f71f30ff0656c8dedd41cc1d7d3d376b0dbe685e2f3",
      "candidate_ids": [
        "5711623e50c2381900b54ab9",
        "572812e74b864d19001643ce",
        "571ce3745efbb31900334e20",
        "57274712708984140094dbad",
        "57287e512ca10214002da3f8",
        "572756fe708984140094dc71",
        "5726a09f708984140094cc39",
        "5727aa413acd2414000de921",
        "5711648850c2381900b54ac3",
        "571166352419e314009555f1",
        "56e16839cd28a01900c67887",
        "5725b76389a1e219009abd4a",
        "5733a45d4776f41900660f23",
        "5733d249d058e614000b6331",
        "572fac17947a6a140053cb54",
        "57373f80c3c5551400e51e8f"
      ],
      "candidate_sha256": [
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "a828c58f3d12ed3b04a42280e3053c818961fb2b99589a63f4145c2a9ab650b5",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174"
      ],
      "example_id": "5711623e50c2381900b54ab9",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        9,
        2,
        8,
        1,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.1849228,
      "neural_order": [
        0,
        8,
        9,
        6,
        10,
        4,
        2,
        3,
        13,
        5,
        14,
        15,
        11,
        1,
        7,
        12
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "021e5538d57dd84bb559aa85b715033ca7f008b87211eff8a80267cb50160b1b",
      "trial_index": 255
    },
    {
      "answer_sha256": "f6557264ff022571d283ac9f546fe4406b747955b7cb547cbd017a59ab263a47",
      "candidate_ids": [
        "573784fa1c45671900574483",
        "5711651050c2381900b54acb",
        "570d4a6bfed7b91900d45e13",
        "571cd703dd7acb1400e4c182",
        "5725bbec271a42140099d0d1",
        "5711628a2419e314009555dd",
        "57265e455951b619008f70bb",
        "5727da564b864d1900163e8f",
        "57302bd0b2c2fd14005689db",
        "572847ff3acd2414000df869",
        "5733dab4d058e614000b6409",
        "57275409708984140094dc35",
        "5725ca4389a1e219009abeb4",
        "56de17f9cffd8e1900b4b5e0",
        "573010fab2c2fd14005687d7",
        "57264cc6dd62a815002e80e4"
      ],
      "candidate_sha256": [
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "8657b03977310d90fc72a13d570f1c07e80c6100a3b2eeb92d2acdff9adb8b16",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "099340918f9359979a06067b9361381a993dbef386a4851ba6bb74d73cd5dc60",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "c44126e5a71b148722d5f363b3fe85277c50806c0cacb2d5d467d0ee1ef9ca2d",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8"
      ],
      "example_id": "5711628a2419e314009555dd",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        1,
        0,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.23274273,
      "neural_order": [
        5,
        1,
        15,
        6,
        0,
        2,
        3,
        4,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "761dc71ac2098f2d5d5a5628c05a10db6f4cbd0169dbb96fd1d5654f1bf811aa",
      "trial_index": 256
    },
    {
      "answer_sha256": "e2bef1b953b28626021c4aba3a6dafc5f78a2615e7a21369851eaaa282c22b87",
      "candidate_ids": [
        "572671a5f1498d1400e8dfe4",
        "572648d1708984140094c15d",
        "5726398589a1e219009ac588",
        "57287b4a4b864d1900164a28",
        "571c83f3dd7acb1400e4c0d8",
        "57107d73b654c5140001f91d",
        "57113ba6b654c5140001fc1f",
        "57332562d058e614000b5730",
        "571144d1a58dae1900cd6d6e",
        "572643de5951261400b5195a",
        "5733dab4d058e614000b6409",
        "5733d3cb4776f419006612ea",
        "5727d0f73acd2414000ded13",
        "571cac5d5efbb31900334dd0",
        "57268066708984140094c821",
        "571163172419e314009555e7"
      ],
      "candidate_sha256": [
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "11e489536cc7ac71a8ee73d6c7e52469ad3ad8807c404fd6cba2907c8189cab7",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1"
      ],
      "example_id": "571163172419e314009555e7",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        4,
        8,
        6,
        14,
        0,
        1,
        2,
        3,
        5,
        7,
        9,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.15130221,
      "neural_order": [
        15,
        8,
        6,
        1,
        14,
        13,
        3,
        9,
        4,
        2,
        12,
        7,
        0,
        5,
        10,
        11
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "056e6ad35a90b24c101abac80ec13bd728fca223ce40426fe33f9cd9de8ab452",
      "trial_index": 257
    },
    {
      "answer_sha256": "0e7ba6727379bcf013767afcba6bbb48fb2ae98e33c86979abac432f7c8a96a1",
      "candidate_ids": [
        "570d4a6bfed7b91900d45e13",
        "5705f13d52bb8914006896f0",
        "572a0bebaf94a219006aa76f",
        "57340a094776f4190066177d",
        "572758e0f1498d1400e8f6ae",
        "5711648850c2381900b54ac3",
        "57274d9bf1498d1400e8f5f6",
        "56e1c3e1e3433e1400423148",
        "5725b5a689a1e219009abd28",
        "56e1c4fce3433e140042314e",
        "57274a1edd62a815002e9a9b",
        "57268527708984140094c8bf",
        "5726c002708984140094d073",
        "57105da9a58dae1900cd699e",
        "571163172419e314009555e7",
        "5727d6faff5b5019007d969a"
      ],
      "candidate_sha256": [
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "d3224d2a7c9b1fb0c025ab44b397ab073a37fb28162d6b3730660c29e767b600",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc"
      ],
      "example_id": "5711648850c2381900b54ac3",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.25336976,
      "neural_order": [
        5,
        14,
        9,
        10,
        2,
        12,
        7,
        1,
        11,
        4,
        13,
        3,
        15,
        0,
        6,
        8
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "ae92377e36836d6e2a02bf7891d9930afb7924a805ce851d5b87328be3dc583d",
      "trial_index": 258
    },
    {
      "answer_sha256": "3dcb7cdffbf132655b288253e8e769e4b094a6b64c106bd7fd4da43627567749",
      "candidate_ids": [
        "571156152419e3140095559b",
        "57286bb84b864d19001649c8",
        "5711651050c2381900b54acb",
        "57264e2f708984140094c1e1",
        "571ccc00dd7acb1400e4c15a",
        "5727ef664b864d1900164060",
        "57060a6e52bb8914006897f8",
        "5730a951069b531400832213",
        "56e1f10ee3433e1400423223",
        "57379ed81c456719005744d5",
        "56e1c720e3433e140042316a",
        "572fca7eb2c2fd1400568473",
        "5711541350c2381900b54a6f",
        "56de4b5c4396321400ee2799",
        "5729f8516aef05140015516c",
        "572883a33acd2414000dfa79"
      ],
      "candidate_sha256": [
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "55bbfa9052d82fc2074807e0795fc8957a358974ef289c3f0e19e4e8dbb35f85"
      ],
      "example_id": "5711651050c2381900b54acb",
      "gated_correct": false,
      "gated_top": 0,
      "gold_index": 2,
      "lexical_correct": false,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.2789946,
      "neural_order": [
        2,
        12,
        0,
        9,
        14,
        4,
        5,
        11,
        8,
        10,
        15,
        3,
        1,
        6,
        7,
        13
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "06409e9d2a49097a73eaf308b351ee47b6f91e9a770d9fb3172017d9ac427cfb",
      "trial_index": 259
    },
    {
      "answer_sha256": "c224e1aa63e5a3a78e65e3ae6c60f45d5d72383f5d7c1d9ba6ff73d18d2d9619",
      "candidate_ids": [
        "5727f05b4b864d1900164068",
        "57060eaf75f01819005e7910",
        "5727f6723acd2414000df0e7",
        "572872dd2ca10214002da37e",
        "56e1b8f3e3433e14004230e6",
        "572f7b33947a6a140053c9a2",
        "5733e8ae4776f419006614a6",
        "572659535951b619008f703f",
        "57274e145951b619008f87e7",
        "5711658e50c2381900b54ad5",
        "572ffb02b2c2fd14005686b7",
        "5727fd123acd2414000df185",
        "572fb059947a6a140053cb80",
        "56e20a3ae3433e140042324a",
        "57285ed5ff5b5019007da1b6",
        "572665ff708984140094c4c3"
      ],
      "candidate_sha256": [
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "e71eab6fb81f391a6f9f6f7ddf6a435221f7159360e4d7a0549e3b969d010395"
      ],
      "example_id": "5711658e50c2381900b54ad5",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.43934501,
      "neural_order": [
        9,
        10,
        7,
        14,
        3,
        15,
        5,
        2,
        0,
        13,
        11,
        1,
        4,
        6,
        8,
        12
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "b9f5c02f98ae6f2d64392729fb1c8cd1784b78b372c5379d41d1ab342e72ee95",
      "trial_index": 260
    },
    {
      "answer_sha256": "0f4168490e38b8447e11ba4bd656aa11b925bd22af30bac464bc153fdb608501",
      "candidate_ids": [
        "5727ff083acd2414000df1ab",
        "5729f06f1d04691400779673",
        "57107d73b654c5140001f91d",
        "571166352419e314009555f1",
        "5729feaf6aef051400155188",
        "571ccc00dd7acb1400e4c15a",
        "572881022ca10214002da416",
        "572a18a4af94a219006aa7df",
        "5729f60caf94a219006aa6ef",
        "572807802ca10214002d9bf8",
        "57286192ff5b5019007da1dc",
        "5706143575f01819005e7950",
        "56de52614396321400ee27fb",
        "5711163bb654c5140001fb13",
        "57268882f1498d1400e8e306",
        "57298ef11d0469140077952d"
      ],
      "candidate_sha256": [
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "6d9776f9c9eca85af80b267fa37562071adf7ed273c2b28bd8a6b9ac745c420e",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "3895e66f384dc9afdd15ffe1c33b4044506755bd22d33e8c94583b00ed06854d",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "e72c427b33209e555b5fc09b718f5d3a1abc94dbdf4f37900c15e3a035c2a954",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68"
      ],
      "example_id": "571166352419e314009555f1",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.35810138,
      "neural_order": [
        3,
        5,
        11,
        10,
        9,
        0,
        12,
        1,
        2,
        4,
        6,
        7,
        8,
        13,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "7292045b016ae1598a38ddd0f5848625c757887b0fba0c46a2835262338057bb",
      "trial_index": 261
    },
    {
      "answer_sha256": "7e1f464afe19e78ee0d22393113b8a9c27eaf972583d9e9d7d4903e9cf2e86c8",
      "candidate_ids": [
        "57339eb9d058e614000b5ef6",
        "5732b6b5328d981900602021",
        "572647d0708984140094c14b",
        "573085ea8ab72b1400f9c54c",
        "57111428b654c5140001faff",
        "572a03086aef0514001551a2",
        "5725c0f289a1e219009abdf2",
        "571cd3b55efbb31900334e03",
        "5725f46289a1e219009ac0fa",
        "572691bedd62a815002e89dc",
        "5711669550c2381900b54adf",
        "5728fc2eaf94a219006a9ec7",
        "57287fec4b864d1900164a3c",
        "572872dd2ca10214002da37e",
        "57274e0d708984140094dbe5",
        "57332f81d058e614000b5776"
      ],
      "candidate_sha256": [
        "de308e41aa84b64f8c0380e4021ce99dfb8b520d42b8a543f77ef7f6f3c88a00",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "25632077ec283d4cc31c80588ea6810e247141a57355bd6b1a28a5238b8acb10"
      ],
      "example_id": "5711669550c2381900b54adf",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        2,
        7,
        3,
        0,
        1,
        4,
        5,
        6,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.56448276,
      "neural_order": [
        10,
        2,
        7,
        9,
        6,
        4,
        1,
        11,
        0,
        3,
        5,
        8,
        12,
        13,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "a59a77633da451b426e1ae1b93c00823ed574b84b9c739c3b4c773961a821a55",
      "trial_index": 262
    },
    {
      "answer_sha256": "cac77afa6447276c543eb76ebefbc3ad54d5115f42df0ec4db4e3d29de3715da",
      "candidate_ids": [
        "5726487b5951b619008f6edd",
        "572fdbb004bcaa1900d76dd9",
        "572f5703a23a5019007fc573",
        "572fad30a23a5019007fc86d",
        "5711651050c2381900b54acb",
        "5726f635dd62a815002e9656",
        "5726a14c708984140094cc51",
        "57302bd0b2c2fd14005689db",
        "571c91c8dd7acb1400e4c10a",
        "5725d16aec44d21400f3d61b",
        "5726a299dd62a815002e8b9e",
        "571a484210f8ca1400304fbe",
        "572757bef1498d1400e8f690",
        "5733314e4776f4190066076a",
        "57094ca7efce8f15003a7dd6",
        "572fc49d04bcaa1900d76ccb"
      ],
      "candidate_sha256": [
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "e1e68b90e348d860a1f3c0cfe6e1367d9c8c8a57ad577d564b36e00900222682",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98"
      ],
      "example_id": "571a484210f8ca1400304fbe",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.21470393,
      "neural_order": [
        11,
        8,
        9,
        1,
        4,
        2,
        13,
        3,
        0,
        12,
        10,
        5,
        6,
        7,
        14,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "a2f98f59f79a3f36a725a685e6f7d6b82b811f2904d0f79fa569bab89684fbbc",
      "trial_index": 263
    },
    {
      "answer_sha256": "97f64d5e88390efc86be5942bfd74fda383450279864731e09d80328b25d4672",
      "candidate_ids": [
        "571a49de4faf5e1900b8a94a",
        "57283db6ff5b5019007d9fbc",
        "5725ca4389a1e219009abeb4",
        "5725f5b1271a42140099d371",
        "5728f6446aef0514001548e4",
        "5726a09f708984140094cc39",
        "56de3efccffd8e1900b4b6fe",
        "573098f38ab72b1400f9c5d2",
        "57302cd004bcaa1900d772d7",
        "56e1ec83cd28a01900c67c0a",
        "5728facd4b864d1900165170",
        "5733f062d058e614000b6633",
        "572821274b864d1900164510",
        "5727500f708984140094dbfd",
        "56e17644e3433e1400422f40",
        "56de4a89cffd8e1900b4b7bd"
      ],
      "candidate_sha256": [
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "40206469ec21c4f98d7c72cb3c313a8bdfc89251b79140b6c63a48b1afe123d0",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "218bcafd4ea9a6566900672f26303adc874834255c2b12646ddef2c72e9093f2",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6"
      ],
      "example_id": "571a49de4faf5e1900b8a94a",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.36581369,
      "neural_order": [
        0,
        4,
        1,
        3,
        13,
        14,
        2,
        7,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "6f11c477199cd500831948e9d96512037b862d75983da3ab4c486e49828a827d",
      "trial_index": 264
    },
    {
      "answer_sha256": "9fca4a83f229467048c90ea66b10fd58cd47759d86e5911d62edb58d7e5457d7",
      "candidate_ids": [
        "5728827b2ca10214002da42a",
        "57338255d058e614000b5c0d",
        "571a4b0f10f8ca1400304fd5",
        "57282358ff5b5019007d9dde",
        "5725fb8138643c19005acf3f",
        "5726577f708984140094c301",
        "572fdd03a23a5019007fca9d",
        "5726a3c6f1498d1400e8e5ae",
        "570d4e6cb3d812140066d66d",
        "572f55e8a23a5019007fc56b",
        "57114f0050c2381900b54a65",
        "571cac5d5efbb31900334dd0",
        "5728501aff5b5019007da166",
        "56de4bb84396321400ee27a2",
        "573399b54776f41900660e64",
        "5727500f708984140094dbfd"
      ],
      "candidate_sha256": [
        "8ec9dcc4c2fe0ea5e8f0e2e102236786bbdd27dc8fb4452718334cf4545b34bf",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "11e489536cc7ac71a8ee73d6c7e52469ad3ad8807c404fd6cba2907c8189cab7",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "184d8e2bfbbe465d571014652c9207a041ec5654986198ec457160d7a42c77dc",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28"
      ],
      "example_id": "571a4b0f10f8ca1400304fd5",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.40583828,
      "neural_order": [
        2,
        11,
        10,
        3,
        13,
        0,
        7,
        12,
        6,
        5,
        15,
        1,
        4,
        8,
        9,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "aedc3b055269737342ea7fa3c99a9be1453f8550af1712d4c942cc1a9feb3bf8",
      "trial_index": 265
    },
    {
      "answer_sha256": "8e1bb2acc359a30b4b01296bcefe35289d5b8d4a8787fd24fcee51c55efb69d3",
      "candidate_ids": [
        "5726487b5951b619008f6edd",
        "573062662461fd1900a9cdf7",
        "5727ff083acd2414000df1ab",
        "572fc49d04bcaa1900d76ccb",
        "57096e1ced30961900e84102",
        "572826634b864d19001645be",
        "572f609ca23a5019007fc5af",
        "572759665951b619008f8883",
        "5728128cff5b5019007d9c76",
        "572fdb17b2c2fd140056851d",
        "57309564069b5314008321a5",
        "571a4d1a4faf5e1900b8a958",
        "57264c42dd62a815002e80c6",
        "5728848cff5b5019007da298",
        "5725d7e438643c19005acdf9",
        "5727dd2e4b864d1900163eba"
      ],
      "candidate_sha256": [
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "928990f37f000a25d64f03019258af004a5c8124a88add2049f7910fea98667f",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a"
      ],
      "example_id": "571a4d1a4faf5e1900b8a958",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.3529282,
      "neural_order": [
        11,
        10,
        14,
        7,
        15,
        8,
        4,
        12,
        0,
        3,
        1,
        2,
        5,
        6,
        9,
        13
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "8b8b6c3ebf8b4d1da683c235b49efef0be8653b624861bb0069dfa2f2f2c9139",
      "trial_index": 266
    },
    {
      "answer_sha256": "23c18946456043eb13921b202843e286bd46e304fc27c54e1e7e8401ad8731fa",
      "candidate_ids": [
        "572a12386aef051400155234",
        "57379829c3c5551400e51f3d",
        "571a4ead10f8ca1400304fdd",
        "5728fd3c3f37b31900477f2b",
        "57282dfb4b864d1900164668",
        "57111ab8a58dae1900cd6c3c",
        "572658daf1498d1400e8dcac",
        "5725d7e438643c19005acdf9",
        "572f76d1b2c2fd1400568153",
        "57269f3ef1498d1400e8e534",
        "572fdd03a23a5019007fca9d",
        "57299021af94a219006aa50b",
        "57287b4a4b864d1900164a28",
        "5729fe5c3f37b3190047862f",
        "5725f46289a1e219009ac0fa",
        "5726431aec44d21400f3dd13"
      ],
      "candidate_sha256": [
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77"
      ],
      "example_id": "571a4ead10f8ca1400304fdd",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        14,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.28915116,
      "neural_order": [
        2,
        12,
        9,
        11,
        8,
        1,
        7,
        15,
        6,
        14,
        4,
        0,
        3,
        5,
        10,
        13
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "1f1659027fa6eeefdf589c616c2a0a7099a727bdeace5ff57d63bb5660b4b7e1",
      "trial_index": 267
    },
    {
      "answer_sha256": "310b5efe6794a5b89157ce51bd943de4897b58710216d818f13865123edd05ad",
      "candidate_ids": [
        "57274971708984140094dbbb",
        "572a135daf94a219006aa79f",
        "57274f67708984140094dbf5",
        "57111ab8a58dae1900cd6c3c",
        "573005b9947a6a140053cf6a",
        "57265ceddd62a815002e82b8",
        "572a03086aef0514001551a2",
        "571a50df4faf5e1900b8a960",
        "573003dd947a6a140053cf42",
        "5725d662ec44d21400f3d687",
        "571ce3745efbb31900334e20",
        "57115b2850c2381900b54a7f",
        "5726398589a1e219009ac588",
        "5733e8ae4776f419006614a6",
        "57114b1a2419e31400955575",
        "572f5875947a6a140053c89a"
      ],
      "candidate_sha256": [
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c"
      ],
      "example_id": "571a50df4faf5e1900b8a960",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        2,
        14,
        0,
        1,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.35692044,
      "neural_order": [
        7,
        14,
        11,
        2,
        10,
        1,
        12,
        6,
        15,
        5,
        4,
        0,
        3,
        8,
        9,
        13
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "affcde8f2f7e9c9f7417337dbb0159cd819c682c25055569e01da67b8f92a664",
      "trial_index": 268
    },
    {
      "answer_sha256": "b982a09b35429d8f87552f1af0a512034e382562cbc792529ed12fd0096e4bef",
      "candidate_ids": [
        "57302faa04bcaa1900d77311",
        "571a52cb4faf5e1900b8a969",
        "573020f7b2c2fd14005688f7",
        "5706111a52bb89140068984c",
        "5725ec8289a1e219009ac0ae",
        "5733647e4776f419006609ae",
        "5726b58f5951b619008f7b55",
        "5705f7c875f01819005e77dc",
        "5733f410d058e614000b6663",
        "57379829c3c5551400e51f3d",
        "57264684708984140094c123",
        "57287b322ca10214002da3be",
        "57265ceddd62a815002e82b8",
        "57274712708984140094dbad",
        "572fc41f947a6a140053cc80",
        "572a07fc6aef0514001551dc"
      ],
      "candidate_sha256": [
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0",
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77"
      ],
      "example_id": "571a52cb4faf5e1900b8a969",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        7,
        11,
        0,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.24867321,
      "neural_order": [
        1,
        9,
        6,
        15,
        4,
        14,
        11,
        0,
        2,
        3,
        5,
        7,
        8,
        10,
        12,
        13
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "b9405b25853275b445f7583215a23622c381ced433c1cf9eb971c223632ea970",
      "trial_index": 269
    },
    {
      "answer_sha256": "c38e8334242ffdf00ec3402b43f3010e5ad2d1550e41805a6c158de1b621370d",
      "candidate_ids": [
        "5733d5704776f4190066130e",
        "572995d46aef051400154fe8",
        "57298ef11d0469140077952d",
        "56de3ebc4396321400ee26e6",
        "57306797396df919000960ee",
        "5727e9523acd2414000def95",
        "57108198b654c5140001f937",
        "57284618ff5b5019007da0a8",
        "56de0f6a4396321400ee257f",
        "57280e1aff5b5019007d9bec",
        "56dde27d9a695914005b9651",
        "571a53d410f8ca1400304fe5",
        "5730b8ca8ab72b1400f9c704",
        "5725f190ec44d21400f3d76f",
        "572745c6708984140094db99",
        "57264cfa708984140094c1c3"
      ],
      "candidate_sha256": [
        "18017326aa99d6478493c6fd9c81f3144c242ec0a8dc61e8a64d3d1ab8e0507e",
        "2a8d2aa463245d980e23aefdfe31db650d5d676f415970557b36b6d0e09480b5",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad"
      ],
      "example_id": "571a53d410f8ca1400304fe5",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.42351852,
      "neural_order": [
        11,
        1,
        2,
        9,
        15,
        12,
        8,
        4,
        0,
        3,
        5,
        6,
        7,
        10,
        13,
        14
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "ce41a3af48993e7d4f45d66b333295256eebe1a5ffa3eb41b0c41a86cda7e952",
      "trial_index": 270
    },
    {
      "answer_sha256": "0774ae1fce97e16108760b77679f9577c201da6e0a57e9b9fb6363b8d3e69c94",
      "candidate_ids": [
        "571c7d55dd7acb1400e4c0c4",
        "5726a14c708984140094cc51",
        "57379829c3c5551400e51f3d",
        "5728eff82ca10214002daadc",
        "572ffb02b2c2fd14005686b7",
        "56e181d9e3433e1400422fa0",
        "5728809f2ca10214002da40c",
        "57293ca73f37b3190047815b",
        "5726a638dd62a815002e8bf6",
        "57300c67947a6a140053cff0",
        "5737898f1c45671900574495",
        "571c91c8dd7acb1400e4c10a",
        "57111428b654c5140001faff",
        "56e1b754cd28a01900c67abc",
        "5725b81b271a42140099d097",
        "57302bd0b2c2fd14005689db"
      ],
      "candidate_sha256": [
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518"
      ],
      "example_id": "571c7d55dd7acb1400e4c0c4",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        4,
        9,
        12,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.44934861,
      "neural_order": [
        0,
        10,
        11,
        2,
        9,
        5,
        8,
        6,
        13,
        1,
        4,
        3,
        7,
        12,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "calibration",
      "question_sha256": "5fd84fddc561c5ec5d8c31b6f3268bd9ca7c68650c498a33173abf3977b366e6",
      "trial_index": 271
    },
    {
      "answer_sha256": "9d30ee393b76c201461473bfe75da7b60a0b97515c9a161f6ad13dfe0f73f6bd",
      "candidate_ids": [
        "57269f3ef1498d1400e8e534",
        "57113f83b654c5140001fc29",
        "572a1046af94a219006aa78d",
        "5729f39a6aef05140015514c",
        "571c8198dd7acb1400e4c0ce",
        "57286d4f2ca10214002da328",
        "5733ea04d058e614000b6594",
        "56e1c3e1e3433e1400423148",
        "5730131c947a6a140053d052",
        "5726559edd62a815002e81c8",
        "5705e63175f01819005e7720",
        "57267640f1498d1400e8e074",
        "57115e532419e314009555af",
        "572861cc4b864d190016495e",
        "56de10b44396321400ee2593",
        "570d4030fed7b91900d45d9f"
      ],
      "candidate_sha256": [
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "a6b7ec7938b966c86b4cc72fa2059dcdbd60bc13208210cff0105baccb082b8b",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "255db98fb0b6e32abd3b8317106e8eeb1841b1f3860f985c69bb911aae85aadc",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "3f83987ebc0c4b6d662d8cc6134fe19bdcd18d63555f50b4c6841486edd3606d",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b"
      ],
      "example_id": "571c8198dd7acb1400e4c0ce",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        11,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.14447432,
      "neural_order": [
        4,
        12,
        11,
        1,
        9,
        0,
        6,
        2,
        14,
        8,
        3,
        5,
        7,
        10,
        13,
        15
      ],
      "neural_top": 4,
      "partition": "calibration",
      "question_sha256": "561b563f83754031f8aefbf16ec71c66ce010ab4e186a0b73f5e0f3cafd670a4",
      "trial_index": 272
    },
    {
      "answer_sha256": "a700e979b1a1a91bbb4c1b082744fc756ac32d738abaf3a744fe84ff5913a093",
      "candidate_ids": [
        "572f59b4a23a5019007fc587",
        "56de17f9cffd8e1900b4b5e0",
        "573403394776f419006616dd",
        "572867d72ca10214002da2fc",
        "57265c10f1498d1400e8dd36",
        "572a0ecb1d04691400779717",
        "571c83f3dd7acb1400e4c0d8",
        "570607f575f01819005e78b4",
        "57115b2850c2381900b54a7f",
        "57264a8cdd62a815002e808c",
        "57377c98c3c5551400e51efb",
        "57299021af94a219006aa50b",
        "570d4e6cb3d812140066d66d",
        "57060df252bb891400689820",
        "57339ad74776f41900660e86",
        "5727cd7dff5b5019007d958e"
      ],
      "candidate_sha256": [
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "1c326ae15d585661849c0a43f71be232f457f11a2a48964b11b0d9172e7cd31b",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374"
      ],
      "example_id": "571c83f3dd7acb1400e4c0d8",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.35777864,
      "neural_order": [
        6,
        4,
        0,
        8,
        14,
        10,
        11,
        15,
        1,
        2,
        3,
        5,
        7,
        9,
        12,
        13
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "9ddcc04558e6a476c4616c8ec76254e9cc07b0a633408e8892f5125aa0c53858",
      "trial_index": 273
    },
    {
      "answer_sha256": "7880bd7fe5dcd28637eade8b95c42961c6c5acffa946a671aa0f01ce3a3479bc",
      "candidate_ids": [
        "571135b8a58dae1900cd6d0e",
        "5729506d6aef051400154cac",
        "5725c95f38643c19005accf3",
        "5726f48df1498d1400e8f0da",
        "57280e1aff5b5019007d9bec",
        "571ccfbadd7acb1400e4c164",
        "57298ef11d0469140077952d",
        "571c8539dd7acb1400e4c0e2",
        "57274712708984140094dbad",
        "57274f49f1498d1400e8f61e",
        "57111380a58dae1900cd6bd6",
        "57115f0a50c2381900b54aa7",
        "57335fcad058e614000b5970",
        "57340d124776f419006617bf",
        "572648d1708984140094c15d",
        "572681c1dd62a815002e8796"
      ],
      "candidate_sha256": [
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "be1e8ef67e19e07dbf69a2cd82b160a507b81ad4a6ae00fb9fbdeda6ebbed945",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8"
      ],
      "example_id": "571c8539dd7acb1400e4c0e2",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.300082,
      "neural_order": [
        7,
        5,
        4,
        10,
        1,
        15,
        2,
        14,
        6,
        12,
        0,
        3,
        8,
        9,
        11,
        13
      ],
      "neural_top": 7,
      "partition": "calibration",
      "question_sha256": "094418652dcecac81ec29db4cb28efd96bd28c9b1cea0ab9024bcf9390442f1f",
      "trial_index": 274
    },
    {
      "answer_sha256": "91aa160df2a35229237ea9a504a0030443a64d5115f5b5fa31195931c43c044e",
      "candidate_ids": [
        "5725c071271a42140099d127",
        "56de49a8cffd8e1900b4b7a7",
        "5727d9c43acd2414000dee17",
        "572ffabf04bcaa1900d76f9f",
        "5733d249d058e614000b6331",
        "5728f7774b864d190016512e",
        "572f5875947a6a140053c89a",
        "5711541350c2381900b54a6f",
        "56e1ce08e3433e14004231a4",
        "57299d1c1d04691400779581",
        "57337f6ad058e614000b5bcb",
        "57274a1edd62a815002e9a9b",
        "57107e6ca58dae1900cd69f2",
        "5711619950c2381900b54aaf",
        "571c879bdd7acb1400e4c0ec",
        "572fffb404bcaa1900d76fef"
      ],
      "candidate_sha256": [
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "ade6ff401c8944661f8dbe0a4e4be3b05995437ccaed1d24de71c038275f80ef",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "511ddcc5ddbeeaf06bef1b76bc323fc7607c5658b94f72c360e5919a5c54656e",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "ee6a40759a8e9342f506db55cd0c40e075a1ce7fd0ccc2da16d91a2be11ea856",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f"
      ],
      "example_id": "571c879bdd7acb1400e4c0ec",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        9,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.4204779,
      "neural_order": [
        14,
        13,
        5,
        7,
        8,
        6,
        0,
        1,
        2,
        3,
        4,
        9,
        10,
        11,
        12,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "8cb237364555c508a502b86c6b68c7b767d14c7f2ddfa13c8d4dba2f2f551905",
      "trial_index": 275
    },
    {
      "answer_sha256": "9fe430e5274621be518924ed90d2355f39eeb1d4d98fb00ea0022f3a26ba5595",
      "candidate_ids": [
        "572f59b4a23a5019007fc587",
        "56de51c64396321400ee27f7",
        "572fcd86947a6a140053ccda",
        "572f5703a23a5019007fc573",
        "572fc5a1947a6a140053cc8a",
        "571093aba58dae1900cd6a5c",
        "571c8eb9dd7acb1400e4c0f6",
        "5727ee372ca10214002d99ec",
        "5733d5704776f4190066130e",
        "571a53d410f8ca1400304fe5",
        "572a070c6aef0514001551c8",
        "5725dd7d89a1e219009abfea",
        "57293f8a6aef051400154bde",
        "57097d63ed30961900e841fc",
        "57294e6b1d04691400779275",
        "570611c475f01819005e793c"
      ],
      "candidate_sha256": [
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "18017326aa99d6478493c6fd9c81f3144c242ec0a8dc61e8a64d3d1ab8e0507e",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "db91c1023a94550f1ffe6e9f87e0b45aaad95f3bef9ada0d0c3872b88286bf7c"
      ],
      "example_id": "571c8eb9dd7acb1400e4c0f6",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.3892483,
      "neural_order": [
        6,
        5,
        14,
        1,
        9,
        0,
        12,
        7,
        10,
        4,
        2,
        3,
        8,
        11,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "b33d2c99410022462c3c2556046aa284cb2ebe9be81b81b4deb6524bb71ab6c1",
      "trial_index": 276
    },
    {
      "answer_sha256": "bb2bd7ca4a327f84e6cd3979f8fa3828a50a08893c1b68f9d6715352c8d07b93",
      "candidate_ids": [
        "572746addd62a815002e9a5e",
        "57377ec7c3c5551400e51f05",
        "5728fb6a1d04691400778ef5",
        "56e1c4fce3433e140042314e",
        "57377083c3c5551400e51edf",
        "5705eccb52bb8914006896b8",
        "57107932a58dae1900cd69ca",
        "5725be0f271a42140099d117",
        "5728284e3acd2414000df5cd",
        "571c9074dd7acb1400e4c100",
        "572fdb17b2c2fd140056851d",
        "57283db6ff5b5019007d9fbc",
        "56e1bd4acd28a01900c67afc",
        "57274971708984140094dbbb",
        "57263b1638643c19005ad333",
        "57286dfa2ca10214002da332"
      ],
      "candidate_sha256": [
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "d1bd9316675d5ac9fa7e69f137619aecc59a46e6cd8398569396ed53c86f02f1",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "069f593a47b1c7d532212a62aba1e27bdd08d0259cb3e0a96bc73f362f4036e4",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "25160eaaf4cedb4feaad143ed4ed40d1a5aeba443aaa0327bbecb05f058934a7",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "87c3a5e21ea40ce9396fd4bd04485fc5c291473b22cdd4b5feb623f3ece0e993"
      ],
      "example_id": "571c9074dd7acb1400e4c100",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.38381171,
      "neural_order": [
        9,
        3,
        12,
        14,
        4,
        1,
        5,
        7,
        13,
        0,
        2,
        6,
        8,
        10,
        11,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "9a2e81471eb321cfa982f6bcbcb13ea8cf9e895a0f2307bfd1892777ed40543e",
      "trial_index": 277
    },
    {
      "answer_sha256": "0b0b98549878207219690ae4d454bd7a350568e81f8c9e214cadd0ce7a12ef05",
      "candidate_ids": [
        "57309446396df919000961b8",
        "5734025d4776f419006616c3",
        "5726a638dd62a815002e8bf6",
        "57264684708984140094c123",
        "56de52614396321400ee27fb",
        "5726a299dd62a815002e8b9e",
        "571c91c8dd7acb1400e4c10a",
        "57339eb9d058e614000b5ef6",
        "57264845f1498d1400e8db0a",
        "56de3ebc4396321400ee26e6",
        "571c97e2dd7acb1400e4c11e",
        "570961aa200fba1400367f15",
        "5726a34bf1498d1400e8e59e",
        "5728eb1a3acd2414000e01c5",
        "57300200b2c2fd1400568729",
        "5730285a04bcaa1900d7726d"
      ],
      "candidate_sha256": [
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "de308e41aa84b64f8c0380e4021ce99dfb8b520d42b8a543f77ef7f6f3c88a00",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065"
      ],
      "example_id": "571c91c8dd7acb1400e4c10a",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        10,
        5,
        2,
        0,
        1,
        3,
        4,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": false,
      "neural_margin": 0.0370024,
      "neural_order": [
        10,
        6,
        2,
        15,
        9,
        7,
        11,
        5,
        0,
        1,
        3,
        4,
        8,
        12,
        13,
        14
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "dbbaf7fbcb01517334d8105b2f24c3be6d81f672674c97470de85af372002e07",
      "trial_index": 278
    },
    {
      "answer_sha256": "88f73d19a2b8a6244d7fdf2a0941020372505c79c29a7bbec174eed111a4dd0e",
      "candidate_ids": [
        "5726a14c708984140094cc51",
        "572fb059947a6a140053cb80",
        "5728848cff5b5019007da298",
        "572756265951b619008f886e",
        "57293e983f37b3190047818b",
        "572f59b4a23a5019007fc587",
        "5727d6faff5b5019007d969a",
        "572a12386aef051400155234",
        "572989846aef051400154fc0",
        "5727f05b4b864d1900164068",
        "571c8198dd7acb1400e4c0ce",
        "5733fb7bd058e614000b66ff",
        "5705e99452bb891400689688",
        "57115e532419e314009555af",
        "5729efab3f37b319004785cf",
        "571c9348dd7acb1400e4c114"
      ],
      "candidate_sha256": [
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "ce72fed7b084941e90079823ae33cc19adb21829e9f5aa99491ba4d96d782570",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31"
      ],
      "example_id": "571c9348dd7acb1400e4c114",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.29748267,
      "neural_order": [
        15,
        10,
        4,
        13,
        2,
        6,
        11,
        8,
        1,
        3,
        0,
        5,
        7,
        9,
        12,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "aa30c3b36661fdf4d6228938ea24bd97057d4f2f9b4694a901845fc97c23ca1a",
      "trial_index": 279
    },
    {
      "answer_sha256": "b982a09b35429d8f87552f1af0a512034e382562cbc792529ed12fd0096e4bef",
      "candidate_ids": [
        "57274e145951b619008f87e7",
        "572a0a391d046914007796df",
        "57267640f1498d1400e8e074",
        "57285ed5ff5b5019007da1b6",
        "5725b92e38643c19005acbd3",
        "572989846aef051400154fc0",
        "5737a0acc3c5551400e51f47",
        "572648d1708984140094c15d",
        "57269e3bf1498d1400e8e516",
        "5728223cff5b5019007d9dc4",
        "56de49a8cffd8e1900b4b7a7",
        "572a1ba46aef05140015528e",
        "572fcb6da23a5019007fc9f1",
        "57263fd138643c19005ad37b",
        "56de4c324396321400ee27ab",
        "571c96095efbb31900334dbc"
      ],
      "candidate_sha256": [
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "aa95ca2f8cb345761597dedb47b8dc46be2773412d98ec4f473e747ae7e16b2a",
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "3ee61e6d7605bacdeb4f37ab798daa449f57668fd923e6b04e1a516237821782"
      ],
      "example_id": "571c96095efbb31900334dbc",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.42624394,
      "neural_order": [
        15,
        6,
        12,
        0,
        8,
        9,
        1,
        7,
        10,
        2,
        11,
        3,
        4,
        5,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "e396001b8f907cf314e7b1dee16683a0449188521d20c7950939da0147cb151b",
      "trial_index": 280
    },
    {
      "answer_sha256": "122856abd4c0f64dfb54d2afc7b8e2f3b68a1e62f540c28ae2ea013a538fce39",
      "candidate_ids": [
        "5733638fd058e614000b59e8",
        "57264efddd62a815002e8134",
        "57283c464b864d19001647c8",
        "57106d2fb654c5140001f8ef",
        "5730285a04bcaa1900d7726d",
        "572a10cd6aef051400155222",
        "573784fa1c45671900574483",
        "572f58d9a23a5019007fc57d",
        "573099ee8ab72b1400f9c5dc",
        "57274971708984140094dbbb",
        "5728eff82ca10214002daadc",
        "571c97e2dd7acb1400e4c11e",
        "572ff935b2c2fd140056869b",
        "57338160d058e614000b5bf9",
        "5727cb4b2ca10214002d9676",
        "572f6a0ba23a5019007fc5eb"
      ],
      "candidate_sha256": [
        "05e6b70a4c996f49687d72f7a373609343967f6fed5be1f3bda6965dcfad141e",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109",
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "46a60c1fb7a3dc1eb8d8008909c9080e1b010b9c75f648c33a07f6ce1e997157",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271"
      ],
      "example_id": "571c97e2dd7acb1400e4c11e",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.49409769,
      "neural_order": [
        11,
        6,
        7,
        9,
        4,
        1,
        3,
        10,
        2,
        0,
        5,
        8,
        12,
        13,
        14,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "6bbbc61d3624002c84573355c6cf69bc83f50846df004e0ec3f1b8dff24099bd",
      "trial_index": 281
    },
    {
      "answer_sha256": "58844c2021e926648b56b9ee2652828b908173eba0eac1e9d937e5fb42e63316",
      "candidate_ids": [
        "5726f868dd62a815002e9684",
        "5726a7ecf1498d1400e8e654",
        "572a12386aef051400155234",
        "5729f799af94a219006aa707",
        "5727cff1ff5b5019007d95c4",
        "57113be3a58dae1900cd6d28",
        "57106185b654c5140001f8db",
        "5726da89dd62a815002e92b2",
        "5705ec1675f01819005e776e",
        "57274beff1498d1400e8f5e4",
        "571caac55efbb31900334dc6",
        "5729fd56af94a219006aa72f",
        "57114b1a2419e31400955575",
        "5726219489a1e219009ac2d0",
        "57284d484b864d1900164900",
        "57263b1638643c19005ad333"
      ],
      "candidate_sha256": [
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "36476aaf2aea4f7927c065f38bbc7e2f8ec013b37ee0a2f1c7f545368f0dc62e",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24"
      ],
      "example_id": "571caac55efbb31900334dc6",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.41573121,
      "neural_order": [
        10,
        5,
        11,
        12,
        1,
        13,
        14,
        6,
        0,
        2,
        3,
        4,
        7,
        8,
        9,
        15
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "cdf1dea3ff30b19717bd784fa45a85a16318da58f7a649cce0c6fe799cdb6604",
      "trial_index": 282
    },
    {
      "answer_sha256": "27756f050e14a1cb1c1ee867f0eace9ea4d9fcb81b8bee089469f1ebd5fd7b17",
      "candidate_ids": [
        "57293e221d046914007791d5",
        "56de17f9cffd8e1900b4b5e0",
        "5733a1854776f41900660f0a",
        "571cac5d5efbb31900334dd0",
        "572f55e8a23a5019007fc56b",
        "57373a9fc3c5551400e51e7b",
        "5725b76389a1e219009abd4a",
        "571cb27fdd7acb1400e4c132",
        "5730126ba23a5019007fcd09",
        "572f6ec7a23a5019007fc621",
        "5730b1022461fd1900a9cfa3",
        "572ff932a23a5019007fcbd5",
        "57376df3c3c5551400e51ed7",
        "571a4ead10f8ca1400304fdd",
        "572659ea5951b619008f7051",
        "572fd264b2c2fd14005684a9"
      ],
      "candidate_sha256": [
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "acd49776c5c867dd78a5c6b915a917f39479b502b183b3b4b0c8d4934fc1536a",
        "11e489536cc7ac71a8ee73d6c7e52469ad3ad8807c404fd6cba2907c8189cab7",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca"
      ],
      "example_id": "571cac5d5efbb31900334dd0",
      "gated_correct": false,
      "gated_top": 7,
      "gold_index": 3,
      "lexical_correct": false,
      "lexical_order": [
        7,
        12,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.11738066,
      "neural_order": [
        3,
        7,
        13,
        5,
        0,
        14,
        6,
        8,
        11,
        12,
        15,
        2,
        1,
        10,
        4,
        9
      ],
      "neural_top": 3,
      "partition": "calibration",
      "question_sha256": "2abf77d8b663eabaac0a94e691aae0b41b30146e5b236a6994d623e1f082574e",
      "trial_index": 283
    },
    {
      "answer_sha256": "c8fa050b733f752aafd4218f72660c15621877620737e2dfa6c78a51e2621c93",
      "candidate_ids": [
        "572659ea5951b619008f7051",
        "56dde1d966d3e219004dad8d",
        "5725cc38ec44d21400f3d5bb",
        "5728fb002ca10214002dab6a",
        "5728809f2ca10214002da40c",
        "5730bb522461fd1900a9d011",
        "57286d7d4b864d19001649e0",
        "571cb27fdd7acb1400e4c132",
        "571cb010dd7acb1400e4c128",
        "56e1ec83cd28a01900c67c0a",
        "56de0f6a4396321400ee257f",
        "5725f00938643c19005aced7",
        "570d26efb3d812140066d48f",
        "5726baf2dd62a815002e8e74",
        "572843304b864d1900164848",
        "57293ca73f37b3190047815b"
      ],
      "candidate_sha256": [
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "2ef6bc8b598c3ae95dea517f30b72d286a3c1ea1c56e1f73f6bd8f2ab9cb63ef",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c"
      ],
      "example_id": "571cb010dd7acb1400e4c128",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        5,
        7,
        11,
        2,
        0,
        1,
        3,
        4,
        6,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.23620425,
      "neural_order": [
        8,
        7,
        0,
        2,
        3,
        12,
        9,
        15,
        5,
        1,
        4,
        14,
        10,
        6,
        11,
        13
      ],
      "neural_top": 8,
      "partition": "calibration",
      "question_sha256": "522b161097059841ea161cdb21521c001af9f5c6e77c2aabfcf211bbbb0104f9",
      "trial_index": 284
    },
    {
      "answer_sha256": "3a6f5deb2a5400a4119ba535865b925cc346bdd8b4620e62281372361937a2af",
      "candidate_ids": [
        "573382d24776f41900660c37",
        "572fc6f204bcaa1900d76cf5",
        "5737a5931c456719005744e7",
        "570d529fb3d812140066d6bb",
        "57268a37f1498d1400e8e33c",
        "56e1c7e2cd28a01900c67b74",
        "57273e50dd62a815002e9a02",
        "570d44abb3d812140066d5fd",
        "56de51244396321400ee27ef",
        "571cb27fdd7acb1400e4c132",
        "57264684708984140094c123",
        "57293d116aef051400154bc8",
        "5728855d3acd2414000dfa8d",
        "572649d8f1498d1400e8db37",
        "5727213c708984140094da35",
        "57268da7f1498d1400e8e39c"
      ],
      "candidate_sha256": [
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "19620cc8402206f6bca616c6a305ca911d96b87a8f3423850512f4f84e4120b1",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f"
      ],
      "example_id": "571cb27fdd7acb1400e4c132",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        15,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.27485204,
      "neural_order": [
        9,
        4,
        15,
        11,
        6,
        1,
        5,
        0,
        12,
        13,
        2,
        3,
        7,
        8,
        10,
        14
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "852384609f3ca3b626d9f43ab81e560b9764fe603c44aa7104917c6c1b005d99",
      "trial_index": 285
    },
    {
      "answer_sha256": "b29ab298651d7a24e439334ed10d8af66eacbc27a07aca39bdadefbe3e9472c1",
      "candidate_ids": [
        "5727515f708984140094dc11",
        "5726577f708984140094c301",
        "57264586f1498d1400e8dac6",
        "57284142ff5b5019007da00a",
        "57280f974b864d1900164370",
        "570d50a5fed7b91900d45e7d",
        "5710f4b8b654c5140001fa45",
        "572758e0f1498d1400e8f6ae",
        "571a49de4faf5e1900b8a94a",
        "572f7b33947a6a140053c9a2",
        "571cbe35dd7acb1400e4c13c",
        "57378c9b1c456719005744a7",
        "5709686c200fba1400367f77",
        "5733f9fa4776f4190066161f",
        "5727e6cbff5b5019007d97ee",
        "572820512ca10214002d9e72"
      ],
      "candidate_sha256": [
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "09dae1e2b2708f999627efe257489cfaad54e793df884b8877c2d638640ffa20",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a"
      ],
      "example_id": "571cbe35dd7acb1400e4c13c",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.34154625,
      "neural_order": [
        10,
        12,
        8,
        2,
        15,
        6,
        9,
        13,
        1,
        0,
        3,
        4,
        5,
        7,
        11,
        14
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "2261be6c4a5532dd034d0abaa541ad5015367e79510c836cd0fb64228bcfd197",
      "trial_index": 286
    },
    {
      "answer_sha256": "ba92e21596cc149adc2ae90c461b804b038bec7fe62cfd59d12e3b51615feda6",
      "candidate_ids": [
        "57264c42dd62a815002e80c6",
        "571135b8a58dae1900cd6d0e",
        "57299ec43f37b3190047850d",
        "573028fa04bcaa1900d77287",
        "57265e11708984140094c3bb",
        "571163172419e314009555e7",
        "571cc3dedd7acb1400e4c146",
        "5726a00cf1498d1400e8e550",
        "5727478cf1498d1400e8f59a",
        "5728809f2ca10214002da40c",
        "572fb059947a6a140053cb80",
        "5728759cff5b5019007da25a",
        "5726ba2c708984140094cf59",
        "572751b4708984140094dc1b",
        "5727f7523acd2414000df10d",
        "572a0bebaf94a219006aa76f"
      ],
      "candidate_sha256": [
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "a6bf93aaa317a4cbe3dd6095edaf3c9cad24ebd2270fecf0196ce024b0880c94",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "9819559c80db18970983cbfdf2bcb3f75443deb410b562890bbe13cea74a03dc",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe"
      ],
      "example_id": "571cc3dedd7acb1400e4c146",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        2,
        0,
        1,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.59075707,
      "neural_order": [
        6,
        9,
        11,
        2,
        14,
        4,
        1,
        7,
        5,
        15,
        0,
        3,
        8,
        10,
        12,
        13
      ],
      "neural_top": 6,
      "partition": "calibration",
      "question_sha256": "24aa7affee6bb631bcfb41128dd2c408a972ad207b90b00ac5c43b994c19f043",
      "trial_index": 287
    },
    {
      "answer_sha256": "2e8c4c94d8a5dd8bf48ff9a9007c148e4a434571e9f32b38cde9f526a9be0580",
      "candidate_ids": [
        "570d418dfed7b91900d45dbf",
        "5727cc15ff5b5019007d9576",
        "572ffd9e04bcaa1900d76fc7",
        "57115dbe2419e314009555a5",
        "572a0bebaf94a219006aa76f",
        "57269016708984140094ca41",
        "571cc3dedd7acb1400e4c146",
        "57265d08708984140094c397",
        "57115bf350c2381900b54a93",
        "57060eaf75f01819005e7910",
        "573406d1d058e614000b6801",
        "5727ea45ff5b5019007d986e",
        "56de4c324396321400ee27ab",
        "571cc5c45efbb31900334dda",
        "5711163bb654c5140001fb13",
        "572807802ca10214002d9bf8"
      ],
      "candidate_sha256": [
        "631ae85a58426197f43bce08d34df2914057034648942e41a9d822902b61b3d3",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a"
      ],
      "example_id": "571cc5c45efbb31900334dda",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.22363096,
      "neural_order": [
        13,
        6,
        5,
        7,
        4,
        12,
        11,
        9,
        1,
        8,
        0,
        2,
        15,
        3,
        10,
        14
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "894fab4f09b40b67b9297b845a06a10d7466a35b5ca339376f8e890ff6f2ddfb",
      "trial_index": 288
    },
    {
      "answer_sha256": "ff3db45f78ec4ef694be15fd77f3092d73573b49e8d6b5283e4eacba4c712b4f",
      "candidate_ids": [
        "57096b66200fba1400367fa7",
        "572fcd86947a6a140053ccda",
        "56de17f9cffd8e1900b4b5e0",
        "572758e0f1498d1400e8f6ae",
        "5725c91e38643c19005acceb",
        "5726e313f1498d1400e8eeb2",
        "573796edc3c5551400e51f33",
        "5728510f4b864d1900164936",
        "57339c16d058e614000b5ec5",
        "570d4606b3d812140066d619",
        "5711623e50c2381900b54ab9",
        "573749741c4567190057445d",
        "572a0bebaf94a219006aa76f",
        "571cc6f85efbb31900334de4",
        "57263ea0271a42140099d7c3",
        "5737a25ac3c5551400e51f51"
      ],
      "candidate_sha256": [
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "b67161cb70767d25408d47a9b2cd15ed23cc66a97c33ff46706b10e7a4772023",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "c456b31b38f7b3d656d571ec7289b0630f5f2d8ce31a3965fddb4291053d73ef",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403"
      ],
      "example_id": "571cc6f85efbb31900334de4",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.5195777,
      "neural_order": [
        13,
        12,
        10,
        8,
        7,
        15,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        9,
        11
      ],
      "neural_top": 13,
      "partition": "calibration",
      "question_sha256": "de416bc4198be845332d2ff2f9b219fc39c6b5071d11289eb817ed518ad4a6ce",
      "trial_index": 289
    },
    {
      "answer_sha256": "0f4168490e38b8447e11ba4bd656aa11b925bd22af30bac464bc153fdb608501",
      "candidate_ids": [
        "56de3e414396321400ee26d8",
        "57339902d058e614000b5e70",
        "57287338ff5b5019007da232",
        "5711607f2419e314009555cd",
        "5725d52f89a1e219009abf78",
        "5725cb33271a42140099d1db",
        "5706149552bb891400689880",
        "571ccd9b5efbb31900334df8",
        "571114cfb654c5140001fb09",
        "57286bf84b864d19001649d2",
        "5733f062d058e614000b6633",
        "571cc8815efbb31900334dee",
        "571ce5055efbb31900334e2a",
        "5725b81b271a42140099d097",
        "56de15104396321400ee25b7",
        "5730005db2c2fd1400568703"
      ],
      "candidate_sha256": [
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "bb7672c097755bb254dcda07123304b126ddc10e9fc8c157410b39caff2b9954",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "9e5f19ae8c4bcea47d2097bb803a6f512f53874779686d5848e1aec6bdadef81",
        "291631d73d5dcdb57b0941017d7b1ab7a9501e869a33f4c995211b70c5487c88",
        "3243e4c3f26f22351bf8c52e8438c396a19b76af187bad37a4ca97532160e33d",
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932"
      ],
      "example_id": "571cc8815efbb31900334dee",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.16219339,
      "neural_order": [
        11,
        12,
        7,
        13,
        3,
        1,
        5,
        14,
        0,
        2,
        4,
        6,
        8,
        9,
        10,
        15
      ],
      "neural_top": 11,
      "partition": "calibration",
      "question_sha256": "417cc1a6ea8af029c0e8b94367c4ba0627d835852a903a0c2af1fa9e7051e159",
      "trial_index": 290
    },
    {
      "answer_sha256": "f29870c00f86674e27ca63194b25f2cb425a1dcf16761b135a507da2e5ba925a",
      "candidate_ids": [
        "571135b8a58dae1900cd6d0e",
        "57309564069b5314008321a5",
        "57297547af94a219006aa45c",
        "5729506d6aef051400154cac",
        "572a07fc6aef0514001551dc",
        "573010fab2c2fd14005687d7",
        "571cda1bdd7acb1400e4c196",
        "5725c69738643c19005accb9",
        "572fbea404bcaa1900d76c5b",
        "572663a9f1498d1400e8ddf2",
        "5726a5525951b619008f78dd",
        "572a064a3f37b3190047865d",
        "5729727baf94a219006aa437",
        "57282358ff5b5019007d9dde",
        "571cca4add7acb1400e4c150",
        "572a12381d0469140077972b"
      ],
      "candidate_sha256": [
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6"
      ],
      "example_id": "571cca4add7acb1400e4c150",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        7,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.21904345,
      "neural_order": [
        14,
        7,
        9,
        4,
        11,
        2,
        3,
        15,
        1,
        13,
        10,
        0,
        5,
        6,
        8,
        12
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "d9eeace7a06ba59c243008ba28d75363c06096eecaca4038238a34f9d9635d20",
      "trial_index": 291
    },
    {
      "answer_sha256": "9c0cf6d970ee6fc4581e1b2855f05773b57c38ee11391334b50c6c0b7400d5bd",
      "candidate_ids": [
        "56e1c2eee3433e1400423134",
        "573007fab2c2fd140056876b",
        "571ccc00dd7acb1400e4c15a",
        "572fdd03a23a5019007fca9d",
        "5737958b1c456719005744c3",
        "5733dd4f4776f419006613aa",
        "570d2c20fed7b91900d45ca7",
        "572683f95951b619008f7525",
        "572ff86004bcaa1900d76f65",
        "5727d9c43acd2414000dee17",
        "56e1ee4de3433e1400423210",
        "5705e33f52bb89140068964c",
        "573750f51c45671900574467",
        "5727f2714b864d1900164072",
        "5725f00938643c19005aced7",
        "5727ec062ca10214002d99b6"
      ],
      "candidate_sha256": [
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "d206c39c98d46bd01f841c82284343c93a112882c53094883d72423bafd6ee01",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "d23507599fde3bea64581b405ee4729f02882f17e7602e4f065a2b2a98228ea6",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2"
      ],
      "example_id": "571ccc00dd7acb1400e4c15a",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        14,
        15,
        13,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.55759147,
      "neural_order": [
        2,
        9,
        7,
        13,
        15,
        10,
        12,
        3,
        4,
        0,
        1,
        5,
        6,
        8,
        11,
        14
      ],
      "neural_top": 2,
      "partition": "calibration",
      "question_sha256": "e8ab29e37fd41747179a52c446eaf0d56deee92025ac46bbf21d84da0bf3a24f",
      "trial_index": 292
    },
    {
      "answer_sha256": "4abe88c54123cf7e171e4870517004fb8c7583212752c21118b593dd485d3290",
      "candidate_ids": [
        "572fdbb004bcaa1900d76dd9",
        "57266193dd62a815002e832a",
        "5730aa52069b53140083221d",
        "57271c235951b619008f860b",
        "573005b9947a6a140053cf6a",
        "57264a74708984140094c18b",
        "572941273f37b319004781ad",
        "57286ead2ca10214002da346",
        "573368e54776f41900660a53",
        "5706094b52bb8914006897de",
        "5730b255396df919000962b0",
        "5726edecdd62a815002e957f",
        "5726431aec44d21400f3dd13",
        "570961aa200fba1400367f15",
        "571ccd9b5efbb31900334df8",
        "5725c69738643c19005accb9"
      ],
      "candidate_sha256": [
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "2b7326d0cd8188e7b1ea38f54e115afbbb5e41e5f6886b800cf1fd31ac49dd48",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "c59e7593c862a0c68cee17ab74270592bac6538a668da2cdfed71c7db8087fec",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "9e5f19ae8c4bcea47d2097bb803a6f512f53874779686d5848e1aec6bdadef81",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e"
      ],
      "example_id": "571ccd9b5efbb31900334df8",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        2,
        3,
        12,
        0,
        1,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.47650482,
      "neural_order": [
        14,
        15,
        0,
        3,
        1,
        2,
        11,
        8,
        10,
        4,
        5,
        6,
        7,
        9,
        12,
        13
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "2aa0f26ed332fb3d01d48346cb23dc7c849f56c6f4b9ec204875c494f855c432",
      "trial_index": 293
    },
    {
      "answer_sha256": "0f4168490e38b8447e11ba4bd656aa11b925bd22af30bac464bc153fdb608501",
      "candidate_ids": [
        "57265200708984140094c237",
        "572648e8dd62a815002e8076",
        "57340549d058e614000b67dd",
        "572643de5951261400b5195a",
        "572fc80b04bcaa1900d76d11",
        "5725d42a89a1e219009abf58",
        "5705e4fe75f01819005e7704",
        "571142b3a58dae1900cd6d5a",
        "5726219489a1e219009ac2d0",
        "5726701add62a815002e84c8",
        "573368e54776f41900660a53",
        "572878942ca10214002da3a2",
        "5729727baf94a219006aa437",
        "5733a1854776f41900660f0a",
        "571ccfbadd7acb1400e4c164",
        "572a02483f37b3190047864b"
      ],
      "candidate_sha256": [
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "acd49776c5c867dd78a5c6b915a917f39479b502b183b3b4b0c8d4934fc1536a",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5"
      ],
      "example_id": "571ccfbadd7acb1400e4c164",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.40631819,
      "neural_order": [
        14,
        7,
        0,
        12,
        4,
        10,
        11,
        1,
        6,
        9,
        3,
        2,
        5,
        8,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "calibration",
      "question_sha256": "74761c7cb21eae7777f9d5dc5f52aa2d095dce2ed234be0833e805c43ce72489",
      "trial_index": 294
    },
    {
      "answer_sha256": "8575e43e52cfebdcf89f5f0fd05d8291b5aa3847836edde2705b86b4ca9f524f",
      "candidate_ids": [
        "572a064a3f37b3190047865d",
        "572647935951b619008f6ec9",
        "572648ed5951b619008f6f03",
        "5726a00cf1498d1400e8e550",
        "572686fc708984140094c8e5",
        "5726f96ddd62a815002e9698",
        "5729fa40af94a219006aa70f",
        "56e1c9bfe3433e1400423192",
        "57115b2850c2381900b54a7f",
        "57340549d058e614000b67dd",
        "57299a6f6aef051400155016",
        "56de49a8cffd8e1900b4b7a7",
        "57269aa65951b619008f77ab",
        "57265e11708984140094c3bb",
        "57111b95a58dae1900cd6c50",
        "571cd11add7acb1400e4c16e"
      ],
      "candidate_sha256": [
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "c426efe1359054866c8d322fd02f908d61dd09289424401de4416d000f338ca6",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387"
      ],
      "example_id": "571cd11add7acb1400e4c16e",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.33992907,
      "neural_order": [
        15,
        4,
        8,
        3,
        0,
        10,
        7,
        6,
        13,
        12,
        5,
        1,
        2,
        9,
        11,
        14
      ],
      "neural_top": 15,
      "partition": "calibration",
      "question_sha256": "43d3bcca9881fb3e48770fdbb0fdd108490b895895f425b3459b6fa27413bfb6",
      "trial_index": 295
    },
    {
      "answer_sha256": "5f902f6bb9a8fb0ad7cb38bc2fffd8f69271c9b1684843583c233e3964c6a333",
      "candidate_ids": [
        "5733dc95d058e614000b644a",
        "570d29c2b3d812140066d4bd",
        "5726a5525951b619008f78dd",
        "5711628a2419e314009555dd",
        "5733fb7bd058e614000b66ff",
        "56de3f784396321400ee26fa",
        "5733ffa7d058e614000b674f",
        "571099b2b654c5140001f9b3",
        "572a0a391d046914007796df",
        "5727dd2e4b864d1900163eba",
        "571cd3b55efbb31900334e03",
        "57337f6ad058e614000b5bcb",
        "572871bd3acd2414000dfa03",
        "570611c475f01819005e793c",
        "57273f27dd62a815002e9a0a",
        "57269bb8708984140094cb95"
      ],
      "candidate_sha256": [
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "099340918f9359979a06067b9361381a993dbef386a4851ba6bb74d73cd5dc60",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "db91c1023a94550f1ffe6e9f87e0b45aaad95f3bef9ada0d0c3872b88286bf7c",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820"
      ],
      "example_id": "571cd3b55efbb31900334e03",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        11,
        2,
        15,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        14
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.29640102,
      "neural_order": [
        10,
        3,
        15,
        8,
        6,
        2,
        14,
        0,
        1,
        4,
        5,
        7,
        9,
        11,
        12,
        13
      ],
      "neural_top": 10,
      "partition": "calibration",
      "question_sha256": "4d2b4ef01cea5d4f8d3ebd35f9acc27700d185bd80a92d73cea27fb22acc0c69",
      "trial_index": 296
    },
    {
      "answer_sha256": "e8f77ab588a87537152f417a854b372055488ca44dd5c44ade8321a0aedbe24b",
      "candidate_ids": [
        "572ff5fcb2c2fd1400568657",
        "572a11663f37b31900478693",
        "56dde27d9a695914005b9651",
        "5733d4c8d058e614000b6353",
        "571ce3745efbb31900334e20",
        "571cd5b1dd7acb1400e4c178",
        "571135b8a58dae1900cd6d0e",
        "572869b84b864d19001649ae",
        "5728742cff5b5019007da246",
        "5710968aa58dae1900cd6a86",
        "5705ec1675f01819005e776e",
        "5726534d708984140094c26d",
        "5725e748ec44d21400f3d733",
        "56de4c324396321400ee27ab",
        "572686ac5951b619008f75a9",
        "5729f9953f37b3190047861f"
      ],
      "candidate_sha256": [
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad"
      ],
      "example_id": "571cd5b1dd7acb1400e4c178",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        4,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.17785247,
      "neural_order": [
        5,
        4,
        6,
        13,
        11,
        0,
        7,
        1,
        8,
        14,
        2,
        3,
        9,
        10,
        12,
        15
      ],
      "neural_top": 5,
      "partition": "calibration",
      "question_sha256": "2ae4fde4a3bb08d26e9555801c9a62101fa77e8ee7dba75f30753333a41a6198",
      "trial_index": 297
    },
    {
      "answer_sha256": "4e23eacb6e0f35d7828149c58a9aca8c5802c403f7fd4a876121ddd195e3c8b4",
      "candidate_ids": [
        "5726b718dd62a815002e8dbe",
        "573362b94776f41900660974",
        "57264a74708984140094c18b",
        "57264991f1498d1400e8db2e",
        "5711119cb654c5140001fae3",
        "5726c19add62a815002e8f8a",
        "572665ff708984140094c4c3",
        "57293e221d046914007791d5",
        "572ffb02b2c2fd14005686b7",
        "571cd703dd7acb1400e4c182",
        "573007fab2c2fd140056876b",
        "572669a9dd62a815002e8416",
        "5728ebcb3acd2414000e01d9",
        "5727d9c43acd2414000dee17",
        "57274a1edd62a815002e9a9b",
        "5726509bdd62a815002e815a"
      ],
      "candidate_sha256": [
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "e71eab6fb81f391a6f9f6f7ddf6a435221f7159360e4d7a0549e3b969d010395",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "8657b03977310d90fc72a13d570f1c07e80c6100a3b2eeb92d2acdff9adb8b16",
        "d206c39c98d46bd01f841c82284343c93a112882c53094883d72423bafd6ee01",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "56342ec864f7c99a6400cb1b743d2de657d247bc5557e34b707c510c768abb1c",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7"
      ],
      "example_id": "571cd703dd7acb1400e4c182",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        10,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.46165193,
      "neural_order": [
        9,
        11,
        6,
        1,
        13,
        12,
        4,
        8,
        2,
        5,
        7,
        0,
        3,
        10,
        14,
        15
      ],
      "neural_top": 9,
      "partition": "calibration",
      "question_sha256": "7014eb2a04a630da1dc6f802ece09e1a2f111a67843bc4864e2fefdf376af672",
      "trial_index": 298
    },
    {
      "answer_sha256": "804cfd0005af7d584fe71fd3cdd95f2a6d156441f0772ce32e33c985f5ed0dd0",
      "candidate_ids": [
        "572944e03f37b319004781e1",
        "571cd88ddd7acb1400e4c18d",
        "572a019f3f37b31900478643",
        "57113be3a58dae1900cd6d28",
        "572fc8a904bcaa1900d76d1f",
        "5728ed94ff5b5019007da97c",
        "5710f4b8b654c5140001fa45",
        "572867212ca10214002da2f2",
        "572ffce5a23a5019007fcc15",
        "5710eca0a58dae1900cd6b3a",
        "572a1f086aef0514001552c0",
        "572655e5f1498d1400e8dc5e",
        "5733fb7bd058e614000b66ff",
        "57338255d058e614000b5c0d",
        "571097baa58dae1900cd6a98",
        "57096b66200fba1400367fa7"
      ],
      "candidate_sha256": [
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352"
      ],
      "example_id": "571cd88ddd7acb1400e4c18d",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        8,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.50663704,
      "neural_order": [
        1,
        3,
        10,
        4,
        15,
        12,
        2,
        5,
        6,
        8,
        0,
        7,
        9,
        11,
        13,
        14
      ],
      "neural_top": 1,
      "partition": "calibration",
      "question_sha256": "836393f5c73c75ffbfe9cbf2e032b8739bce8515c4e06b073e71230c3fd9ca49",
      "trial_index": 299
    },
    {
      "answer_sha256": "49a25f9feefaffecad0fcd30c50dc9331cff8b55ece53def6285c09e17e6f5d7",
      "candidate_ids": [
        "5733f1784776f41900661575",
        "5727515f708984140094dc11",
        "5727de862ca10214002d9860",
        "572681c1dd62a815002e8796",
        "5727fd123acd2414000df185",
        "570d4c3bfed7b91900d45e31",
        "572a213e6aef0514001552ee",
        "572f6a0ba23a5019007fc5eb",
        "5726deed5951b619008f80c5",
        "5727ef664b864d1900164060",
        "5733faaf4776f4190066162f",
        "5728855d3acd2414000dfa8d",
        "5726ea985951b619008f8261",
        "571cda1bdd7acb1400e4c196",
        "57340549d058e614000b67dd",
        "5733cf61d058e614000b62e9"
      ],
      "candidate_sha256": [
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "7a964e744589792d6e944294f3245ad4605a6cfd1edb01ff0a55eaa561f86c05",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20"
      ],
      "example_id": "571cda1bdd7acb1400e4c196",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.61476915,
      "neural_order": [
        13,
        3,
        6,
        1,
        2,
        8,
        0,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "987c4d1d8408e199cd715877e5bd6149a7bfbdb3a425828c490aabab60277f46",
      "trial_index": 300
    },
    {
      "answer_sha256": "8020e6435580e82ea61d4d7ab89db8785205d6e981224101353e8d16b232ebbf",
      "candidate_ids": [
        "5727d88b4b864d1900163e5e",
        "572f64ccb2c2fd14005680b7",
        "5732b6b5328d981900602021",
        "5733faaf4776f4190066162f",
        "57378e311c456719005744af",
        "5725cbb289a1e219009abed2",
        "572fd8efb2c2fd14005684fa",
        "5726dcbddd62a815002e9320",
        "571161092419e314009555d7",
        "57264228ec44d21400f3dcf5",
        "56de0f6a4396321400ee257f",
        "572fc5a1947a6a140053cc8a",
        "57274a8ff1498d1400e8f5bc",
        "57264845f1498d1400e8db0a",
        "571cdcb85efbb31900334e0c",
        "57108ee6a58dae1900cd6a1a"
      ],
      "candidate_sha256": [
        "4aaa6faf86477d3be34fa2e2263e88b5c583216a96351af8fd82b9f4924bd81e",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "0c60fccfa0a5891c548480d13173fbf66ab99f403f068010ba84b6402ed6daff",
        "040a5627ef7bb5d949592bbcd7381d5c49b141f464054a8ba8c03c8253480ddb",
        "9e095a3ecca7dadb435ed37e7d41a6678b5100836b272ad4c274c6d1374e9011",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6"
      ],
      "example_id": "571cdcb85efbb31900334e0c",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.66408982,
      "neural_order": [
        14,
        5,
        9,
        10,
        2,
        8,
        4,
        0,
        1,
        3,
        6,
        7,
        11,
        12,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "fef22f4ea53d3561b5dcf494fc8ba92152fee41253e5465bcc219446442090a3",
      "trial_index": 301
    },
    {
      "answer_sha256": "c67893848505cfe4f28431e77cd7741aa9872358ca376c5bec526b0fbfded715",
      "candidate_ids": [
        "5725cf3238643c19005acd61",
        "571cdcb85efbb31900334e0c",
        "571cde695efbb31900334e16",
        "56e1ded7cd28a01900c67bd4",
        "57115ac550c2381900b54a77",
        "572683075951b619008f7513",
        "572647d0708984140094c14b",
        "5727db85ff5b5019007d96fc",
        "57097c8fed30961900e841f2",
        "572665ff708984140094c4c3",
        "5730b541396df919000962c2",
        "572883a33acd2414000dfa79",
        "572a05eb3f37b31900478653",
        "57273d8bf1498d1400e8f4fe",
        "57263ea0271a42140099d7c3",
        "5728fb6a1d04691400778ef5"
      ],
      "candidate_sha256": [
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "e71eab6fb81f391a6f9f6f7ddf6a435221f7159360e4d7a0549e3b969d010395",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "55bbfa9052d82fc2074807e0795fc8957a358974ef289c3f0e19e4e8dbb35f85",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "0303b992326c30ab7517bcf7737bdc345dbbbc7eae232baca3af64f05f6565a4",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260"
      ],
      "example_id": "571cde695efbb31900334e16",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        1,
        4,
        6,
        9,
        0,
        3,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.02962756,
      "neural_order": [
        2,
        1,
        6,
        5,
        3,
        4,
        0,
        14,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "fe00380fde28adce3edbcb0d927889b78e3bc1be8ca90d8b6b0c73a5af0bbbaf",
      "trial_index": 302
    },
    {
      "answer_sha256": "61a0dbc3a2a3460819e4f73cf467634ca22dd05cac4549c5b5d6e7dca6c3421d",
      "candidate_ids": [
        "5725fabc89a1e219009ac128",
        "572825a92ca10214002d9efe",
        "570d50a5fed7b91900d45e7d",
        "57265746dd62a815002e8218",
        "570d529fb3d812140066d6bb",
        "5726581fdd62a815002e823a",
        "572ff7ab04bcaa1900d76f51",
        "5726baf2dd62a815002e8e74",
        "571ce3745efbb31900334e20",
        "57109275b654c5140001f99f",
        "5711119cb654c5140001fae3",
        "57338255d058e614000b5c0d",
        "571c8198dd7acb1400e4c0ce",
        "572faf74b2c2fd1400568347",
        "5727aa413acd2414000de921",
        "57309564069b5314008321a5"
      ],
      "candidate_sha256": [
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2"
      ],
      "example_id": "571ce3745efbb31900334e20",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.43626553,
      "neural_order": [
        8,
        3,
        12,
        4,
        1,
        6,
        9,
        7,
        5,
        0,
        2,
        10,
        11,
        13,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "a1dc5d5dfb87ffbc39b8d63cb36db5400e0c442b7fb6d16a95c2877128c07a09",
      "trial_index": 303
    },
    {
      "answer_sha256": "adf1e69257cbe74e46952acf36d24f9981c2d7b7b8111214b49529011abc073a",
      "candidate_ids": [
        "5727e9523acd2414000def95",
        "570d50a5fed7b91900d45e7d",
        "5730aef38ab72b1400f9c682",
        "571ce5055efbb31900334e2a",
        "56e1a564cd28a01900c67a48",
        "5725bbec271a42140099d0d1",
        "5710eb6fb654c5140001fa17",
        "56de3f784396321400ee26fa",
        "572ff760b2c2fd1400568677",
        "5726f36cdd62a815002e95fe",
        "572fffb404bcaa1900d76fef",
        "572a04d51d046914007796cd",
        "5727db85ff5b5019007d96fc",
        "572f5703a23a5019007fc573",
        "56de49a8cffd8e1900b4b7a7",
        "5725b81b271a42140099d097"
      ],
      "candidate_sha256": [
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "bedd9f5d0a5db4c8fc9999b0ed320ab5e6d56704b9e02e5331f2c1973ba98941",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "4cb08df8d00c330db23afb5af129580ac1c4437b39cc7557de63a8b111d7abe4",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f",
        "215da96f975d61a3dc7af5d1e2b75129453d3b8e5a3b4af20a46de0b25ce3030",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3"
      ],
      "example_id": "571ce5055efbb31900334e2a",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.51578705,
      "neural_order": [
        3,
        11,
        12,
        8,
        2,
        7,
        15,
        9,
        10,
        0,
        1,
        4,
        5,
        6,
        13,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "0a8ae8112206c619c5f16888090b9c5fc5726efee2fbeea856c81da9bec2e7c5",
      "trial_index": 304
    },
    {
      "answer_sha256": "368426a8c09eed4e81004d1c365031c2baa9687c94cf855be1f0e650d252c37d",
      "candidate_ids": [
        "5733d3cb4776f419006612ea",
        "5727f2714b864d1900164072",
        "5726eb4b5951b619008f826b",
        "5726d8bd708984140094d35b",
        "5727490bdd62a815002e9a82",
        "5728f50baf94a219006a9e55",
        "570d50a5fed7b91900d45e7d",
        "5711119cb654c5140001fae3",
        "5730b6592461fd1900a9cfcf",
        "5726a09f708984140094cc39",
        "5725c0f289a1e219009abdf2",
        "5727f7523acd2414000df10d",
        "57340549d058e614000b67dd",
        "573786b51c4567190057448d",
        "571ce6655efbb31900334e34",
        "5726ea985951b619008f8261"
      ],
      "candidate_sha256": [
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "9819559c80db18970983cbfdf2bcb3f75443deb410b562890bbe13cea74a03dc",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "7a964e744589792d6e944294f3245ad4605a6cfd1edb01ff0a55eaa561f86c05"
      ],
      "example_id": "571ce6655efbb31900334e34",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.55585235,
      "neural_order": [
        14,
        10,
        9,
        5,
        3,
        4,
        0,
        1,
        2,
        6,
        7,
        8,
        11,
        12,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "eaa09ad7a865766e9471617b57424a994adb89b56c2df43ecac1de7e634e3776",
      "trial_index": 305
    },
    {
      "answer_sha256": "faa4d6a8e5a7014752d7c82081b49cbbef7dde62b86994f05e7c6ba3ae710bdb",
      "candidate_ids": [
        "572982e66aef051400154f92",
        "572a070c6aef0514001551c8",
        "57115e532419e314009555af",
        "5728659f4b864d190016498a",
        "5729fa40af94a219006aa70f",
        "56de11154396321400ee25aa",
        "5730069004bcaa1900d7702d",
        "57264f18f1498d1400e8dbae",
        "572faec7b2c2fd1400568333",
        "56e16839cd28a01900c67887",
        "5727e9523acd2414000def95",
        "5728202c4b864d19001644ec",
        "572fd7b8947a6a140053cd3a",
        "57284456ff5b5019007da05c",
        "5737aafd1c456719005744fb",
        "5725b33f6a3fe71400b8952d"
      ],
      "candidate_sha256": [
        "93620703d50b4155fab9038240e64a854d1329e60d76b3887ba89b6b28625e9e",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "0427123719c1fd7ac7026780bf08a83e759f63ccba9a9425627df9633f21559d",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc"
      ],
      "example_id": "5725b33f6a3fe71400b8952d",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.53323569,
      "neural_order": [
        15,
        2,
        7,
        1,
        6,
        5,
        13,
        0,
        10,
        12,
        3,
        4,
        8,
        9,
        11,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "5416513c7a4e1f0de5c8d191d8118214399898ba49007aa6c47bd0559ed93a74",
      "trial_index": 306
    },
    {
      "answer_sha256": "4dd2ef8907e48f756486560f70e764a88469ab945132373ea85bcca1e20a1d65",
      "candidate_ids": [
        "573011de04bcaa1900d770f9",
        "572fbea404bcaa1900d76c5b",
        "57264586f1498d1400e8dac6",
        "5728659f4b864d190016498a",
        "5725b5a689a1e219009abd28",
        "57284456ff5b5019007da05c",
        "57300580b2c2fd140056874d",
        "57265642f1498d1400e8dc68",
        "57286c8cff5b5019007da218",
        "5725d34089a1e219009abf50",
        "5725e748ec44d21400f3d733",
        "572f59b4a23a5019007fc587",
        "56e1b169cd28a01900c67a72",
        "57337f6ad058e614000b5bcb",
        "56e1c0f6cd28a01900c67b2c",
        "572a07c11d046914007796d5"
      ],
      "candidate_sha256": [
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "0427123719c1fd7ac7026780bf08a83e759f63ccba9a9425627df9633f21559d",
        "8e983a2167b912359aa8e1110455d27a279e9b3baaac76337616f57b19396e19",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27"
      ],
      "example_id": "5725b5a689a1e219009abd28",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        3,
        9,
        13,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.26205896,
      "neural_order": [
        4,
        2,
        5,
        0,
        8,
        13,
        7,
        9,
        15,
        11,
        3,
        1,
        6,
        10,
        12,
        14
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "68e82ffad90fb6468a5dbeaf4ea8057410b1fb98a561fa036e08a17e8dfe6f99",
      "trial_index": 307
    },
    {
      "answer_sha256": "83690b33fc30e90cdd7013bbb4b59070a98209f93ab1ec066c4f18019a09e96f",
      "candidate_ids": [
        "5725b76389a1e219009abd4a",
        "572ff83ab2c2fd1400568681",
        "57377aac1c45671900574479",
        "5727d9c43acd2414000dee17",
        "5705eb3375f01819005e7764",
        "572856beff5b5019007da190",
        "57293ca73f37b3190047815b",
        "57282358ff5b5019007d9dde",
        "572870b2ff5b5019007da222",
        "5728759cff5b5019007da25a",
        "572759665951b619008f8883",
        "5733fb7bd058e614000b66ff",
        "57338255d058e614000b5c0d",
        "572863363acd2414000df985",
        "5725f190ec44d21400f3d76f",
        "56e1aba0e3433e1400423094"
      ],
      "candidate_sha256": [
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "75ac617b40cb38bf492969cedae24066684167be82eea9788bf5bc732da485ac",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8"
      ],
      "example_id": "5725b76389a1e219009abd4a",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        10,
        9,
        5,
        6,
        1,
        2,
        3,
        4,
        7,
        8,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.2787684,
      "neural_order": [
        0,
        10,
        6,
        14,
        4,
        8,
        1,
        13,
        11,
        5,
        9,
        3,
        2,
        7,
        12,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "0f27b7dcd5dfbea31ec06c9a0465e911584d2612ecae4151882e668de95d28e6",
      "trial_index": 308
    },
    {
      "answer_sha256": "35568a20b7c10e8227b5d4423e37b825797bf390ca5f8f4f7335189af32e2f13",
      "candidate_ids": [
        "5725b76389a1e219009abd4a",
        "56e1c2eee3433e1400423134",
        "5710f4b8b654c5140001fa45",
        "5725c57a89a1e219009abe5e",
        "5730a951069b531400832213",
        "5727cebc2ca10214002d96f0",
        "5710f114a58dae1900cd6b60",
        "5728fb002ca10214002dab6a",
        "571caac55efbb31900334dc6",
        "572726c9708984140094da7b",
        "57265bdfdd62a815002e829e",
        "56de1645cffd8e1900b4b5d0",
        "572a13841d0469140077973b",
        "57377ec7c3c5551400e51f05",
        "5725b92e38643c19005acbd3",
        "56de52614396321400ee27fb"
      ],
      "candidate_sha256": [
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "36476aaf2aea4f7927c065f38bbc7e2f8ec013b37ee0a2f1c7f545368f0dc62e",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066"
      ],
      "example_id": "5725b92e38643c19005acbd3",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        10,
        0,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.12501276,
      "neural_order": [
        14,
        0,
        10,
        12,
        8,
        2,
        6,
        4,
        11,
        5,
        13,
        1,
        3,
        7,
        9,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "f67e38007d87b198cca075eadd7aba16668e91f3505897b50cd7243ee458f627",
      "trial_index": 309
    },
    {
      "answer_sha256": "3e122bfe7b674c0e5112a12023a0a45881424b5f4b16e9d62c6dc42649c81d9f",
      "candidate_ids": [
        "57298ef11d0469140077952d",
        "57269cc3dd62a815002e8b12",
        "572881022ca10214002da416",
        "56de51c64396321400ee27f7",
        "5725bad5271a42140099d0bd",
        "57300a25a23a5019007fcc97",
        "5733f1784776f41900661575",
        "5725c071271a42140099d127",
        "572f76d1b2c2fd1400568153",
        "5706111a52bb89140068984c",
        "5726fa525951b619008f83f7",
        "572fadcbb2c2fd1400568329",
        "573062662461fd1900a9cdf7",
        "572867543acd2414000df9a1",
        "5730126ba23a5019007fcd09",
        "57108198b654c5140001f937"
      ],
      "candidate_sha256": [
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "6d2627b1297d1087a3e3c836fbeab911593b2c58f4b2ae45d6c39dd4ba695bea",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "b11a179d53281b2c32154de7d72bc5f8e3c0c35320a1c9d0f9b20fb0e505678d",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6"
      ],
      "example_id": "5725bad5271a42140099d0bd",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.07846635,
      "neural_order": [
        4,
        14,
        7,
        5,
        10,
        0,
        6,
        13,
        11,
        1,
        2,
        3,
        8,
        9,
        12,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "b2c3537a749803417ea7ab39aaf85f2bc0664c4089f3f958e2131c62aeeec5e0",
      "trial_index": 310
    },
    {
      "answer_sha256": "d8cf3b195f3ea163de57cc642a56f59ff952d507d7f106dd193cdeee0419b1d2",
      "candidate_ids": [
        "5725e748ec44d21400f3d733",
        "5725bcb6271a42140099d0eb",
        "573749741c4567190057445d",
        "5737477bc3c5551400e51ea3",
        "572a096e6aef0514001551e4",
        "5733f1784776f41900661575",
        "57294e6b1d04691400779275",
        "5728ed94ff5b5019007da97c",
        "57308f6b8ab72b1400f9c580",
        "5734025d4776f419006616c3",
        "57267640f1498d1400e8e074",
        "57265a58dd62a815002e8270",
        "56e1dc62cd28a01900c67bcb",
        "57339555d058e614000b5df3",
        "56e1aba0e3433e1400423094",
        "5710968aa58dae1900cd6a86"
      ],
      "candidate_sha256": [
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "5b8656a663456a61c1d9692e2b9d3858aa19a6b21b538a75b82941eb5206617c",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca"
      ],
      "example_id": "5725bcb6271a42140099d0eb",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        11,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.23502279,
      "neural_order": [
        1,
        6,
        11,
        5,
        8,
        9,
        12,
        0,
        2,
        3,
        4,
        7,
        10,
        13,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "9570ab1a1917b24332bdef649ecb67347a20e6461dd566bb9c0d89059c2e9f97",
      "trial_index": 311
    },
    {
      "answer_sha256": "e9c85351747c8da851cbdf8207512430f5eded86a2ef588b84aee74134bea613",
      "candidate_ids": [
        "57267ebfdd62a815002e872c",
        "5726a638dd62a815002e8bf6",
        "5726577f708984140094c301",
        "57288428ff5b5019007da28e",
        "5728f2e26aef051400154896",
        "57265746dd62a815002e8218",
        "571a484210f8ca1400304fbe",
        "5726398589a1e219009ac588",
        "5711619950c2381900b54aaf",
        "56de40da4396321400ee2708",
        "57300911947a6a140053cfb6",
        "571153422419e3140095557d",
        "57300f8504bcaa1900d770d1",
        "57264e455951b619008f6f65",
        "571a50df4faf5e1900b8a960",
        "57261dab38643c19005ad037"
      ],
      "candidate_sha256": [
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "e1e68b90e348d860a1f3c0cfe6e1367d9c8c8a57ad577d564b36e00900222682",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc"
      ],
      "example_id": "57261dab38643c19005ad037",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        12,
        8,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.20501421,
      "neural_order": [
        15,
        12,
        2,
        9,
        3,
        13,
        8,
        6,
        14,
        5,
        11,
        0,
        1,
        4,
        7,
        10
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "bb281f67570bfe76aa803aeaefc4cbde7fda40a58e6f1a8de4608986c86823f5",
      "trial_index": 312
    },
    {
      "answer_sha256": "32973a5f415645845ddf4138255483edd8b8e40ba738a87dbf540557d4850365",
      "candidate_ids": [
        "5725bcb6271a42140099d0eb",
        "57284618ff5b5019007da0a8",
        "56e1a38de3433e140042305c",
        "57268a37f1498d1400e8e33c",
        "572a1f086aef0514001552c0",
        "572fffb404bcaa1900d76fef",
        "5725e748ec44d21400f3d733",
        "572fdd03a23a5019007fca9d",
        "5728710c3acd2414000df9ef",
        "57336755d058e614000b5a3c",
        "570d2c20fed7b91900d45ca7",
        "56e1c7e2cd28a01900c67b74",
        "57297d421d046914007794e5",
        "572ffa79a23a5019007fcbe9",
        "57261f9f271a42140099d4a9",
        "56de4c324396321400ee27ab"
      ],
      "candidate_sha256": [
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "576db96228f229ebd5188bfb187b0285ce8e7a6eae31bd74cad56712e9e871a5",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913"
      ],
      "example_id": "57261f9f271a42140099d4a9",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        4,
        0,
        8,
        2,
        9,
        1,
        3,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.07615196,
      "neural_order": [
        14,
        4,
        0,
        5,
        1,
        13,
        6,
        10,
        12,
        2,
        3,
        15,
        7,
        8,
        9,
        11
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "485915b980f128849717ebdc8722a1c082853d65a10faffe59c0e1792ab0d329",
      "trial_index": 313
    },
    {
      "answer_sha256": "7276d25c8fa32302e7d032f94b792dfaa1e9de5c830ceaf1f3e8ae641c5cd43e",
      "candidate_ids": [
        "570d2d68fed7b91900d45cbb",
        "572648ed5951b619008f6f03",
        "572900f73f37b31900477f69",
        "572fd7b8947a6a140053cd3a",
        "57115c7450c2381900b54a9d",
        "572759665951b619008f8883",
        "5726f96ddd62a815002e9698",
        "5727dd2e4b864d1900163eba",
        "56e19557e3433e1400422fee",
        "57111992b654c5140001fb43",
        "56de4c324396321400ee27ab",
        "56de0ffd4396321400ee258d",
        "57264cac708984140094c1b3",
        "57109180a58dae1900cd6a40",
        "5726241189a1e219009ac2de",
        "5711119cb654c5140001fae3"
      ],
      "candidate_sha256": [
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "4e041656788e4036188e81c0a35175217b17488e8ea0585f48b4719b1fac1708",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "8d0f136ca3e21898494994833851f5c381fc610d00c5a286083f4b15d65caad4",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "61af1803b0730548c2562a214669d95520c7f20b1ca36c0cd3e8cbc90c590f5b",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438"
      ],
      "example_id": "5726241189a1e219009ac2de",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.22713425,
      "neural_order": [
        14,
        12,
        0,
        5,
        9,
        2,
        15,
        10,
        13,
        7,
        4,
        1,
        8,
        3,
        6,
        11
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "6832f12e879d794bc8fab87c7bc8b2b16fbc5e851e3333af4d4b75fd385b176a",
      "trial_index": 314
    },
    {
      "answer_sha256": "63c8807c2fd92b19aaf72de8e65052e29fb918197ee33e3aafcbfe1d0347a436",
      "candidate_ids": [
        "5727500f708984140094dbfd",
        "572fd6aa04bcaa1900d76d9f",
        "57266193dd62a815002e832a",
        "5725db98ec44d21400f3d6c5",
        "5726bcde708984140094cfbf",
        "57296f293f37b319004783a3",
        "57336755d058e614000b5a3c",
        "570d3468b3d812140066d543",
        "5726487b5951b619008f6edd",
        "5733834ed058e614000b5c26",
        "5728eff82ca10214002daadc",
        "572a0b101d046914007796e9",
        "57293d116aef051400154bc8",
        "57268066708984140094c821",
        "57265c10f1498d1400e8dd36",
        "5737a9afc3c5551400e51f61"
      ],
      "candidate_sha256": [
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "329e2dd3e90bd25984d9c186e564a1d7ab4e07d619284ce70de287269ef4a9a8",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31"
      ],
      "example_id": "5726487b5951b619008f6edd",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        3,
        9,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.35273771,
      "neural_order": [
        8,
        4,
        9,
        11,
        2,
        13,
        14,
        1,
        6,
        12,
        10,
        7,
        0,
        3,
        5,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "52877ca0816d2c76cf8b5b84c2c95d91ff011a005613c72c5068d091f4cffcd4",
      "trial_index": 315
    },
    {
      "answer_sha256": "15b407c8e881d47deda76cf56caf6705774ee41f02378b0bd1d711c0a00201ac",
      "candidate_ids": [
        "5733ef47d058e614000b662b",
        "57286b003acd2414000df9c1",
        "57097c8fed30961900e841f2",
        "5733a45d4776f41900660f23",
        "56dddf4066d3e219004dad5f",
        "571cca4add7acb1400e4c150",
        "572973f76aef051400154f0b",
        "572fae4b04bcaa1900d76bdf",
        "56de0f6a4396321400ee257f",
        "5727d6154b864d1900163e34",
        "571a4ead10f8ca1400304fdd",
        "57274a1edd62a815002e9a9b",
        "57114667a58dae1900cd6d80",
        "572649d8f1498d1400e8db37",
        "572863c72ca10214002da2d4",
        "5705f09e75f01819005e77a4"
      ],
      "candidate_sha256": [
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e"
      ],
      "example_id": "572649d8f1498d1400e8db37",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.36502053,
      "neural_order": [
        13,
        7,
        4,
        8,
        0,
        10,
        5,
        12,
        2,
        14,
        15,
        11,
        1,
        6,
        3,
        9
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "1b4d2f1c17052e27bf4ed154d2c77c2b9cd9300e1e65d978adf3520a228a2b00",
      "trial_index": 316
    },
    {
      "answer_sha256": "269e44da1f5d52de5eaa4cfc59df1b06f125dec008bc70fc0236d9c18ed97df9",
      "candidate_ids": [
        "5727482bf1498d1400e8f5a4",
        "56de4a474396321400ee2786",
        "5725c95f38643c19005accf3",
        "571cc8815efbb31900334dee",
        "572864542ca10214002da2de",
        "572996c73f37b319004784b3",
        "571166352419e314009555f1",
        "56de49564396321400ee277a",
        "5726462b708984140094c117",
        "57265200708984140094c237",
        "57264cac708984140094c1b3",
        "5726e08e5951b619008f810f",
        "571caac55efbb31900334dc6",
        "5725c743ec44d21400f3d549",
        "572fca7eb2c2fd1400568473",
        "5725b33f6a3fe71400b8952d"
      ],
      "candidate_sha256": [
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "e5f87e55985983815a9352a8e3c2413035c4ebf26e931d9d4dbccd2d10f98bf9",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9",
        "36476aaf2aea4f7927c065f38bbc7e2f8ec013b37ee0a2f1c7f545368f0dc62e",
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc"
      ],
      "example_id": "57264cac708984140094c1b3",
      "gated_correct": false,
      "gated_top": 15,
      "gold_index": 10,
      "lexical_correct": false,
      "lexical_order": [
        15,
        10,
        13,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        11,
        12,
        14
      ],
      "lexical_top": 15,
      "neural_correct": false,
      "neural_margin": 0.03033881,
      "neural_order": [
        15,
        10,
        9,
        6,
        12,
        3,
        5,
        2,
        7,
        0,
        1,
        4,
        8,
        11,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "79be1f3e389682d799d8aabb2c38ed60ebc0083e5fb6605c939add6bfef8730d",
      "trial_index": 317
    },
    {
      "answer_sha256": "ccf443450e4fa5a2a579cad282b339d863b03b82093ab449bd883d3ef10fa0b1",
      "candidate_ids": [
        "572ff935b2c2fd140056869b",
        "5737a25ac3c5551400e51f51",
        "57280fd3ff5b5019007d9c26",
        "56de40da4396321400ee2708",
        "57339ad74776f41900660e86",
        "5727aeac3acd2414000de987",
        "57264d9edd62a815002e80fe",
        "5728848cff5b5019007da298",
        "56de52614396321400ee27fb",
        "57337ea24776f41900660bd0",
        "572951f16aef051400154cce",
        "571136b8a58dae1900cd6d22",
        "5726a7ecf1498d1400e8e654",
        "56e1c2eee3433e1400423134",
        "572a02483f37b3190047864b",
        "570d28bdb3d812140066d4a3"
      ],
      "candidate_sha256": [
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9"
      ],
      "example_id": "57264d9edd62a815002e80fe",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        15,
        2,
        0,
        1,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.31043229,
      "neural_order": [
        6,
        10,
        5,
        7,
        11,
        8,
        4,
        2,
        3,
        15,
        13,
        1,
        0,
        9,
        12,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "0c5e6ac8f3b4f169ab048f2f0df24e811a05b34312a42d0cdae9bd3c035caec2",
      "trial_index": 318
    },
    {
      "answer_sha256": "5483494b8a37b7c6115affcad584acce99a341fe424ef72669bdde36c2ea6bdf",
      "candidate_ids": [
        "56de4a89cffd8e1900b4b7bd",
        "572941273f37b319004781ad",
        "5725daa8ec44d21400f3d6b3",
        "5733d68ed058e614000b637f",
        "5726baf2dd62a815002e8e74",
        "572fc8a904bcaa1900d76d1f",
        "5726722bdd62a815002e8528",
        "57273a465951b619008f86ff",
        "57376c50c3c5551400e51ecd",
        "5706074552bb8914006897d4",
        "5733db8dd058e614000b6428",
        "57108d69b654c5140001f983",
        "57264efddd62a815002e8134",
        "57339c16d058e614000b5ec5",
        "5711669550c2381900b54adf",
        "5728501aff5b5019007da166"
      ],
      "candidate_sha256": [
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "0fe05bd1833002131a7e69e040cc114a9dfc4409000d5d0674152325ab91cb19",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "9807afc08eed1c309af6758565c9fb73e71562db167a6f2beaae892f76dd712d",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "7544f3f342f82852b6fd8a74d46285e0772381cc4cd0b61c47fea00590e8938e",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab"
      ],
      "example_id": "57264efddd62a815002e8134",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        5,
        8,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.35319554,
      "neural_order": [
        12,
        14,
        6,
        5,
        2,
        1,
        4,
        11,
        9,
        0,
        3,
        7,
        8,
        10,
        13,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "a2f0d6517a7e1ec13643971b8b4e186678cabefe1f2a89d98d8a209a1fe28efc",
      "trial_index": 319
    },
    {
      "answer_sha256": "105e1c07f38cb21b8c6e1bfe102dfd958ea479bdbce8045b720c5679ca0fdb0e",
      "candidate_ids": [
        "572940246aef051400154bec",
        "57114dfb50c2381900b54a53",
        "570d4030fed7b91900d45d9f",
        "57275409708984140094dc35",
        "5728ec6a4b864d19001650aa",
        "56e1cbe2cd28a01900c67bac",
        "5728659f4b864d190016498a",
        "572742bd5951b619008f8785",
        "57274f67708984140094dbf5",
        "572f57c704bcaa1900d7686d",
        "57274e975951b619008f87f9",
        "572853e8ff5b5019007da188",
        "572650325951b619008f6fa9",
        "573749741c4567190057445d",
        "572a05eb3f37b31900478653",
        "57264d9edd62a815002e80fe"
      ],
      "candidate_sha256": [
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "4b2a2a7e8bce520e5e42162e14db64c90808895b36ef2edfc527bbc3df66971c",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "e83edd2f6eb1fa0981acc0119542b4a54c226b0e0249686e43082425f6d974b0",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "e6e9688ac6ad143a28b8c993b4b8abe4fd62becbdc3ffdc61af49f215cd67b21",
        "ce8d2359f23498cb006a1bf2f91d5494c463bc986a83749d5f9208b28fa63c51",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "b0008033f8b97b0cf458d8e505384798d796a2fc78a3fced41f39a2591ef64cd",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2"
      ],
      "example_id": "572650325951b619008f6fa9",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        15,
        14,
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        13
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.13968594,
      "neural_order": [
        12,
        15,
        14,
        1,
        0,
        7,
        8,
        10,
        5,
        9,
        3,
        2,
        4,
        6,
        11,
        13
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "2a0267d9a5f9712f8b5438440b50ad1decf26a8ec6d0b16b6dae5a21141a7161",
      "trial_index": 320
    },
    {
      "answer_sha256": "a075c474f3e7f0cb0283485808512c9df91f875916a3b02a861266bcd644b649",
      "candidate_ids": [
        "5726ba2c708984140094cf59",
        "572a020f6aef051400155198",
        "572648d1708984140094c15d",
        "57377862c3c5551400e51ef3",
        "572a0d21af94a219006aa783",
        "572a1046af94a219006aa78d",
        "5729ffda1d046914007796af",
        "57265200708984140094c237",
        "5733d249d058e614000b6331",
        "57287b322ca10214002da3be",
        "5728eef92ca10214002daab0",
        "5726a638dd62a815002e8bf6",
        "5728eb1a3acd2414000e01c5",
        "5730ab63396df91900096260",
        "573735e8c3c5551400e51e71",
        "570d2417fed7b91900d45c3d"
      ],
      "candidate_sha256": [
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "b557caa6a1a9ebdc6ec705acc8c1078fbde1cf27379b5961913015d24cbd8809",
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d",
        "80e800db40294efc728c4dcbf06c36437000cef741a99c5f9ed99d6d0756badc",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3"
      ],
      "example_id": "57265200708984140094c237",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        4,
        13,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.35672637,
      "neural_order": [
        7,
        8,
        4,
        11,
        13,
        9,
        14,
        5,
        0,
        6,
        10,
        1,
        2,
        3,
        12,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "78e5a96b2b8320a5dc2d4a13b8a321380354ea42ca881c3dcd84fcae52cb2f63",
      "trial_index": 321
    },
    {
      "answer_sha256": "59ae856cd788d0f57e39fdd66d421ba930cd89be4682de3aa36c22a2021a710d",
      "candidate_ids": [
        "572856beff5b5019007da190",
        "57268da7f1498d1400e8e39c",
        "57106d2fb654c5140001f8ef",
        "572a0bafaf94a219006aa765",
        "572941273f37b319004781ad",
        "5726a638dd62a815002e8bf6",
        "57338160d058e614000b5bf9",
        "57265360dd62a815002e819a",
        "5728fd3c3f37b31900477f2b",
        "57264684708984140094c123",
        "5727fd123acd2414000df185",
        "572a1fe16aef0514001552d0",
        "57335c20d058e614000b58f9",
        "5733d68ed058e614000b637f",
        "570d2af0fed7b91900d45c93",
        "5733d7cbd058e614000b63ab"
      ],
      "candidate_sha256": [
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "55e3030ac51ce0d76d4fcad6c72bf5b57a9192da5256c5e338b03fe44bf2bb59",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "0fe05bd1833002131a7e69e040cc114a9dfc4409000d5d0674152325ab91cb19",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202"
      ],
      "example_id": "57265360dd62a815002e819a",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        11,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.192316,
      "neural_order": [
        7,
        10,
        4,
        11,
        9,
        13,
        3,
        5,
        14,
        12,
        15,
        2,
        0,
        1,
        6,
        8
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "52f07e60ba891f2fdaee92d3121fd61e00e11c9268901d9e8c321f6c35d9fe68",
      "trial_index": 322
    },
    {
      "answer_sha256": "7c22251e7f589932f61e6d5539740110be1290c2ab87b9ce0d7a4203ee71562b",
      "candidate_ids": [
        "5727515f708984140094dc11",
        "5727500f708984140094dbfd",
        "571c96095efbb31900334dbc",
        "57265526708984140094c2bd",
        "57308cf88ab72b1400f9c576",
        "5725e1c4271a42140099d2d7",
        "571c8eb9dd7acb1400e4c0f6",
        "572f76d1b2c2fd1400568153",
        "5727e21e4b864d1900163f34",
        "570d28bdb3d812140066d4a3",
        "572643de5951261400b5195a",
        "5727f05b4b864d1900164068",
        "5727ef664b864d1900164060",
        "57060cc352bb89140068980e",
        "5705e99452bb891400689688",
        "5726472bdd62a815002e8042"
      ],
      "candidate_sha256": [
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "3ee61e6d7605bacdeb4f37ab798daa449f57668fd923e6b04e1a516237821782",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "874533677a269c5d5fb8bd3a3213307ea1632c2236ea48b8da8182d07905fcc8",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5"
      ],
      "example_id": "57265526708984140094c2bd",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        4,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.31745088,
      "neural_order": [
        3,
        4,
        7,
        11,
        9,
        14,
        13,
        10,
        0,
        1,
        2,
        5,
        6,
        8,
        12,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "4d20af30e3c81421c82955773c9d783c67e5bfaaa1f355fb9c1e85793f5bf720",
      "trial_index": 323
    },
    {
      "answer_sha256": "8f45569ea8b53a4a3c0538c25e087a497874d3edbf910ac6825f10462278fac0",
      "candidate_ids": [
        "5727cd0f4b864d1900163d72",
        "57293ca73f37b3190047815b",
        "5727da564b864d1900163e8f",
        "572a11663f37b31900478693",
        "5711475ca58dae1900cd6d8b",
        "572869b84b864d19001649ae",
        "571ce3745efbb31900334e20",
        "5725c41eec44d21400f3d50b",
        "5728151b4b864d1900164428",
        "57377c98c3c5551400e51efb",
        "57377083c3c5551400e51edf",
        "57337ea24776f41900660bd0",
        "5726581fdd62a815002e823a",
        "57264c42dd62a815002e80c6",
        "570d26efb3d812140066d48f",
        "572a058aaf94a219006aa751"
      ],
      "candidate_sha256": [
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "ef42577253be3fe07baa71af3cfeb5941c249b771cedcf680fa4a087509e1d2c",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "892da1bea8067c68eb6c0076aabb9951ef6cdc4cc76275dd8412ade20dfeca2c",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "1c326ae15d585661849c0a43f71be232f457f11a2a48964b11b0d9172e7cd31b",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "2ef6bc8b598c3ae95dea517f30b72d286a3c1ea1c56e1f73f6bd8f2ab9cb63ef",
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701"
      ],
      "example_id": "5726581fdd62a815002e823a",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.39479653,
      "neural_order": [
        12,
        3,
        5,
        11,
        4,
        10,
        7,
        14,
        0,
        13,
        1,
        2,
        6,
        8,
        9,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "29cc2fa7aa7a7b4e871dc757278b670a914cb10fc9cf096372d75de7c025e20a",
      "trial_index": 324
    },
    {
      "answer_sha256": "6dee91b78812aa23d9619e0d74471cd9eb2088cda26ac83de17fab7b416ae78b",
      "candidate_ids": [
        "572753335951b619008f8853",
        "5730126ba23a5019007fcd09",
        "571098f8a58dae1900cd6ab2",
        "57306797396df919000960ee",
        "572fc043a23a5019007fc95d",
        "573098f38ab72b1400f9c5d2",
        "572659535951b619008f703f",
        "5733d13e4776f419006612c4",
        "57302efe04bcaa1900d772f5",
        "5726577f708984140094c301",
        "572fac17947a6a140053cb54",
        "57094d489928a8140047150a",
        "572a0e0e1d04691400779707",
        "5730aef38ab72b1400f9c682",
        "572f7b33947a6a140053c9a2",
        "572a0ecb1d04691400779717"
      ],
      "candidate_sha256": [
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "f3e994c58b1e4906937c035c80fb9519764a0f4f42ff12a50c7aeac3190ebd39",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "4b184251045ff68cd470a8330d18c3848a5524d5ac96c1ad4b4b92ae5d9d5694",
        "bedd9f5d0a5db4c8fc9999b0ed320ab5e6d56704b9e02e5331f2c1973ba98941",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835"
      ],
      "example_id": "572659535951b619008f703f",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.44938283,
      "neural_order": [
        6,
        0,
        10,
        4,
        13,
        11,
        2,
        7,
        14,
        1,
        3,
        5,
        8,
        9,
        12,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "077b923dd34a85bbbc5c400233e60e88fa94b7466fe0a4f2314fa7ae2c316db3",
      "trial_index": 325
    },
    {
      "answer_sha256": "ecb0eef76639d741ec4e1fd0d185fbeef8966549a7df570bc07d54b4fb0e6357",
      "candidate_ids": [
        "5706143575f01819005e7950",
        "57265e11708984140094c3bb",
        "571c83f3dd7acb1400e4c0d8",
        "572643de5951261400b5195a",
        "57265a58dd62a815002e8270",
        "5706139252bb891400689864",
        "5728683b3acd2414000df9af",
        "572683075951b619008f7513",
        "5729f4b41d0469140077968b",
        "572754fff1498d1400e8f660",
        "57274beff1498d1400e8f5e4",
        "5728661e2ca10214002da2e8",
        "57094a79efce8f15003a7dc4",
        "57265d86f1498d1400e8dd50",
        "5727dd2e4b864d1900163eba",
        "5733e5a14776f4190066145b"
      ],
      "candidate_sha256": [
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "cabcb2e17697c1ca5e1bf56512bbf3851f990033a3f5d302aa90e0f3bfed2236",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92"
      ],
      "example_id": "57265a58dd62a815002e8270",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.34838648,
      "neural_order": [
        4,
        3,
        5,
        0,
        13,
        7,
        12,
        10,
        8,
        9,
        2,
        1,
        6,
        11,
        14,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "a01266878895f606b4aa875f18cc2987272e6f1be071e0aca96a04123ec40be8",
      "trial_index": 326
    },
    {
      "answer_sha256": "2e0b6634d610d18703807120aaa3f56af1ace36818cd223af5402eb2bb001f64",
      "candidate_ids": [
        "572faf74b2c2fd1400568347",
        "57281940ff5b5019007d9d44",
        "57265bdfdd62a815002e829e",
        "56de15dbcffd8e1900b4b5c8",
        "56de1563cffd8e1900b4b5c2",
        "57308cf88ab72b1400f9c576",
        "572943ab1d04691400779219",
        "5725f46289a1e219009ac0fa",
        "572fda6fb2c2fd140056850b",
        "56de3cd0cffd8e1900b4b6be",
        "571cc3dedd7acb1400e4c146",
        "5728661e2ca10214002da2e8",
        "570d3468b3d812140066d543",
        "57265c10f1498d1400e8dd36",
        "573368e54776f41900660a53",
        "57264e2f708984140094c1e1"
      ],
      "candidate_sha256": [
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "5b1d7779ff634f8b4049e70e99b5eaadf538e5a3d3a2aa5069b096c68cb79e5d",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75"
      ],
      "example_id": "57265bdfdd62a815002e829e",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        12,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.25835492,
      "neural_order": [
        2,
        12,
        7,
        1,
        6,
        5,
        4,
        10,
        14,
        0,
        3,
        8,
        9,
        11,
        13,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "620d3ea9df0177ebb3902f6dc4edc9975e76b1efb341eac1e180c7993ced904b",
      "trial_index": 327
    },
    {
      "answer_sha256": "11d1ff4edd53407199c92b00cbc3abe8c9c5d5bed4414ad07b55c896b0e7a713",
      "candidate_ids": [
        "572883a33acd2414000dfa79",
        "5733d13e4776f419006612c4",
        "5727515f708984140094dc11",
        "5725ec8289a1e219009ac0ae",
        "5726dcbddd62a815002e9320",
        "572815d9ff5b5019007d9cbc",
        "57265ceddd62a815002e82b8",
        "5706149552bb891400689880",
        "571cca4add7acb1400e4c150",
        "572a07a86aef0514001551d2",
        "56e1b62ecd28a01900c67aa3",
        "573312fcd058e614000b56e6",
        "57299c2c6aef051400155020",
        "572905ce1d04691400778f83",
        "56de0f6a4396321400ee257f",
        "571117d4a58dae1900cd6c0a"
      ],
      "candidate_sha256": [
        "55bbfa9052d82fc2074807e0795fc8957a358974ef289c3f0e19e4e8dbb35f85",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032"
      ],
      "example_id": "57265ceddd62a815002e82b8",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.57938476,
      "neural_order": [
        6,
        3,
        5,
        9,
        8,
        2,
        11,
        12,
        7,
        4,
        0,
        1,
        10,
        13,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "893f5091532504c46d98f2500941ed9b0b93f18f7a2179e43e6855c3ea35ad7d",
      "trial_index": 328
    },
    {
      "answer_sha256": "a78f19952edd18bf02b3c9eb704b088e2120941d6acb22f6f795c42796e60252",
      "candidate_ids": [
        "57264b3edd62a815002e80aa",
        "57113f83b654c5140001fc29",
        "5727ee372ca10214002d99ec",
        "5706149552bb891400689880",
        "5733f5264776f419006615a3",
        "56e16182e3433e1400422e28",
        "57294baaaf94a219006aa26b",
        "57280f974b864d1900164370",
        "572fdbb004bcaa1900d76dd9",
        "57060cc352bb89140068980e",
        "5711541350c2381900b54a6f",
        "5727cc15ff5b5019007d9576",
        "57114aceb654c5140001fc47",
        "573027d6a23a5019007fce9d",
        "57265e11708984140094c3bb",
        "5737a9afc3c5551400e51f61"
      ],
      "candidate_sha256": [
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "a6b7ec7938b966c86b4cc72fa2059dcdbd60bc13208210cff0105baccb082b8b",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "9384b2ae8d06f600126ecfff9a80168fbb32d4cda2c1d70e200f7ab4d0578b47",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31"
      ],
      "example_id": "57265e11708984140094c3bb",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.32507744,
      "neural_order": [
        14,
        1,
        6,
        10,
        13,
        12,
        2,
        0,
        3,
        4,
        5,
        7,
        8,
        9,
        11,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "44d71c040901683c9c888715d13cc715bf329aafb2837890c49baca8603bb284",
      "trial_index": 329
    },
    {
      "answer_sha256": "fff47ba2fb52a5832e0e4f0fb1f63cd062a41b48d01622633030e408c77c578b",
      "candidate_ids": [
        "57332e48d058e614000b5762",
        "5727aeac3acd2414000de987",
        "56de10b44396321400ee2593",
        "572945b11d0469140077922f",
        "5733fb7bd058e614000b66ff",
        "57293e983f37b3190047818b",
        "572f58d9a23a5019007fc57d",
        "571c8eb9dd7acb1400e4c0f6",
        "57266193dd62a815002e832a",
        "573085ea8ab72b1400f9c54c",
        "56e1a0dccd28a01900c67a2e",
        "5727d88b4b864d1900163e5e",
        "57273e50dd62a815002e9a02",
        "5725b7f389a1e219009abd5d",
        "56e1b169cd28a01900c67a72",
        "572856beff5b5019007da190"
      ],
      "candidate_sha256": [
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "46a60c1fb7a3dc1eb8d8008909c9080e1b010b9c75f648c33a07f6ce1e997157",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "4aaa6faf86477d3be34fa2e2263e88b5c583216a96351af8fd82b9f4924bd81e",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796"
      ],
      "example_id": "5725b7f389a1e219009abd5d",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        5,
        15,
        7,
        11,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        12,
        14
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.51175181,
      "neural_order": [
        13,
        6,
        5,
        3,
        0,
        4,
        12,
        7,
        9,
        10,
        14,
        2,
        15,
        8,
        1,
        11
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "41b589741e5f65575a9eab8a0035006ef55d1288899b2847c2d13713b0a8308c",
      "trial_index": 330
    },
    {
      "answer_sha256": "66352a1da7979409506737d0f43eff8d5587b86edb00c051209172368ba520ab",
      "candidate_ids": [
        "57114dfb50c2381900b54a53",
        "5706139252bb891400689864",
        "571099b2b654c5140001f9b3",
        "5726b718dd62a815002e8dbe",
        "5733a1854776f41900660f0a",
        "572fd6aa04bcaa1900d76d9f",
        "5705fc3a52bb89140068976a",
        "56de4bb84396321400ee27a2",
        "57264f18f1498d1400e8dbae",
        "5725bbec271a42140099d0d1",
        "5705f36452bb891400689718",
        "57274ca8dd62a815002e9aa4",
        "571cc8815efbb31900334dee",
        "5737a5931c456719005744e7",
        "5733a560d058e614000b5f77",
        "57377ec7c3c5551400e51f05"
      ],
      "candidate_sha256": [
        "4b2a2a7e8bce520e5e42162e14db64c90808895b36ef2edfc527bbc3df66971c",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "acd49776c5c867dd78a5c6b915a917f39479b502b183b3b4b0c8d4934fc1536a",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71"
      ],
      "example_id": "5725bbec271a42140099d0d1",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        3,
        15,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.31982854,
      "neural_order": [
        9,
        3,
        5,
        2,
        0,
        8,
        14,
        15,
        10,
        13,
        12,
        4,
        1,
        6,
        7,
        11
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "08e3ea470d67177ad0fe14f7a7931f8fd9b8a8c18228df881a0d07f8ff5d3692",
      "trial_index": 331
    },
    {
      "answer_sha256": "d3393054b41a90831e29a432303bc72650c0197d2c7fe915bb5bc7902c797fa5",
      "candidate_ids": [
        "5730042804bcaa1900d77011",
        "5725c743ec44d21400f3d549",
        "5737a4511c456719005744df",
        "57332c1e4776f4190066073a",
        "573027d6a23a5019007fce9d",
        "57273f27dd62a815002e9a0a",
        "572991943f37b319004784a1",
        "5727ea45ff5b5019007d986e",
        "572905ce1d04691400778f83",
        "5727fc37ff5b5019007d9a18",
        "57269e3bf1498d1400e8e516",
        "572ff83ab2c2fd1400568681",
        "57274046dd62a815002e9a12",
        "572fffb1b2c2fd14005686f9",
        "5726398589a1e219009ac588",
        "57336755d058e614000b5a3c"
      ],
      "candidate_sha256": [
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "0392a2097b98d54cbb0accb20f821be719c1c3bbb9aa73d9293a66c7480a8de6",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "809ab002069708b05d7022073e5692a5d269e21b863617ea0d1573256c8af15f",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702"
      ],
      "example_id": "5725c743ec44d21400f3d549",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.23352527,
      "neural_order": [
        1,
        10,
        6,
        9,
        11,
        5,
        0,
        7,
        2,
        3,
        12,
        15,
        4,
        8,
        13,
        14
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "940cd67b2fc27c3c4060cfa8194baa0fa3974aa79408c0943ba903422e958d5f",
      "trial_index": 332
    },
    {
      "answer_sha256": "0924c7f0857904d516a85077dccb58f0df8d4920ae20a35e5abab4d93d8cc4b6",
      "candidate_ids": [
        "5726c3da708984140094d0d9",
        "56e1b355e3433e14004230b0",
        "5728eef92ca10214002daab0",
        "571161092419e314009555d7",
        "572a07fc6aef0514001551dc",
        "5710eca0a58dae1900cd6b3a",
        "57097c8fed30961900e841f2",
        "57096b66200fba1400367fa7",
        "572fdb17b2c2fd140056851d",
        "5725ca4389a1e219009abeb4",
        "572fd6aa04bcaa1900d76d9f",
        "5733638fd058e614000b59e8",
        "5705fc3a52bb89140068976a",
        "57286f373acd2414000df9db",
        "5733a2a9d058e614000b5f28",
        "572882242ca10214002da420"
      ],
      "candidate_sha256": [
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc",
        "62ee7897368284927c822dd6d4fe87c9b80a087753928b980b6d5f9f64e71efb",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "05e6b70a4c996f49687d72f7a373609343967f6fed5be1f3bda6965dcfad141e",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c"
      ],
      "example_id": "5725ca4389a1e219009abeb4",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        8,
        13,
        2,
        1,
        3,
        4,
        5,
        6,
        7,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.37098206,
      "neural_order": [
        9,
        0,
        8,
        2,
        5,
        10,
        12,
        4,
        13,
        6,
        1,
        3,
        7,
        11,
        14,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "0d3ff4074368fb6153b1063d18f8b34247ce984a59ae0740a6c48e333681d236",
      "trial_index": 333
    },
    {
      "answer_sha256": "84a4ed89bc846d9e0c6b27a613c5487cfb266ba6c056037e0e66f07b7052c188",
      "candidate_ids": [
        "570d50a5fed7b91900d45e7d",
        "570d29c2b3d812140066d4bd",
        "57285841ff5b5019007da19b",
        "572fae4b04bcaa1900d76bdf",
        "572a1a5c6aef051400155284",
        "571c8eb9dd7acb1400e4c0f6",
        "5725f8f5ec44d21400f3d7b1",
        "5725bbec271a42140099d0d1",
        "572746addd62a815002e9a5e",
        "5726219489a1e219009ac2d0",
        "5730b255396df919000962b0",
        "572f6c85947a6a140053c940",
        "5727cff1ff5b5019007d95c4",
        "572fbf21a23a5019007fc937",
        "571090abb654c5140001f995",
        "5725cc38ec44d21400f3d5bb"
      ],
      "candidate_sha256": [
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "0762a1bc5feb74b6631bfc70b8e16724e09b6a70c7043591f2417fd39f23f6e0",
        "bb3727c243a6cb744686ba6886c54791cd7745eca592eef8baf9c5b06c996b3e",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "93fbba530b12b34ec00e68920885b0413568d15ff0159deaeffa23795ab32c51"
      ],
      "example_id": "5725cc38ec44d21400f3d5bb",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        7,
        10,
        1,
        5,
        0,
        2,
        3,
        4,
        6,
        8,
        9,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.16981192,
      "neural_order": [
        15,
        7,
        3,
        10,
        11,
        5,
        4,
        13,
        9,
        14,
        0,
        1,
        2,
        6,
        8,
        12
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "e103864bfed5e1e6714bc74a910426941a8f85f9afc44233185f1a2e4565ab0c",
      "trial_index": 334
    },
    {
      "answer_sha256": "51654f170a83aab0104a6d1e8535fd3f3179cd35b78858d19d0ed804612fbc2d",
      "candidate_ids": [
        "57300c67947a6a140053cff0",
        "571cc5c45efbb31900334dda",
        "57265d08708984140094c397",
        "57339dd94776f41900660ecc",
        "5737821cc3c5551400e51f19",
        "57332a734776f41900660726",
        "5733dd4f4776f419006613aa",
        "570967c4ed30961900e840ba",
        "5727e9523acd2414000def95",
        "57281940ff5b5019007d9d44",
        "571a53d410f8ca1400304fe5",
        "57264865dd62a815002e8062",
        "57109180a58dae1900cd6a40",
        "57286d7d4b864d19001649e0",
        "5725e748ec44d21400f3d733",
        "57097c8fed30961900e841f2"
      ],
      "candidate_sha256": [
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336"
      ],
      "example_id": "57264865dd62a815002e8062",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.36852935,
      "neural_order": [
        11,
        6,
        5,
        15,
        0,
        4,
        7,
        13,
        8,
        2,
        3,
        12,
        1,
        9,
        10,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "b37198ba46c9e08945348fa3f387b62fc1bfff4814f60dfe4bc6d5e4e044903a",
      "trial_index": 335
    },
    {
      "answer_sha256": "cd9166dc02c0d17bbfebd28e24f62134eb7375820280c0cc12f6024695aac6c8",
      "candidate_ids": [
        "57114667a58dae1900cd6d80",
        "57340a094776f4190066177d",
        "57306797396df919000960ee",
        "56e19724cd28a01900c679f6",
        "5728e8212ca10214002daa6c",
        "5726c002708984140094d073",
        "571099b2b654c5140001f9b3",
        "572750df5951b619008f882f",
        "572980f9af94a219006aa4d1",
        "56e1efa0e3433e140042321a",
        "57265ceddd62a815002e82b8",
        "57264a8cdd62a815002e808c",
        "5725cb33271a42140099d1db",
        "571cc8815efbb31900334dee",
        "572885023acd2414000dfa83",
        "5727cff1ff5b5019007d95c4"
      ],
      "candidate_sha256": [
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6"
      ],
      "example_id": "57264a8cdd62a815002e808c",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.33168306,
      "neural_order": [
        11,
        5,
        12,
        7,
        6,
        2,
        15,
        8,
        4,
        10,
        0,
        9,
        3,
        13,
        1,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "0a2137b37eeb14398dd13d2f1a1d686cbe44074a11bff8313f5fc65e5b837688",
      "trial_index": 336
    },
    {
      "answer_sha256": "5071de4278186556fac030f5b5b587ffb515ed13f4638f25a4e0bd82067b1831",
      "candidate_ids": [
        "5706094b52bb8914006897de",
        "5726ddf6f1498d1400e8ee04",
        "5705ec1675f01819005e776e",
        "57340b1bd058e614000b6869",
        "572867212ca10214002da2f2",
        "57265e11708984140094c3bb",
        "572f59b4a23a5019007fc587",
        "5728f3724b864d1900165116",
        "571163172419e314009555e7",
        "5728fc2eaf94a219006a9ec7",
        "57264b3edd62a815002e80aa",
        "5737821cc3c5551400e51f19",
        "572824f13acd2414000df58f",
        "57302faa04bcaa1900d77311",
        "57264e455951b619008f6f65",
        "572f7b33947a6a140053c9a2"
      ],
      "candidate_sha256": [
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "d0c3193100a2c9cefa5efa046ca711a3a7943ddbb731e1aa29b247fe5eb27c7c",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc"
      ],
      "example_id": "57264e455951b619008f6f65",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        8,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.42451197,
      "neural_order": [
        14,
        3,
        13,
        7,
        15,
        0,
        12,
        11,
        5,
        8,
        2,
        1,
        4,
        6,
        9,
        10
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "223b88a5fc1c0f736f75ef6dcaa0d093b5237bbbadc36923e3d612de78129883",
      "trial_index": 337
    },
    {
      "answer_sha256": "c3b32516bb734b5a49216e732c50d75c7ff169cab5bbe3cc6f31c7b0a70f7828",
      "candidate_ids": [
        "5728f3724b864d1900165116",
        "5727c55bff5b5019007d94ce",
        "57297547af94a219006aa45c",
        "572fadcbb2c2fd1400568329",
        "5726545f708984140094c2a5",
        "57379829c3c5551400e51f3d",
        "56e20a27cd28a01900c67c24",
        "5733fb7bd058e614000b66ff",
        "5733dc95d058e614000b644a",
        "57280fd3ff5b5019007d9c26",
        "57379ed81c456719005744d5",
        "5727db85ff5b5019007d96fc",
        "573766251c45671900574471",
        "573009a004bcaa1900d7704f",
        "57108ee6a58dae1900cd6a1a",
        "5725dd7d89a1e219009abfea"
      ],
      "candidate_sha256": [
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "f4a941b8fb7b8c4a1f8010d912eaa080c3e3ddf68b132d4a96fc1da34d4ea0d2",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448"
      ],
      "example_id": "5726545f708984140094c2a5",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.47163944,
      "neural_order": [
        4,
        6,
        3,
        10,
        12,
        14,
        13,
        15,
        0,
        1,
        2,
        5,
        7,
        8,
        9,
        11
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "cfc89c48b392e216bfc558ad36c5a996479f0a716bf58d361e9827bf10d7e630",
      "trial_index": 338
    },
    {
      "answer_sha256": "2a03bd94a939993ea49c4b2053650db9355adca6122d51394c34a4aac9d46f4e",
      "candidate_ids": [
        "56de1563cffd8e1900b4b5c2",
        "572a070c6aef0514001551c8",
        "572656e4dd62a815002e81fa",
        "5705ec1675f01819005e776e",
        "56e1c9bfe3433e1400423192",
        "5728fc9e1d04691400778f11",
        "57264efddd62a815002e8134",
        "571a4ead10f8ca1400304fdd",
        "57264c42dd62a815002e80c6",
        "57376df3c3c5551400e51ed7",
        "57111992b654c5140001fb43",
        "57293d116aef051400154bc8",
        "57265360dd62a815002e819a",
        "5726da89dd62a815002e92b2",
        "571cbe35dd7acb1400e4c13c",
        "56e1ec83cd28a01900c67c0a"
      ],
      "candidate_sha256": [
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "3c1fdff7bdb3cfd3ccc8ba02719a8a942b0f26957f044cbeb9e587d6f71ae0cd",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6"
      ],
      "example_id": "572656e4dd62a815002e81fa",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.44358662,
      "neural_order": [
        2,
        6,
        4,
        11,
        5,
        9,
        15,
        13,
        0,
        1,
        3,
        7,
        8,
        10,
        12,
        14
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "4e345c157b226eeac56f4ea876891074c058aa4abff7a02bd5b147b8b989464c",
      "trial_index": 339
    },
    {
      "answer_sha256": "baf69dfea52dd0e22505761386e97f3a8a4323283540b1695985c052a9bd37be",
      "candidate_ids": [
        "570d29c2b3d812140066d4bd",
        "5733266d4776f41900660712",
        "572f5703a23a5019007fc573",
        "57309446396df919000961b8",
        "572759665951b619008f8883",
        "5726e5c95951b619008f81cb",
        "572658435951b619008f7025",
        "57097b50200fba140036804d",
        "5728151b4b864d1900164428",
        "5726fa525951b619008f83f7",
        "57274e0d708984140094dbe5",
        "5728fd3c3f37b31900477f2b",
        "5729f5a03f37b31900478605",
        "570607f575f01819005e78b4",
        "5725ff8238643c19005acf49",
        "5729fefbaf94a219006aa739"
      ],
      "candidate_sha256": [
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "81ba9c103184caabf51fa7a6f0ba2856254a93dd30897f55cc1cd699bd203a42"
      ],
      "example_id": "572658435951b619008f7025",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        8,
        4,
        11,
        0,
        1,
        2,
        3,
        5,
        7,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.30756945,
      "neural_order": [
        6,
        8,
        11,
        3,
        1,
        4,
        2,
        7,
        9,
        10,
        12,
        0,
        15,
        5,
        13,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "f993ae865d748172e03209c919265e8e291178b17f7c9e805d87dc4def0d326f",
      "trial_index": 340
    },
    {
      "answer_sha256": "d32bdb7bb261e881c7c0a26a54a7ab30000f977c727e6968f3de0142940b2899",
      "candidate_ids": [
        "5728710c3acd2414000df9ef",
        "5730aa52069b53140083221d",
        "571a4d1a4faf5e1900b8a958",
        "570607f575f01819005e78b4",
        "571090abb654c5140001f995",
        "572677e7708984140094c723",
        "57287fec4b864d1900164a3c",
        "57308cf88ab72b1400f9c576",
        "572978f91d046914007794d3",
        "572744aff1498d1400e8f588",
        "5725b81b271a42140099d097",
        "57265e455951b619008f70bb",
        "5728151b4b864d1900164428",
        "5727dd2e4b864d1900163eba",
        "572a1ba46aef05140015528e",
        "572fc41f947a6a140053cc80"
      ],
      "candidate_sha256": [
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "aa95ca2f8cb345761597dedb47b8dc46be2773412d98ec4f473e747ae7e16b2a",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965"
      ],
      "example_id": "57265e455951b619008f70bb",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        14,
        12,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.15734759,
      "neural_order": [
        11,
        12,
        14,
        7,
        10,
        15,
        13,
        1,
        8,
        0,
        2,
        3,
        4,
        5,
        6,
        9
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "c6ae4725fbf542c449dac9cd7437f27821ca7e45168975ba3e020940a2e54916",
      "trial_index": 341
    },
    {
      "answer_sha256": "d32bdb7bb261e881c7c0a26a54a7ab30000f977c727e6968f3de0142940b2899",
      "candidate_ids": [
        "573755afc3c5551400e51eb3",
        "572699db5951b619008f7799",
        "57264cfa708984140094c1c3",
        "571c9348dd7acb1400e4c114",
        "57286bb84b864d19001649c8",
        "572a0ce11d046914007796fd",
        "57265aaf5951b619008f706b",
        "57263b1638643c19005ad333",
        "5711488ab654c5140001fc3d",
        "57293d116aef051400154bc8",
        "571cc8815efbb31900334dee",
        "5725e45689a1e219009ac048",
        "57285213ff5b5019007da182",
        "572a12386aef051400155234",
        "5726f635dd62a815002e9656",
        "57287b4a4b864d1900164a28"
      ],
      "candidate_sha256": [
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca"
      ],
      "example_id": "572699db5951b619008f7799",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        14,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.22199116,
      "neural_order": [
        1,
        0,
        12,
        14,
        2,
        8,
        9,
        10,
        7,
        6,
        5,
        3,
        4,
        11,
        13,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "7cd934e08e70ddde23e4ded23aa3424a17a511f2fc74fcbdbd16daff65eecb1f",
      "trial_index": 342
    },
    {
      "answer_sha256": "cc72a6d8a6e4767586d552223b90fd0d12cdb440ff19f900f0b9fd3d0103a81b",
      "candidate_ids": [
        "57265a58dd62a815002e8270",
        "572885023acd2414000dfa83",
        "5733d13e4776f419006612c4",
        "572869b84b864d19001649ae",
        "57269aa65951b619008f77ab",
        "56de3ebc4396321400ee26e6",
        "572a0bf96aef051400155204",
        "5733e8ae4776f419006614a6",
        "572827fc3acd2414000df5bb",
        "5705f13d52bb8914006896f0",
        "5725fcbe271a42140099d3ad",
        "570d32e4fed7b91900d45d01",
        "572a12386aef051400155234",
        "572853e8ff5b5019007da188",
        "571cd88ddd7acb1400e4c18d",
        "57265746dd62a815002e8218"
      ],
      "candidate_sha256": [
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "ac61ee33bbfbaddef1dc6828628cc2fc4f4972316e9fd55501c40fcc62475a94",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7"
      ],
      "example_id": "57269aa65951b619008f77ab",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        6,
        0,
        1,
        2,
        3,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.48594792,
      "neural_order": [
        4,
        3,
        15,
        13,
        11,
        14,
        6,
        1,
        5,
        12,
        0,
        2,
        7,
        8,
        9,
        10
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "7ba3909b50bc09ef8e75ab5833912c01424d73da2b837796eae041e0c4874d8c",
      "trial_index": 343
    },
    {
      "answer_sha256": "e706016fbdd5ed18d669f4503c58ec0bdc57e233112d3bfd3dca9c65ee8b4899",
      "candidate_ids": [
        "572fcd86947a6a140053ccda",
        "572a0ce11d046914007796fd",
        "57269bb8708984140094cb95",
        "571ce6655efbb31900334e34",
        "572a13841d0469140077973b",
        "57265d86f1498d1400e8dd50",
        "57298ef11d0469140077952d",
        "57265bdfdd62a815002e829e",
        "5733fc6ed058e614000b670f",
        "5727fd123acd2414000df185",
        "57266e72f1498d1400e8df8c",
        "572a0b0b6aef0514001551f6",
        "57302ad804bcaa1900d772af",
        "572822233acd2414000df555",
        "57299021af94a219006aa50b",
        "572a12381d0469140077972b"
      ],
      "candidate_sha256": [
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6"
      ],
      "example_id": "57269bb8708984140094cb95",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.31917249,
      "neural_order": [
        2,
        13,
        12,
        0,
        4,
        3,
        15,
        6,
        10,
        5,
        9,
        11,
        14,
        1,
        7,
        8
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "1ab0530643d4c89b6680d9757e39f719def69d99faec161356672dda456a0fff",
      "trial_index": 344
    },
    {
      "answer_sha256": "c032d87b14925f326903c890af27ecce48a1a8a8d1a5cede07ec2eeb7f020676",
      "candidate_ids": [
        "56e1c4fce3433e140042314e",
        "57269cc3dd62a815002e8b12",
        "572648e8dd62a815002e8076",
        "57264f18f1498d1400e8dbae",
        "5728742cff5b5019007da246",
        "5733647e4776f419006609ae",
        "5733e771d058e614000b6545",
        "5726d8bd708984140094d35b",
        "5733d68ed058e614000b637f",
        "572fff45947a6a140053cf26",
        "5733e8ae4776f419006614a6",
        "57109180a58dae1900cd6a40",
        "572ffb02b2c2fd14005686b7",
        "5727e21e4b864d1900163f34",
        "573083dc2461fd1900a9ce6d",
        "572759dbdd62a815002e9b94"
      ],
      "candidate_sha256": [
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "6d2627b1297d1087a3e3c836fbeab911593b2c58f4b2ae45d6c39dd4ba695bea",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "0fe05bd1833002131a7e69e040cc114a9dfc4409000d5d0674152325ab91cb19",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "874533677a269c5d5fb8bd3a3213307ea1632c2236ea48b8da8182d07905fcc8",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12"
      ],
      "example_id": "57269cc3dd62a815002e8b12",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        14,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.50757144,
      "neural_order": [
        1,
        14,
        6,
        13,
        10,
        15,
        2,
        11,
        4,
        7,
        8,
        0,
        3,
        5,
        9,
        12
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "3eab5c5398347f1a79daedca0b29520b69c12923c13e4f716ce106c6ff6b68a5",
      "trial_index": 345
    },
    {
      "answer_sha256": "84efbb0cd72a09fa34ef0d11e5fbf2ee1fb7b151d9a920fdf90fa97322a0ddcf",
      "candidate_ids": [
        "572f55e8a23a5019007fc56b",
        "571c83f3dd7acb1400e4c0d8",
        "5725f7cd38643c19005acf23",
        "572a135daf94a219006aa79f",
        "57269cc3dd62a815002e8b12",
        "572fd73e947a6a140053cd32",
        "5726a299dd62a815002e8b9e",
        "572fc659b2c2fd1400568447",
        "57115ff82419e314009555c3",
        "57107932a58dae1900cd69ca",
        "56e1c2eee3433e1400423134",
        "56de3efccffd8e1900b4b6fe",
        "572818f54b864d190016446c",
        "57269e3bf1498d1400e8e516",
        "5730285a04bcaa1900d7726d",
        "572fdb17b2c2fd140056851d"
      ],
      "candidate_sha256": [
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "6d2627b1297d1087a3e3c836fbeab911593b2c58f4b2ae45d6c39dd4ba695bea",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "218bcafd4ea9a6566900672f26303adc874834255c2b12646ddef2c72e9093f2",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427"
      ],
      "example_id": "57269e3bf1498d1400e8e516",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        5,
        4,
        9,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": false,
      "neural_margin": 0.01567627,
      "neural_order": [
        4,
        13,
        5,
        7,
        10,
        15,
        8,
        6,
        14,
        12,
        11,
        1,
        0,
        2,
        3,
        9
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "7d41231d2a31deab08bc73869a5daf8d981c426fccf05dd2ef7cc782adce6fb0",
      "trial_index": 346
    },
    {
      "answer_sha256": "a963c2646a30b2f21cfffd093a28481f6fd89cf65a664eeabc42241c3d65758f",
      "candidate_ids": [
        "5725cf3238643c19005acd61",
        "57269f3ef1498d1400e8e534",
        "57264e455951b619008f6f65",
        "5737a9afc3c5551400e51f61",
        "57282dfb4b864d1900164668",
        "571cc5c45efbb31900334dda",
        "57265285708984140094c25b",
        "572658435951b619008f7025",
        "572fcc43b2c2fd140056847d",
        "5725c41eec44d21400f3d50b",
        "5727dd2e4b864d1900163eba",
        "57373a9fc3c5551400e51e7b",
        "5733cf61d058e614000b62e9",
        "57284618ff5b5019007da0a8",
        "5728661e2ca10214002da2e8",
        "56e181d9e3433e1400422fa0"
      ],
      "candidate_sha256": [
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "5c1f57c736e7567a63ad3b0fe8dd60fb3b7aaa53a754a732221188180c8fcf30",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797"
      ],
      "example_id": "57269f3ef1498d1400e8e534",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        7,
        3,
        9,
        0,
        2,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.09742405,
      "neural_order": [
        1,
        7,
        8,
        2,
        4,
        11,
        14,
        15,
        0,
        10,
        3,
        5,
        6,
        9,
        12,
        13
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "331601a479dd2db5504c64d45c01d7a19a65990ffc05279d9011fb29fd07490b",
      "trial_index": 347
    },
    {
      "answer_sha256": "3912280c4e559b04b8c2ab5beef4bd32f3d4d7765cf0999d394debbc937e5430",
      "candidate_ids": [
        "5726a00cf1498d1400e8e550",
        "573060b48ab72b1400f9c4c6",
        "572683075951b619008f7513",
        "5733fd66d058e614000b6735",
        "56e1c2eee3433e1400423134",
        "572944e03f37b319004781e1",
        "57284b904b864d19001648e2",
        "57113be3a58dae1900cd6d28",
        "570d4329fed7b91900d45dd7",
        "571090abb654c5140001f995",
        "57287fec4b864d1900164a3c",
        "570d2f5bfed7b91900d45ccf",
        "572fd264b2c2fd14005684a9",
        "57265746dd62a815002e8218",
        "57287d4a2ca10214002da3e4",
        "572663a9f1498d1400e8ddf2"
      ],
      "candidate_sha256": [
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "b7d78d9fd858b52f42115f7144f2b2a6844dffc86f3a087c6833c6f78182e39e",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae"
      ],
      "example_id": "5726a00cf1498d1400e8e550",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.42425291,
      "neural_order": [
        0,
        11,
        12,
        14,
        1,
        6,
        7,
        5,
        4,
        3,
        13,
        8,
        15,
        2,
        9,
        10
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "fd0f0e7d409a0edeb8aca15e6503ad72e71bde46b3d89b0f191ef586a0c3faba",
      "trial_index": 348
    },
    {
      "answer_sha256": "7f5d840643f9de7e48d5ac157ee5a3dd17d61317fc868d31bd5289885e2cca77",
      "candidate_ids": [
        "57293c246aef051400154bb8",
        "57111b95a58dae1900cd6c50",
        "57300911947a6a140053cfb6",
        "572667e2f1498d1400e8de90",
        "57286b003acd2414000df9c1",
        "5726887e708984140094c917",
        "56e1cbe2cd28a01900c67bac",
        "5726c3da708984140094d0d9",
        "57263c78ec44d21400f3dc7b",
        "572746addd62a815002e9a5e",
        "5726a09f708984140094cc39",
        "572a096e6aef0514001551e4",
        "572825a92ca10214002d9efe",
        "57287c142ca10214002da3d0",
        "57379ed81c456719005744d5",
        "5728170d3acd2414000df443"
      ],
      "candidate_sha256": [
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "e554472c5bbbb76c640ff328994e3b5a7756f78559314f7bba89f2e1f6c2316e",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "3374d6a44b9178f296fdc341f31140576f5017dd266ba22fa141b973e9734bd5",
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "3a95085253039f4bf191f1037c44a2641c44c877382c059105507c459d53eaba"
      ],
      "example_id": "5726a09f708984140094cc39",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        15,
        5,
        1,
        7,
        12,
        0,
        2,
        3,
        4,
        6,
        8,
        9,
        11,
        13,
        14
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.47265545,
      "neural_order": [
        10,
        7,
        1,
        12,
        6,
        2,
        0,
        4,
        9,
        5,
        3,
        8,
        11,
        13,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "13287689ec1def61522a8e39a44fdf4de85f091e43ae260d8e108e174910e026",
      "trial_index": 349
    },
    {
      "answer_sha256": "c3e7a4f8e72000b2bc00e781ce869bfc8f300c191056fe4169c1984cbf1cdf58",
      "candidate_ids": [
        "5729fd111d046914007796a5",
        "57111992b654c5140001fb43",
        "56e1b62ecd28a01900c67aa3",
        "5725ce4d38643c19005acd4d",
        "5727d3843acd2414000ded69",
        "5726a14c708984140094cc51",
        "57282358ff5b5019007d9dde",
        "571cc3dedd7acb1400e4c146",
        "5727afa82ca10214002d93c2",
        "572821274b864d1900164510",
        "57274971708984140094dbbb",
        "57377aac1c45671900574479",
        "572881704b864d1900164a50",
        "5725f190ec44d21400f3d76f",
        "57338255d058e614000b5c0d",
        "571cdcb85efbb31900334e0c"
      ],
      "candidate_sha256": [
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "eb6de5520e4ff5dd74226484694777930247235a992261581ce74e58cdb39246",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "8636e94058ce0f55e10d0eaeb35617ebb9fd6ad1e019585a005e41e4592bcac8",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75"
      ],
      "example_id": "5726a14c708984140094cc51",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        11,
        13,
        8,
        2,
        4,
        0,
        1,
        3,
        6,
        7,
        9,
        10,
        12,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.61822095,
      "neural_order": [
        5,
        2,
        11,
        12,
        8,
        14,
        4,
        0,
        1,
        3,
        6,
        7,
        9,
        10,
        13,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "91f14746a8b993b9736fcc55f4bbd950a2294d8d06dbc49f402c6e819773a002",
      "trial_index": 350
    },
    {
      "answer_sha256": "0334dcb56bb35e277d537226831ca9257c71f1001489eecf38cc2b876478f047",
      "candidate_ids": [
        "5726c19add62a815002e8f8a",
        "5729a3716aef05140015506a",
        "57269e3bf1498d1400e8e516",
        "5727ea45ff5b5019007d986e",
        "572f609ca23a5019007fc5af",
        "56e1febfe3433e1400423236",
        "5733d2dbd058e614000b6339",
        "57096c95200fba1400367fbb",
        "57297a276aef051400154f88",
        "5733d7cbd058e614000b63ab",
        "5733fd66d058e614000b6735",
        "57280fd3ff5b5019007d9c26",
        "5726a1e5dd62a815002e8b86",
        "572872822ca10214002da374",
        "5706111a52bb89140068984c",
        "572a1046af94a219006aa78d"
      ],
      "candidate_sha256": [
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb"
      ],
      "example_id": "5726a1e5dd62a815002e8b86",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        2,
        11,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.13875726,
      "neural_order": [
        12,
        0,
        2,
        4,
        1,
        8,
        6,
        15,
        11,
        13,
        10,
        5,
        14,
        7,
        3,
        9
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "0e9ce9f6d416949558444f9775cd63f599fac5a7490280c8b6090bd097bf6a60",
      "trial_index": 351
    },
    {
      "answer_sha256": "140bedbf9c3f6d56a9846d2ba7088798683f4da0c248231336e6a05679e4fdfe",
      "candidate_ids": [
        "57274f49f1498d1400e8f61e",
        "56e20a27cd28a01900c67c24",
        "572f57c704bcaa1900d7686d",
        "572643de5951261400b5195a",
        "57285ed5ff5b5019007da1b6",
        "570967c4ed30961900e840ba",
        "5726ed6cf1498d1400e8f00c",
        "5725fabc89a1e219009ac128",
        "5728827b2ca10214002da42a",
        "573010fab2c2fd14005687d7",
        "56de179dcffd8e1900b4b5da",
        "56e1aba0e3433e1400423094",
        "5725b33f6a3fe71400b8952d",
        "5726dcbddd62a815002e9320",
        "571163172419e314009555e7",
        "5726a299dd62a815002e8b9e"
      ],
      "candidate_sha256": [
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "8ec9dcc4c2fe0ea5e8f0e2e102236786bbdd27dc8fb4452718334cf4545b34bf",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095"
      ],
      "example_id": "5726a299dd62a815002e8b9e",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        14,
        8,
        6,
        2,
        12,
        10,
        0,
        1,
        3,
        4,
        5,
        7,
        9,
        11,
        13
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.54813556,
      "neural_order": [
        15,
        1,
        4,
        12,
        2,
        8,
        13,
        5,
        0,
        10,
        3,
        6,
        7,
        9,
        11,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "677d2b1ab8e918e78908d8a7b8a7156f97e2fc74766dde82e937bd687f716527",
      "trial_index": 352
    },
    {
      "answer_sha256": "f1cfa5ebb149e8099d561aae57beed6c68f990f45a910ea9d7b460dbcc5350be",
      "candidate_ids": [
        "57286f373acd2414000df9db",
        "57300a25a23a5019007fcc97",
        "57336755d058e614000b5a3c",
        "5726a34bf1498d1400e8e59e",
        "5706149552bb891400689880",
        "571a484210f8ca1400304fbe",
        "572671a5f1498d1400e8dfe4",
        "5733f062d058e614000b6633",
        "57287d4a2ca10214002da3e4",
        "5728848cff5b5019007da298",
        "5730a0778ab72b1400f9c60a",
        "5705fec152bb89140068977a",
        "5726431aec44d21400f3dd13",
        "572655e5f1498d1400e8dc5e",
        "5727c8863acd2414000dec49",
        "56e1c4fce3433e140042314e"
      ],
      "candidate_sha256": [
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "e1e68b90e348d860a1f3c0cfe6e1367d9c8c8a57ad577d564b36e00900222682",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928"
      ],
      "example_id": "5726a34bf1498d1400e8e59e",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        8,
        1,
        0,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.26036553,
      "neural_order": [
        3,
        1,
        7,
        13,
        5,
        10,
        8,
        0,
        2,
        4,
        6,
        9,
        11,
        12,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "38d9a135c4e079b817e763b43758e3c5ffbc9153034758aea24849f298fbbb70",
      "trial_index": 353
    },
    {
      "answer_sha256": "eb3debc270d9348906547ca4711c6d29d74392e6ac5e1e9b6f9e283846fb0b0d",
      "candidate_ids": [
        "5726a3c6f1498d1400e8e5ae",
        "5733da01d058e614000b63f9",
        "57337ea24776f41900660bd0",
        "57274046dd62a815002e9a12",
        "56e17a7ccd28a01900c679a1",
        "56de49564396321400ee277a",
        "571a53d410f8ca1400304fe5",
        "572a005f1d046914007796b7",
        "5727cc15ff5b5019007d9576",
        "572807802ca10214002d9bf8",
        "572659535951b619008f703f",
        "5725fb8138643c19005acf3f",
        "572ff760b2c2fd1400568677",
        "572647935951b619008f6ec9",
        "572fc6f204bcaa1900d76cf5",
        "572a04d51d046914007796cd"
      ],
      "candidate_sha256": [
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "003096af8bd0b8d02214753872917ecb683e3c67185edc99abc12163b9741faf",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "29fcf6218856cfd64395c81351e873a0769b0f291577dd5933cc1ac378f40976",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "19620cc8402206f6bca616c6a305ca911d96b87a8f3423850512f4f84e4120b1",
        "215da96f975d61a3dc7af5d1e2b75129453d3b8e5a3b4af20a46de0b25ce3030"
      ],
      "example_id": "5726a3c6f1498d1400e8e5ae",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.35554614,
      "neural_order": [
        0,
        1,
        9,
        5,
        14,
        13,
        12,
        10,
        8,
        2,
        3,
        4,
        6,
        7,
        11,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "61bdc5f3184f94dc378926dc4679559d9006c26643b19fa0fda86cc914622120",
      "trial_index": 354
    },
    {
      "answer_sha256": "a5a282fde3c1bc4d17c65612518e33c5ffb197cb5038ca11a6f9ce42f05b082e",
      "candidate_ids": [
        "5728fd3c3f37b31900477f2b",
        "5705f13d52bb8914006896f0",
        "572ffd75b2c2fd14005686e5",
        "57114b1a2419e31400955575",
        "5726a46cdd62a815002e8bd0",
        "572755b7708984140094dc4d",
        "5726398589a1e219009ac588",
        "572659535951b619008f703f",
        "572ff935b2c2fd140056869b",
        "57094d489928a8140047150a",
        "572a07fc6aef0514001551dc",
        "57109180a58dae1900cd6a40",
        "56e1b754cd28a01900c67abc",
        "5725b81b271a42140099d097",
        "57293e983f37b3190047818b",
        "5727d4922ca10214002d977c"
      ],
      "candidate_sha256": [
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "b6f799a7fbe964e85d020346753d8182351cdec9710d1c891b7da4642c30fdc0"
      ],
      "example_id": "5726a46cdd62a815002e8bd0",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        5,
        10,
        13,
        15,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        11,
        12,
        14
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.35449892,
      "neural_order": [
        4,
        1,
        7,
        14,
        0,
        15,
        11,
        9,
        5,
        8,
        2,
        10,
        3,
        6,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "08312962c37e92253f620933fc635d1319aa9f3c34e9ad64c5d9ffdb3d1c3979",
      "trial_index": 355
    },
    {
      "answer_sha256": "70249838c151f70a386501c682a0c9f9c2cb3323c8737e1620a6feb630c3d4d1",
      "candidate_ids": [
        "57265200708984140094c237",
        "572744aff1498d1400e8f588",
        "573755afc3c5551400e51eb3",
        "57274e0d708984140094dbe5",
        "5726219489a1e219009ac2d0",
        "571cbe35dd7acb1400e4c13c",
        "57274ca8dd62a815002e9aa4",
        "5730005db2c2fd1400568703",
        "572f64ccb2c2fd14005680b7",
        "5726a5525951b619008f78dd",
        "5725f7cd38643c19005acf23",
        "5725ff8238643c19005acf49",
        "5705e33f52bb89140068964c",
        "573784fa1c45671900574483",
        "57376a1bc3c5551400e51ec3",
        "572995d46aef051400154fe8"
      ],
      "candidate_sha256": [
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a",
        "2a8d2aa463245d980e23aefdfe31db650d5d676f415970557b36b6d0e09480b5"
      ],
      "example_id": "5726a5525951b619008f78dd",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.47344201,
      "neural_order": [
        9,
        0,
        6,
        3,
        7,
        5,
        13,
        4,
        8,
        1,
        2,
        10,
        11,
        12,
        14,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "288e997fffed18b41537e02624d98d28e262cc538c7fab1fba1b015cd3714b1d",
      "trial_index": 356
    },
    {
      "answer_sha256": "eaa0c312ec1d12315c6a68db6c2047e4be14082b0cb574d92a35e16613fcea76",
      "candidate_ids": [
        "572a07a86aef0514001551d2",
        "571c8eb9dd7acb1400e4c0f6",
        "57280e1aff5b5019007d9bec",
        "572fca7eb2c2fd1400568473",
        "5725c743ec44d21400f3d549",
        "5725f2c838643c19005aceed",
        "5737898f1c45671900574495",
        "5730b4282461fd1900a9cfc5",
        "572a1fe16aef0514001552d0",
        "57111ab8a58dae1900cd6c3c",
        "57285ed5ff5b5019007da1b6",
        "572ffa79a23a5019007fcbe9",
        "5726a638dd62a815002e8bf6",
        "572ff673b2c2fd1400568669",
        "57373f80c3c5551400e51e8f",
        "570603c475f01819005e7882"
      ],
      "candidate_sha256": [
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "b471e50d3722fdd0e8fac7b90182b37de8e917f2b6ef94945670aafaf625f2bc",
        "55e3030ac51ce0d76d4fcad6c72bf5b57a9192da5256c5e338b03fe44bf2bb59",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "752151b83536aa2b39e17ee8104540f3b94aa10671b826eb5b19cc6d1e68a8ae"
      ],
      "example_id": "5726a638dd62a815002e8bf6",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        4,
        1,
        7,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.26228143,
      "neural_order": [
        12,
        4,
        13,
        1,
        11,
        2,
        9,
        10,
        7,
        8,
        0,
        6,
        3,
        5,
        14,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "633d538c1cf24e0ad2e6e74dc67d3db48657979258aa13e7a0829f85e7c4d0f5",
      "trial_index": 357
    },
    {
      "answer_sha256": "71a67c46927d11854aafef7b670726058a1c21072cc34172c47b518cbe3df583",
      "candidate_ids": [
        "571cc3dedd7acb1400e4c146",
        "5726249538643c19005ad07f",
        "572a07a86aef0514001551d2",
        "57293d116aef051400154bc8",
        "5730b7ce069b5314008322c3",
        "5726ed6cf1498d1400e8f00c",
        "573361404776f4190066093c",
        "5730088e947a6a140053cfac",
        "572a1dbb3f37b319004786f5",
        "5726a7ecf1498d1400e8e654",
        "57265e455951b619008f70bb",
        "5726f868dd62a815002e9684",
        "5725c91e38643c19005acceb",
        "571a50df4faf5e1900b8a960",
        "572f5703a23a5019007fc573",
        "5733a560d058e614000b5f77"
      ],
      "candidate_sha256": [
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "a4008fb587348767c254824b7e5bb46aaec5eaa762404c37c3e9e2f52fc93eac",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "66cbf5ef7ba18a058b7e88c4c66ba9e11f4ef0efae05c7495c2ee6e756648699",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "b67161cb70767d25408d47a9b2cd15ed23cc66a97c33ff46706b10e7a4772023",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b"
      ],
      "example_id": "5726a7ecf1498d1400e8e654",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.2874983,
      "neural_order": [
        9,
        10,
        4,
        3,
        0,
        2,
        1,
        5,
        8,
        15,
        13,
        12,
        7,
        6,
        11,
        14
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "de3dec40157757d5a889fb24b0022e4b071364cff64cf1bcbe17fced834d8e86",
      "trial_index": 358
    },
    {
      "answer_sha256": "f1cfa5ebb149e8099d561aae57beed6c68f990f45a910ea9d7b460dbcc5350be",
      "candidate_ids": [
        "5726b58f5951b619008f7b55",
        "5728fc2eaf94a219006a9ec7",
        "57268da7f1498d1400e8e39c",
        "570d30fdfed7b91900d45ce3",
        "573011de04bcaa1900d770f9",
        "5727b0892ca10214002d93e8",
        "5733d2dbd058e614000b6339",
        "57340b1bd058e614000b6869",
        "57340a094776f4190066177d",
        "572ffa79a23a5019007fcbe9",
        "5725ce4d38643c19005acd4d",
        "57332a734776f41900660726",
        "5705f09e75f01819005e77a4",
        "57115c7450c2381900b54a9d",
        "57097b50200fba140036804d",
        "57265285708984140094c25b"
      ],
      "candidate_sha256": [
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936"
      ],
      "example_id": "5726b58f5951b619008f7b55",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        7,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.37628241,
      "neural_order": [
        0,
        14,
        9,
        11,
        12,
        15,
        4,
        7,
        5,
        8,
        3,
        13,
        1,
        2,
        6,
        10
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "67e7ce1b45d4110a16c4e7239c0d183bc225bedd0fa8fc41ab05d9a93eb7f18e",
      "trial_index": 359
    },
    {
      "answer_sha256": "56a3d2dfe6a9cfbb63dbd7ce068318f2571876aea0164c2386686673453df70b",
      "candidate_ids": [
        "57115f0a50c2381900b54aa7",
        "57107d73b654c5140001f91d",
        "5725d16aec44d21400f3d61b",
        "5710f2e2a58dae1900cd6b72",
        "572654e2708984140094c2b7",
        "57302bd0b2c2fd14005689db",
        "5709686c200fba1400367f77",
        "57332442d058e614000b5720",
        "5726b718dd62a815002e8dbe",
        "57306797396df919000960ee",
        "573007fab2c2fd140056876b",
        "572699db5951b619008f7799",
        "5725b76389a1e219009abd4a",
        "56e19724cd28a01900c679f6",
        "57284d484b864d1900164900",
        "57094d489928a8140047150a"
      ],
      "candidate_sha256": [
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "9d0852e7a492edfb020138aeed2e89901eaf6cbc0450e6da752320d2c7c024e0",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "fcb9ab642fd54564ba8561d66afa42c7a340302550f69419acc9e102acd011a5",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "d206c39c98d46bd01f841c82284343c93a112882c53094883d72423bafd6ee01",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57"
      ],
      "example_id": "5726b718dd62a815002e8dbe",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.10853692,
      "neural_order": [
        8,
        9,
        11,
        12,
        0,
        3,
        1,
        15,
        13,
        10,
        5,
        2,
        4,
        6,
        7,
        14
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "d3ed2329b0506a3a4aaf237491202cfe3ddb119c5b8a1e353957edcaf03cb7ef",
      "trial_index": 360
    },
    {
      "answer_sha256": "2c598436e5575a5769b69986014588d52c2698414b623e81b2e776766c30eaba",
      "candidate_ids": [
        "57300f8504bcaa1900d770d1",
        "5725b5a689a1e219009abd28",
        "5733d4c8d058e614000b6353",
        "57338007d058e614000b5bda",
        "57377083c3c5551400e51edf",
        "57273cca708984140094db33",
        "57294209af94a219006aa201",
        "57264cc6dd62a815002e80e4",
        "5726a34bf1498d1400e8e59e",
        "57060f3e75f01819005e7922",
        "5726b929f1498d1400e8e8e9",
        "5725c57a89a1e219009abe5e",
        "57265d08708984140094c397",
        "5733314e4776f4190066076a",
        "572989846aef051400154fc0",
        "56de179dcffd8e1900b4b5da"
      ],
      "candidate_sha256": [
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "1629b48b9aae07c00bf8bd9ec7d76d9982b5e645f832551c105d868a0f34f81e",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2"
      ],
      "example_id": "5726b929f1498d1400e8e8e9",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.14595826,
      "neural_order": [
        10,
        1,
        3,
        13,
        6,
        2,
        0,
        7,
        5,
        4,
        8,
        9,
        11,
        12,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "6a717290cb90ec93ca8740c0611b54b3063f502fdfd429387d292c38494fba4f",
      "trial_index": 361
    },
    {
      "answer_sha256": "6e9ce21354a0bb1f82fe1bfd9c5e5f6fec48bab43fcf3632f858b7a68cb0d99d",
      "candidate_ids": [
        "572885023acd2414000dfa83",
        "571c91c8dd7acb1400e4c10a",
        "572757bef1498d1400e8f690",
        "56de4c324396321400ee27ab",
        "5728f2e26aef051400154896",
        "573750f51c45671900574467",
        "57060a6e52bb8914006897f8",
        "5726516a708984140094c223",
        "57338160d058e614000b5bf9",
        "56e1c0f6cd28a01900c67b2c",
        "572860e03acd2414000df977",
        "572810ec2ca10214002d9d06",
        "572951f16aef051400154cce",
        "5726ba2c708984140094cf59",
        "5726f96ddd62a815002e9698",
        "572a213e6aef0514001552ee"
      ],
      "candidate_sha256": [
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "ecce67f5ebd93f7f51596b468b36d1918c0f3d320dd7e6bd4fa7ac0ca537f812",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0"
      ],
      "example_id": "5726ba2c708984140094cf59",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        5,
        10,
        11,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        12,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.48326753,
      "neural_order": [
        13,
        0,
        11,
        14,
        3,
        6,
        2,
        10,
        5,
        12,
        7,
        1,
        4,
        8,
        9,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "7fc25d19b3fa3cc665c0b881be3e44a6ed2ef07dbe157cf9f00f0d2c3c517012",
      "trial_index": 362
    },
    {
      "answer_sha256": "a96f32f5940e68af7d88ea0ba9b873433cb01589329c5abdfc852f55c10fa4ec",
      "candidate_ids": [
        "572fce12a23a5019007fca11",
        "572a0d21af94a219006aa783",
        "57115ac550c2381900b54a77",
        "57269e3bf1498d1400e8e516",
        "57274ca8dd62a815002e9aa4",
        "57284d484b864d1900164900",
        "5726baf2dd62a815002e8e74",
        "5727c94bff5b5019007d954a",
        "5727d3843acd2414000ded69",
        "57332c1e4776f4190066073a",
        "572a09abaf94a219006aa75b",
        "57293e221d046914007791d5",
        "56de4a89cffd8e1900b4b7bd",
        "572a0bebaf94a219006aa76f",
        "57115bf350c2381900b54a93",
        "572826634b864d19001645be"
      ],
      "candidate_sha256": [
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "eb6de5520e4ff5dd74226484694777930247235a992261581ce74e58cdb39246",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "ced0f58ca39721ecf4eef6e96cbd5c7920c5e262af997dc8d26a7efb5a90b9a6",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2"
      ],
      "example_id": "5726baf2dd62a815002e8e74",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.01327151,
      "neural_order": [
        6,
        3,
        0,
        11,
        10,
        1,
        12,
        8,
        2,
        14,
        4,
        13,
        5,
        7,
        9,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "f8ee208f2f33a4b8895f0c84b4468b7ddac45cb9b0992ec3c4b37c54887ee397",
      "trial_index": 363
    },
    {
      "answer_sha256": "067c357d15ff1000510a3a42746d8ab8c2f99df72f25ae7661d4cb6cb97d23f6",
      "candidate_ids": [
        "5705fec152bb89140068977a",
        "572fc5a1947a6a140053cc8a",
        "572facb0a23a5019007fc863",
        "572f6ec7a23a5019007fc621",
        "5728759cff5b5019007da25a",
        "571c9074dd7acb1400e4c100",
        "5705e63175f01819005e7720",
        "5727f2714b864d1900164072",
        "571a53d410f8ca1400304fe5",
        "5727fd123acd2414000df185",
        "572fbea404bcaa1900d76c5b",
        "5728fc2eaf94a219006a9ec7",
        "56e17e6ee3433e1400422f7f",
        "5726bc1add62a815002e8ea6",
        "573092088ab72b1400f9c594",
        "572683f95951b619008f7525"
      ],
      "candidate_sha256": [
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "65a4f29a52c31659b66baa59545297d8af56b2b1f884801f998c7d4c6e8396d8",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "069f593a47b1c7d532212a62aba1e27bdd08d0259cb3e0a96bc73f362f4036e4",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "00b76f48835bfc2434242a5dea1681e9e4d9c363cc74180aa0b9b9fec6f02bef",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616"
      ],
      "example_id": "5726bc1add62a815002e8ea6",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        7,
        1,
        2,
        0,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.16201178,
      "neural_order": [
        13,
        7,
        2,
        9,
        1,
        4,
        11,
        10,
        12,
        14,
        5,
        0,
        3,
        6,
        8,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "596ecf72977a998416393221475b692fbe27d255c418326e8c48bcafd31e7000",
      "trial_index": 364
    },
    {
      "answer_sha256": "f583df0d31b6f27d784cec43451adc5f465004800dbfa2650dd83cee54ee65de",
      "candidate_ids": [
        "5727ed2e3acd2414000df001",
        "5705ffde52bb891400689784",
        "5727b8df3acd2414000dea99",
        "572824f13acd2414000df58f",
        "5705e26d75f01819005e76d4",
        "57263cfcec44d21400f3dc8d",
        "5732b6b5328d981900602021",
        "5725fabc89a1e219009ac128",
        "57300200b2c2fd1400568729",
        "5726a299dd62a815002e8b9e",
        "57115b8b50c2381900b54a89",
        "572757bef1498d1400e8f690",
        "5733dd4f4776f419006613aa",
        "5726bcde708984140094cfbf",
        "5727afa82ca10214002d93c2",
        "5729f39a6aef05140015514c"
      ],
      "candidate_sha256": [
        "b38835b0da15953f81e7fa39f6745a69d2e09f953b85ebc4995b8b8252bb8940",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "286230116fa743f77871c0721b3874ae2889664d4113752586c5f53755626f21",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "8636e94058ce0f55e10d0eaeb35617ebb9fd6ad1e019585a005e41e4592bcac8",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726"
      ],
      "example_id": "5726bcde708984140094cfbf",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        9,
        8,
        14,
        4,
        11,
        3,
        0,
        1,
        2,
        5,
        6,
        7,
        10,
        12,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.10859119,
      "neural_order": [
        13,
        9,
        11,
        2,
        4,
        12,
        0,
        8,
        7,
        1,
        14,
        3,
        5,
        6,
        10,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "0cbb6a36f085c5821a1ad8dcda54a369bd21dfae50f82c6da1dfed7122ad18be",
      "trial_index": 365
    },
    {
      "answer_sha256": "082bfba0470fc8667d69e47557c6a0650669507645923f87bffa34db9e57855e",
      "candidate_ids": [
        "572a213e6aef0514001552ee",
        "5725d183271a42140099d23d",
        "56e1ddfce3433e14004231d5",
        "5725f2c838643c19005aceed",
        "5733a2a9d058e614000b5f28",
        "57271c235951b619008f860b",
        "572ffabf04bcaa1900d76f9f",
        "5726c002708984140094d073",
        "572f76d1b2c2fd1400568153",
        "5725d7e438643c19005acdf9",
        "57264cfa708984140094c1c3",
        "57113639a58dae1900cd6d18",
        "56dde0ba66d3e219004dad75",
        "57294279af94a219006aa209",
        "57114e8d50c2381900b54a5b",
        "572a02483f37b3190047864b"
      ],
      "candidate_sha256": [
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe",
        "ade6ff401c8944661f8dbe0a4e4be3b05995437ccaed1d24de71c038275f80ef",
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5"
      ],
      "example_id": "5726c002708984140094d073",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        12,
        0,
        3,
        1,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.19226603,
      "neural_order": [
        7,
        0,
        5,
        12,
        8,
        2,
        15,
        4,
        14,
        1,
        3,
        6,
        9,
        10,
        11,
        13
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "4a792b77510517e831fe31eb4a8d33b6169b4cd40df424c2e6cc44d47f377dff",
      "trial_index": 366
    },
    {
      "answer_sha256": "6f6a4e56098cfd9af29e3ae549503b370211a4e94421457fe4cfd39a38a1fa08",
      "candidate_ids": [
        "5726c19add62a815002e8f8a",
        "573004bf947a6a140053cf56",
        "57297ed93f37b3190047845f",
        "5737477bc3c5551400e51ea3",
        "571127a5a58dae1900cd6cc4",
        "5733a32bd058e614000b5f32",
        "572fcc43b2c2fd140056847d",
        "570d44abb3d812140066d5fd",
        "572f7588947a6a140053c984",
        "5726f0e35951b619008f82f7",
        "57302efe04bcaa1900d772f5",
        "5728ebcb3acd2414000e01d9",
        "5725ca4389a1e219009abeb4",
        "570614ff52bb89140068988a",
        "57096b66200fba1400367fa7",
        "573368044776f41900660a29"
      ],
      "candidate_sha256": [
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "5b8656a663456a61c1d9692e2b9d3858aa19a6b21b538a75b82941eb5206617c",
        "8394917ed3c6abd9f899d73ce839433e519acb0219643e9bfa9a48f92940892c",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "56342ec864f7c99a6400cb1b743d2de657d247bc5557e34b707c510c768abb1c",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482"
      ],
      "example_id": "5726c19add62a815002e8f8a",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.40014203,
      "neural_order": [
        0,
        6,
        12,
        9,
        8,
        11,
        4,
        5,
        14,
        1,
        2,
        3,
        7,
        10,
        13,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "9ca2d6558c6fac05a497505ea5c11062994992a7111115c1cc0c22540e5fc0aa",
      "trial_index": 367
    },
    {
      "answer_sha256": "c250b5fbe101b1d46dde267f81d5183e14d9151925ff7311393b2cc1f9ac422c",
      "candidate_ids": [
        "5728667eff5b5019007da1fc",
        "56de51244396321400ee27ef",
        "5706143575f01819005e7950",
        "5726c3da708984140094d0d9",
        "572683075951b619008f7513",
        "57379ed81c456719005744d5",
        "572863c72ca10214002da2d4",
        "56e1fc57e3433e140042322c",
        "572863363acd2414000df985",
        "572871bd3acd2414000dfa03",
        "5725b92e38643c19005acbd3",
        "56de10b44396321400ee2593",
        "56de4b5c4396321400ee2799",
        "572f57c704bcaa1900d7686d",
        "56e1aff7cd28a01900c67a68",
        "572f6ec7a23a5019007fc621"
      ],
      "candidate_sha256": [
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6"
      ],
      "example_id": "5726c3da708984140094d0d9",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        11,
        12,
        1,
        14,
        7,
        0,
        2,
        4,
        5,
        6,
        8,
        9,
        10,
        13,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.44276922,
      "neural_order": [
        3,
        0,
        4,
        11,
        8,
        7,
        2,
        9,
        14,
        5,
        1,
        6,
        10,
        12,
        13,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "621f4055c18d5548dd813214367ea77c4174fecc318bb8a8601aa3bb1e08b3d8",
      "trial_index": 368
    },
    {
      "answer_sha256": "653cf423e986e605cd166292e550b821806b39c5d7df1299dd0ebccf19e3f849",
      "candidate_ids": [
        "5730ab63396df91900096260",
        "572f7b33947a6a140053c9a2",
        "572a11663f37b31900478693",
        "5706139252bb891400689864",
        "571cde695efbb31900334e16",
        "5726c5a9f1498d1400e8eac4",
        "572f6a0ba23a5019007fc5eb",
        "57302faa04bcaa1900d77311",
        "57302a3aa23a5019007fcecf",
        "57268882f1498d1400e8e306",
        "57308f6b8ab72b1400f9c580",
        "5730909d8ab72b1400f9c58a",
        "5711488ab654c5140001fc3d",
        "573009a004bcaa1900d7704f",
        "57379a4b1c456719005744cd",
        "5711623e50c2381900b54ab9"
      ],
      "candidate_sha256": [
        "574d0f6a2b0526e5af0a55cbe40d842b1cbdfb2d1ab8e4ca96d6cc232bbc289d",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "6638daf3357f97afba40505b33af08c76b24c6d5399a55772351804230526ef8",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "37c7853eb3100481bf8eb03dc20a277af3908ee8f3eb55e8315e451b7ed2d724",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2"
      ],
      "example_id": "5726c5a9f1498d1400e8eac4",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        13,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.36227615,
      "neural_order": [
        5,
        13,
        1,
        2,
        11,
        7,
        10,
        15,
        14,
        6,
        0,
        3,
        4,
        8,
        9,
        12
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "f76201fbf3a01b389640caa514f46d0121c84391776cf47c102c39a2654ec7c6",
      "trial_index": 369
    },
    {
      "answer_sha256": "46317a4c8bd3b326ee4a19ab4c028aeb41a521464e5e1aecc2791fc6b4cb131f",
      "candidate_ids": [
        "56e17644e3433e1400422f40",
        "5733e5a14776f4190066145b",
        "57335c20d058e614000b58f9",
        "572751b4708984140094dc1b",
        "56de17f9cffd8e1900b4b5e0",
        "5726ba2c708984140094cf59",
        "572a1ba46aef05140015528e",
        "5727ed2e3acd2414000df001",
        "572657d9dd62a815002e8230",
        "572853e8ff5b5019007da188",
        "5728501aff5b5019007da166",
        "57107d73b654c5140001f91d",
        "57303048947a6a140053d254",
        "57060a6e52bb8914006897f8",
        "57113639a58dae1900cd6d18",
        "5725b81b271a42140099d097"
      ],
      "candidate_sha256": [
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "aa95ca2f8cb345761597dedb47b8dc46be2773412d98ec4f473e747ae7e16b2a",
        "b38835b0da15953f81e7fa39f6745a69d2e09f953b85ebc4995b8b8252bb8940",
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "13ad58ca96a59b05a605a8e8fd1cadd65b62ca392d3e04f125096c9b8294e267",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3"
      ],
      "example_id": "5725b81b271a42140099d097",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        14,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.63527506,
      "neural_order": [
        15,
        13,
        4,
        1,
        0,
        8,
        3,
        6,
        2,
        5,
        7,
        9,
        10,
        11,
        12,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "fe0969ac1888900a015ce58c2421b1444ca91861849507369e456bead55e6623",
      "trial_index": 370
    },
    {
      "answer_sha256": "6d693642ce05d52eeb20ef94f5586ad19c131abc639d6e9154fda52a0deb9ed6",
      "candidate_ids": [
        "5725edfe38643c19005ace9f",
        "57287b4a4b864d1900164a28",
        "57273f27dd62a815002e9a0a",
        "572849b4ff5b5019007da0f2",
        "56e1bc3ae3433e1400423104",
        "5730bb522461fd1900a9d011",
        "573081c2069b531400832133",
        "57332442d058e614000b5720",
        "5705eee952bb8914006896de",
        "5726e3c4dd62a815002e9404",
        "571c83f3dd7acb1400e4c0d8",
        "5725be0f271a42140099d117",
        "5705fec152bb89140068977a",
        "57302e45947a6a140053d22e",
        "572ffee1947a6a140053cf14",
        "572a0e0e1d04691400779707"
      ],
      "candidate_sha256": [
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "b2195ad0e9b3f93273c11d1e0fdad467bc4a4f14db96c3044f0ff84b36bfde6f",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "fcb9ab642fd54564ba8561d66afa42c7a340302550f69419acc9e102acd011a5",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "1d470bf791028e4f456c100c44c5b3410fac821035b36b868d821eb23e1f2a74",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "2d5af9a4d02fc579e6e15f098bcffbcc6459cdee51fd51b38ea0075eee7a728a",
        "804a1a2d7b5d5f57198945102d763975f6edcf55d1fa34d39bc21af2a7e82d7e",
        "4b184251045ff68cd470a8330d18c3848a5524d5ac96c1ad4b4b92ae5d9d5694"
      ],
      "example_id": "5725be0f271a42140099d117",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.45877328,
      "neural_order": [
        11,
        6,
        12,
        5,
        8,
        1,
        15,
        0,
        2,
        3,
        4,
        7,
        9,
        10,
        13,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "bbbb509e51e02e09c601c85121ff62bc3257371a5fe92aab34bd159176997d64",
      "trial_index": 371
    },
    {
      "answer_sha256": "3f909f1e0de26608396dc33081d097f713b7bea0a559b309953652b6c970e208",
      "candidate_ids": [
        "5727490bdd62a815002e9a82",
        "572987e46aef051400154fa2",
        "572a12381d0469140077972b",
        "57298ef11d0469140077952d",
        "57376828c3c5551400e51eb9",
        "573027d6a23a5019007fce9d",
        "57263cfcec44d21400f3dc8d",
        "5725f5b1271a42140099d371",
        "5705e33f52bb89140068964c",
        "5725c071271a42140099d127",
        "57097b50200fba140036804d",
        "57265ceddd62a815002e82b8",
        "5737958b1c456719005744c3",
        "572941273f37b319004781ad",
        "572f5875947a6a140053c89a",
        "572677e7708984140094c723"
      ],
      "candidate_sha256": [
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "8075afe5649850130d6277b4d42a263b335970200c99b9786462feb745bd73d5",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "bb8a76f08ce73953c6a06eda3ddc44cab30380cc4b6a0f4d40f518df582eefb4",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db"
      ],
      "example_id": "5725c071271a42140099d127",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.4927107,
      "neural_order": [
        9,
        15,
        14,
        13,
        8,
        6,
        2,
        5,
        1,
        3,
        0,
        4,
        7,
        10,
        11,
        12
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "d5b433c4fc8f3e6169e6083ca07a45b14cefc9a221b3da360628f268babd68f9",
      "trial_index": 372
    },
    {
      "answer_sha256": "3742009605885c243d365065cd0cb46dad5b6fc0e3c979344e58bfaf78fbf149",
      "candidate_ids": [
        "5726b929f1498d1400e8e8e9",
        "57106185b654c5140001f8db",
        "572686fc708984140094c8e5",
        "572827fc3acd2414000df5bb",
        "5726deed5951b619008f80c5",
        "57097051ed30961900e84132",
        "5728501aff5b5019007da166",
        "570d4030fed7b91900d45d9f",
        "5725c41eec44d21400f3d50b",
        "572f59b4a23a5019007fc587",
        "57264865dd62a815002e8062",
        "5729ffda1d046914007796af",
        "5733fb7bd058e614000b66ff",
        "57265aaf5951b619008f706b",
        "5727de862ca10214002d9860",
        "5733f309d058e614000b6648"
      ],
      "candidate_sha256": [
        "1629b48b9aae07c00bf8bd9ec7d76d9982b5e645f832551c105d868a0f34f81e",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "b557caa6a1a9ebdc6ec705acc8c1078fbde1cf27379b5961913015d24cbd8809",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "5b469d64ced1175805ddbb53037730159f46f770f6697c84c8b9e8738d9bbb19"
      ],
      "example_id": "5725c41eec44d21400f3d50b",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": false,
      "neural_margin": 0.00515497,
      "neural_order": [
        7,
        0,
        6,
        1,
        4,
        11,
        14,
        5,
        13,
        2,
        8,
        3,
        9,
        10,
        12,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "4a05edfd7a11e91134d66dc83ab5b7b3d3643a52c1cb62e4298ae96b1d3a3793",
      "trial_index": 373
    },
    {
      "answer_sha256": "a15178720e397dfc0dce0db2ca133a5e66eff8c1ff1212a2beb4d4cae3628748",
      "candidate_ids": [
        "57274a1edd62a815002e9a9b",
        "5729f4e46aef051400155156",
        "57269bb8708984140094cb95",
        "5711119cb654c5140001fae3",
        "5728683b3acd2414000df9af",
        "57275411dd62a815002e9b30",
        "5709686c200fba1400367f77",
        "57267947f1498d1400e8e0ec",
        "57264f18f1498d1400e8dbae",
        "5730042804bcaa1900d77011",
        "5725c0f289a1e219009abdf2",
        "5725c63438643c19005acc9f",
        "571a4b0f10f8ca1400304fd5",
        "572745c6708984140094db99",
        "572684365951b619008f753f",
        "5726b58f5951b619008f7b55"
      ],
      "candidate_sha256": [
        "67ab57fbbb00278c88580a71043b1831e27d8c1f8d9697b56e8c9fc715fb8ec3",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "4bfde5c5f53f69927f5f235510fbbe2969feac3237b78782ea69a8ebe395cbde",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0"
      ],
      "example_id": "5725c63438643c19005acc9f",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.34002962,
      "neural_order": [
        11,
        12,
        7,
        6,
        4,
        8,
        9,
        10,
        14,
        5,
        0,
        1,
        2,
        3,
        13,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "2691d4ae500cf33276c00abbb917ce63c250e61eb18d27b9a4b7e46b3dd74866",
      "trial_index": 374
    },
    {
      "answer_sha256": "bebd5adfb300ec59c219af31480ed2572b0aa6dbe443787663d633b613c90ddc",
      "candidate_ids": [
        "572882242ca10214002da420",
        "5711475ca58dae1900cd6d8b",
        "5725c63438643c19005acc9f",
        "57286bb84b864d19001649c8",
        "5725c95f38643c19005accf3",
        "5726a7ecf1498d1400e8e654",
        "573383d0d058e614000b5c35",
        "5733a560d058e614000b5f77",
        "5728848cff5b5019007da298",
        "572ffabf04bcaa1900d76f9f",
        "5730b1022461fd1900a9cfa3",
        "572f609ca23a5019007fc5af",
        "57377ec7c3c5551400e51f05",
        "57264fe65951b619008f6fa1",
        "572658daf1498d1400e8dcac",
        "570d47b8b3d812140066d62d"
      ],
      "candidate_sha256": [
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "ade6ff401c8944661f8dbe0a4e4be3b05995437ccaed1d24de71c038275f80ef",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77"
      ],
      "example_id": "5725c95f38643c19005accf3",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": false,
      "neural_margin": 0.02587825,
      "neural_order": [
        7,
        4,
        0,
        8,
        13,
        14,
        2,
        9,
        11,
        1,
        3,
        5,
        6,
        10,
        12,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "6df0c00ff9be3d3a4019e0a1a0bfd8daff47d1de6dbd2079ae896bd54dc9a3a1",
      "trial_index": 375
    },
    {
      "answer_sha256": "de413b8e885da8771e2f8afa5a99517504e900d8943ae66e3b442face8525a1c",
      "candidate_ids": [
        "572a0bafaf94a219006aa765",
        "57094d489928a8140047150a",
        "572ffa79a23a5019007fcbe9",
        "570603c475f01819005e7882",
        "5727fd123acd2414000df185",
        "5725cbb289a1e219009abed2",
        "56dddf4066d3e219004dad5f",
        "5725b92e38643c19005acbd3",
        "5729f12e3f37b319004785e1",
        "570611c475f01819005e793c",
        "572ffbaa947a6a140053cee6",
        "57286d7d4b864d19001649e0",
        "570961aa200fba1400367f15",
        "5726701add62a815002e84c8",
        "5728683b3acd2414000df9af",
        "56de52614396321400ee27fb"
      ],
      "candidate_sha256": [
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "752151b83536aa2b39e17ee8104540f3b94aa10671b826eb5b19cc6d1e68a8ae",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb",
        "040a5627ef7bb5d949592bbcd7381d5c49b141f464054a8ba8c03c8253480ddb",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "db91c1023a94550f1ffe6e9f87e0b45aaad95f3bef9ada0d0c3872b88286bf7c",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066"
      ],
      "example_id": "5725cbb289a1e219009abed2",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        4,
        6,
        0,
        1,
        2,
        3,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.38040795,
      "neural_order": [
        5,
        15,
        2,
        11,
        9,
        6,
        4,
        7,
        0,
        1,
        3,
        8,
        10,
        12,
        13,
        14
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "5641609f1fdc624536698f2d250a65e99a3b688f42e4d0f46012f9b86db3b97d",
      "trial_index": 376
    },
    {
      "answer_sha256": "d683e109a88a02dd3755de8b3e39f7fe750df853ec12e5d04be88c8f93f1db50",
      "candidate_ids": [
        "5726e313f1498d1400e8eeb2",
        "57300e2604bcaa1900d770b7",
        "5726398589a1e219009ac588",
        "57283db6ff5b5019007d9fbc",
        "57274e145951b619008f87e7",
        "5725cf3238643c19005acd61",
        "570d2f5bfed7b91900d45ccf",
        "5711488ab654c5140001fc3d",
        "57269016708984140094ca41",
        "57265e97708984140094c3c3",
        "57264a0ef1498d1400e8db40",
        "572659535951b619008f703f",
        "5705e4fe75f01819005e7704",
        "57286b003acd2414000df9c1",
        "572a070c6aef0514001551c8",
        "57105da9a58dae1900cd699e"
      ],
      "candidate_sha256": [
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "403b83bcd02b8204d4620813147424fdb54ec54909eb54e867858a16ee0028ec",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c"
      ],
      "example_id": "5725cf3238643c19005acd61",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        4,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.32058763,
      "neural_order": [
        5,
        9,
        10,
        8,
        12,
        11,
        4,
        1,
        13,
        3,
        14,
        2,
        6,
        7,
        0,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "306edba5bc50af2aabdd3f787b0040e1ac92165a63400d8f468e15b740956453",
      "trial_index": 377
    },
    {
      "answer_sha256": "cc863e11d847d2d460abee7b48ae2a49f74a159b969471f0efe9a17dd5d07a82",
      "candidate_ids": [
        "57373a9fc3c5551400e51e7b",
        "56e1c3e1e3433e1400423148",
        "572a1dbb3f37b319004786f5",
        "5728f50baf94a219006a9e55",
        "57264cac708984140094c1b3",
        "57106644b654c5140001f8e5",
        "57340549d058e614000b67dd",
        "5725d16aec44d21400f3d61b",
        "573027d6a23a5019007fce9d",
        "570d4329fed7b91900d45dd7",
        "5711628a2419e314009555dd",
        "57287ee3ff5b5019007da274",
        "5727c3b02ca10214002d95ba",
        "5729fefbaf94a219006aa739",
        "57302cd004bcaa1900d772d7",
        "57264b1ddd62a815002e80a0"
      ],
      "candidate_sha256": [
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "099340918f9359979a06067b9361381a993dbef386a4851ba6bb74d73cd5dc60",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b",
        "81ba9c103184caabf51fa7a6f0ba2856254a93dd30897f55cc1cd699bd203a42",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a"
      ],
      "example_id": "5725d16aec44d21400f3d61b",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        1,
        14,
        4,
        9,
        0,
        2,
        3,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.5683194,
      "neural_order": [
        7,
        15,
        1,
        9,
        0,
        13,
        14,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        12
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "dd9a584ff37a01e429c6146ae9021f9ae547e95e4630bf2649aaf675af2940db",
      "trial_index": 378
    },
    {
      "answer_sha256": "25cc51c462bbaf314c3e7ac7f8c1ccb9a41e6d35fdddf787a4e7bccefb9635a1",
      "candidate_ids": [
        "5726c19add62a815002e8f8a",
        "57106644b654c5140001f8e5",
        "57287b4a4b864d1900164a28",
        "572fbf21a23a5019007fc937",
        "57274e975951b619008f87f9",
        "5727482bf1498d1400e8f5a4",
        "572fd7b8947a6a140053cd3a",
        "5728fd3c3f37b31900477f2b",
        "571a4ead10f8ca1400304fdd",
        "5725cfd0271a42140099d225",
        "5706111a52bb89140068984c",
        "57332e48d058e614000b5762",
        "571c91c8dd7acb1400e4c10a",
        "572faf74b2c2fd1400568347",
        "572867212ca10214002da2f2",
        "5726722bdd62a815002e8528"
      ],
      "candidate_sha256": [
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c"
      ],
      "example_id": "5726722bdd62a815002e8528",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        4,
        9,
        8,
        2,
        0,
        1,
        3,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.59701407,
      "neural_order": [
        15,
        12,
        13,
        0,
        5,
        6,
        9,
        10,
        1,
        2,
        3,
        4,
        7,
        8,
        11,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "7c09780d6fd984d08117b874eefdae38a572842186889997c78c5d2442d4762b",
      "trial_index": 379
    },
    {
      "answer_sha256": "e216b51bd0e7749bf3f5a988214dcfbed44da0a4e7c4841c27af7f53fc6def43",
      "candidate_ids": [
        "571135b8a58dae1900cd6d0e",
        "57269f3ef1498d1400e8e534",
        "57377aac1c45671900574479",
        "5729f3831d0469140077967b",
        "5726b58f5951b619008f7b55",
        "57264d9edd62a815002e80fe",
        "570961aa200fba1400367f15",
        "57284142ff5b5019007da00a",
        "56e16839cd28a01900c67887",
        "572856beff5b5019007da190",
        "57299a6f6aef051400155016",
        "56de1563cffd8e1900b4b5c2",
        "56e1c720e3433e140042316a",
        "57266193dd62a815002e832a",
        "57274ca8dd62a815002e9aa4",
        "56de49a8cffd8e1900b4b7a7"
      ],
      "candidate_sha256": [
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "c426efe1359054866c8d322fd02f908d61dd09289424401de4416d000f338ca6",
        "d352fae900cfd1fb07e900ce8c975e46f39748c746696de6bdeb051c5bc2580d",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f"
      ],
      "example_id": "5729f3831d0469140077967b",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.60997963,
      "neural_order": [
        3,
        10,
        15,
        5,
        0,
        1,
        2,
        4,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "94125137521b08ee8ffcbc8c8c19eee6d67ae6212685a8e21f7178402d06c071",
      "trial_index": 380
    },
    {
      "answer_sha256": "d1d2451ac107a345f6611a80ed9fa0d6cb61cdb8fde3cf97babc732a0fad4a4d",
      "candidate_ids": [
        "5726509bdd62a815002e815a",
        "572f57c704bcaa1900d7686d",
        "57309bfb8ab72b1400f9c5e6",
        "57115b8b50c2381900b54a89",
        "57265e97708984140094c3c3",
        "56dde0ba66d3e219004dad75",
        "5728284e3acd2414000df5cd",
        "572f76d1b2c2fd1400568153",
        "57109275b654c5140001f99f",
        "5737a0acc3c5551400e51f47",
        "56e1bc3ae3433e1400423104",
        "57300a25a23a5019007fcc97",
        "57285ed5ff5b5019007da1b6",
        "571a52cb4faf5e1900b8a969",
        "56de3cd0cffd8e1900b4b6be",
        "5729fd56af94a219006aa72f"
      ],
      "candidate_sha256": [
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "e7ec0cd3aac9781316be9d6ab44c0c548df3690a3d5194acc26a123ff6e85d5c",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3"
      ],
      "example_id": "5729fd56af94a219006aa72f",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        14,
        4,
        10,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.2842929,
      "neural_order": [
        15,
        4,
        7,
        1,
        0,
        10,
        5,
        14,
        11,
        9,
        13,
        6,
        2,
        8,
        3,
        12
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "e50c6c2a92f7e721f170f4c4d24f00a4e6f5b81036486f0529a6ce00ceee30ec",
      "trial_index": 381
    },
    {
      "answer_sha256": "aadeacfb57a0c14813e2cab6c0952bd5609d25b9eb7e56e057dc88d7c9296094",
      "candidate_ids": [
        "5728f6446aef0514001548e4",
        "5733f7b9d058e614000b66a9",
        "573766251c45671900574471",
        "5729feaf6aef051400155188",
        "5726509bdd62a815002e815a",
        "5729fb003f37b31900478627",
        "5726db5add62a815002e92d4",
        "56e19557e3433e1400422fee",
        "57263ea0271a42140099d7c3",
        "57336755d058e614000b5a3c",
        "572756fe708984140094dc71",
        "5705edcd52bb8914006896ca",
        "571c7d55dd7acb1400e4c0c4",
        "5733fc6ed058e614000b670f",
        "5725f00938643c19005aced7",
        "56e1b62ecd28a01900c67aa3"
      ],
      "candidate_sha256": [
        "40206469ec21c4f98d7c72cb3c313a8bdfc89251b79140b6c63a48b1afe123d0",
        "5d46e9f7c620b7d8f42a0c90216d4bb7625ee95e4d9af99b663434ae1e08a483",
        "f4a941b8fb7b8c4a1f8010d912eaa080c3e3ddf68b132d4a96fc1da34d4ea0d2",
        "3895e66f384dc9afdd15ffe1c33b4044506755bd22d33e8c94583b00ed06854d",
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7",
        "ffaee1c83ab04918603b9940620c0c8b52e675143bc31a91f1fccac00d2b1bf7",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379"
      ],
      "example_id": "5729feaf6aef051400155188",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        6,
        0,
        1,
        2,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.48171186,
      "neural_order": [
        3,
        4,
        11,
        14,
        12,
        10,
        13,
        7,
        1,
        9,
        2,
        0,
        5,
        6,
        8,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "39a91826def0bbfc4569dc696cef58ceb3e35057fc571b9caadc4674077ae83e",
      "trial_index": 382
    },
    {
      "answer_sha256": "34917e991ff65846d0d104351f9f789eea147d42c91363d691efb9f64c563b94",
      "candidate_ids": [
        "5711651050c2381900b54acb",
        "5726e313f1498d1400e8eeb2",
        "572a005f1d046914007796b7",
        "57274712708984140094dbad",
        "570963a5200fba1400367f33",
        "572855973acd2414000df925",
        "5726a7ecf1498d1400e8e654",
        "572a1046af94a219006aa78d",
        "5726e08e5951b619008f810f",
        "5705ec1675f01819005e776e",
        "57293c246aef051400154bb8",
        "57293f8a6aef051400154bde",
        "5728510f4b864d1900164936",
        "572885c44b864d1900164a78",
        "5725cfd0271a42140099d225",
        "571099b2b654c5140001f9b3"
      ],
      "candidate_sha256": [
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "40f970a92f69db6815bd67e7934fc725f910e6a6e08c5ff4e55067dcc721b938",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "7c74fc3f547a3dd5535ba498d466d980e22cbb4bd99161de9aa21fe70cf743c7",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776"
      ],
      "example_id": "572a005f1d046914007796b7",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.49385172,
      "neural_order": [
        2,
        9,
        14,
        13,
        0,
        7,
        11,
        10,
        3,
        4,
        1,
        5,
        6,
        8,
        12,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "e266b5c92437dc4421cb3a7b1bc5ba354379c50e97461d9d4fe978cc52fd16f8",
      "trial_index": 383
    },
    {
      "answer_sha256": "e86480db8e360439f6ded61dbbd69bc2c00933ca893719e1aaafb50b18924bf3",
      "candidate_ids": [
        "5725e1c4271a42140099d2d7",
        "57302cd004bcaa1900d772d7",
        "570d4a6bfed7b91900d45e13",
        "57111713a58dae1900cd6c00",
        "572686fc708984140094c8e5",
        "5726ed6cf1498d1400e8f00c",
        "56e1febfe3433e1400423236",
        "572a020f6aef051400155198",
        "57300911947a6a140053cfb6",
        "572ff83ab2c2fd1400568681",
        "57265e455951b619008f70bb",
        "5725c41eec44d21400f3d50b",
        "56e1aff7cd28a01900c67a68",
        "56e1b169cd28a01900c67a72",
        "5725c0f289a1e219009abdf2",
        "572a05eb3f37b31900478653"
      ],
      "candidate_sha256": [
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "a5fca4c3e39db7dc72170cfcb6b3abd6a58b735cbb9cce2d5f5f6e553669722a",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c"
      ],
      "example_id": "572a020f6aef051400155198",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.12606969,
      "neural_order": [
        7,
        11,
        4,
        9,
        2,
        14,
        8,
        15,
        6,
        1,
        0,
        3,
        5,
        10,
        12,
        13
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "25d42de0651dd37fa84cc3e12f40bcdfccd44ae51e95d005c42797f07332853d",
      "trial_index": 384
    },
    {
      "answer_sha256": "365c3e314205325c2d79d9b715e0faf6050ff85b32f1a595f3ea562d2dba5cc6",
      "candidate_ids": [
        "570d4030fed7b91900d45d9f",
        "5725fcbe271a42140099d3ad",
        "573406d1d058e614000b6801",
        "5729727baf94a219006aa437",
        "573755afc3c5551400e51eb3",
        "57339555d058e614000b5df3",
        "5710e8c8a58dae1900cd6b26",
        "5733e771d058e614000b6545",
        "572a03a06aef0514001551aa",
        "5730131c947a6a140053d052",
        "56e17e6ee3433e1400422f7f",
        "573784fa1c45671900574483",
        "56e16839cd28a01900c67887",
        "570953a7efce8f15003a7dfe",
        "572650325951b619008f6fa9",
        "572fcc43b2c2fd140056847d"
      ],
      "candidate_sha256": [
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "00b76f48835bfc2434242a5dea1681e9e4d9c363cc74180aa0b9b9fec6f02bef",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "b0008033f8b97b0cf458d8e505384798d796a2fc78a3fced41f39a2591ef64cd",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1"
      ],
      "example_id": "572a03a06aef0514001551aa",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.52229944,
      "neural_order": [
        8,
        15,
        10,
        14,
        12,
        6,
        11,
        4,
        13,
        0,
        1,
        2,
        3,
        5,
        7,
        9
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "535398c08803c223bc3d47e84504c384ab69f67f1d426b1e44a6fc9c87cac9e0",
      "trial_index": 385
    },
    {
      "answer_sha256": "f282f227a6efeca9a2d851881df978738dfdf49e39308c19accd627608c598b9",
      "candidate_ids": [
        "5726a7ecf1498d1400e8e654",
        "5727c0402ca10214002d9564",
        "5737a0acc3c5551400e51f47",
        "572a064a3f37b3190047865d",
        "57269e3bf1498d1400e8e516",
        "570961aa200fba1400367f15",
        "57286ead2ca10214002da346",
        "572658daf1498d1400e8dcac",
        "571155ae2419e31400955591",
        "573784fa1c45671900574483",
        "572648ed5951b619008f6f03",
        "571c879bdd7acb1400e4c0ec",
        "5728fd206aef05140015494c",
        "56de3d594396321400ee26ca",
        "57263fd138643c19005ad37b",
        "572745c6708984140094db99"
      ],
      "candidate_sha256": [
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "2b7326d0cd8188e7b1ea38f54e115afbbb5e41e5f6886b800cf1fd31ac49dd48",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3"
      ],
      "example_id": "572a064a3f37b3190047865d",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.2743302,
      "neural_order": [
        3,
        10,
        13,
        7,
        0,
        12,
        11,
        6,
        9,
        4,
        5,
        1,
        2,
        8,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "f9fc81970aca0dc6d03a4b09d83a78660e3c4fabcfad6a9c2e8434dd59b11b0a",
      "trial_index": 386
    },
    {
      "answer_sha256": "7a7480bb9a1ad53cb7af3ec53be3467cf16297de4b198d3addc1a68a68c6f74a",
      "candidate_ids": [
        "57268220f1498d1400e8e216",
        "5726a299dd62a815002e8b9e",
        "57060a1175f01819005e78d2",
        "57297bc9af94a219006aa4c7",
        "57302efe04bcaa1900d772f5",
        "56e1b8f3e3433e14004230e6",
        "572629c6271a42140099d6a3",
        "572fd1c4947a6a140053cd02",
        "570d28bdb3d812140066d4a3",
        "572a07c11d046914007796d5",
        "5725ce4d38643c19005acd4d",
        "57097051ed30961900e84132",
        "572ffd9e04bcaa1900d76fc7",
        "571095a8a58dae1900cd6a76",
        "57275409708984140094dc35",
        "5729f69caf94a219006aa6f9"
      ],
      "candidate_sha256": [
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "e154bf723ad4445d0e204018449b16d521e83192271d4e25bbadf5ea7feca6e4",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "f4dabeabfa17830c3d2d8edab044167839ef11bb080d2f91289d80dffa01dc27",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "0b80ec4b4d01cc5efd889368e776efe6eb305942873d06582992445d85649487",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "d5f4c4227c07c52cff10ca527f73efedf7e6c5aa9c01a3d9bf8a9468a6ae6939"
      ],
      "example_id": "572a07c11d046914007796d5",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.43767854,
      "neural_order": [
        9,
        2,
        13,
        11,
        4,
        15,
        12,
        6,
        0,
        8,
        1,
        3,
        5,
        7,
        10,
        14
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "04a9f32069e781b4f1c6ee6660af2c0f019593de11380d116a13a698cbf131c5",
      "trial_index": 387
    },
    {
      "answer_sha256": "4a3f74e10b2876f24465224421324afe61f5385edece52dfadceff03ea851af2",
      "candidate_ids": [
        "5728284e3acd2414000df5cd",
        "57269aa65951b619008f77ab",
        "5705e4fe75f01819005e7704",
        "57267b3c5951b619008f7423",
        "57286c8cff5b5019007da218",
        "57286ec63acd2414000df9d1",
        "572a09abaf94a219006aa75b",
        "5729f39a6aef05140015514c",
        "572821274b864d1900164510",
        "572843304b864d1900164848",
        "56e1ded7cd28a01900c67bd4",
        "57308f6b8ab72b1400f9c580",
        "570d391fb3d812140066d575",
        "5710eca0a58dae1900cd6b3a",
        "56de10b44396321400ee2593",
        "5730a4d02461fd1900a9cf29"
      ],
      "candidate_sha256": [
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "3344ef93052b85063f75bebf68306da4193c68170ffc88ba5cf12a5bf36faa69"
      ],
      "example_id": "572a09abaf94a219006aa75b",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.37738783,
      "neural_order": [
        6,
        3,
        2,
        11,
        12,
        10,
        1,
        15,
        4,
        5,
        13,
        0,
        7,
        8,
        9,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "84b74ebe8e0d1fe858a268a6bcce84c79173c33c6b371ff9a8f3258d327e1253",
      "trial_index": 388
    },
    {
      "answer_sha256": "a20a2b7bb0842d5cf8a0c06c626421fd51ec103925c1819a51271f2779afa730",
      "candidate_ids": [
        "5729ffda1d046914007796af",
        "57335ddbd058e614000b592f",
        "572824f13acd2414000df58f",
        "5726722bdd62a815002e8528",
        "5727f16c3acd2414000df05b",
        "5727c94bff5b5019007d954a",
        "573361404776f4190066093c",
        "5725b5a689a1e219009abd28",
        "572a10cd6aef051400155222",
        "57271f125951b619008f8635",
        "573796edc3c5551400e51f33",
        "572a0b101d046914007796e9",
        "573011de04bcaa1900d770f9",
        "5729f4e46aef051400155156",
        "5727565ef1498d1400e8f674",
        "572fcd86947a6a140053ccda"
      ],
      "candidate_sha256": [
        "b557caa6a1a9ebdc6ec705acc8c1078fbde1cf27379b5961913015d24cbd8809",
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "a8497768ed7455269c8c0e43100892fa8dc1ffff0a532613071c4cb9e3f05a18",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "9460dc76dfa16d9b21d2dc60d3a3fa080ab76498642af342b5185d05e1d3d668",
        "329e2dd3e90bd25984d9c186e564a1d7ab4e07d619284ce70de287269ef4a9a8",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300"
      ],
      "example_id": "572a0b101d046914007796e9",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        14,
        4,
        3,
        8,
        0,
        1,
        2,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.25364429,
      "neural_order": [
        11,
        3,
        4,
        7,
        2,
        10,
        13,
        9,
        14,
        0,
        1,
        5,
        6,
        8,
        12,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "638ed7e843ceaecc279bcf91793c3da4b913703c60afa70b3904c3a8e9aeccd5",
      "trial_index": 389
    },
    {
      "answer_sha256": "7d12ba56e9f8b3dc64f77c87318c4f37bc12cfbf1a37573cdf3e4fa683f20155",
      "candidate_ids": [
        "572a0bebaf94a219006aa76f",
        "57285841ff5b5019007da19b",
        "57106185b654c5140001f8db",
        "5727d9c43acd2414000dee17",
        "572822233acd2414000df555",
        "57297ed93f37b3190047845f",
        "571166352419e314009555f1",
        "57293b843f37b31900478133",
        "5727ee372ca10214002d99ec",
        "5726a14c708984140094cc51",
        "57300f8504bcaa1900d770d1",
        "5726a00cf1498d1400e8e550",
        "5733cf61d058e614000b62e9",
        "5705e3f252bb89140068966a",
        "571c8eb9dd7acb1400e4c0f6",
        "5725f190ec44d21400f3d76f"
      ],
      "candidate_sha256": [
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "12614d82ce4ab7673831e5a8b91a94c2835cd9456fa1bf01dc62c16ab070086e",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "e6df4440ff2b8d131a28947a6e46e63948ebc1b4b9c7c987a90b69f59c0c3e1d",
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd"
      ],
      "example_id": "572a0bebaf94a219006aa76f",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        2,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.57469106,
      "neural_order": [
        0,
        8,
        7,
        10,
        5,
        13,
        3,
        14,
        12,
        1,
        15,
        2,
        4,
        6,
        9,
        11
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "35b74cf205adb7ceb64b8b46ca1c6f0238469f77eaf2f30d4f1115ed0bd630ab",
      "trial_index": 390
    },
    {
      "answer_sha256": "4225cd6e32e96d97f2cd7eb87cf0e75003a8c02135105728b710ea118d6040f4",
      "candidate_ids": [
        "57378c9b1c456719005744a7",
        "5711119cb654c5140001fae3",
        "572ffa79a23a5019007fcbe9",
        "57267640f1498d1400e8e074",
        "57275409708984140094dc35",
        "5729f4b41d0469140077968b",
        "5706143575f01819005e7950",
        "572f59b4a23a5019007fc587",
        "5733647e4776f419006609ae",
        "572a0c541d046914007796f3",
        "5726e5c95951b619008f81cb",
        "572878942ca10214002da3a2",
        "5725c0f289a1e219009abdf2",
        "57263ea0271a42140099d7c3",
        "57274e145951b619008f87e7",
        "5728e8212ca10214002daa6c"
      ],
      "candidate_sha256": [
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "cabcb2e17697c1ca5e1bf56512bbf3851f990033a3f5d302aa90e0f3bfed2236",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea"
      ],
      "example_id": "5725c0f289a1e219009abdf2",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.25931633,
      "neural_order": [
        12,
        2,
        7,
        15,
        3,
        1,
        10,
        13,
        8,
        11,
        9,
        0,
        5,
        6,
        4,
        14
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "8dd8a03b7cbba274987feeaf605d6be86312ca3a3ede0f3597af4d6c44a34415",
      "trial_index": 391
    },
    {
      "answer_sha256": "11db4ea9f2b8d884da15e1121586766440367d1313024e20d5cabe8259ee66f4",
      "candidate_ids": [
        "56de51244396321400ee27ef",
        "570d2d68fed7b91900d45cbb",
        "5710e8c8a58dae1900cd6b26",
        "57377ec7c3c5551400e51f05",
        "572982e66aef051400154f92",
        "5730b6592461fd1900a9cfcf",
        "56e1b62ecd28a01900c67aa3",
        "57108ee6a58dae1900cd6a1a",
        "571cda1bdd7acb1400e4c196",
        "57287f6a3acd2414000dfa51",
        "572742bd5951b619008f8785",
        "573011de04bcaa1900d770f9",
        "572677e7708984140094c723",
        "5725c337271a42140099d163",
        "57096e1ced30961900e84102",
        "5710eb6fb654c5140001fa17"
      ],
      "candidate_sha256": [
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "93620703d50b4155fab9038240e64a854d1329e60d76b3887ba89b6b28625e9e",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3",
        "ce8d2359f23498cb006a1bf2f91d5494c463bc986a83749d5f9208b28fa63c51",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "ee73f593c26ad38e58d299361ba118fcf077591f71e9d4ed9515efb8b2fc75f2",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4"
      ],
      "example_id": "5725c337271a42140099d163",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        9,
        0,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.35107836,
      "neural_order": [
        13,
        2,
        0,
        12,
        1,
        15,
        5,
        4,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        14
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "0925d4abd403376a8d1e8fa4239e9312faeb6fdff1d51d36de327ffc88f226da",
      "trial_index": 392
    },
    {
      "answer_sha256": "a1c9c81940d6d4c253b738960600d244488f65404ae2dd0ba08f530fd8211a58",
      "candidate_ids": [
        "5726577f708984140094c301",
        "5737a5931c456719005744e7",
        "5729f12e3f37b319004785e1",
        "573403394776f419006616dd",
        "57115dbe2419e314009555a5",
        "570d529fb3d812140066d6bb",
        "5725dd7d89a1e219009abfea",
        "57264e2f708984140094c1e1",
        "5725c57a89a1e219009abe5e",
        "57286010ff5b5019007da1ca",
        "56de4b074396321400ee2793",
        "5733d7cbd058e614000b63ab",
        "57286d4f2ca10214002da328",
        "5725c69738643c19005accb9",
        "56de0f6a4396321400ee257f",
        "571142b3a58dae1900cd6d5a"
      ],
      "candidate_sha256": [
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "4b9ccabd043e8d3efbbb53f781f4ab7c568a48406248ef91fa7c55b5a9531b9b",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a"
      ],
      "example_id": "5725c57a89a1e219009abe5e",
      "gated_correct": false,
      "gated_top": 10,
      "gold_index": 8,
      "lexical_correct": false,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.32307375,
      "neural_order": [
        8,
        13,
        2,
        5,
        7,
        1,
        11,
        0,
        3,
        4,
        6,
        9,
        10,
        12,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "0a1d6bbf2a7923ffd67c95e3fbab3f49aaafc7e5ea164ee4744648d93db90736",
      "trial_index": 393
    },
    {
      "answer_sha256": "ac0ace9604718be1d10d7d0e2ec8f42c08c4641256a3e077dcf4eb0e7db0a6ba",
      "candidate_ids": [
        "572fd7b8947a6a140053cd3a",
        "57338007d058e614000b5bda",
        "57108ee6a58dae1900cd6a1a",
        "5726701add62a815002e84c8",
        "572ff5fcb2c2fd1400568657",
        "572f55e8a23a5019007fc56b",
        "57287ee3ff5b5019007da274",
        "57273a465951b619008f86ff",
        "5725c69738643c19005accb9",
        "5726559edd62a815002e81c8",
        "5726545f708984140094c2a5",
        "572847ff3acd2414000df869",
        "570d529fb3d812140066d6bb",
        "57060f3e75f01819005e7922",
        "5726bcde708984140094cfbf",
        "57288428ff5b5019007da28e"
      ],
      "candidate_sha256": [
        "9a51dc8913bed32890777c8d8e326667913afeaefcb20485f6c2fcf8e94a57d2",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "c44126e5a71b148722d5f363b3fe85277c50806c0cacb2d5d467d0ee1ef9ca2d",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19"
      ],
      "example_id": "5725c69738643c19005accb9",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.3409273,
      "neural_order": [
        8,
        12,
        10,
        0,
        2,
        3,
        13,
        9,
        4,
        1,
        5,
        6,
        7,
        11,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "e8a98cf053c61d8a968e72162b05d90538321ea7a7cc840210f7199e2aa34c2c",
      "trial_index": 394
    },
    {
      "answer_sha256": "3d66170bdab90a4948b77ea2f62d36f414e8dd9e1f63f15be4698ef65d116046",
      "candidate_ids": [
        "5726a7ecf1498d1400e8e654",
        "57114667a58dae1900cd6d80",
        "570967c4ed30961900e840ba",
        "572a02483f37b3190047864b",
        "5725c91e38643c19005acceb",
        "56de0f6a4396321400ee257f",
        "572655e5f1498d1400e8dc5e",
        "572989846aef051400154fc0",
        "57274f49f1498d1400e8f61e",
        "572fc49d04bcaa1900d76ccb",
        "572fbf21a23a5019007fc937",
        "5711648850c2381900b54ac3",
        "572f6ec7a23a5019007fc621",
        "5728223cff5b5019007d9dc4",
        "57282358ff5b5019007d9dde",
        "5727afa82ca10214002d93c2"
      ],
      "candidate_sha256": [
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "88ebefee76460e60ceee4d95b9c02b9473aec92dfa15668f865eb35a055afba5",
        "b67161cb70767d25408d47a9b2cd15ed23cc66a97c33ff46706b10e7a4772023",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "8636e94058ce0f55e10d0eaeb35617ebb9fd6ad1e019585a005e41e4592bcac8"
      ],
      "example_id": "5725c91e38643c19005acceb",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.35801414,
      "neural_order": [
        4,
        12,
        5,
        6,
        3,
        14,
        2,
        13,
        0,
        1,
        7,
        8,
        9,
        10,
        11,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "9f172e5dc48f2d28248741f8635b4c44bb85b4b3a8a10f2f1a80527ac203688d",
      "trial_index": 395
    },
    {
      "answer_sha256": "bf2202ca6a849663d3dedb970e75bffd6560c57dd2f17b5b482f4cb8c02ed5a8",
      "candidate_ids": [
        "5730876a396df9190009617a",
        "573027d6a23a5019007fce9d",
        "57097c8fed30961900e841f2",
        "57339eb9d058e614000b5ef6",
        "572f58d9a23a5019007fc57d",
        "57113c6da58dae1900cd6d32",
        "570d29c2b3d812140066d4bd",
        "57097141200fba1400367ff9",
        "5706149552bb891400689880",
        "5733e8ae4776f419006614a6",
        "57266193dd62a815002e832a",
        "57339a554776f41900660e74",
        "5725c63438643c19005acc9f",
        "57269f3ef1498d1400e8e534",
        "57264f18f1498d1400e8dbae",
        "5725cb33271a42140099d1db"
      ],
      "candidate_sha256": [
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "d3384a30ecacbbee697fe3d01deb29486590cdb9655118cfdca4ea3e68081336",
        "de308e41aa84b64f8c0380e4021ce99dfb8b520d42b8a543f77ef7f6f3c88a00",
        "46a60c1fb7a3dc1eb8d8008909c9080e1b010b9c75f648c33a07f6ce1e997157",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "1b6b54e8c8b9ed8a8cd2a8ba1a4fecc7094aad721964a7ddb65b5eb788d029ec",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b"
      ],
      "example_id": "5725cb33271a42140099d1db",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.39893862,
      "neural_order": [
        15,
        0,
        14,
        4,
        12,
        1,
        6,
        11,
        8,
        9,
        3,
        13,
        2,
        5,
        7,
        10
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "fe43f0a04e95dea2bb96ce2f244d8c25bf5bc0d82fbdc717a3a6acbd1755c0d8",
      "trial_index": 396
    },
    {
      "answer_sha256": "7441ddcbae09972170d2d11c160583e90304211545f15c63e886553a1dcc6715",
      "candidate_ids": [
        "5729a3716aef05140015506a",
        "56e1b00ce3433e140042309e",
        "57115b2850c2381900b54a7f",
        "57060a1175f01819005e78d2",
        "5727cd7dff5b5019007d958e",
        "5726b58f5951b619008f7b55",
        "572750e8dd62a815002e9af0",
        "572647d0708984140094c14b",
        "5711119cb654c5140001fae3",
        "5733a560d058e614000b5f77",
        "57265746dd62a815002e8218",
        "5726d8bd708984140094d35b",
        "57274beff1498d1400e8f5e4",
        "572755b7708984140094dc4d",
        "571099b2b654c5140001f9b3",
        "572a096e6aef0514001551e4"
      ],
      "candidate_sha256": [
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374",
        "380ed5d16b3e18e5bda2da4ba87c2f5203099504f6e9d67415a4907077e2d2c0",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "8edb4137eebb6b23098b0dc1e5f01d1ab0dfb3b1806b0eedceb92bd651c64438",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea"
      ],
      "example_id": "572647d0708984140094c14b",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        15,
        4,
        12,
        11,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        13,
        14
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.00337638,
      "neural_order": [
        7,
        10,
        11,
        15,
        4,
        12,
        2,
        1,
        9,
        8,
        0,
        14,
        5,
        3,
        6,
        13
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "f1e9c828e15340d264132a334e3fdec210b8611abcc65e3bcd85b67420869ed1",
      "trial_index": 397
    },
    {
      "answer_sha256": "9e23c6fd79edc57f9765d878feb3cc06cec4c281b6ad9abacae56406b24ad417",
      "candidate_ids": [
        "573776eec3c5551400e51ee9",
        "572a06866aef0514001551be",
        "56de15dbcffd8e1900b4b5c8",
        "572940246aef051400154bec",
        "572989846aef051400154fc0",
        "57268c01dd62a815002e8912",
        "57112686b654c5140001fbd3",
        "572853e8ff5b5019007da188",
        "572648e8dd62a815002e8076",
        "572881022ca10214002da416",
        "572879574b864d1900164a14",
        "572f55e8a23a5019007fc56b",
        "5725f6e138643c19005acf19",
        "5725f00938643c19005aced7",
        "56e16839cd28a01900c67887",
        "573750f51c45671900574467"
      ],
      "candidate_sha256": [
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd"
      ],
      "example_id": "572648e8dd62a815002e8076",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        6,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.39971463,
      "neural_order": [
        8,
        5,
        6,
        11,
        9,
        2,
        13,
        7,
        0,
        1,
        3,
        4,
        10,
        12,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "8f8b0b0950e204dd447e2c95447aea24918f0ee1d25c41768126e264de79acc2",
      "trial_index": 398
    },
    {
      "answer_sha256": "8d01bc206566b8f2ca8c1647e09ac544eebd6848f85ceeaa4d8bcfb39b0214db",
      "candidate_ids": [
        "573312fcd058e614000b56e6",
        "57264a74708984140094c18b",
        "572867212ca10214002da2f2",
        "57264a0ef1498d1400e8db40",
        "57281ab63acd2414000df493",
        "56de3e414396321400ee26d8",
        "5733dc95d058e614000b644a",
        "570610b275f01819005e792a",
        "5725d662ec44d21400f3d687",
        "57332e48d058e614000b5762",
        "56e1ddfce3433e14004231d5",
        "5705fec152bb89140068977a",
        "573361404776f4190066093c",
        "57302faa04bcaa1900d77311",
        "5726a34bf1498d1400e8e59e",
        "56de51c64396321400ee27f7"
      ],
      "candidate_sha256": [
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4"
      ],
      "example_id": "57264a0ef1498d1400e8db40",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.27301439,
      "neural_order": [
        3,
        1,
        15,
        5,
        6,
        14,
        8,
        12,
        10,
        0,
        2,
        4,
        7,
        9,
        11,
        13
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "953da0499c823ab4995159ab408c3f4350e6709818c428614d5242cb99be5323",
      "trial_index": 399
    },
    {
      "answer_sha256": "4e143bdc93dae1a4a81be76b1b93d198ecc77d4a3eeaaadde672403b52ef6d8a",
      "candidate_ids": [
        "5733f8dc4776f419006615f7",
        "572fd73e947a6a140053cd32",
        "5726a5525951b619008f78dd",
        "5725daa8ec44d21400f3d6b3",
        "5730a4d02461fd1900a9cf29",
        "56e200e4cd28a01900c67c14",
        "5726dcbddd62a815002e9320",
        "573766251c45671900574471",
        "5728f7774b864d190016512e",
        "5725d42a89a1e219009abf58",
        "5706139252bb891400689864",
        "57096c95200fba1400367fbb",
        "572fc41f947a6a140053cc80",
        "57264b1ddd62a815002e80a0",
        "57111992b654c5140001fb43",
        "57338007d058e614000b5bda"
      ],
      "candidate_sha256": [
        "c2c1868add9d6aa35780c0c8d4492524306055ae928ea56b315e6e9e6969d494",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "3344ef93052b85063f75bebf68306da4193c68170ffc88ba5cf12a5bf36faa69",
        "f7140e68e270abe4b07098bebe1f59cce949b799b8237e1c88451a74c65cd4a0",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "f4a941b8fb7b8c4a1f8010d912eaa080c3e3ddf68b132d4a96fc1da34d4ea0d2",
        "511ddcc5ddbeeaf06bef1b76bc323fc7607c5658b94f72c360e5919a5c54656e",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "5b38927a00cca8f78729fc0a7e1783aa49382db22072a7337823fda2888c3a8c",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805"
      ],
      "example_id": "57264b1ddd62a815002e80a0",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.29641844,
      "neural_order": [
        13,
        14,
        0,
        8,
        1,
        2,
        12,
        10,
        6,
        4,
        7,
        3,
        5,
        9,
        11,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "9e7aa88bf6c462fd4562f06a4864fc3f00afbde0cb358ffea6efa88225d115bf",
      "trial_index": 400
    },
    {
      "answer_sha256": "72cfd72bcab0a37fec5cd4797ee105667f2365aabccaeb3fc1e71b148f5d7e6d",
      "candidate_ids": [
        "5710e9f8a58dae1900cd6b30",
        "572659ea5951b619008f7051",
        "572f6ec7a23a5019007fc621",
        "56e1aff7cd28a01900c67a68",
        "5711163bb654c5140001fb13",
        "56de3d594396321400ee26ca",
        "5728510f4b864d1900164936",
        "5728742cff5b5019007da246",
        "57264cfa708984140094c1c3",
        "572885c44b864d1900164a78",
        "57107a3ea58dae1900cd69de",
        "572686fc708984140094c8e5",
        "572fc043a23a5019007fc95d",
        "57309446396df919000961b8",
        "572654e2708984140094c2b7",
        "5730aef38ab72b1400f9c682"
      ],
      "candidate_sha256": [
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "f3e994c58b1e4906937c035c80fb9519764a0f4f42ff12a50c7aeac3190ebd39",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "9d0852e7a492edfb020138aeed2e89901eaf6cbc0450e6da752320d2c7c024e0",
        "bedd9f5d0a5db4c8fc9999b0ed320ab5e6d56704b9e02e5331f2c1973ba98941"
      ],
      "example_id": "57264cfa708984140094c1c3",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        1,
        9,
        3,
        4,
        0,
        2,
        5,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.06238828,
      "neural_order": [
        8,
        1,
        3,
        9,
        13,
        11,
        12,
        6,
        2,
        14,
        10,
        4,
        0,
        5,
        7,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "2d4bd5044e3f73dc123a7b689060ba0a0020d51c8ec1a865e62fb3e836b8e044",
      "trial_index": 401
    },
    {
      "answer_sha256": "adb3147f3aff15c92f9e70ec15f0d82482c48d19e3032449abcbe186450a9a37",
      "candidate_ids": [
        "572754fff1498d1400e8f660",
        "573028fa04bcaa1900d77287",
        "570967c4ed30961900e840ba",
        "57109180a58dae1900cd6a40",
        "5728151b4b864d1900164428",
        "57264e66dd62a815002e811a",
        "5737477bc3c5551400e51ea3",
        "572843304b864d1900164848",
        "5726a00cf1498d1400e8e550",
        "57274d1cdd62a815002e9aae",
        "57265200708984140094c237",
        "5728804b4b864d1900164a46",
        "571163172419e314009555e7",
        "5706155352bb891400689894",
        "5727cff1ff5b5019007d95c4",
        "5726db5add62a815002e92d4"
      ],
      "candidate_sha256": [
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "5b8656a663456a61c1d9692e2b9d3858aa19a6b21b538a75b82941eb5206617c",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "2a88a6e435244f1025c628b748e860ab699fad659a4359c6fb5c85d3ee365576",
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "f9d0ba77dd8dff4446935bfec391a5986633d68d8bdf2319ad27e1fd5890d87c",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530"
      ],
      "example_id": "57264e66dd62a815002e811a",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.64147112,
      "neural_order": [
        5,
        12,
        4,
        8,
        15,
        6,
        0,
        14,
        2,
        1,
        3,
        7,
        9,
        10,
        11,
        13
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "6dc3ceec7b8b0e9609fca469b4c9512873c38ef67d8902dc3d2d06085c308842",
      "trial_index": 402
    },
    {
      "answer_sha256": "624142c73217640bd719b73e870f7ff6f0136c5fbe1942f16a03d2c3d56a1e3c",
      "candidate_ids": [
        "572987e46aef051400154fa2",
        "5728759cff5b5019007da25a",
        "5727cd0f4b864d1900163d72",
        "57108d69b654c5140001f983",
        "57377ec7c3c5551400e51f05",
        "572655e5f1498d1400e8dc5e",
        "571cc8815efbb31900334dee",
        "57269016708984140094ca41",
        "57261dab38643c19005ad037",
        "57297547af94a219006aa45c",
        "57300200b2c2fd1400568729",
        "57338007d058e614000b5bda",
        "57283c464b864d19001647c8",
        "5725f6e138643c19005acf19",
        "57269e3bf1498d1400e8e516",
        "5726f635dd62a815002e9656"
      ],
      "candidate_sha256": [
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "45a0b4e5e93112c3796f5644ea9a736bcd8a141fdef37358e59af981df454813",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436"
      ],
      "example_id": "572655e5f1498d1400e8dc5e",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        3,
        7,
        0,
        1,
        2,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.25616471,
      "neural_order": [
        5,
        7,
        6,
        11,
        3,
        14,
        2,
        1,
        0,
        4,
        8,
        9,
        10,
        12,
        13,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "94b68456f3c1997a6218870cfbe1a2e9d55e1f10f880024edb538e45c297655d",
      "trial_index": 403
    },
    {
      "answer_sha256": "c1ebb782062ad8571a80a1b9c8d3835c92f140038bf79970bf4fb74f660aa7c3",
      "candidate_ids": [
        "572fc8a904bcaa1900d76d1f",
        "571a4d1a4faf5e1900b8a958",
        "57265746dd62a815002e8218",
        "572945b11d0469140077922f",
        "5725b7f389a1e219009abd5d",
        "572812e74b864d19001643ce",
        "572726c9708984140094da7b",
        "5728ebcb3acd2414000e01d9",
        "5737a5931c456719005744e7",
        "57335ddbd058e614000b592f",
        "57308ddc396df919000961a4",
        "570d30fdfed7b91900d45ce3",
        "572fbb04a23a5019007fc8f7",
        "572867212ca10214002da2f2",
        "5725ce4d38643c19005acd4d",
        "5705f36452bb891400689718"
      ],
      "candidate_sha256": [
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "56342ec864f7c99a6400cb1b743d2de657d247bc5557e34b707c510c768abb1c",
        "46331ca74c00c63b58196c7c8beb570d989a8d2270612f509af4b3e4499236ed",
        "d16cfd194068a566c150d73414f420784023764bb0b49639fd879fedf9396397",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "06dabbfdce88f733bf8b7c056c89d87a5ec555147201d86e7703ff7355c1b756",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba"
      ],
      "example_id": "57265746dd62a815002e8218",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        5,
        1,
        0,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.25259072,
      "neural_order": [
        2,
        9,
        15,
        3,
        10,
        11,
        4,
        6,
        8,
        5,
        14,
        0,
        1,
        7,
        12,
        13
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "68143bc1719ffca9008861ce9e2e35b7f8c24b0cfc8b5db7ffff0bfb9c56b0d5",
      "trial_index": 404
    },
    {
      "answer_sha256": "4d1ba5826af02fabb12464a069d73076a0a9bac9d016464a2d2b2aa042ae61c3",
      "candidate_ids": [
        "570d29c2b3d812140066d4bd",
        "571098f8a58dae1900cd6ab2",
        "572658daf1498d1400e8dcac",
        "56e2042ecd28a01900c67c1e",
        "572949306aef051400154c68",
        "57264586f1498d1400e8dac6",
        "57096f37200fba1400367fe5",
        "571c8eb9dd7acb1400e4c0f6",
        "5733f9fa4776f4190066161f",
        "572a0bafaf94a219006aa765",
        "5733d4c8d058e614000b6353",
        "5726887e708984140094c917",
        "5730069004bcaa1900d7702d",
        "5726487b5951b619008f6edd",
        "5726a00cf1498d1400e8e550",
        "570d50a5fed7b91900d45e7d"
      ],
      "candidate_sha256": [
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "0771c41591ce859133397b70f7fe295ca178804c352b68cc34f3f35caa1c92ab",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed"
      ],
      "example_id": "572658daf1498d1400e8dcac",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        11,
        10,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.14742665,
      "neural_order": [
        2,
        11,
        12,
        1,
        5,
        13,
        7,
        3,
        8,
        4,
        0,
        6,
        9,
        10,
        14,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "c40f4164d0410156b517aa1aed5c95d3150e6429d43ec425c78039bb639cd38c",
      "trial_index": 405
    },
    {
      "answer_sha256": "9fedfb6b1375e714c9d7921fa8c9ad49b182dabd928eb909e64481c7006c9aaf",
      "candidate_ids": [
        "5728eef92ca10214002daab0",
        "572996c73f37b319004784b3",
        "57300ec0947a6a140053d004",
        "571099b2b654c5140001f9b3",
        "57264efddd62a815002e8134",
        "573003dd947a6a140053cf42",
        "57379ed81c456719005744d5",
        "5733dc95d058e614000b644a",
        "572659ea5951b619008f7051",
        "5733cff84776f419006612b4",
        "5705f36452bb891400689718",
        "5725d183271a42140099d23d",
        "572ffbaa947a6a140053cee6",
        "5726887e708984140094c917",
        "5705ffde52bb891400689784",
        "57274a8ff1498d1400e8f5bc"
      ],
      "candidate_sha256": [
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "33e49ae942c39aefd29d877e8334707b666747d16641fd8392ae988a08b47f20",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "ea8fcbfc81f8e0914b4fda68a85f3de1d3c5b4dfcd3e1c1417c68ab3413da669",
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667",
        "7c956034b70248bfe50536e6b9edd2fbe5fb3ec6ed3a1e01287ad5ab54a4f2ba",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "6bba6ea4f692054f16ed06c2ac597ed5e099bbecf957936e9ac945b3b788e01d",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45"
      ],
      "example_id": "572659ea5951b619008f7051",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        14,
        3,
        11,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.41226295,
      "neural_order": [
        8,
        1,
        6,
        9,
        10,
        5,
        11,
        13,
        0,
        2,
        3,
        4,
        7,
        12,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "da08ef9ec9fd3db0f1a2b735f8206d86269118cd32b59a1470fecf1b662d5321",
      "trial_index": 406
    },
    {
      "answer_sha256": "20515122aea24acadf929626d297becd5d3d12cf8d79c9b22b6f6accef9ac3ab",
      "candidate_ids": [
        "5725d34089a1e219009abf50",
        "572853e8ff5b5019007da188",
        "572ff86004bcaa1900d76f65",
        "56de49a8cffd8e1900b4b7a7",
        "5726a5525951b619008f78dd",
        "570614ff52bb89140068988a",
        "57264586f1498d1400e8dac6",
        "5729f5a03f37b31900478605",
        "57300f8504bcaa1900d770d1",
        "571142b3a58dae1900cd6d5a",
        "5727b0892ca10214002d93e8",
        "57273e50dd62a815002e9a02",
        "57293bc91d0469140077919b",
        "572900f73f37b31900477f69",
        "57265aaf5951b619008f706b",
        "572ff7ab04bcaa1900d76f51"
      ],
      "candidate_sha256": [
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89",
        "4e041656788e4036188e81c0a35175217b17488e8ea0585f48b4719b1fac1708",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0"
      ],
      "example_id": "57265aaf5951b619008f706b",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.39746387,
      "neural_order": [
        14,
        15,
        2,
        5,
        9,
        11,
        12,
        1,
        6,
        0,
        3,
        4,
        7,
        8,
        10,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "1b9c638a01724f36c0f22fa5bd794042f5b22cb79b792cadbc0bb389d15dffba",
      "trial_index": 407
    },
    {
      "answer_sha256": "866d2a746c7e671f83214b3cd623a0ecce0566d72423003564d6ee2ea387e840",
      "candidate_ids": [
        "572821274b864d1900164510",
        "5710f114a58dae1900cd6b60",
        "56e1bc3ae3433e1400423104",
        "5729f9953f37b3190047861f",
        "5730040f947a6a140053cf4c",
        "573098f38ab72b1400f9c5d2",
        "57300911947a6a140053cfb6",
        "57265c10f1498d1400e8dd36",
        "570614ff52bb89140068988a",
        "5726446a89cfff1900a8404d",
        "57294e6b1d04691400779275",
        "5726431aec44d21400f3dd13",
        "571c879bdd7acb1400e4c0ec",
        "5726f96ddd62a815002e9698",
        "57281d494b864d19001644be",
        "5737aafd1c456719005744fb"
      ],
      "candidate_sha256": [
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9"
      ],
      "example_id": "57265c10f1498d1400e8dd36",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.15526895,
      "neural_order": [
        7,
        12,
        3,
        6,
        10,
        1,
        0,
        2,
        4,
        5,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "941ae7250901337355785f50b6bf507390fa3924d1ebe228fbf8ec92ca8f0dba",
      "trial_index": 408
    },
    {
      "answer_sha256": "9371a4bbd6150615d06d27225102c2cf0702a503ec29331a8cb89fb3b36d3b40",
      "candidate_ids": [
        "56de40da4396321400ee2708",
        "572879574b864d1900164a14",
        "5726fa525951b619008f83f7",
        "5726b929f1498d1400e8e8e9",
        "572754dd708984140094dc3c",
        "5727d4922ca10214002d977c",
        "5726e179dd62a815002e93ae",
        "572663a9f1498d1400e8ddf2",
        "57265700dd62a815002e820e",
        "57286b003acd2414000df9c1",
        "57108c95b654c5140001f979",
        "570d2af0fed7b91900d45c93",
        "572885023acd2414000dfa83",
        "5726bc1add62a815002e8ea6",
        "57265d86f1498d1400e8dd50",
        "57105da9a58dae1900cd699e"
      ],
      "candidate_sha256": [
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "1629b48b9aae07c00bf8bd9ec7d76d9982b5e645f832551c105d868a0f34f81e",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "b6f799a7fbe964e85d020346753d8182351cdec9710d1c891b7da4642c30fdc0",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "cc86f9fb2768ce9b19832d6a94a9d712d07f8de895c58b6bda5d40bf486c1203",
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c"
      ],
      "example_id": "57265d86f1498d1400e8dd50",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        9,
        4,
        15,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.25703478,
      "neural_order": [
        14,
        2,
        6,
        4,
        8,
        10,
        5,
        15,
        7,
        9,
        0,
        1,
        3,
        11,
        12,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "3929e1e2178dec7300615944b34624a1714bf381ee89415a3622500f252ad8bf",
      "trial_index": 409
    },
    {
      "answer_sha256": "f7fb680b05d0ab7ce4e41d26d4bfc796719bb5b9ba1f3170d8419805d0323757",
      "candidate_ids": [
        "571a4d1a4faf5e1900b8a958",
        "57265aaf5951b619008f706b",
        "57114f0050c2381900b54a65",
        "5727e0474b864d1900163f08",
        "57300911947a6a140053cfb6",
        "5725b81b271a42140099d097",
        "5733ffa7d058e614000b674f",
        "5733d3cb4776f419006612ea",
        "57097d63ed30961900e841fc",
        "5725e1c4271a42140099d2d7",
        "5705e63175f01819005e7720",
        "57339ad74776f41900660e86",
        "57265e97708984140094c3c3",
        "57107c24a58dae1900cd69e8",
        "573330444776f41900660758",
        "57373d0cc3c5551400e51e85"
      ],
      "candidate_sha256": [
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "d404471721922f4e521c100992e50a7d7424e6de36fb7f727122c1b4b42aaf6a",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "c7c7ec362f0439a4c9ad1edead0e0f7505b22fd24ce7be23b8711356cdcda6a3",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "dba710cd82bb65136087ed545607820a13e196f28ad71ddb50f8213f409b789b",
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c"
      ],
      "example_id": "57265e97708984140094c3c3",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.12920551,
      "neural_order": [
        12,
        1,
        4,
        5,
        10,
        3,
        11,
        0,
        2,
        6,
        7,
        8,
        9,
        13,
        14,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "8027c163cb887a3840d8bca0f077ef8e2aa72c2fb517a94fa5ad1c39b5942a74",
      "trial_index": 410
    },
    {
      "answer_sha256": "f5e76e90d65dc9b4a7df7a152e7bf37ed528ecf8aee25f3f2c495965aab83f8f",
      "candidate_ids": [
        "5711607f2419e314009555cd",
        "572754dd708984140094dc3c",
        "573362b94776f41900660974",
        "571155ae2419e31400955591",
        "572681c1dd62a815002e8796",
        "5730069004bcaa1900d7702d",
        "5726a299dd62a815002e8b9e",
        "573749741c4567190057445d",
        "5726ed6cf1498d1400e8f00c",
        "57265c10f1498d1400e8dd36",
        "57271f125951b619008f8635",
        "572fce12a23a5019007fca11",
        "573020f7b2c2fd14005688f7",
        "57294279af94a219006aa209",
        "57300a25a23a5019007fcc97",
        "571c8198dd7acb1400e4c0ce"
      ],
      "candidate_sha256": [
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119"
      ],
      "example_id": "572681c1dd62a815002e8796",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        9,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.36974982,
      "neural_order": [
        4,
        9,
        15,
        10,
        2,
        5,
        8,
        0,
        1,
        3,
        13,
        6,
        7,
        11,
        12,
        14
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "b99f73f218a0e5be22d001e5672b50da03ec2c3ba3fe9825de551b976d8780b6",
      "trial_index": 411
    },
    {
      "answer_sha256": "025367a652177cec4953304701c8d73018a1e58b5f62b6493119551e4cc8f054",
      "candidate_ids": [
        "572871bd3acd2414000dfa03",
        "572691bedd62a815002e89dc",
        "5726559edd62a815002e81c8",
        "57286d7d4b864d19001649e0",
        "5728170d3acd2414000df443",
        "5726398589a1e219009ac588",
        "57114dfb50c2381900b54a53",
        "5730069004bcaa1900d7702d",
        "5728fb6a1d04691400778ef5",
        "573027d6a23a5019007fce9d",
        "56e20a27cd28a01900c67c24",
        "5730088e947a6a140053cfac",
        "57281c594b864d19001644a2",
        "572753335951b619008f8853",
        "572683075951b619008f7513",
        "57264e2f708984140094c1e1"
      ],
      "candidate_sha256": [
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "3a95085253039f4bf191f1037c44a2641c44c877382c059105507c459d53eaba",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "4b2a2a7e8bce520e5e42162e14db64c90808895b36ef2edfc527bbc3df66971c",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75"
      ],
      "example_id": "572683075951b619008f7513",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.41107565,
      "neural_order": [
        14,
        6,
        1,
        7,
        5,
        3,
        9,
        13,
        15,
        2,
        8,
        0,
        4,
        10,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "f0fee278ecbc08f89653bcaec19a9919bf4cff84935c2b843514bca79fa358fd",
      "trial_index": 412
    },
    {
      "answer_sha256": "1e83938cb76eccf52cb75b2373e62bef86e09d6ad73871c528cfdac0aa92c876",
      "candidate_ids": [
        "57269016708984140094ca41",
        "572944e03f37b319004781e1",
        "572905ce1d04691400778f83",
        "56e1b754cd28a01900c67abc",
        "57113ba6b654c5140001fc1f",
        "5726c002708984140094d073",
        "57275409708984140094dc35",
        "57287d4a2ca10214002da3e4",
        "572973f76aef051400154f0b",
        "5728742cff5b5019007da246",
        "572684365951b619008f753f",
        "5730040f947a6a140053cf4c",
        "56e16182e3433e1400422e28",
        "5726d9935951b619008f7fed",
        "57271f125951b619008f8635",
        "570602fa52bb89140068979e"
      ],
      "candidate_sha256": [
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "d7f4a15ad5e5aa6dbb0e1b77cd3857b0dd33e47c936805bb4c12b2d4038deb93",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "510883d4b2a3ae325bd49893df0384493db2a83b15829bd11692c175088d38fb",
        "b814394ce33a2687a04a7dfdc912728917560900888085bd23b9b566d65ed7e4",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d"
      ],
      "example_id": "572684365951b619008f753f",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        5,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.34448332,
      "neural_order": [
        10,
        6,
        14,
        5,
        15,
        0,
        4,
        2,
        8,
        11,
        1,
        3,
        7,
        9,
        12,
        13
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "63e57a069af596ec3d4e71f15688fb61dc3d366e47207c35df2c0c7408ea2465",
      "trial_index": 413
    },
    {
      "answer_sha256": "e91b2f06c7c93af4d6fa2d4738f7a6db0512b7323649f276003dd625e1e4e42f",
      "candidate_ids": [
        "57300f8504bcaa1900d770d1",
        "570d29c2b3d812140066d4bd",
        "5733dd4f4776f419006613aa",
        "5733a2a9d058e614000b5f28",
        "57336755d058e614000b5a3c",
        "572900f73f37b31900477f69",
        "5725f5b1271a42140099d371",
        "571cb27fdd7acb1400e4c132",
        "57268220f1498d1400e8e216",
        "57376a1bc3c5551400e51ec3",
        "57261f9f271a42140099d4a9",
        "5725d183271a42140099d23d",
        "5726e985dd62a815002e94da",
        "57268a37f1498d1400e8e33c",
        "572686fc708984140094c8e5",
        "57264d58f1498d1400e8db7a"
      ],
      "candidate_sha256": [
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "4e041656788e4036188e81c0a35175217b17488e8ea0585f48b4719b1fac1708",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "683b342d0ffcfc6fe43a44ff9a999b6c5cf8e41e48c559c45ade86e4512db2be",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d"
      ],
      "example_id": "572686fc708984140094c8e5",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        5,
        15,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.30075759,
      "neural_order": [
        14,
        13,
        4,
        7,
        5,
        9,
        8,
        15,
        6,
        3,
        12,
        1,
        0,
        2,
        10,
        11
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "44fa12bbc0d7ffbc0fe155076d8ad8f4c48edbe89803970c901e207774e64103",
      "trial_index": 414
    },
    {
      "answer_sha256": "ff8813921fc6a6c0638c8ff78a3cbc18dc197f64bc9c934c4e0b21b08383fdc6",
      "candidate_ids": [
        "5733f9fa4776f4190066161f",
        "5726deed5951b619008f80c5",
        "570d3468b3d812140066d543",
        "56dde0ba66d3e219004dad75",
        "56de1645cffd8e1900b4b5d0",
        "5726472bdd62a815002e8042",
        "57378862c3c5551400e51f21",
        "57265e455951b619008f70bb",
        "5705fd8475f01819005e7840",
        "57339dd94776f41900660ecc",
        "57332562d058e614000b5730",
        "57373d0cc3c5551400e51e85",
        "56e1c0f6cd28a01900c67b2c",
        "5726887e708984140094c917",
        "5705fec152bb89140068977a",
        "5705f09e75f01819005e77a4"
      ],
      "candidate_sha256": [
        "7675d4a09eddad84999d4fcae9082509b0a254495c1552f93e0c585d8139bde0",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "06a36c034cb1681bb677178da6125ae4b9b63cd95527bdc90f5a7af7da958523",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "b850237f43814691bedb635f5b07cea17e5bd3a2be801e564b61359a1746c8c6",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e"
      ],
      "example_id": "5726887e708984140094c917",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.54227053,
      "neural_order": [
        13,
        0,
        3,
        7,
        8,
        11,
        10,
        9,
        1,
        12,
        4,
        2,
        5,
        6,
        14,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "536861cee8cd94c2793702f34d77d4a4e8a504a154aaadf9ab7c4326afa69847",
      "trial_index": 415
    },
    {
      "answer_sha256": "d4ff7c57ba016514be7d18922cef24a96af00fecd6ea879e7ae1d6755e23e195",
      "candidate_ids": [
        "57378c9b1c456719005744a7",
        "5726534d708984140094c26d",
        "5726559edd62a815002e81c8",
        "57060eaf75f01819005e7910",
        "5725db98ec44d21400f3d6c5",
        "57268a37f1498d1400e8e33c",
        "57265e97708984140094c3c3",
        "571161092419e314009555d7",
        "572a06af3f37b31900478667",
        "56de40da4396321400ee2708",
        "572f6c85947a6a140053c940",
        "57302efe04bcaa1900d772f5",
        "56e1ec83cd28a01900c67c0a",
        "572815d9ff5b5019007d9cbc",
        "571cca4add7acb1400e4c150",
        "5725d34089a1e219009abf50"
      ],
      "candidate_sha256": [
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "651298a168110f388d3688f8765ff1a70797c36a014e4c3a03d0bf31007236d7",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d",
        "bb3727c243a6cb744686ba6886c54791cd7745eca592eef8baf9c5b06c996b3e",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8"
      ],
      "example_id": "57268a37f1498d1400e8e33c",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        6,
        8,
        1,
        0,
        2,
        3,
        4,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.25676324,
      "neural_order": [
        5,
        6,
        1,
        2,
        8,
        14,
        13,
        9,
        10,
        0,
        3,
        4,
        7,
        11,
        12,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "d794955b533a7ac1cd35d2d3d50979a48c88b22e76c37c9ee8751e88d76aea56",
      "trial_index": 416
    },
    {
      "answer_sha256": "35e4654b9a6ac0f2197d7105b8d0574204b07a8cc0668fad4b1b0e9a3774df33",
      "candidate_ids": [
        "56dde27d9a695914005b9651",
        "57264d9edd62a815002e80fe",
        "5726516a708984140094c223",
        "5730035e04bcaa1900d77001",
        "5727e9523acd2414000def95",
        "570d3468b3d812140066d543",
        "571166352419e314009555f1",
        "572fbfa504bcaa1900d76c73",
        "57282358ff5b5019007d9dde",
        "5729517d6aef051400154cc8",
        "5725b5a689a1e219009abd28",
        "5727c94bff5b5019007d954a",
        "5727ec062ca10214002d99b6",
        "5705eb3375f01819005e7764",
        "57268c01dd62a815002e8912",
        "57302bd0b2c2fd14005689db"
      ],
      "candidate_sha256": [
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "58de06c671d2cd29f3b8bde136326194d51c92607fc4a624d1d5a6e90dd5cf33",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "0fd3a0b720dd6f6468a6501fe9388d8e892e0b780957975d0ce5f25eec028be6",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "e72f4d9a7040228d30bf4ff23cb64178c78f45cc775b615c629ed3bea1763b91",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518"
      ],
      "example_id": "57268c01dd62a815002e8912",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.34742667,
      "neural_order": [
        14,
        10,
        2,
        12,
        9,
        4,
        15,
        0,
        3,
        8,
        5,
        1,
        6,
        7,
        11,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "a279890de30037ee7a7ee274e1e476379ab6178037bfdb4003a1d6d5e02e2817",
      "trial_index": 417
    },
    {
      "answer_sha256": "a7eff4779adfb274788303cb9d5da829e37351907e4efb60f7c02facae1c79b4",
      "candidate_ids": [
        "5733a1854776f41900660f0a",
        "5726ed6cf1498d1400e8f00c",
        "5705ffde52bb891400689784",
        "573020f7b2c2fd14005688f7",
        "57113c6da58dae1900cd6d32",
        "572655e5f1498d1400e8dc5e",
        "5730a951069b531400832213",
        "56de4b074396321400ee2793",
        "5706149552bb891400689880",
        "57268da7f1498d1400e8e39c",
        "572951f16aef051400154cce",
        "572755b7708984140094dc4d",
        "5727cb4b2ca10214002d9676",
        "5729f799af94a219006aa707",
        "57107c24a58dae1900cd69e8",
        "572f65e9b2c2fd14005680cb"
      ],
      "candidate_sha256": [
        "acd49776c5c867dd78a5c6b915a917f39479b502b183b3b4b0c8d4934fc1536a",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "3bafbc5d56c1857ae4a5346827458b7900e775e4ebe79997c59538a3013f8c57"
      ],
      "example_id": "57268da7f1498d1400e8e39c",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.40398723,
      "neural_order": [
        9,
        5,
        0,
        10,
        4,
        6,
        14,
        13,
        15,
        1,
        7,
        11,
        2,
        3,
        8,
        12
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "d11b41494f6b072dc818ec9bd6d3152d1631cd58f0fd69b8e460c9fa44e5bf0b",
      "trial_index": 418
    },
    {
      "answer_sha256": "ea8e5ee57c8f80ad1c83b4fa04adcd5b7b15161427874a1d1b8d42b06af4d28a",
      "candidate_ids": [
        "573027d6a23a5019007fce9d",
        "5727478cf1498d1400e8f59a",
        "570966e0200fba1400367f4f",
        "5729f06f1d04691400779673",
        "570d2f5bfed7b91900d45ccf",
        "57268f05dd62a815002e8990",
        "5729fa40af94a219006aa70f",
        "572681c1dd62a815002e8796",
        "572a0a686aef0514001551ec",
        "57287c2bff5b5019007da26c",
        "572826634b864d19001645be",
        "57280f974b864d1900164370",
        "5711648850c2381900b54ac3",
        "572683f95951b619008f7525",
        "572fbea404bcaa1900d76c5b",
        "57308cf88ab72b1400f9c576"
      ],
      "candidate_sha256": [
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "a6bf93aaa317a4cbe3dd6095edaf3c9cad24ebd2270fecf0196ce024b0880c94",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "6d9776f9c9eca85af80b267fa37562071adf7ed273c2b28bd8a6b9ac745c420e",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4"
      ],
      "example_id": "57268f05dd62a815002e8990",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        8,
        13,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.05098479,
      "neural_order": [
        5,
        13,
        7,
        15,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        14
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "efb5b6c9c1a7f611d2cccc27b65732fe17722b32c50f8d1d1365982463096c07",
      "trial_index": 419
    },
    {
      "answer_sha256": "56ce485055059813c63a4a33d4e46315cea7ba0742895140bce7820b377916eb",
      "candidate_ids": [
        "572885023acd2414000dfa83",
        "5727b0892ca10214002d93e8",
        "5727dd2e4b864d1900163eba",
        "57293c246aef051400154bb8",
        "5711648850c2381900b54ac3",
        "5725d7e438643c19005acdf9",
        "572871bc4b864d1900164a04",
        "57265a58dd62a815002e8270",
        "57288428ff5b5019007da28e",
        "5733647e4776f419006609ae",
        "5728f7774b864d190016512e",
        "57264684708984140094c123",
        "57269016708984140094ca41",
        "572ff935b2c2fd140056869b",
        "5730a951069b531400832213",
        "57287ddf3acd2414000dfa3f"
      ],
      "candidate_sha256": [
        "961f2bbff5ebb4ccdaa901097a0bffde9754f25b2405a8a8486effbefd2b3cef",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "2270569cb36db404362cc8b2e4403766e6676bbdb8406dcc985687c2d3109534",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "511ddcc5ddbeeaf06bef1b76bc323fc7607c5658b94f72c360e5919a5c54656e",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91"
      ],
      "example_id": "57269016708984140094ca41",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.48922218,
      "neural_order": [
        12,
        6,
        10,
        11,
        13,
        1,
        2,
        14,
        0,
        15,
        9,
        3,
        4,
        5,
        7,
        8
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "a85857e9b163266e5f05f70cd5aa29ef6b9c97acb3e85906d64012e8bb6e4463",
      "trial_index": 420
    },
    {
      "answer_sha256": "be115a88e981a61bbbf9ecbb1f2c714028b616f22de2922309418e6c46445bff",
      "candidate_ids": [
        "57107e6ca58dae1900cd69f2",
        "572882242ca10214002da420",
        "56e1c9bfe3433e1400423192",
        "57288428ff5b5019007da28e",
        "572691bedd62a815002e89dc",
        "57114f0050c2381900b54a65",
        "572855973acd2414000df925",
        "56de51c64396321400ee27f7",
        "572970c11d04691400779463",
        "572683075951b619008f7513",
        "57332b66d058e614000b5758",
        "570d2af0fed7b91900d45c93",
        "5728710c3acd2414000df9ef",
        "5733fb7bd058e614000b66ff",
        "5726ddf6f1498d1400e8ee04",
        "57268882f1498d1400e8e306"
      ],
      "candidate_sha256": [
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "aebcfcf5e92f90749af94c364522a8495b52fb7cb88fe5c156e674b8a823b5de",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "d0c3193100a2c9cefa5efa046ca711a3a7943ddbb731e1aa29b247fe5eb27c7c",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe"
      ],
      "example_id": "572691bedd62a815002e89dc",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        9,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.3452845,
      "neural_order": [
        4,
        9,
        2,
        15,
        3,
        7,
        1,
        0,
        14,
        6,
        5,
        8,
        10,
        11,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "66c540962d6f6be78becea7407cf12bff2860cf83bdbd889b2f4d7ccd17fef55",
      "trial_index": 421
    },
    {
      "answer_sha256": "a4676031b3575984777edccf14c1b405817e976a2200905d047009c1d34046e5",
      "candidate_ids": [
        "57308ddc396df919000961a4",
        "57296d571d04691400779413",
        "57114aceb654c5140001fc47",
        "573368044776f41900660a29",
        "57299ec43f37b3190047850d",
        "5733266d4776f41900660712",
        "57300200b2c2fd1400568729",
        "5729fe5c3f37b3190047862f",
        "57108073b654c5140001f925",
        "5725ce4d38643c19005acd4d",
        "5728510f4b864d1900164936",
        "5737a25ac3c5551400e51f51",
        "56e17a7ccd28a01900c679a1",
        "56de3ebc4396321400ee26e6",
        "5728ebcb3acd2414000e01d9",
        "572fc5a1947a6a140053cc8a"
      ],
      "candidate_sha256": [
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "29fcf6218856cfd64395c81351e873a0769b0f291577dd5933cc1ac378f40976",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "56342ec864f7c99a6400cb1b743d2de657d247bc5557e34b707c510c768abb1c",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4"
      ],
      "example_id": "5725ce4d38643c19005acd4d",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        3,
        5,
        0,
        1,
        2,
        4,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.46580674,
      "neural_order": [
        9,
        13,
        3,
        12,
        4,
        10,
        8,
        5,
        0,
        1,
        2,
        6,
        7,
        11,
        14,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "9e1c22fd71c8181949db2b9604ad20c27f3ce5a177f97b9ccbed68272d406f06",
      "trial_index": 422
    },
    {
      "answer_sha256": "3f613c55d57f95fe15534edae86f6863d67d2df186eb6721cd70a037b03cf7fa",
      "candidate_ids": [
        "5727cd0f4b864d1900163d72",
        "572fffb1b2c2fd14005686f9",
        "5725cfd0271a42140099d225",
        "57268a37f1498d1400e8e33c",
        "5706149552bb891400689880",
        "5725fabc89a1e219009ac128",
        "5725d52f89a1e219009abf78",
        "5711648850c2381900b54ac3",
        "5737a9afc3c5551400e51f61",
        "57302cd004bcaa1900d772d7",
        "5725c743ec44d21400f3d549",
        "5729727baf94a219006aa437",
        "572853e8ff5b5019007da188",
        "5705eee952bb8914006896de",
        "56e1ded7cd28a01900c67bd4",
        "5727e0474b864d1900163f08"
      ],
      "candidate_sha256": [
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "809ab002069708b05d7022073e5692a5d269e21b863617ea0d1573256c8af15f",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "8aa9f86abdd349279b06baad059345781064b54eee95261b74945baf17d0dbc9",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "d404471721922f4e521c100992e50a7d7424e6de36fb7f727122c1b4b42aaf6a"
      ],
      "example_id": "5725cfd0271a42140099d225",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        13,
        12,
        5,
        10,
        1,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.26120499,
      "neural_order": [
        2,
        13,
        6,
        12,
        14,
        4,
        15,
        0,
        1,
        3,
        5,
        7,
        8,
        9,
        10,
        11
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "522691a4c3422fd292e9e9adaff370cf02d03ebf53755dcbd0a0c9de2b169a4d",
      "trial_index": 423
    },
    {
      "answer_sha256": "e8d16bc18243647de5f34e218810f72d38dbb126569f30c9ea3436dcf446623e",
      "candidate_ids": [
        "5725d183271a42140099d23d",
        "572fb059947a6a140053cb80",
        "5730a951069b531400832213",
        "57337ddc4776f41900660bbb",
        "5727de862ca10214002d9860",
        "56de51c64396321400ee27f7",
        "572a04d51d046914007796cd",
        "571cb010dd7acb1400e4c128",
        "57287ddf3acd2414000dfa3f",
        "56e20a3ae3433e140042324a",
        "57338160d058e614000b5bf9",
        "570614ff52bb89140068988a",
        "5727dc473acd2414000dee43",
        "572a0a686aef0514001551ec",
        "5705e26d75f01819005e76d4",
        "57293e983f37b3190047818b"
      ],
      "candidate_sha256": [
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a",
        "a314db2abd80831cb1dd0b897e40dbdfe8b9581f7f7fcc78bbeadcf5387d89bb",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "215da96f975d61a3dc7af5d1e2b75129453d3b8e5a3b4af20a46de0b25ce3030",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957"
      ],
      "example_id": "5725d183271a42140099d23d",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.39383279,
      "neural_order": [
        0,
        14,
        11,
        8,
        2,
        9,
        4,
        6,
        1,
        3,
        5,
        7,
        10,
        12,
        13,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "8ac80bd565d21e54135dd12d8a956db27f08f0f215d429fb3a254bc03a5d56e1",
      "trial_index": 424
    },
    {
      "answer_sha256": "0d6cd34a972264c59ab90d1fb980546d848721c051497c5671b2b261f09ab4dc",
      "candidate_ids": [
        "571136b8a58dae1900cd6d22",
        "572fc80b04bcaa1900d76d11",
        "56e202e9e3433e1400423240",
        "5725d34aec44d21400f3d639",
        "572a05eb3f37b31900478653",
        "57264b3edd62a815002e80aa",
        "56de10b44396321400ee2593",
        "57060f3e75f01819005e7922",
        "570d50a5fed7b91900d45e7d",
        "57096b66200fba1400367fa7",
        "5733ea04d058e614000b6594",
        "572fd6aa04bcaa1900d76d9f",
        "57268220f1498d1400e8e216",
        "570d44abb3d812140066d5fd",
        "56e1c0f6cd28a01900c67b2c",
        "57264a74708984140094c18b"
      ],
      "candidate_sha256": [
        "65a81c8feae191ef7a152e1dec85f18a0afabb2e7598553f33c096459f1405d1",
        "aa435df4d2a72fd757ff2d11baa9f4e4b4206e8b9af85915efff222fa87fed44",
        "03eb6accc47f8f37613f49787d90a7cde1808cd11e3eca574b283b3e63a9aa56",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352",
        "255db98fb0b6e32abd3b8317106e8eeb1841b1f3860f985c69bb911aae85aadc",
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "7a38de1a68d52da105b6f4e1e5cb3afc3d76d75df6b2ba3d16b97a20f1574260",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91"
      ],
      "example_id": "5725d34aec44d21400f3d639",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.52496646,
      "neural_order": [
        3,
        0,
        9,
        8,
        4,
        2,
        5,
        1,
        6,
        7,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "b121fcd2df21692a99d95d2e7ecb5d0a78f8fda316a3d7ba2ab96ed0de92f00b",
      "trial_index": 425
    },
    {
      "answer_sha256": "b0a76de259a3326b25706e2d1f85296c2510a4443d301cfcc1230801a27ed876",
      "candidate_ids": [
        "57373f80c3c5551400e51e8f",
        "5725d42a89a1e219009abf58",
        "5705e33f52bb89140068964c",
        "57286bb84b864d19001649c8",
        "57115bf350c2381900b54a93",
        "5727f05b4b864d1900164068",
        "572a070c6aef0514001551c8",
        "572a0a686aef0514001551ec",
        "57300a25a23a5019007fcc97",
        "572699db5951b619008f7799",
        "5725ce4d38643c19005acd4d",
        "5726ed6cf1498d1400e8f00c",
        "571126dfa58dae1900cd6cb2",
        "5733a32bd058e614000b5f32",
        "57107a3ea58dae1900cd69de",
        "5725d52f89a1e219009abf78"
      ],
      "candidate_sha256": [
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "6fdda2b40ed2d1eb102dbaebbb876c30f89a03d2cf4dc06edd24f4c14fcae328",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299"
      ],
      "example_id": "5725d42a89a1e219009abf58",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        10,
        3,
        0,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.09364924,
      "neural_order": [
        1,
        10,
        3,
        0,
        2,
        12,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "466fc69e10a222d9f48241dbe8ab1bcad7760082cce1d57b7cc2e9971237bb28",
      "trial_index": 426
    },
    {
      "answer_sha256": "8b5b9db0c13db24256c829aa364aa90c6d2eba318b9232a4ab9313b954d3555f",
      "candidate_ids": [
        "5728759cff5b5019007da25a",
        "5733a32bd058e614000b5f32",
        "5733314e4776f4190066076a",
        "572759dbdd62a815002e9b94",
        "56de4c324396321400ee27ab",
        "5725ec8289a1e219009ac0ae",
        "57332562d058e614000b5730",
        "56de51244396321400ee27ef",
        "5730035e04bcaa1900d77001",
        "5733e771d058e614000b6545",
        "57300888b2c2fd1400568775",
        "56e1bc3ae3433e1400423104",
        "570967c4ed30961900e840ba",
        "5725d662ec44d21400f3d687",
        "572ffd9e04bcaa1900d76fc7",
        "572821274b864d1900164510"
      ],
      "candidate_sha256": [
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "3bd1082a529b9d984a09852df94240c24b0e05d5f22cb1b10e6d95d51b9e1392",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "8635441a30aa01ffd3ca07852a7f30884b92d001afc637c3cc456220e5edefd2",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62"
      ],
      "example_id": "5725d662ec44d21400f3d687",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        5,
        7,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.35729127,
      "neural_order": [
        13,
        5,
        3,
        15,
        10,
        2,
        1,
        14,
        11,
        8,
        7,
        0,
        4,
        6,
        9,
        12
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "337913a2305005b79ac0ae593831b4c30bfdb0a60bd3e30f1f4a8ffe0a0e5923",
      "trial_index": 427
    },
    {
      "answer_sha256": "38a8b06bd4732a0017b8c01606868e53e50e886c1ea6573d4f7b13bd73f980b5",
      "candidate_ids": [
        "57280e1aff5b5019007d9bec",
        "57287c2bff5b5019007da26c",
        "5729f3831d0469140077967b",
        "5725d7e438643c19005acdf9",
        "5726431aec44d21400f3dd13",
        "56e16839cd28a01900c67887",
        "572818f54b864d190016446c",
        "56de148dcffd8e1900b4b5bc",
        "57264cfa708984140094c1c3",
        "5730126ba23a5019007fcd09",
        "572684365951b619008f753f",
        "572815d9ff5b5019007d9cbc",
        "572faec7b2c2fd1400568333",
        "572fffb404bcaa1900d76fef",
        "57265e97708984140094c3c3",
        "5726516a708984140094c223"
      ],
      "candidate_sha256": [
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "536e195415da4934032881cb450b11e570caec121170eb2072ecccea49d36996",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "072cae5328a3bbbbba93f5b2e21167abdbd25abb7bbc7d61fd8f22c8a29ff41c",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db"
      ],
      "example_id": "5725d7e438643c19005acdf9",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.40656264,
      "neural_order": [
        3,
        11,
        4,
        0,
        1,
        13,
        9,
        7,
        12,
        5,
        15,
        2,
        6,
        8,
        10,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "3d4977b08cd3be824cff3519689b11a344f8f260e52164c967c41c92ef411f48",
      "trial_index": 428
    },
    {
      "answer_sha256": "ed823ec32c5d4e9ca9dd968bb0fe9366b7d904ce0cae615308ddd5b89f0e6a3a",
      "candidate_ids": [
        "5733f062d058e614000b6633",
        "57281ab63acd2414000df493",
        "573749741c4567190057445d",
        "5725daa8ec44d21400f3d6b3",
        "57337ea24776f41900660bd0",
        "57337ddc4776f41900660bbb",
        "5725db98ec44d21400f3d6c5",
        "572a0b0b6aef0514001551f6",
        "56de4b5c4396321400ee2799",
        "57264fe65951b619008f6fa1",
        "57281c594b864d19001644a2",
        "56e1fc57e3433e140042322c",
        "57293e983f37b3190047818b",
        "57268f05dd62a815002e8990",
        "572657d9dd62a815002e8230",
        "5727ea45ff5b5019007d986e"
      ],
      "candidate_sha256": [
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "d45650da8c886afa2e19ad51465d44f3cd13497458bef050944602c9e5448957",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9"
      ],
      "example_id": "5725daa8ec44d21400f3d6b3",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        6,
        8,
        0,
        1,
        2,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.32366428,
      "neural_order": [
        3,
        6,
        1,
        4,
        0,
        2,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "935790abe97451e028ea0f1400fbd626a5bb96dcf77c39f4e771217aaacd8560",
      "trial_index": 429
    },
    {
      "answer_sha256": "29889554cb005799686aa2ff82c856f843b72c321ea2c2f31477b821e0734ef5",
      "candidate_ids": [
        "57060f3e75f01819005e7922",
        "57265360dd62a815002e819a",
        "57097051ed30961900e84132",
        "57275411dd62a815002e9b30",
        "5730ac782461fd1900a9cf73",
        "572fdd03a23a5019007fca9d",
        "572fd73e947a6a140053cd32",
        "57274e975951b619008f87f9",
        "5727ff083acd2414000df1ab",
        "5725d34aec44d21400f3d639",
        "5733f062d058e614000b6633",
        "571094b7a58dae1900cd6a66",
        "5725db98ec44d21400f3d6c5",
        "57265642f1498d1400e8dc68",
        "573003dd947a6a140053cf42",
        "5729f4b41d0469140077968b"
      ],
      "candidate_sha256": [
        "5edf6aded128ebbbf800aac0b81c39c677b7387bf981608b76f5b3f572854f2e",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "4bfde5c5f53f69927f5f235510fbbe2969feac3237b78782ea69a8ebe395cbde",
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "42997822223b3f6df17a4a37bedc887ccfda8c2b66b8ab6520fa18d8b3ecc788",
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "cabcb2e17697c1ca5e1bf56512bbf3851f990033a3f5d302aa90e0f3bfed2236"
      ],
      "example_id": "5725db98ec44d21400f3d6c5",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.45177163,
      "neural_order": [
        12,
        0,
        2,
        10,
        9,
        14,
        13,
        7,
        1,
        3,
        4,
        5,
        6,
        8,
        11,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "bbde383e8c18538f375bfe21afce92854b9b47ad19e73abb2968514dfd5355f2",
      "trial_index": 430
    },
    {
      "answer_sha256": "5a79597a1e86363d5d5f9094fff5453d6e06b05c91a0a76a83fa2f895e0fdb6f",
      "candidate_ids": [
        "57264efddd62a815002e8134",
        "5733fe73d058e614000b673d",
        "5725dd7d89a1e219009abfea",
        "5726a638dd62a815002e8bf6",
        "572629c6271a42140099d6a3",
        "57284142ff5b5019007da00a",
        "5727fc37ff5b5019007d9a18",
        "5737a0acc3c5551400e51f47",
        "572f609ca23a5019007fc5af",
        "571153422419e3140095557d",
        "57274b35f1498d1400e8f5d4",
        "56e1c720e3433e140042316a",
        "570d2556fed7b91900d45c47",
        "572f76d1b2c2fd1400568153",
        "5729a03f1d04691400779593",
        "56e1d9fee3433e14004231cb"
      ],
      "candidate_sha256": [
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42"
      ],
      "example_id": "5725dd7d89a1e219009abfea",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.4926864,
      "neural_order": [
        2,
        5,
        10,
        12,
        1,
        9,
        11,
        0,
        3,
        4,
        6,
        7,
        8,
        13,
        14,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "0540f69767fdabb82e858e527ca7131714fd29f983424102515e75f4efe90525",
      "trial_index": 431
    },
    {
      "answer_sha256": "1ac611951b55071cc2130e320830cf2519791592bd5c9eac95a6b1bf2a6ccdf0",
      "candidate_ids": [
        "5725e1c4271a42140099d2d7",
        "572867212ca10214002da2f2",
        "57332562d058e614000b5730",
        "573085ea8ab72b1400f9c54c",
        "56de52614396321400ee27fb",
        "5726e313f1498d1400e8eeb2",
        "5737aafd1c456719005744fb",
        "57293d6d1d046914007791b7",
        "5733fb7bd058e614000b66ff",
        "573368e54776f41900660a53",
        "57273e50dd62a815002e9a02",
        "57115b2850c2381900b54a7f",
        "5727ea45ff5b5019007d986e",
        "5737a25ac3c5551400e51f51",
        "5726ed6cf1498d1400e8f00c",
        "5727c3b02ca10214002d95ba"
      ],
      "candidate_sha256": [
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "791c219cd9c502ccefcd69101aaa7d1840dfd05d897de2cd9161208b8499eee4",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b"
      ],
      "example_id": "5725e1c4271a42140099d2d7",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.33983092,
      "neural_order": [
        0,
        2,
        4,
        8,
        11,
        15,
        10,
        13,
        1,
        3,
        5,
        6,
        7,
        9,
        12,
        14
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "a81663193772796589d899f0e9de4893babe4cb5c135a280135056fc224de30c",
      "trial_index": 432
    },
    {
      "answer_sha256": "f4e2ebe5eb9f56a7e6549ddf31996478a99ced6ef2dc5fb66b9c6ed43fdc892a",
      "candidate_ids": [
        "572a0ecb1d04691400779717",
        "57287c2bff5b5019007da26c",
        "572f567cb2c2fd140056803f",
        "5726ba2c708984140094cf59",
        "572663a9f1498d1400e8ddf2",
        "572649d8f1498d1400e8db37",
        "5727db85ff5b5019007d96fc",
        "56de0f6a4396321400ee257f",
        "5725e45689a1e219009ac048",
        "57105da9a58dae1900cd699e",
        "5733faaf4776f4190066162f",
        "572fd1c4947a6a140053cd02",
        "572683f95951b619008f7525",
        "573027d6a23a5019007fce9d",
        "5733fb7bd058e614000b66ff",
        "57287ddf3acd2414000dfa3f"
      ],
      "candidate_sha256": [
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "75bcc54c3f28b2056c91642f34ed043daecd110b921b6d80aee1698da2f1af22",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91"
      ],
      "example_id": "5725e45689a1e219009ac048",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.30831116,
      "neural_order": [
        8,
        6,
        11,
        4,
        1,
        0,
        3,
        15,
        2,
        5,
        7,
        9,
        10,
        12,
        13,
        14
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "fc4a5fa3e27bbb0f98c28830b2fa3b53a4334b84f9a63abae055a9f1c8e11bd3",
      "trial_index": 433
    },
    {
      "answer_sha256": "8726411ad777368d0fd02d80318ce786514884ca9e33431d6e6f33e74308e2d8",
      "candidate_ids": [
        "5725e748ec44d21400f3d733",
        "57287b4a4b864d1900164a28",
        "57275411dd62a815002e9b30",
        "57264cc6dd62a815002e80e4",
        "5726509bdd62a815002e815a",
        "572f5875947a6a140053c89a",
        "56de49564396321400ee277a",
        "5726577f708984140094c301",
        "5726f7715951b619008f838d",
        "571a4b0f10f8ca1400304fd5",
        "5728f50baf94a219006a9e55",
        "57340549d058e614000b67dd",
        "571a50df4faf5e1900b8a960",
        "573098f38ab72b1400f9c5d2",
        "57281bb84b864d190016449a",
        "57298ef11d0469140077952d"
      ],
      "candidate_sha256": [
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "3468eac1513c8eee07810913259de5cde890dfb553ee40fcc2b4a14a30ccc6ca",
        "4bfde5c5f53f69927f5f235510fbbe2969feac3237b78782ea69a8ebe395cbde",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "34b3cf8b2ee4c5e03d253cbd14d037704ed4a2f0fbffba0798b1eb00604b8c7e",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68"
      ],
      "example_id": "5725e748ec44d21400f3d733",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.54371128,
      "neural_order": [
        0,
        1,
        14,
        15,
        7,
        5,
        2,
        13,
        3,
        4,
        6,
        8,
        9,
        10,
        11,
        12
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "82a55ae19f141037776857c549ccfed260c857a96d64579e83c83dc7731db8ee",
      "trial_index": 434
    },
    {
      "answer_sha256": "a3efcd2966d2d8cd1a4964082bff17457c04ac94399276c90e93c1872a955410",
      "candidate_ids": [
        "56e1ba41cd28a01900c67ae0",
        "572a0e0e1d04691400779707",
        "5730aa52069b53140083221d",
        "57302bd0b2c2fd14005689db",
        "57265d86f1498d1400e8dd50",
        "57373a9fc3c5551400e51e7b",
        "5730088e947a6a140053cfac",
        "56de51c64396321400ee27f7",
        "5725ec8289a1e219009ac0ae",
        "57332562d058e614000b5730",
        "57097051ed30961900e84132",
        "57281ab63acd2414000df493",
        "572ffa79a23a5019007fcbe9",
        "5726472bdd62a815002e8042",
        "5730040f947a6a140053cf4c",
        "57339ad74776f41900660e86"
      ],
      "candidate_sha256": [
        "a21efd80668b227ef68dd022f77a31b092d02d626d2e188520af3bf6059d57dc",
        "4b184251045ff68cd470a8330d18c3848a5524d5ac96c1ad4b4b92ae5d9d5694",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "db3be505bd193819cce8b4ae5d16873a702ab02aa2690f1f0fa04d0d3f092a5f",
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843"
      ],
      "example_id": "5725ec8289a1e219009ac0ae",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        6,
        15,
        2,
        3,
        0,
        1,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.18984227,
      "neural_order": [
        8,
        11,
        15,
        9,
        7,
        13,
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        12,
        14
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "19b793aaa2b68130a792b849af897d16a0059eebe5e3fd525c48c2926e328ed0",
      "trial_index": 435
    },
    {
      "answer_sha256": "3c1470ae2e6c94a7ef3b8ab655cc73ca723e08a68bd744075a3c49753aaaed9c",
      "candidate_ids": [
        "57309bfb8ab72b1400f9c5e6",
        "571117d4a58dae1900cd6c0a",
        "5711658e50c2381900b54ad5",
        "571a50df4faf5e1900b8a960",
        "5727213c708984140094da35",
        "572fac17947a6a140053cb54",
        "5725edfe38643c19005ace9f",
        "572821274b864d1900164510",
        "57300c67947a6a140053cff0",
        "5711163bb654c5140001fb13",
        "573362b94776f41900660974",
        "5727ee372ca10214002d99ec",
        "572648e8dd62a815002e8076",
        "5727b8df3acd2414000dea99",
        "572870b2ff5b5019007da222",
        "572881704b864d1900164a50"
      ],
      "candidate_sha256": [
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "46b22fbb7cb725f7dbd41758b31d1211080f97d10bc7534a4a7e8bf4a1bb8c62",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "286230116fa743f77871c0721b3874ae2889664d4113752586c5f53755626f21",
        "75ac617b40cb38bf492969cedae24066684167be82eea9788bf5bc732da485ac",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa"
      ],
      "example_id": "5725edfe38643c19005ace9f",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.66845787,
      "neural_order": [
        6,
        15,
        9,
        10,
        7,
        11,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        8,
        13,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "d2d9b404dca6c6c9ee914cb8110ec99138816499e35cd7a7712e5744f7d2522b",
      "trial_index": 436
    },
    {
      "answer_sha256": "8f43ecc284cd1b593d336d8fe38662da24632b005e49a384c658368b3218c47a",
      "candidate_ids": [
        "57115b8b50c2381900b54a89",
        "5729f1fcaf94a219006aa6d3",
        "5733dab4d058e614000b6409",
        "57263dcd89a1e219009ac5a2",
        "57111380a58dae1900cd6bd6",
        "5728ed94ff5b5019007da97c",
        "572faec7b2c2fd1400568333",
        "5730aa52069b53140083221d",
        "57274ca8dd62a815002e9aa4",
        "571144d1a58dae1900cd6d6e",
        "572849b4ff5b5019007da0f2",
        "572fce12a23a5019007fca11",
        "56de3d594396321400ee26ca",
        "56de3f784396321400ee26fa",
        "5725f00938643c19005aced7",
        "56e1febfe3433e1400423236"
      ],
      "candidate_sha256": [
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "99afd74c371b051e0dd2704df0676d8589e7c060ad964a7cae71b5a5bde23c1e",
        "8151eb2fd23c89744c97deee3f909d1ed56661bfdadf4e610d49ee205734fba8",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf"
      ],
      "example_id": "5725f00938643c19005aced7",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.43588645,
      "neural_order": [
        14,
        6,
        12,
        13,
        10,
        3,
        5,
        11,
        0,
        1,
        2,
        4,
        7,
        8,
        9,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "04fc7cddae091463b6b4fc319f06b1e1bc4d9a6443c31402f20b5627f12008f3",
      "trial_index": 437
    },
    {
      "answer_sha256": "781d140662235cc2e44caa82542c36e84d4575a4e094a0a2dd17c65691cab269",
      "candidate_ids": [
        "573085ea8ab72b1400f9c54c",
        "57108ee6a58dae1900cd6a1a",
        "571c8198dd7acb1400e4c0ce",
        "573408ef4776f41900661757",
        "57264c42dd62a815002e80c6",
        "57271f125951b619008f8635",
        "571093aba58dae1900cd6a5c",
        "56e1c9bfe3433e1400423192",
        "571a4d1a4faf5e1900b8a958",
        "57273a465951b619008f86ff",
        "5726887e708984140094c917",
        "572fb42904bcaa1900d76c15",
        "5729efab3f37b319004785cf",
        "572658435951b619008f7025",
        "57309bfb8ab72b1400f9c5e6",
        "5725f190ec44d21400f3d76f"
      ],
      "candidate_sha256": [
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6",
        "fbc28ab618de17aa9bd46ab3f1997022bc6a9d236b4f2f2c21adae550e178119",
        "52dde7db662fc424e04bd1ed3cd943e7d11c0868e7f691a8c6cac05cb561b057",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd"
      ],
      "example_id": "5725f190ec44d21400f3d76f",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        9,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.12901642,
      "neural_order": [
        15,
        9,
        1,
        2,
        8,
        7,
        12,
        3,
        4,
        6,
        0,
        5,
        10,
        11,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "575cf5fa3c22f769f3acf39f10f846d3d5152ac9d2228138e469a51f377e8856",
      "trial_index": 438
    },
    {
      "answer_sha256": "9481db2e2b1229d97d81c29c3d8a6fea2ba6fe42667fec13ddb6fb2647fc6bf0",
      "candidate_ids": [
        "570966e0200fba1400367f4f",
        "57265bdfdd62a815002e829e",
        "56dde0379a695914005b9636",
        "572871bd3acd2414000dfa03",
        "57282358ff5b5019007d9dde",
        "573383d0d058e614000b5c35",
        "57111380a58dae1900cd6bd6",
        "572655e5f1498d1400e8dc5e",
        "57293e221d046914007791d5",
        "5725f7cd38643c19005acf23",
        "572ff673b2c2fd1400568669",
        "5725f2c838643c19005aceed",
        "57286ab72ca10214002da31e",
        "5705e99452bb891400689688",
        "57299c2c6aef051400155020",
        "573083dc2461fd1900a9ce6d"
      ],
      "candidate_sha256": [
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "72457fa0c9150fb2a7fed021650358b9295b523f55397d61363a047683297908",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d"
      ],
      "example_id": "5725f2c838643c19005aceed",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.43748247,
      "neural_order": [
        11,
        12,
        15,
        2,
        13,
        5,
        0,
        1,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "eb0e13e491f57737980dbcd90527b69ccdecde47fd699da8b4e17d711037e82e",
      "trial_index": 439
    },
    {
      "answer_sha256": "8ba5ef4e282bf7bc5cf13f731a1b9f525bd3b1f69cbfa24c1c69c303c0ac7019",
      "candidate_ids": [
        "57373d0cc3c5551400e51e85",
        "572656e4dd62a815002e81fa",
        "571cbe35dd7acb1400e4c13c",
        "56e1dc62cd28a01900c67bcb",
        "5725f46289a1e219009ac0fa",
        "570d50a5fed7b91900d45e7d",
        "57293bc91d0469140077919b",
        "57287338ff5b5019007da232",
        "5733314e4776f4190066076a",
        "57298ef11d0469140077952d",
        "570d4c3bfed7b91900d45e31",
        "57373f80c3c5551400e51e8f",
        "572f6ec7a23a5019007fc621",
        "572a03a06aef0514001551aa",
        "5733d7cbd058e614000b63ab",
        "5705fec152bb89140068977a"
      ],
      "candidate_sha256": [
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "3c1fdff7bdb3cfd3ccc8ba02719a8a942b0f26957f044cbeb9e587d6f71ae0cd",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "c90a7044afff31821c0805fe52dad349ac2ee01551ee8e380a62d59837644b2f",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89",
        "bb7672c097755bb254dcda07123304b126ddc10e9fc8c157410b39caff2b9954",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "50310ec9fff1a082120e6ce71994825c1a3eb5590a871e3a69350c7d407b9202",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5"
      ],
      "example_id": "5725f46289a1e219009ac0fa",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.70543354,
      "neural_order": [
        4,
        2,
        6,
        13,
        7,
        0,
        14,
        10,
        15,
        9,
        11,
        1,
        3,
        5,
        8,
        12
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "80f5d8a644d3d2e00bd4beb23e5a8c5ca311447a88a1f5d60ac98ad953c24655",
      "trial_index": 440
    },
    {
      "answer_sha256": "776d41debd37e2ce6f896e5688523496ec1dca22fbf40ed569897db1b32c72e8",
      "candidate_ids": [
        "571cd3b55efbb31900334e03",
        "572940246aef051400154bec",
        "57300c67947a6a140053cff0",
        "5725f5b1271a42140099d371",
        "5706111a52bb89140068984c",
        "57284142ff5b5019007da00a",
        "5705f09e75f01819005e77a4",
        "57299326af94a219006aa515",
        "57339555d058e614000b5df3",
        "572691bedd62a815002e89dc",
        "57268220f1498d1400e8e216",
        "571a4d1a4faf5e1900b8a958",
        "5729fa40af94a219006aa70f",
        "5729506d6aef051400154cac",
        "572fadcbb2c2fd1400568329",
        "57115dbe2419e314009555a5"
      ],
      "candidate_sha256": [
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "f54da4edb14741599b317e246ffe56becbfd450806ace7773701841b9f5b06c4",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806"
      ],
      "example_id": "5725f5b1271a42140099d371",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        8,
        15,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.44830808,
      "neural_order": [
        3,
        4,
        6,
        15,
        12,
        11,
        13,
        5,
        0,
        1,
        10,
        2,
        7,
        8,
        9,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "7e694156b232bda21fc6defbeef879f46ab2d0587cecf297c5d986a3bef1768d",
      "trial_index": 441
    },
    {
      "answer_sha256": "79a23c437beb2c9a60afc16110eb514b3ed208dc29d8959f566cb25d50076efd",
      "candidate_ids": [
        "5725c95f38643c19005accf3",
        "572fc8a904bcaa1900d76d1f",
        "573093598ab72b1400f9c5ae",
        "5725b7f389a1e219009abd5d",
        "56dddf4066d3e219004dad5f",
        "573003dd947a6a140053cf42",
        "572a213e6aef0514001552ee",
        "572812e74b864d19001643ce",
        "57297ed93f37b3190047845f",
        "5725f6e138643c19005acf19",
        "57114f0050c2381900b54a65",
        "57115c7450c2381900b54a9d",
        "572987e46aef051400154fa2",
        "5728151b4b864d1900164428",
        "5726545f708984140094c2a5",
        "57105da9a58dae1900cd699e"
      ],
      "candidate_sha256": [
        "32aa68d3a51961f2fc85d1117b79448f87b2591002c8300196d5ce966afb629c",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "ccfb450b288c15536e52e7e1c54aa458c0e2c2936babc78f6cab1b418fa9e1db",
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "9294fafaa3256dfeae51d7bef0a6f25791c067fb43c9150099381f45dc7e8f78",
        "ffed69e1f4655d6e30b95cc1e27d9a549c4abc117e34ed3b1d8029ffec8f87b6",
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c"
      ],
      "example_id": "5725f6e138643c19005acf19",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.53879756,
      "neural_order": [
        9,
        11,
        5,
        15,
        0,
        10,
        14,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        12,
        13
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "13a5c82a18f1c9a0ba18b6cda15866b6a4fb8e1f0bcd8e41a235eb07b6f44a9a",
      "trial_index": 442
    },
    {
      "answer_sha256": "6472e697561128bed8bdd74878adc3055d86839065732fdab7a79191e4f3d728",
      "candidate_ids": [
        "5727d9c43acd2414000dee17",
        "5725f7cd38643c19005acf23",
        "572754fff1498d1400e8f660",
        "56e1ee4de3433e1400423210",
        "56de3e414396321400ee26d8",
        "573062662461fd1900a9cdf7",
        "5729f799af94a219006aa707",
        "57263fd138643c19005ad37b",
        "57286c8cff5b5019007da218",
        "572970c11d04691400779463",
        "5725edfe38643c19005ace9f",
        "57293f353f37b3190047819b",
        "57299d1c1d04691400779581",
        "5730aef38ab72b1400f9c682",
        "571c9348dd7acb1400e4c114",
        "5705e33f52bb89140068964c"
      ],
      "candidate_sha256": [
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "d23507599fde3bea64581b405ee4729f02882f17e7602e4f065a2b2a98228ea6",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "03ea982f284d948986e37c976fdae843ae60b16e9bc2b2f30c5d51a7c1a4117f",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "dea705fdf75f529664435816398522b553e31b0965a1287f3375c31a630e304e",
        "ee6a40759a8e9342f506db55cd0c40e075a1ce7fd0ccc2da16d91a2be11ea856",
        "bedd9f5d0a5db4c8fc9999b0ed320ab5e6d56704b9e02e5331f2c1973ba98941",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76"
      ],
      "example_id": "5725f7cd38643c19005acf23",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        10,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.2753426,
      "neural_order": [
        1,
        10,
        15,
        8,
        7,
        3,
        11,
        9,
        6,
        2,
        4,
        0,
        5,
        12,
        13,
        14
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "6e71c378717507fa410e7ba4305f1f112954746d76ef631804c5b27944ebb13b",
      "trial_index": 443
    },
    {
      "answer_sha256": "3e79eb5a8b69ec7222cb685c43dda56582768bca2f276615801ade9358fce099",
      "candidate_ids": [
        "57286f373acd2414000df9db",
        "572f567cb2c2fd140056803f",
        "5734025d4776f419006616c3",
        "572940973f37b319004781a5",
        "571c7d55dd7acb1400e4c0c4",
        "573403394776f419006616dd",
        "57271f125951b619008f8635",
        "571cd3b55efbb31900334e03",
        "5727cff1ff5b5019007d95c4",
        "5728facd4b864d1900165170",
        "57265ceddd62a815002e82b8",
        "57268066708984140094c821",
        "57285841ff5b5019007da19b",
        "57273cca708984140094db33",
        "5711163bb654c5140001fb13",
        "5725f8f5ec44d21400f3d7b1"
      ],
      "candidate_sha256": [
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "75bcc54c3f28b2056c91642f34ed043daecd110b921b6d80aee1698da2f1af22",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "db91f2599d75c551b0b759a111af3526cadd1aaf2de507a25b546a5c61b6d412",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "ee860af1558390e8f6b7b1a0d64e334296f4af9fd2716e0a65c5eb5c88fa4d3e",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "8936635bf73401c433870cb063f6320a4ef8777db9a366c03e4dbb42de622dd0",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d"
      ],
      "example_id": "5725f8f5ec44d21400f3d7b1",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        5,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.51364508,
      "neural_order": [
        15,
        10,
        8,
        3,
        4,
        14,
        13,
        12,
        1,
        11,
        0,
        2,
        5,
        6,
        7,
        9
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "94af674b9b0d72853c7ddd720c9a7d97cc2e5e13f450f4d8fa7ce1a48565951a",
      "trial_index": 444
    },
    {
      "answer_sha256": "0a266f7591bed2841c9c2bac8d8aeacf9ef75cf17268e252c7790f7e3e83d596",
      "candidate_ids": [
        "572881022ca10214002da416",
        "573382d24776f41900660c37",
        "572fcd86947a6a140053ccda",
        "57297781af94a219006aa4a3",
        "57114e8d50c2381900b54a5b",
        "57332b66d058e614000b5758",
        "571a53d410f8ca1400304fe5",
        "57299326af94a219006aa515",
        "57340a094776f4190066177d",
        "5730b541396df919000962c2",
        "5737898f1c45671900574495",
        "571cc3dedd7acb1400e4c146",
        "573092088ab72b1400f9c594",
        "5727ff083acd2414000df1ab",
        "5725fabc89a1e219009ac128",
        "572753335951b619008f8853"
      ],
      "candidate_sha256": [
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "1bb8a054229137a9005ea877e0aa27b556891e3521788927c58348127564b154",
        "f54da4edb14741599b317e246ffe56becbfd450806ace7773701841b9f5b06c4",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b",
        "064e6e2465bbb0aee7fd1b10f9d0d93bd19e9d5c2dc16a730053cdc4940edc83",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3"
      ],
      "example_id": "5725fabc89a1e219009ac128",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        13,
        8,
        1,
        12,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.37583605,
      "neural_order": [
        14,
        1,
        8,
        4,
        13,
        7,
        5,
        2,
        0,
        3,
        6,
        9,
        10,
        11,
        12,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "f26e037c5033b0f8cbe615273fa89c5acd427b7a173da6e67f24495e8b43a40a",
      "trial_index": 445
    },
    {
      "answer_sha256": "d2fc9b2aff5121ead87cec08a9f51927ab8925f8bceb8ecef18006b612f1a500",
      "candidate_ids": [
        "5711651050c2381900b54acb",
        "572a0f073f37b3190047867f",
        "572fc5a1947a6a140053cc8a",
        "5733ea04d058e614000b6594",
        "570d29c2b3d812140066d4bd",
        "5726a299dd62a815002e8b9e",
        "5725f7cd38643c19005acf23",
        "571c9348dd7acb1400e4c114",
        "570d2556fed7b91900d45c47",
        "5725fb8138643c19005acf3f",
        "572f7588947a6a140053c984",
        "572985011d04691400779501",
        "5711623e50c2381900b54ab9",
        "572fc41f947a6a140053cc80",
        "572827fc3acd2414000df5bb",
        "57264cfa708984140094c1c3"
      ],
      "candidate_sha256": [
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "255db98fb0b6e32abd3b8317106e8eeb1841b1f3860f985c69bb911aae85aadc",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "d13f4c8801b254424ebba66bd53992eaa1d07ff15b51cbc45a38d3388ca7a095",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "61c9f55d3cf908f983f76f4c2cd78ee34354857005ed7934adfc6655f6c44b2a",
        "ce701ae7bea9bf9e47c5972ca7624bc90227bd2c59f41f67a73bf89c12c44680",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad"
      ],
      "example_id": "5725fb8138643c19005acf3f",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.24273928,
      "neural_order": [
        9,
        6,
        14,
        10,
        4,
        0,
        8,
        11,
        12,
        2,
        1,
        3,
        5,
        7,
        13,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "cd91b997d252f357e67f2e01078b3d49dabf19550a1648760494de4154d60e54",
      "trial_index": 446
    },
    {
      "answer_sha256": "1ce9746d6f3cc7cf336efd296dccf8882ad7ef2c94cdce3e8bac031d3912598b",
      "candidate_ids": [
        "5727d0f73acd2414000ded13",
        "56de3f784396321400ee26fa",
        "572a03a06aef0514001551aa",
        "5728fd206aef05140015494c",
        "5710f4b8b654c5140001fa45",
        "5726c19add62a815002e8f8a",
        "5725fcbe271a42140099d3ad",
        "57269bb8708984140094cb95",
        "572861cc4b864d190016495e",
        "57115b2850c2381900b54a7f",
        "5730a0778ab72b1400f9c60a",
        "5710ed7bb654c5140001fa2b",
        "570d2d68fed7b91900d45cbb",
        "571c8eb9dd7acb1400e4c0f6",
        "57300a9a04bcaa1900d77063",
        "5725edfe38643c19005ace9f"
      ],
      "candidate_sha256": [
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "2ed7b7eae99e7056f5662b63272052233c62837610dd0ec46f3f2b941d8b9112",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "3f83987ebc0c4b6d662d8cc6134fe19bdcd18d63555f50b4c6841486edd3606d",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "99ba13b83b00cb792f32172352e751adb373fb038760c686b2cbb9543de8c934",
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275"
      ],
      "example_id": "5725fcbe271a42140099d3ad",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.18001693,
      "neural_order": [
        6,
        15,
        14,
        8,
        12,
        3,
        1,
        2,
        0,
        4,
        5,
        7,
        9,
        10,
        11,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "1c3b355c9523225730a68452b094fb2c31037aa9f535f0dabf71ea39ff9bf039",
      "trial_index": 447
    },
    {
      "answer_sha256": "a4676031b3575984777edccf14c1b405817e976a2200905d047009c1d34046e5",
      "candidate_ids": [
        "5730a40f396df91900096234",
        "5729fd56af94a219006aa72f",
        "56de4c324396321400ee27ab",
        "571097baa58dae1900cd6a98",
        "570d2c20fed7b91900d45ca7",
        "56dde0ba66d3e219004dad75",
        "5730005db2c2fd1400568703",
        "5725fe63ec44d21400f3d7dd",
        "57302efe04bcaa1900d772f5",
        "571cd11add7acb1400e4c16e",
        "571155ae2419e31400955591",
        "57261f9f271a42140099d4a9",
        "57264b1ddd62a815002e80a0",
        "5733e8ae4776f419006614a6",
        "5733f410d058e614000b6663",
        "57287c2bff5b5019007da26c"
      ],
      "candidate_sha256": [
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "7816ed95a3a075f7465b32f2629c5ffe989081c15b7e3e5d43c8253591c70cfd",
        "dbbb33306f8f352995a1d3fd2732b580913659def1453f2b1f9ebc7902eda387",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "7e6cd28894d7c2d4bce2f35293457cb14f0582ffa98614ffd1b3aa751d7e5a6b",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749"
      ],
      "example_id": "5725fe63ec44d21400f3d7dd",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.43506126,
      "neural_order": [
        7,
        11,
        3,
        12,
        5,
        4,
        13,
        8,
        0,
        1,
        2,
        6,
        9,
        10,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "e4a988293cbf371cf2eb352b50301f9a46ab935c1179c100cfe5a62e6649a14b",
      "trial_index": 448
    },
    {
      "answer_sha256": "be69ea1a05c0902618f6c6ee9d42d2b0198bec6d7dd9acd2db4efea7bede6069",
      "candidate_ids": [
        "57340b1bd058e614000b6869",
        "572759665951b619008f8883",
        "5728fc2eaf94a219006a9ec7",
        "5711488ab654c5140001fc3d",
        "57265bdfdd62a815002e829e",
        "571a50df4faf5e1900b8a960",
        "5705eb3375f01819005e7764",
        "573004bf947a6a140053cf56",
        "57264865dd62a815002e8062",
        "5726ba2c708984140094cf59",
        "5726f2375951b619008f830f",
        "57097d63ed30961900e841fc",
        "57284b904b864d19001648e2",
        "57096505ed30961900e84082",
        "5725ff8238643c19005acf49",
        "57111713a58dae1900cd6c00"
      ],
      "candidate_sha256": [
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "2892ce1ee90d7e758e8bd4b12fd7bebc4276caad2f0d0af29d0444e47b6a4a05",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3"
      ],
      "example_id": "5725ff8238643c19005acf49",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.52674283,
      "neural_order": [
        14,
        13,
        11,
        6,
        4,
        3,
        10,
        9,
        0,
        1,
        2,
        5,
        7,
        8,
        12,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "b0af8dbeb41f46212f5f07eda2aea382246d385db287d449af7ac42cdcecb745",
      "trial_index": 449
    },
    {
      "answer_sha256": "b88a9bf5b76e974941c8caf7c4491f12a4127a0c1177292344cc6498a0ee1722",
      "candidate_ids": [
        "5727d4922ca10214002d977c",
        "5705e63175f01819005e7720",
        "57113f83b654c5140001fc29",
        "570d391fb3d812140066d575",
        "572810ec2ca10214002d9d06",
        "5705ec1675f01819005e776e",
        "5725daa8ec44d21400f3d6b3",
        "5725b76389a1e219009abd4a",
        "57265a58dd62a815002e8270",
        "5729f3831d0469140077967b",
        "572ffc99947a6a140053cef6",
        "572ffd75b2c2fd14005686e5",
        "572fad30a23a5019007fc86d",
        "57335fcad058e614000b5970",
        "5725d34089a1e219009abf50",
        "5727d0f73acd2414000ded13"
      ],
      "candidate_sha256": [
        "b6f799a7fbe964e85d020346753d8182351cdec9710d1c891b7da4642c30fdc0",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "a6b7ec7938b966c86b4cc72fa2059dcdbd60bc13208210cff0105baccb082b8b",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "1251911230ef039a4b2c5626af38721c8740071e0d9fdaabbc60165f4b237585",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "9a0ea788d4b66d4412369821a33e96287b8b91a59dbbe258cb494dfacfb20ad4",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "781a0469e96a630984364360256ef446a9a4f8ad14eb5552f82e6f2b813a457a"
      ],
      "example_id": "5725d34089a1e219009abf50",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.1492662,
      "neural_order": [
        14,
        12,
        15,
        10,
        2,
        13,
        11,
        6,
        5,
        7,
        0,
        1,
        3,
        4,
        8,
        9
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "2749f596f06e2b1bd02fc0f8347838724f1f2491a252af7e36c88f715a9feaa8",
      "trial_index": 450
    },
    {
      "answer_sha256": "402c5f1fcdbb7062d1e98815f76090766c3a8b8195a753c97f4fb2a03bce04b0",
      "candidate_ids": [
        "572867543acd2414000df9a1",
        "57302a3aa23a5019007fcecf",
        "5727fc37ff5b5019007d9a18",
        "57296d571d04691400779413",
        "572a135daf94a219006aa79f",
        "5737a84dc3c5551400e51f59",
        "572973f76aef051400154f0b",
        "57274f49f1498d1400e8f61e",
        "571142b3a58dae1900cd6d5a",
        "57096c95200fba1400367fbb",
        "5728501aff5b5019007da166",
        "56de52614396321400ee27fb",
        "572ffd75b2c2fd14005686e5",
        "5725d52f89a1e219009abf78",
        "57106644b654c5140001f8e5",
        "571114cfb654c5140001fb09"
      ],
      "candidate_sha256": [
        "b11a179d53281b2c32154de7d72bc5f8e3c0c35320a1c9d0f9b20fb0e505678d",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "5bae1cf69d61dbf0b59d2020a2b8aaf2c59dfea5d8efc8e4b560e663cbaefcbd",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "308326ce8e8a00212b35957ebc9a655b0ea0f166d9dfe56b9c3159dfb8d95066",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "291631d73d5dcdb57b0941017d7b1ab7a9501e869a33f4c995211b70c5487c88"
      ],
      "example_id": "5725d52f89a1e219009abf78",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.45837672,
      "neural_order": [
        13,
        1,
        9,
        7,
        2,
        12,
        3,
        4,
        5,
        10,
        0,
        6,
        8,
        11,
        14,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "ce6c87c89b6dd703adabe34c9c6f296e5ee7ed671888c93a985bbd568d32a20c",
      "trial_index": 451
    },
    {
      "answer_sha256": "816c400c94034a87124d2111fca2db8712922c1286a3d7d3c9d726f8c8a2c940",
      "candidate_ids": [
        "5726219489a1e219009ac2d0",
        "57274b35f1498d1400e8f5d4",
        "572755b7708984140094dc4d",
        "57294209af94a219006aa201",
        "5733faaf4776f4190066162f",
        "573362b94776f41900660974",
        "57299a6f6aef051400155016",
        "572900f73f37b31900477f69",
        "572681c1dd62a815002e8796",
        "57294e6b1d04691400779275",
        "572856beff5b5019007da190",
        "572a09abaf94a219006aa75b",
        "571a50df4faf5e1900b8a960",
        "572ff626947a6a140053ce8e",
        "5726542ff1498d1400e8dc28",
        "5733da01d058e614000b63f9"
      ],
      "candidate_sha256": [
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "e2cd131fecabce3cefdc2927bfc27c97f16241235e3b471062914b36c5041531",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "c426efe1359054866c8d322fd02f908d61dd09289424401de4416d000f338ca6",
        "4e041656788e4036188e81c0a35175217b17488e8ea0585f48b4719b1fac1708",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "003096af8bd0b8d02214753872917ecb683e3c67185edc99abc12163b9741faf"
      ],
      "example_id": "5726219489a1e219009ac2d0",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        10,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.32344886,
      "neural_order": [
        0,
        10,
        1,
        8,
        11,
        12,
        3,
        7,
        2,
        4,
        5,
        6,
        9,
        13,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "7d4cd094a83c800ec079ffa0e968245ac95d83dc8e35c6957f0b6cfacb36ace2",
      "trial_index": 452
    },
    {
      "answer_sha256": "5a1db5bef78848eeaba8ff00693e4b27c377f9f88fb0278c77244dcd985ff713",
      "candidate_ids": [
        "5725c69738643c19005accb9",
        "5730126ba23a5019007fcd09",
        "5726219489a1e219009ac2d0",
        "572ffb02b2c2fd14005686b7",
        "5733d3cb4776f419006612ea",
        "571cc8815efbb31900334dee",
        "5733266d4776f41900660712",
        "5729727baf94a219006aa437",
        "5726249538643c19005ad07f",
        "57338007d058e614000b5bda",
        "5733a32bd058e614000b5f32",
        "57286010ff5b5019007da1ca",
        "57281d494b864d19001644be",
        "57267947f1498d1400e8e0ec",
        "5728e8212ca10214002daa6c",
        "57107a3ea58dae1900cd69de"
      ],
      "candidate_sha256": [
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "c3e063b58dd288e80a70b1270dda3415db07151f95bf15c5acc7536204913546",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "a4008fb587348767c254824b7e5bb46aaec5eaa762404c37c3e9e2f52fc93eac",
        "f590868c5774f9fe9ea786373f0abcc9129efd13a38596e59b6a6f6da7c59805",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "2fd2db7f9ebb0764f0702a549c4f477adbd55ed996cfd70886abb1c7a8ed9560",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb"
      ],
      "example_id": "5726249538643c19005ad07f",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.14690192,
      "neural_order": [
        8,
        11,
        1,
        12,
        2,
        7,
        15,
        13,
        10,
        14,
        0,
        3,
        4,
        5,
        6,
        9
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "f365f46192998843e46a591f041d01e0f0630c941d85bd7e99a57f4b9840d171",
      "trial_index": 453
    },
    {
      "answer_sha256": "f13461419f44673141129bd638cb7c1bee577bed7ad559ca12e60c140cf0a1d9",
      "candidate_ids": [
        "5729517d6aef051400154cc8",
        "5726a00cf1498d1400e8e550",
        "5705fec152bb89140068977a",
        "5726722bdd62a815002e8528",
        "572883153acd2414000dfa6f",
        "5727c94bff5b5019007d954a",
        "56de3ebc4396321400ee26e6",
        "572a0bf96aef051400155204",
        "5725f190ec44d21400f3d76f",
        "572629c6271a42140099d6a3",
        "5728855d3acd2414000dfa8d",
        "5727c55bff5b5019007d94ce",
        "5727c3b02ca10214002d95ba",
        "572998673f37b319004784d5",
        "57265c10f1498d1400e8dd36",
        "57097d63ed30961900e841fc"
      ],
      "candidate_sha256": [
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "1eab59159d4e8655de645254d1011cd4393701716f63f5e98c49c4c34ab70dd5",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "26b6d4ddb4d6e011bd478efd35ae856f92c6e8ada70ccb719e77366c934a5062",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b",
        "56c5a58a20c281345ad1b156034653805b4b37502a76613ea7935761d3263ade",
        "07329d2931c0ce6c3dad2ef45fb8898b17becf72c1aea534e734c98af62ad377",
        "b2f898736ced32b61db0cde936335f49239327adee65e1a41c575fc7d1f644c5"
      ],
      "example_id": "572629c6271a42140099d6a3",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.37583985,
      "neural_order": [
        9,
        8,
        11,
        4,
        12,
        0,
        1,
        14,
        2,
        3,
        5,
        6,
        7,
        10,
        13,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "cd4f8bf29245c7aca87b2bac379194500ddda363a997c36e6106b89a0f6b9d1b",
      "trial_index": 454
    },
    {
      "answer_sha256": "18e7e087e672d1ee223b8af7fc64cf840f4b624546cf531e5cc568dbc5598ee3",
      "candidate_ids": [
        "571097baa58dae1900cd6a98",
        "57112686b654c5140001fbd3",
        "5730069004bcaa1900d7702d",
        "572905ce1d04691400778f83",
        "5726e5c95951b619008f81cb",
        "571cc3dedd7acb1400e4c146",
        "5727cd7dff5b5019007d958e",
        "5733e5a14776f4190066145b",
        "570d2d68fed7b91900d45cbb",
        "573081c2069b531400832133",
        "5727f16c3acd2414000df05b",
        "57339dd94776f41900660ecc",
        "572fcd86947a6a140053ccda",
        "57268a37f1498d1400e8e33c",
        "5726385e271a42140099d797",
        "5729506d6aef051400154cac"
      ],
      "candidate_sha256": [
        "0a118a818a197b12016ca27d276ed3e7b816a1cf21ec5c34ff2fa99974a9c9f2",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "01704857530cee2a6f99c9214d415216f7d781377d1e50bbc701534fcb152564",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "a8497768ed7455269c8c0e43100892fa8dc1ffff0a532613071c4cb9e3f05a18",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5"
      ],
      "example_id": "5726385e271a42140099d797",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.64791453,
      "neural_order": [
        14,
        5,
        1,
        2,
        4,
        13,
        12,
        6,
        9,
        3,
        10,
        0,
        7,
        8,
        11,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "29ac0b6d2acef938d6408a1df4c62afe84619e3db962cacc67d23de993846cb2",
      "trial_index": 455
    },
    {
      "answer_sha256": "9faaee073d42f003e26ce6fe69c141c16a1340b706dd638402a58903d56e2325",
      "candidate_ids": [
        "57060df252bb891400689820",
        "5728759cff5b5019007da25a",
        "5725d7e438643c19005acdf9",
        "5706111a52bb89140068984c",
        "56de1645cffd8e1900b4b5d0",
        "5726398589a1e219009ac588",
        "5725edfe38643c19005ace9f",
        "5730a40f396df91900096234",
        "572882242ca10214002da420",
        "57107c24a58dae1900cd69e8",
        "5729a3716aef05140015506a",
        "5725ce4d38643c19005acd4d",
        "57265200708984140094c237",
        "572a03086aef0514001551a2",
        "5725be0f271a42140099d117",
        "57274ca8dd62a815002e9aa4"
      ],
      "candidate_sha256": [
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "273477661eba1a30cf5348558342b1b77bb236085aa369e14e196d446acf8186",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "211cfda70063ab1e0e07d5d81f97e9429340864c0156851deda9af6d7cea2759"
      ],
      "example_id": "5726398589a1e219009ac588",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.36657356,
      "neural_order": [
        5,
        3,
        6,
        7,
        1,
        4,
        10,
        0,
        2,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "f7262b697ce8b164fc442ab6bc6511ba882d6e96ae18a5b30ed889dbcf1af597",
      "trial_index": 456
    },
    {
      "answer_sha256": "8fea257dc851f52dda8194d29df43a4fe1b43183bc6635468853aeb97ab58c9c",
      "candidate_ids": [
        "572686fc708984140094c8e5",
        "5727502f708984140094dc07",
        "56e1bc3ae3433e1400423104",
        "570d4a6bfed7b91900d45e13",
        "571ccc00dd7acb1400e4c15a",
        "573380e0d058e614000b5be9",
        "572826634b864d19001645be",
        "57273cca708984140094db33",
        "57377862c3c5551400e51ef3",
        "572940246aef051400154bec",
        "5726545f708984140094c2a5",
        "57265360dd62a815002e819a",
        "572a03086aef0514001551a2",
        "5711651050c2381900b54acb",
        "57263b1638643c19005ad333",
        "57265f605951b619008f70db"
      ],
      "candidate_sha256": [
        "ee37f6d5e8ae5ffa71d04ba73f6605a502e38e1deddf19c27c6cdcd7e5b27db0",
        "e32237eb13f46727755bd53347a189c0620b3ffff778cecbcd627a94fbebc022",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "c8a147d9ec8838084460fc3f8acab7d1b838aedbe76e6c6f66908200bfc0bf6e",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "6e526c735afbeadcd44a1d8d841a40b60992ac9325959c55154f81bb07a80b07",
        "8d580236fd76ded81ee333dc09963839e4d30f9d6bcacfa3220cdacf79b49a24",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31"
      ],
      "example_id": "57263b1638643c19005ad333",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.48181745,
      "neural_order": [
        14,
        13,
        12,
        0,
        11,
        3,
        9,
        5,
        15,
        7,
        10,
        1,
        2,
        4,
        6,
        8
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "82722847a981fd6aa5c2d9952bf21f498be8cf7441ca0cf6f4faae1f5964003c",
      "trial_index": 457
    },
    {
      "answer_sha256": "fd21cc3cb5062bc4ac714c489e1ca0e37a577c19ba23b0d00e9767f598d37636",
      "candidate_ids": [
        "57263c78ec44d21400f3dc7b",
        "5730126ba23a5019007fcd09",
        "572869b84b864d19001649ae",
        "56de16ca4396321400ee25c5",
        "572744aff1498d1400e8f588",
        "57266c015951b619008f7237",
        "572f609ca23a5019007fc5af",
        "57283c464b864d19001647c8",
        "572872dd2ca10214002da37e",
        "57265f605951b619008f70db",
        "572755b7708984140094dc4d",
        "57297a276aef051400154f88",
        "5737898f1c45671900574495",
        "571156152419e3140095559b",
        "5711648850c2381900b54ac3",
        "570d2f5bfed7b91900d45ccf"
      ],
      "candidate_sha256": [
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "8a85cd818746eda4a579cf0b01dadabe239a7a0f3424abef5e5a29b224d72859",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "384864e4e0fdf5d9878d660ef1fcbdd574fe8eb85ca0ccefb6977bc75502a0ea",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b"
      ],
      "example_id": "57263c78ec44d21400f3dc7b",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.33301211,
      "neural_order": [
        0,
        1,
        12,
        4,
        10,
        11,
        15,
        5,
        8,
        2,
        3,
        6,
        7,
        9,
        13,
        14
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "dbf578838a146b5ae4e6d4c8535ed6fc6e3277950732258d081805355d88561d",
      "trial_index": 458
    },
    {
      "answer_sha256": "acc8425c2bbf90886105695caf3e010cdd59e712a8946ea299fdce0a6e429ad1",
      "candidate_ids": [
        "573011de04bcaa1900d770f9",
        "57265ceddd62a815002e82b8",
        "5705eb3375f01819005e7764",
        "56e1c7e2cd28a01900c67b74",
        "573020f7b2c2fd14005688f7",
        "56de4bb84396321400ee27a2",
        "57263cfcec44d21400f3dc8d",
        "572744aff1498d1400e8f588",
        "57302700a23a5019007fce89",
        "56e1a0dccd28a01900c67a2e",
        "571c9348dd7acb1400e4c114",
        "57274f49f1498d1400e8f61e",
        "5726a46cdd62a815002e8bd0",
        "571a4b0f10f8ca1400304fd5",
        "5733cf61d058e614000b62e9",
        "5737aafd1c456719005744fb"
      ],
      "candidate_sha256": [
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "0b9e372a1ede4fad3e2e3e533dabfda50ac0e4edb94f6e58693ab349419aab00",
        "30b7a829adbfdb906de66d9485f19fc60e679fb1f8bd9c9f1d2ce62b19587dd1",
        "3e323e5c6d09bfcbdf60c1f613c0c9360d6a95200b947d3698c789308e00d3d6",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "62130be8ca9be96234ce2d1001ed5f92b28c71efa655f09e83432829cf934fd9",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9"
      ],
      "example_id": "57263cfcec44d21400f3dc8d",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.5581415,
      "neural_order": [
        6,
        0,
        10,
        12,
        14,
        15,
        1,
        8,
        2,
        3,
        4,
        5,
        7,
        9,
        11,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "98e7e3707becc0e05acce5f69d8e107060bcd96accc6bce64a59f676ff752968",
      "trial_index": 459
    },
    {
      "answer_sha256": "b3483d97b9e9b9f60d5692f52a8027a22f393b4093348fbba5040f0b7ba15b80",
      "candidate_ids": [
        "572663a9f1498d1400e8ddf2",
        "57299326af94a219006aa515",
        "571cca4add7acb1400e4c150",
        "57293d6d1d046914007791b7",
        "56de1645cffd8e1900b4b5d0",
        "57273f27dd62a815002e9a0a",
        "57297547af94a219006aa45c",
        "572f55e8a23a5019007fc56b",
        "57274a8ff1498d1400e8f5bc",
        "5727ef664b864d1900164060",
        "5729f8516aef05140015516c",
        "5737a9afc3c5551400e51f61",
        "57297ed93f37b3190047845f",
        "5726e3c4dd62a815002e9404",
        "57263dcd89a1e219009ac5a2",
        "5710ed7bb654c5140001fa2b"
      ],
      "candidate_sha256": [
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "f54da4edb14741599b317e246ffe56becbfd450806ace7773701841b9f5b06c4",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "e5d79b0b05eee021a524a6dd62e33af2c2360f9063ec0f600a1cd0995c8e90d5",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "1d470bf791028e4f456c100c44c5b3410fac821035b36b868d821eb23e1f2a74",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0"
      ],
      "example_id": "57263dcd89a1e219009ac5a2",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        7,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.38491508,
      "neural_order": [
        14,
        7,
        15,
        8,
        1,
        6,
        13,
        5,
        9,
        3,
        0,
        2,
        4,
        10,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "64237cb833eddb91a18eb2b7f7bc8996b860fa2de9a13768634ebb11fbd09e5f",
      "trial_index": 460
    },
    {
      "answer_sha256": "6bdab92e9eecb32b2fc97c9578426dd62b3d6078cf08e3994c82da0e93d69eca",
      "candidate_ids": [
        "5727ee372ca10214002d99ec",
        "57107d73b654c5140001f91d",
        "57111428b654c5140001faff",
        "57273a465951b619008f86ff",
        "57265526708984140094c2bd",
        "5706139252bb891400689864",
        "57114aceb654c5140001fc47",
        "5728284e3acd2414000df5cd",
        "57109275b654c5140001f99f",
        "5730126ba23a5019007fcd09",
        "5725d183271a42140099d23d",
        "5729f799af94a219006aa707",
        "571ce5055efbb31900334e2a",
        "571a52cb4faf5e1900b8a969",
        "57263ea0271a42140099d7c3",
        "5711619950c2381900b54aaf"
      ],
      "candidate_sha256": [
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "9ff1cf99ff46b4125584133f2aa228433fde124d4049eec612a520abe22e99bb",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "d58b1a4d6bf601f6d89c2d7d1e8973e368c0b2c976fc6c565cb1649d3183c7f8",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b"
      ],
      "example_id": "57263ea0271a42140099d7c3",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.65141775,
      "neural_order": [
        14,
        7,
        6,
        15,
        5,
        9,
        3,
        0,
        1,
        2,
        4,
        8,
        10,
        11,
        12,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "d043efd145c7cc2407193dd165284c62d54361bf4f0a3e153f04a9bbe1ec7036",
      "trial_index": 461
    },
    {
      "answer_sha256": "f2e3f501a48339ebd5ecf22dc4992e75d5d08be59fb1e0ae8f6a24bd11f062ff",
      "candidate_ids": [
        "570d4a6bfed7b91900d45e13",
        "572fd73e947a6a140053cd32",
        "57281bb84b864d190016449a",
        "57264586f1498d1400e8dac6",
        "572a005f1d046914007796b7",
        "573009a004bcaa1900d7704f",
        "57302faa04bcaa1900d77311",
        "57263fd138643c19005ad37b",
        "572f5703a23a5019007fc573",
        "5730b541396df919000962c2",
        "572a0bf96aef051400155204",
        "5733eb34d058e614000b65cc",
        "57288428ff5b5019007da28e",
        "573383d0d058e614000b5c35",
        "571cb010dd7acb1400e4c128",
        "5729efab3f37b319004785cf"
      ],
      "candidate_sha256": [
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "074c6c799ea81f9b852852b627e00aa8593f51a3f37f3a8ec43f0920c5759379",
        "34b3cf8b2ee4c5e03d253cbd14d037704ed4a2f0fbffba0798b1eb00604b8c7e",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "dd7857f10f7e487d744935798a225792743a3c103b1d4db8a14b1172c9682c3f",
        "50a9382232ac3ca465e69da205479dd780df82bee329a4657750cceca976b925",
        "c1a5d009b8c85c5dde0e45cc691dd8c27b600879cfd03afe12b6f61869979552",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "14e1246d6c57cac98cd47a35fe5c0801832d2bfe7bc8d39af8efc222d99387f9",
        "4442999103cde32d1a33d79202bf06335155059787f9dbf41c83f72719474b19",
        "ccddcce0a3cdaa222a653d19721b5a6a6943274bbe9d87e4ff56cc6a6a1792dd",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce"
      ],
      "example_id": "57263fd138643c19005ad37b",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.17737341,
      "neural_order": [
        7,
        11,
        5,
        2,
        13,
        9,
        12,
        0,
        1,
        3,
        4,
        6,
        8,
        10,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "5dd6bddfedd0a697ecee680cb273eff335efa6a936b5bf46108cde53411e4d16",
      "trial_index": 462
    },
    {
      "answer_sha256": "17aea53c3fa6613c576faebb0b2033ea9ef1e0702ea397f485fe8aa81eaeff5f",
      "candidate_ids": [
        "57263ea0271a42140099d7c3",
        "57096505ed30961900e84082",
        "57268da7f1498d1400e8e39c",
        "5727cff1ff5b5019007d95c4",
        "57267b3c5951b619008f7423",
        "5706094b52bb8914006897de",
        "572fc5a1947a6a140053cc8a",
        "57378c9b1c456719005744a7",
        "57332562d058e614000b5730",
        "5728eff82ca10214002daadc",
        "5710ed7bb654c5140001fa2b",
        "56de3f784396321400ee26fa",
        "570d2af0fed7b91900d45c93",
        "5726414e271a42140099d7e5",
        "5729f799af94a219006aa707",
        "56e1ded7cd28a01900c67bd4"
      ],
      "candidate_sha256": [
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "2892ce1ee90d7e758e8bd4b12fd7bebc4276caad2f0d0af29d0444e47b6a4a05",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "2f4205298894876b510bd47b3ec642ab534c15428a4b1df1010ece22c9c17979",
        "8680c2d1511073dfa6129eaab02ad788c74a59c1cb87f2123d2a4768527f9002",
        "0a529ab76e79b45b08980b9b865c76d84eaa81ef7636f3bfee749b2c574af85e",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5"
      ],
      "example_id": "5726414e271a42140099d7e5",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.50117789,
      "neural_order": [
        13,
        0,
        1,
        6,
        3,
        9,
        8,
        5,
        2,
        4,
        7,
        10,
        11,
        12,
        14,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "b3291144da51ed85652940e9dcda71b8cf8b1f50ea050f4f8b8acefc78f01cb1",
      "trial_index": 463
    },
    {
      "answer_sha256": "5e73e4bc7fced6b06f2b0b29027910c82fbebb599d3c90744a4e552334ef55f6",
      "candidate_ids": [
        "572a064a3f37b3190047865d",
        "5729a3716aef05140015506a",
        "572ffce5a23a5019007fcc15",
        "57377aac1c45671900574479",
        "5726eb4b5951b619008f826b",
        "572fbea404bcaa1900d76c5b",
        "57303048947a6a140053d254",
        "57264228ec44d21400f3dcf5",
        "5725bbec271a42140099d0d1",
        "571cd5b1dd7acb1400e4c178",
        "573368044776f41900660a29",
        "57339555d058e614000b5df3",
        "56de4c324396321400ee27ab",
        "56dde27d9a695914005b9651",
        "572fc5a1947a6a140053cc8a",
        "572a1f086aef0514001552c0"
      ],
      "candidate_sha256": [
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "13ad58ca96a59b05a605a8e8fd1cadd65b62ca392d3e04f125096c9b8294e267",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93"
      ],
      "example_id": "57264228ec44d21400f3dcf5",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.49243897,
      "neural_order": [
        7,
        1,
        6,
        2,
        12,
        9,
        11,
        4,
        13,
        0,
        3,
        5,
        8,
        10,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "9b777601f3a8d09a2ce5ce80e2189286261d978601771489fbf2856f6468f3ca",
      "trial_index": 464
    },
    {
      "answer_sha256": "6338fd0bf10ac9ed44e47efc271ad0fa352c42efa786a70c7f16d43fd398193d",
      "candidate_ids": [
        "5726a3c6f1498d1400e8e5ae",
        "571153422419e3140095557d",
        "5725f8f5ec44d21400f3d7b1",
        "571cd88ddd7acb1400e4c18d",
        "57294279af94a219006aa209",
        "572a0bebaf94a219006aa76f",
        "5726431aec44d21400f3dd13",
        "572824f13acd2414000df58f",
        "57264fe65951b619008f6fa1",
        "56e1c720e3433e140042316a",
        "5737821cc3c5551400e51f19",
        "56e1c3e1e3433e1400423148",
        "57335fcad058e614000b5970",
        "57378c9b1c456719005744a7",
        "57273e50dd62a815002e9a02",
        "570d47b8b3d812140066d62d"
      ],
      "candidate_sha256": [
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "b83bbd98d7d0ce0ede11408022de62f3ba986c786fa87f5d0b95200a67164f77",
        "7164ec3ee8b4a82bbf8610d05c97b443293538f3c698d1afbb72c29b0088afd8",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "c99e5888a8f8ccf5a0160d0c4ef2b8d0a431e4373b2f68f19ea25de8cb7913a9",
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77"
      ],
      "example_id": "5726431aec44d21400f3dd13",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.5841774,
      "neural_order": [
        6,
        14,
        8,
        4,
        9,
        0,
        10,
        1,
        12,
        11,
        13,
        2,
        3,
        5,
        7,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "f741badf54c1629c3ae7a17a489146a692777457703ea511075fc726039ce091",
      "trial_index": 465
    },
    {
      "answer_sha256": "8faab3c116f5d3b9e61b198914aa1ea303669d320947f9d4326d75c2c4f72e9a",
      "candidate_ids": [
        "57096f37200fba1400367fe5",
        "57107e6ca58dae1900cd69f2",
        "572647935951b619008f6ec9",
        "57283f014b864d19001647e8",
        "573380e0d058e614000b5be9",
        "5728151b4b864d1900164428",
        "572759665951b619008f8883",
        "57111713a58dae1900cd6c00",
        "5728ec6a4b864d19001650aa",
        "572812e74b864d19001643ce",
        "572643de5951261400b5195a",
        "57302e45947a6a140053d22e",
        "572757bef1498d1400e8f690",
        "57107a3ea58dae1900cd69de",
        "572a06866aef0514001551be",
        "56e1a564cd28a01900c67a48"
      ],
      "candidate_sha256": [
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3",
        "e83edd2f6eb1fa0981acc0119542b4a54c226b0e0249686e43082425f6d974b0",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "2d5af9a4d02fc579e6e15f098bcffbcc6459cdee51fd51b38ea0075eee7a728a",
        "891950705a403bc6367740ae6d2db4901c0b1890a28fc1726ed1f707a2909c32",
        "7f50192eb7a5686105d400314c808341ce6c56c9f1ca7c4671b3ac6ae7badacb",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4"
      ],
      "example_id": "572643de5951261400b5195a",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        12,
        1,
        7,
        13,
        2,
        0,
        3,
        4,
        5,
        6,
        8,
        9,
        11,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.55150115,
      "neural_order": [
        10,
        3,
        15,
        0,
        11,
        7,
        14,
        2,
        12,
        1,
        4,
        5,
        6,
        8,
        9,
        13
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "b099e822ea342c2028521e2d570c4b70718eb611b213881fc52adf0e1f4b01c1",
      "trial_index": 466
    },
    {
      "answer_sha256": "cbf488ca5c7d08cabd51ae0504ff9cf89919050f644d417b17a31b2dcba93881",
      "candidate_ids": [
        "57268527708984140094c8bf",
        "5727f16c3acd2414000df05b",
        "571cda1bdd7acb1400e4c196",
        "573382d24776f41900660c37",
        "5726446a89cfff1900a8404d",
        "5726e179dd62a815002e93ae",
        "572815d9ff5b5019007d9cbc",
        "5730aa52069b53140083221d",
        "572659ea5951b619008f7051",
        "573005b9947a6a140053cf6a",
        "5726a34bf1498d1400e8e59e",
        "56e1b62ecd28a01900c67aa3",
        "572fbf21a23a5019007fc937",
        "571135b8a58dae1900cd6d0e",
        "5727f6723acd2414000df0e7",
        "572a0ce11d046914007796fd"
      ],
      "candidate_sha256": [
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "a8497768ed7455269c8c0e43100892fa8dc1ffff0a532613071c4cb9e3f05a18",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "1f239cb46f0c6bc97c1b290e8ae59c90ddabab0351d296bec756f787061f0007",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "f0037223955d5106b09114e08d378b3e847a1dd15682970d2ba9c87224b56add",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "693713d8eb004f9b2925b8e7fb7701fce4b4894173dae3fe1b852c4362e287c4",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869"
      ],
      "example_id": "5726446a89cfff1900a8404d",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.5775407,
      "neural_order": [
        4,
        0,
        14,
        9,
        10,
        12,
        8,
        5,
        15,
        6,
        13,
        1,
        2,
        3,
        7,
        11
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "4ef359e5d073f7b09776e1ffdb7839fcb263849b58f089a9fc1b9fc1ff97755e",
      "trial_index": 467
    },
    {
      "answer_sha256": "c5347c1a0ac07f19121f8d24436ef4fb154f0c976d41a16d4cb8f3b26e1d7c0d",
      "candidate_ids": [
        "57300580b2c2fd140056874d",
        "5727ec062ca10214002d99b6",
        "572973f76aef051400154f0b",
        "572658435951b619008f7025",
        "5711619950c2381900b54aaf",
        "57286ab72ca10214002da31e",
        "572fcd86947a6a140053ccda",
        "57268220f1498d1400e8e216",
        "57264586f1498d1400e8dac6",
        "56de11154396321400ee25aa",
        "572744aff1498d1400e8f588",
        "5726516a708984140094c223",
        "571a4ead10f8ca1400304fdd",
        "571ce5055efbb31900334e2a",
        "57097141200fba1400367ff9",
        "5725c337271a42140099d163"
      ],
      "candidate_sha256": [
        "8e983a2167b912359aa8e1110455d27a279e9b3baaac76337616f57b19396e19",
        "df8e14256213521b4719d529b80802d5cb28c8cef90309ec70d1cee907004fc2",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "584f3e0b53ad471ed774b85bafb1bcb103872014ef751859a9d2170ef0b13f3b",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "14212c40b27a0e19bf32e180538be5414b13cedcfeecb9acc491564a62569f91",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "1eabc60bd67e282008972d9719c0467f1212ff70fc4080e4658c6586cf589027",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "ee73f593c26ad38e58d299361ba118fcf077591f71e9d4ed9515efb8b2fc75f2"
      ],
      "example_id": "57264586f1498d1400e8dac6",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.54498467,
      "neural_order": [
        8,
        14,
        4,
        5,
        12,
        11,
        0,
        13,
        2,
        6,
        9,
        7,
        1,
        3,
        10,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "32c5b97eb6e6a12ea23be030b74cd14b48f08f27b24738e4b458f828a6f03153",
      "trial_index": 468
    },
    {
      "answer_sha256": "d03b6d91ab775adf1e285eb41471df9e0b9f71c2149d4f8af09b20b43e960a26",
      "candidate_ids": [
        "5727d6154b864d1900163e34",
        "572fc659b2c2fd1400568447",
        "57300911947a6a140053cfb6",
        "5706094b52bb8914006897de",
        "571cde695efbb31900334e16",
        "571144d1a58dae1900cd6d6e",
        "57106644b654c5140001f8e5",
        "56e20a3ae3433e140042324a",
        "5726d9935951b619008f7fed",
        "571a49de4faf5e1900b8a94a",
        "5725fabc89a1e219009ac128",
        "5726462b708984140094c117",
        "572a11663f37b31900478693",
        "57060cc352bb89140068980e",
        "57286d7d4b864d19001649e0",
        "5726b718dd62a815002e8dbe"
      ],
      "candidate_sha256": [
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "85280982cc4ee3fe931d5767fcaa5f1dfbab2804f71c4ac8ac953b8ad303f0d0",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8",
        "6b5baac861c36b5fd13e7a181e848be90fe9291892c24fcaa534999aeb639149",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4"
      ],
      "example_id": "5726462b708984140094c117",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.62312348,
      "neural_order": [
        11,
        7,
        15,
        1,
        12,
        14,
        5,
        3,
        8,
        6,
        0,
        2,
        4,
        9,
        10,
        13
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "531fc45a63098fb4474ff2deee12bc53ac7bde05ce7ca11fc667532b25b35766",
      "trial_index": 469
    },
    {
      "answer_sha256": "a4c85b5b4e4a2b625bff6fd5fbcd506e9fd4157874fbcc76f654d74ba99e25b7",
      "candidate_ids": [
        "57268882f1498d1400e8e306",
        "5726472bdd62a815002e8042",
        "57302cd004bcaa1900d772d7",
        "57287c2bff5b5019007da26c",
        "573380e0d058e614000b5be9",
        "57377ec7c3c5551400e51f05",
        "570d29c2b3d812140066d4bd",
        "57266e72f1498d1400e8df8c",
        "572940973f37b319004781a5",
        "573361404776f4190066093c",
        "572867543acd2414000df9a1",
        "57060a1175f01819005e78d2",
        "57269bb8708984140094cb95",
        "572ff673b2c2fd1400568669",
        "57299c2c6aef051400155020",
        "5730a0778ab72b1400f9c60a"
      ],
      "candidate_sha256": [
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "db91f2599d75c551b0b759a111af3526cadd1aaf2de507a25b546a5c61b6d412",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "b11a179d53281b2c32154de7d72bc5f8e3c0c35320a1c9d0f9b20fb0e505678d",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e"
      ],
      "example_id": "5726472bdd62a815002e8042",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.64516916,
      "neural_order": [
        1,
        2,
        8,
        14,
        5,
        3,
        15,
        0,
        13,
        12,
        4,
        6,
        7,
        9,
        10,
        11
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "5f2a2248a57803fea0c76745fa0fbc079efc69455f26d2dbb4ee15ffe5329c3d",
      "trial_index": 470
    },
    {
      "answer_sha256": "8dbb94919f39024cfa452a0103e75e7d45ca3103d401502a3c5ce21c6514809f",
      "candidate_ids": [
        "572a213e6aef0514001552ee",
        "572663a9f1498d1400e8ddf2",
        "5725d34aec44d21400f3d639",
        "572fffb404bcaa1900d76fef",
        "57115b8b50c2381900b54a89",
        "5730a40f396df91900096234",
        "5728223cff5b5019007d9dc4",
        "5705ec1675f01819005e776e",
        "5725bad5271a42140099d0bd",
        "5730876a396df9190009617a",
        "572647e2dd62a815002e805c",
        "57296d571d04691400779413",
        "57378b141c4567190057449f",
        "570d26efb3d812140066d48f",
        "57269bb8708984140094cb95",
        "57339902d058e614000b5e70"
      ],
      "candidate_sha256": [
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "714fee269da5bf0ab19edacac6b230457f2e61909cc2f67ed6942e365a3cba1f",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "2ef6bc8b598c3ae95dea517f30b72d286a3c1ea1c56e1f73f6bd8f2ab9cb63ef",
        "c9162cb30bac727b8d2f7d94c4e21384424a026f05c8240a3a256c71ad02d820",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8"
      ],
      "example_id": "572647e2dd62a815002e805c",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.65533842,
      "neural_order": [
        10,
        0,
        12,
        2,
        11,
        13,
        1,
        15,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        14
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "c3dda330912f7b667c94c9844886a6678014c79c58a47b2ac0fb397a32e9627e",
      "trial_index": 471
    },
    {
      "answer_sha256": "437c150c579e070f2aad0afc7e665b48e08ecc2256664c5d7a329f00278ca5ec",
      "candidate_ids": [
        "5725ec8289a1e219009ac0ae",
        "57284e9fff5b5019007da150",
        "572f7b33947a6a140053c9a2",
        "56de15dbcffd8e1900b4b5c8",
        "573081c2069b531400832133",
        "56dde0ba66d3e219004dad75",
        "571112ada58dae1900cd6bcc",
        "5726e313f1498d1400e8eeb2",
        "572648d1708984140094c15d",
        "5728f50baf94a219006a9e55",
        "572a142e3f37b319004786b9",
        "56e1f10ee3433e1400423223",
        "56e200e4cd28a01900c67c14",
        "572ffce5a23a5019007fcc15",
        "57302ad804bcaa1900d772af",
        "57287b322ca10214002da3be"
      ],
      "candidate_sha256": [
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "2040b02c1c6b01d5338300b80a1c0ae71f9b266eb95d141e6f008328e7ad6251",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "f7140e68e270abe4b07098bebe1f59cce949b799b8237e1c88451a74c65cd4a0",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "fd8dc45596c41569da99ac58e0d5bcbadf3f002b35f3fd5816c3a3fa03003927"
      ],
      "example_id": "572648d1708984140094c15d",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.35413174,
      "neural_order": [
        8,
        15,
        9,
        7,
        1,
        10,
        12,
        11,
        0,
        2,
        3,
        4,
        5,
        6,
        13,
        14
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "334e300005e56e31006a542bb66a51fc475c67e9a3cd94d50e1514721792325c",
      "trial_index": 472
    },
    {
      "answer_sha256": "62c0ce18fae623dee0d21217ed43ceccb5c526523f6b4d1f6c0e7e735b40abe0",
      "candidate_ids": [
        "572657d9dd62a815002e8230",
        "57264684708984140094c123",
        "5730208fa23a5019007fcded",
        "57106185b654c5140001f8db",
        "572659535951b619008f703f",
        "5727e9523acd2414000def95",
        "5729fd111d046914007796a5",
        "572a1a5c6aef051400155284",
        "56e1f10ee3433e1400423223",
        "572855973acd2414000df925",
        "56e1cbe2cd28a01900c67bac",
        "572a09abaf94a219006aa75b",
        "57286ec63acd2414000df9d1",
        "5727db85ff5b5019007d96fc",
        "572a0b0b6aef0514001551f6",
        "5727fd123acd2414000df185"
      ],
      "candidate_sha256": [
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "3ecc89fde7f003712351e0a999835c7c7b1eb29643c0cb5ee5b0cec5de607ceb"
      ],
      "example_id": "57264684708984140094c123",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.48464922,
      "neural_order": [
        1,
        3,
        14,
        6,
        2,
        7,
        5,
        0,
        4,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "9b0cf2789b1fcdf77cb7c0dd8757c888217206f6e1ffb03facebb1a9e3c88160",
      "trial_index": 473
    },
    {
      "answer_sha256": "876c7eefe8acb114c22f21351c75f3291b93a7df2b0d92d789c2bf57a012d630",
      "candidate_ids": [
        "57264cfa708984140094c1c3",
        "570d4329fed7b91900d45dd7",
        "56e1c720e3433e140042316a",
        "5729fd111d046914007796a5",
        "572fc659b2c2fd1400568447",
        "57113639a58dae1900cd6d18",
        "5725fb8138643c19005acf3f",
        "5729f60caf94a219006aa6ef",
        "571ce6655efbb31900334e34",
        "5729f39a6aef05140015514c",
        "572fdbb004bcaa1900d76dd9",
        "5705f13d52bb8914006896f0",
        "572fd1c4947a6a140053cd02",
        "572a0a391d046914007796df",
        "572647935951b619008f6ec9",
        "57264228ec44d21400f3dcf5"
      ],
      "candidate_sha256": [
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "3c91050d11e7bf64fd4ca62f7dedea1d4bfbf339da5be4a455898c6807e8d955",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a",
        "6e5410ef5b1cf7aa294d0905e9bb3458c6bbdd873d39aefcd61606cb2be59f78",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "8a9810070ed666290169acfdbaeb51a8a73dcf7b0055d2f1812c5e06d3166b9f",
        "f85a29eb0186d32f90fce7f7df6c7ef2c96feab2c5e43d2346243e0507c5ee9c",
        "fa0feedc832b261e84790b92d403185cafe2458b7c2be43092115d6d482e83d1",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "61ba934084baf3b8a93fd6f9021a0589c6bcd1470742bda45e13232b3b0a656a",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb"
      ],
      "example_id": "572647935951b619008f6ec9",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": false,
      "neural_margin": 0.02985265,
      "neural_order": [
        3,
        14,
        9,
        0,
        13,
        5,
        8,
        7,
        12,
        2,
        10,
        1,
        4,
        6,
        11,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "d17ec3c72a2cd0cfba5020d4692db7bd3435f8650d47e9c1d97dfcb4fde85c36",
      "trial_index": 474
    },
    {
      "answer_sha256": "bb05790f40dd3ba15e22191056c48a00c8cebb4eefef9053a421bb5b48662175",
      "candidate_ids": [
        "571cc8815efbb31900334dee",
        "57332c1e4776f4190066073a",
        "571c879bdd7acb1400e4c0ec",
        "572872822ca10214002da374",
        "57296d571d04691400779413",
        "5730035e04bcaa1900d77001",
        "5733eb34d058e614000b65cc",
        "57264845f1498d1400e8db0a",
        "5710f4b8b654c5140001fa45",
        "5730126ba23a5019007fcd09",
        "5730a0778ab72b1400f9c60a",
        "57274d1cdd62a815002e9aae",
        "56de49a8cffd8e1900b4b7a7",
        "5733647e4776f419006609ae",
        "57264a8cdd62a815002e808c",
        "56e1fc57e3433e140042322c"
      ],
      "candidate_sha256": [
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "b1d48329392cfda37a8a0a338e6cac3fbf697eed8309358031f0fb2baff989f5",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "14e1246d6c57cac98cd47a35fe5c0801832d2bfe7bc8d39af8efc222d99387f9",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "2a88a6e435244f1025c628b748e860ab699fad659a4359c6fb5c85d3ee365576",
        "95379f334c7d36ba1f093516d6dd76bea434b5242cb83707f6166b127a657c0f",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace"
      ],
      "example_id": "57264845f1498d1400e8db0a",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.41428916,
      "neural_order": [
        7,
        8,
        10,
        9,
        5,
        12,
        1,
        3,
        13,
        14,
        6,
        11,
        0,
        2,
        4,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "f3e4fed8ccedc47dedfda4bcfde6a36f397255225547b943d1eae505a832f846",
      "trial_index": 475
    },
    {
      "answer_sha256": "23e099627ef38d7b7ede341837951390fad171e604cc595662a6509c946608d2",
      "candidate_ids": [
        "5733fd66d058e614000b6735",
        "572a03a06aef0514001551aa",
        "57094ca7efce8f15003a7dd6",
        "56de4b5c4396321400ee2799",
        "5705ec1675f01819005e776e",
        "5728f2e26aef051400154896",
        "572a0a686aef0514001551ec",
        "572943ab1d04691400779219",
        "57300888b2c2fd1400568775",
        "5730b1022461fd1900a9cfa3",
        "57309446396df919000961b8",
        "56e1e9dfe3433e14004231fc",
        "57115ff82419e314009555c3",
        "572fff45947a6a140053cf26",
        "572648ed5951b619008f6f03",
        "57060a1175f01819005e78d2"
      ],
      "candidate_sha256": [
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "4223eae6be53331c4f2721e1f906ea76788c87ae5c29c199441afe79332024cb",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "32756d0f25725e1a1c9c4e7203090175f0c3f7aa5a2b0f4bbb0c71e2d1d4c752",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "8b49ba4a19fcac7c6887e0bae45f362bacbf551eac347713868b8674715123dc",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897"
      ],
      "example_id": "572648ed5951b619008f6f03",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        5,
        3,
        9,
        0,
        1,
        2,
        4,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.28152138,
      "neural_order": [
        14,
        6,
        5,
        10,
        13,
        8,
        9,
        3,
        15,
        0,
        7,
        1,
        2,
        4,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "db8ead34bb8e4b9e5f70af468d00b8f50e85d7918c524837d1eac71abf96b7b0",
      "trial_index": 476
    },
    {
      "answer_sha256": "edc0b6d60383a3879b91e819a07a02269091402bdc71c66728bf8cd075f6ca7d",
      "candidate_ids": [
        "572fd6aa04bcaa1900d76d9f",
        "57264991f1498d1400e8db2e",
        "56e1c3e1e3433e1400423148",
        "5727aec03acd2414000de991",
        "5729efab3f37b319004785cf",
        "572669a9dd62a815002e8416",
        "57286ead2ca10214002da346",
        "5706094b52bb8914006897de",
        "572a03a06aef0514001551aa",
        "570d418dfed7b91900d45dbf",
        "570d4606b3d812140066d619",
        "572759dbdd62a815002e9b94",
        "5727db85ff5b5019007d96fc",
        "572a07a86aef0514001551d2",
        "57113c6da58dae1900cd6d32",
        "57107c24a58dae1900cd69e8"
      ],
      "candidate_sha256": [
        "8796b2318cf8e6050ff8ba905f88d551f8360a13cfbf8739e1234d4ed4ac2c25",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "afa714a72ebd441495497201ca8e5b34e7410ce35809dcecc075c5b3d84568f3",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "2b7326d0cd8188e7b1ea38f54e115afbbb5e41e5f6886b800cf1fd31ac49dd48",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "631ae85a58426197f43bce08d34df2914057034648942e41a9d822902b61b3d3",
        "810d6a73710dfe1aac1ce28b652be8d0a715e0146a5e5a5d3791227cf6741757",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "73afbad581995770bafda2ce0aebd8167d519734331003f9017bf02a85371773",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604"
      ],
      "example_id": "57264991f1498d1400e8db2e",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.47099024,
      "neural_order": [
        1,
        13,
        8,
        10,
        14,
        7,
        6,
        4,
        15,
        11,
        0,
        2,
        3,
        5,
        9,
        12
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "0850a9fe828d1a28eb36639438a81a2fb6cccc62380555a622c7729f18d6fcac",
      "trial_index": 477
    },
    {
      "answer_sha256": "b433f99b611060b63ed17d5bdaba59cab16b5db335ddb3933a5412618106c387",
      "candidate_ids": [
        "5725c91e38643c19005acceb",
        "5729fe5c3f37b3190047862f",
        "5710eb6fb654c5140001fa17",
        "5727e6cbff5b5019007d97ee",
        "57261f9f271a42140099d4a9",
        "572878942ca10214002da3a2",
        "57265aaf5951b619008f706b",
        "570d28bdb3d812140066d4a3",
        "57273cca708984140094db33",
        "56de0f6a4396321400ee257f",
        "57274046dd62a815002e9a12",
        "5726ba2c708984140094cf59",
        "57378e311c456719005744af",
        "57107c24a58dae1900cd69e8",
        "57264a74708984140094c18b",
        "57268f05dd62a815002e8990"
      ],
      "candidate_sha256": [
        "b67161cb70767d25408d47a9b2cd15ed23cc66a97c33ff46706b10e7a4772023",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "09dae1e2b2708f999627efe257489cfaad54e793df884b8877c2d638640ffa20",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "89494d97715f55668860a0d80728f6cfb351c1eaed8068e4319ca353f7e4a9a9",
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "0c60fccfa0a5891c548480d13173fbf66ab99f403f068010ba84b6402ed6daff",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b"
      ],
      "example_id": "57264a74708984140094c18b",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        9,
        7,
        11,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.40971693,
      "neural_order": [
        14,
        13,
        12,
        15,
        2,
        0,
        9,
        5,
        11,
        1,
        3,
        4,
        6,
        7,
        8,
        10
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "b3a97ee09b24d4ba175fe0598d97ae6024a2f8c0aa939533200d12c7ead31573",
      "trial_index": 478
    },
    {
      "answer_sha256": "fe93a8f0452db59110a7828cb447784e1c588ea02076b4b804b7df9f6af9b453",
      "candidate_ids": [
        "5728223cff5b5019007d9dc4",
        "57264b3edd62a815002e80aa",
        "572a18a4af94a219006aa7df",
        "572658daf1498d1400e8dcac",
        "571a50df4faf5e1900b8a960",
        "5727fc37ff5b5019007d9a18",
        "5729fa40af94a219006aa70f",
        "56dde0ba66d3e219004dad75",
        "56de3d594396321400ee26ca",
        "5726ba2c708984140094cf59",
        "571cc3dedd7acb1400e4c146",
        "572726c9708984140094da7b",
        "5726542ff1498d1400e8dc28",
        "57274f67708984140094dbf5",
        "572a0b0b6aef0514001551f6",
        "572669a9dd62a815002e8416"
      ],
      "candidate_sha256": [
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "e72c427b33209e555b5fc09b718f5d3a1abc94dbdf4f37900c15e3a035c2a954",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "231a015181e1e06c442f571a8baa25ce2ec844e939e02b2955b0aee6945c648b",
        "5ff3ae0e9f5f03c7c4795cd17b5dcba5e7554d6e0137d3133bd31c7039a115f0",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7"
      ],
      "example_id": "57264b3edd62a815002e80aa",
      "gated_correct": false,
      "gated_top": 12,
      "gold_index": 1,
      "lexical_correct": false,
      "lexical_order": [
        12,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": false,
      "neural_margin": 0.05050957,
      "neural_order": [
        12,
        1,
        8,
        4,
        6,
        14,
        11,
        15,
        13,
        7,
        5,
        3,
        2,
        9,
        0,
        10
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "1ac7e4316a1f66fb1078219cacfad719a64e204fea3eae57078a6e3f684d8f96",
      "trial_index": 479
    },
    {
      "answer_sha256": "bdd4dc9d3b23cd711b8cdde2038dd6c074701b3f6e0f461e9b68baa76ab040ce",
      "candidate_ids": [
        "572fd47fa23a5019007fca55",
        "56e19724cd28a01900c679f6",
        "57264a0ef1498d1400e8db40",
        "572ff890a23a5019007fcbcd",
        "5725f2c838643c19005aceed",
        "57300a25a23a5019007fcc97",
        "572fdb17b2c2fd140056851d",
        "572fc78d04bcaa1900d76d07",
        "57273f27dd62a815002e9a0a",
        "57264c42dd62a815002e80c6",
        "56e17644e3433e1400422f40",
        "56e1ded7cd28a01900c67bd4",
        "5726e985dd62a815002e94da",
        "572fb059947a6a140053cb80",
        "573312fcd058e614000b56e6",
        "571cd88ddd7acb1400e4c18d"
      ],
      "candidate_sha256": [
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "1d7256dd5cb1409ee57137874102372657e6487019158a45b7eb0cf824b798d8",
        "1c7faebb317f0204acbe7dc29fd2a0485558974e592b85d53d495bfd21592427",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "f96ead1ca495fa12c62b2d9c79649efa0226e58ed0cdf5ce59bf46077b95c66a",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51"
      ],
      "example_id": "57264c42dd62a815002e80c6",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        14,
        2,
        4,
        0,
        1,
        3,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.49155952,
      "neural_order": [
        9,
        3,
        11,
        15,
        2,
        4,
        5,
        8,
        0,
        1,
        6,
        7,
        10,
        12,
        13,
        14
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "fa3361227f38e1e68f830e4f49997ee18416010212119b534c4381dc35afd21b",
      "trial_index": 480
    },
    {
      "answer_sha256": "1cee44ef685b07b79bf92e0d43b8fae6dff05f459fac0edca3c1eb49b9be819e",
      "candidate_ids": [
        "57284d484b864d1900164900",
        "572f5533a23a5019007fc55c",
        "57299ec43f37b3190047850d",
        "5726542ff1498d1400e8dc28",
        "5726c19add62a815002e8f8a",
        "57286951ff5b5019007da20e",
        "57264cc6dd62a815002e80e4",
        "5733a6ac4776f41900660f58",
        "57268da7f1498d1400e8e39c",
        "5711628a2419e314009555dd",
        "57265526708984140094c2bd",
        "5726f96ddd62a815002e9698",
        "56e200e4cd28a01900c67c14",
        "572643de5951261400b5195a",
        "572a09abaf94a219006aa75b",
        "56de4bb84396321400ee27a2"
      ],
      "candidate_sha256": [
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "7c28011f4adb5bc9518e20aa534136a5ab7d9e920f0263cb0d482a6ce7859ede",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "a88c4ac0de00e7997e2a1b06e1cb9264ebde325344bb4578efc14b8e824224c4",
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "099340918f9359979a06067b9361381a993dbef386a4851ba6bb74d73cd5dc60",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "f7140e68e270abe4b07098bebe1f59cce949b799b8237e1c88451a74c65cd4a0",
        "b04a1b4486692807dd767cb5a92dd6ad22056e725f2af81c0800045aa31137f7",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31"
      ],
      "example_id": "57264cc6dd62a815002e80e4",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.22919803,
      "neural_order": [
        6,
        12,
        7,
        15,
        9,
        3,
        5,
        4,
        0,
        8,
        14,
        2,
        13,
        1,
        10,
        11
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "575e8f5e8f289c70a3f7ab7c67a01e1f26865d8f8f37764681224f19e681078b",
      "trial_index": 481
    },
    {
      "answer_sha256": "62145ece4eedfe32f3fbd4d87c47512c1e37ac28b618ee761fc89edbb36f34bd",
      "candidate_ids": [
        "56e20a27cd28a01900c67c24",
        "5728eb1a3acd2414000e01c5",
        "5727cd0f4b864d1900163d72",
        "572f609ca23a5019007fc5af",
        "57283c464b864d19001647c8",
        "5729f60caf94a219006aa6ef",
        "57264d58f1498d1400e8db7a",
        "5706111a52bb89140068984c",
        "5725db98ec44d21400f3d6c5",
        "572fac17947a6a140053cb54",
        "5727fc37ff5b5019007d9a18",
        "5726701add62a815002e84c8",
        "57282dfb4b864d1900164668",
        "5725c57a89a1e219009abe5e",
        "572658daf1498d1400e8dcac",
        "57060cc352bb89140068980e"
      ],
      "candidate_sha256": [
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "0ec0e6b5618c1738b1c93ac9054e167d46f12278dfbe8b106e3f529e1e64a7d8",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "9a83164fe22168da561ee3f57125658d4ef64b378429778d40c67af0fbc052d7",
        "50a30c5fdb5dd581c62da7eeaa8fafd7a5bac341994ed7f7df2a717fda4c9bca",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "6c46f023fb08a6f62653e6be395d8b3528a5b473c155fcc84208ed0dab0eb3f8"
      ],
      "example_id": "57264d58f1498d1400e8db7a",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.45381285,
      "neural_order": [
        6,
        5,
        12,
        3,
        10,
        4,
        14,
        15,
        1,
        0,
        2,
        7,
        8,
        9,
        11,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "94bd3603e6b0a8cb7044bbcbc26f78c7e26bf1ebc374437cdb8de875151cb8bc",
      "trial_index": 482
    },
    {
      "answer_sha256": "c03a311ecd378bc945e2d9c4d6cfec973f4ddf97c9ec2c7637d299479e7c80f2",
      "candidate_ids": [
        "57286d4f2ca10214002da328",
        "56e1e9dfe3433e14004231fc",
        "57107932a58dae1900cd69ca",
        "5730208fa23a5019007fcded",
        "5729081d3f37b31900477fab",
        "5730b7ce069b5314008322c3",
        "572fd47fa23a5019007fca55",
        "57302cd004bcaa1900d772d7",
        "572fc659b2c2fd1400568447",
        "572fc78d04bcaa1900d76d07",
        "56de15104396321400ee25b7",
        "57267640f1498d1400e8e074",
        "572980f9af94a219006aa4d1",
        "570966e0200fba1400367f4f",
        "57377862c3c5551400e51ef3",
        "57264e2f708984140094c1e1"
      ],
      "candidate_sha256": [
        "64123ebe7dcece5e2ae488efbb42144772e07ad8e132aee9ff56b96db3711b24",
        "a2cf630a7f1289cfaadc365903fadad7bae668fdb2735e77dcaea5a85de175a4",
        "702b27fc97e08ab514f3c58b2a2f6a8afddd58c1701fe5d11c1af9c85471d783",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "db6de70b42aaae5f96618f4be9863dbd2b0618f69ea12eeb5dd9dbfb1a8bcef5",
        "66cbf5ef7ba18a058b7e88c4c66ba9e11f4ef0efae05c7495c2ee6e756648699",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "c18cd9bcda557cf94732269b3307905fbdae084432b3379a0d573849288ee094",
        "a0ea05ac7403bcfecbe81b7ad6a235a9fc20ef72e87f618566850160b797957f",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "14726515a519c37e4446d133619e2a5896b98c848623ed10650db04407a29c75"
      ],
      "example_id": "57264e2f708984140094c1e1",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.35289608,
      "neural_order": [
        15,
        3,
        12,
        9,
        4,
        1,
        7,
        11,
        5,
        2,
        14,
        10,
        13,
        0,
        6,
        8
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "879caf8286df37eb83997db4dd9f3b1c944191cdd940a5793bacf55fc705948a",
      "trial_index": 483
    },
    {
      "answer_sha256": "14472307f9be857eb12fbcc7f9ea8b4219a7a41134680e66e0a94fa5dae8e827",
      "candidate_ids": [
        "5733647e4776f419006609ae",
        "572669a9dd62a815002e8416",
        "570d3468b3d812140066d543",
        "57287fec4b864d1900164a3c",
        "56e16839cd28a01900c67887",
        "57337ddc4776f41900660bbb",
        "572fd47fa23a5019007fca55",
        "5729fd56af94a219006aa72f",
        "57284b904b864d19001648e2",
        "573368e54776f41900660a53",
        "57096c95200fba1400367fbb",
        "56de10b44396321400ee2593",
        "57114e8d50c2381900b54a5b",
        "57264f18f1498d1400e8dbae",
        "5705f7c875f01819005e77dc",
        "5726398589a1e219009ac588"
      ],
      "candidate_sha256": [
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "6fd6522d4ecd5fc13437e466b8409c90aeecf62508493d6cd828ec6d82cec94b",
        "642222e452cce1e39c0eb8188e89a14d975ebc4c4ff837d19560868bd871042a",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "d9538b16aa9b6d2253faa5214e2ae46fe8c12c15499fedbe064fb2fc35d11f71",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1"
      ],
      "example_id": "57264f18f1498d1400e8dbae",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": false,
      "neural_margin": 0.21918963,
      "neural_order": [
        1,
        13,
        9,
        8,
        6,
        5,
        7,
        11,
        4,
        2,
        15,
        0,
        3,
        10,
        12,
        14
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "4c335f9b2063c54546b22318feff86d92618bc2682033ccb40671b5bb1831019",
      "trial_index": 484
    },
    {
      "answer_sha256": "920678de3a2a9338b78dad021378d58506363856f981e58772bb7e677000b93f",
      "candidate_ids": [
        "57113be3a58dae1900cd6d28",
        "572a0a391d046914007796df",
        "57264fe65951b619008f6fa1",
        "57280fd3ff5b5019007d9c26",
        "5726a00cf1498d1400e8e550",
        "5733e8ae4776f419006614a6",
        "572f7b33947a6a140053c9a2",
        "572ffa79a23a5019007fcbe9",
        "572f64ccb2c2fd14005680b7",
        "57268527708984140094c8bf",
        "572820512ca10214002d9e72",
        "5733a2a9d058e614000b5f28",
        "5727aeac3acd2414000de987",
        "5730005db2c2fd1400568703",
        "573361404776f4190066093c",
        "5733cff84776f419006612b4"
      ],
      "candidate_sha256": [
        "6f0c22572b7c29b54c20175449ef0491d897bce3356ab95cb96a162fb70d3403",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "5723097f5c1a03b86b78cb5e77be7c8ee39ce75baf102b5a199e74dcbcbc32bb",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "804c25a5922e15803ee38548ababd1125916e39e9cd0153b28e6755ec55e9edc",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "dcc81677898074e44be2a7eff5c32d8289a558c94741d4acadce83b97ba38260",
        "af4aff7a2b634d5898861af6928a2f837aa4adca50d23171a9ed122ea3e2b932",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667"
      ],
      "example_id": "57264fe65951b619008f6fa1",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.27453608,
      "neural_order": [
        2,
        1,
        9,
        12,
        8,
        4,
        14,
        6,
        13,
        11,
        7,
        3,
        0,
        5,
        10,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "262922fba6cce02ddaa74b4401108cc8bc6630729eb654d92a29a7dfdf51231d",
      "trial_index": 485
    },
    {
      "answer_sha256": "5e5bb3ca110d825e0e03944bf5289dc002765db848751044a12d658d308b6eff",
      "candidate_ids": [
        "5726c5a9f1498d1400e8eac4",
        "572818f54b864d190016446c",
        "57111428b654c5140001faff",
        "5728f3724b864d1900165116",
        "571c91c8dd7acb1400e4c10a",
        "5727d6154b864d1900163e34",
        "572fb059947a6a140053cb80",
        "57281edd3acd2414000df4eb",
        "57265bdfdd62a815002e829e",
        "572726c9708984140094da7b",
        "573735e8c3c5551400e51e71",
        "5726509bdd62a815002e815a",
        "57269e3bf1498d1400e8e516",
        "57294209af94a219006aa201",
        "5733ffa7d058e614000b674f",
        "573362b94776f41900660974"
      ],
      "candidate_sha256": [
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "f390cbcc7bd65fb1d7a393f33688551d69b62380ab207dd27d85bce34922a884",
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "2271df8c8c1b2d712c122939c9c53f699509a751bc040f6c45e9660ceb9c2177",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "80e800db40294efc728c4dcbf06c36437000cef741a99c5f9ed99d6d0756badc",
        "89715c36aaa470396ac75f4c2e1aa11887c372bb25420039c14d4417323c24e7",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2"
      ],
      "example_id": "5726509bdd62a815002e815a",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        3,
        10,
        14,
        15,
        13,
        12,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.13055936,
      "neural_order": [
        11,
        3,
        10,
        2,
        13,
        14,
        1,
        9,
        8,
        12,
        0,
        4,
        5,
        6,
        7,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "0cf1c2330bdbcd60a8c8164475dd5171b04a7a686d97193cb188adf3aea6a355",
      "trial_index": 486
    },
    {
      "answer_sha256": "64e89e19876011a266db23230346a3fd6a604fe572f7da7b100ed48e41ddc155",
      "candidate_ids": [
        "572827fc3acd2414000df5bb",
        "5730b541396df919000962c2",
        "5710f4b8b654c5140001fa45",
        "5729ffda1d046914007796af",
        "571c7d55dd7acb1400e4c0c4",
        "571161092419e314009555d7",
        "571135b8a58dae1900cd6d0e",
        "573792ee1c456719005744b9",
        "5725f8f5ec44d21400f3d7b1",
        "5726516a708984140094c223",
        "571ccd9b5efbb31900334df8",
        "56de0ffd4396321400ee258d",
        "573004bf947a6a140053cf56",
        "56de49564396321400ee277a",
        "572655e5f1498d1400e8dc5e",
        "5726b718dd62a815002e8dbe"
      ],
      "candidate_sha256": [
        "1d2a85c2f037a152cb0baebc5a70de8d798a0348c89163667f68afa981d9afd1",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "b557caa6a1a9ebdc6ec705acc8c1078fbde1cf27379b5961913015d24cbd8809",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "b5aed2dd9b558399cacafe86ab644b1de7e671e6c9a21e13cd89fb7a0e04e7db",
        "9e5f19ae8c4bcea47d2097bb803a6f512f53874779686d5848e1aec6bdadef81",
        "8d0f136ca3e21898494994833851f5c381fc610d00c5a286083f4b15d65caad4",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "c6c943ad8d937131ec164bb4c6bdeb900da7261119bc84eb598b8d715c378b1d",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4"
      ],
      "example_id": "5726516a708984140094c223",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.38994174,
      "neural_order": [
        9,
        14,
        3,
        0,
        2,
        8,
        13,
        1,
        11,
        4,
        5,
        6,
        7,
        10,
        12,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "65796f3e5975261b215eed17e402b38c60140b29cb522a366957d7c49a2dde7d",
      "trial_index": 487
    },
    {
      "answer_sha256": "65ed5cd6c3a9a138891094032f811fbfadb37c0407a2eab9a8455655ae60f7a5",
      "candidate_ids": [
        "5726887e708984140094c917",
        "572fc5a1947a6a140053cc8a",
        "56de10b44396321400ee2593",
        "57265285708984140094c25b",
        "56e1bd4acd28a01900c67afc",
        "572807802ca10214002d9bf8",
        "572856beff5b5019007da190",
        "572a03a06aef0514001551aa",
        "5728eff82ca10214002daadc",
        "5730a0778ab72b1400f9c60a",
        "5729a3716aef05140015506a",
        "572ff86004bcaa1900d76f65",
        "5733e5a14776f4190066145b",
        "5725cb33271a42140099d1db",
        "5737a9afc3c5551400e51f61",
        "5726bc1add62a815002e8ea6"
      ],
      "candidate_sha256": [
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "60527ca1cce54d340513d3ebe3086d4f283d0953ce0dd1d62f7c93de4d1730e4",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "a4949b4d47aa099e44c4ad20733b839d1a26bbb979486bcc915df667685fc936",
        "25160eaaf4cedb4feaad143ed4ed40d1a5aeba443aaa0327bbecb05f058934a7",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "2c1b073fd38cdbe7a83f4730db39e216b51a39e47e536562eeac27a25d7cfc31",
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56"
      ],
      "example_id": "57265285708984140094c25b",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.37007761,
      "neural_order": [
        3,
        8,
        9,
        13,
        12,
        0,
        7,
        10,
        1,
        2,
        4,
        5,
        6,
        11,
        14,
        15
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "8b956d7f64657dab25067b9d74e92bd256df726b61cc2c5d19b907689455d532",
      "trial_index": 488
    },
    {
      "answer_sha256": "9f62d053577d9569f2bf0415406656bf1e34d44ad1ad0eeffcf54765326a3cce",
      "candidate_ids": [
        "572647d0708984140094c14b",
        "572ffce5a23a5019007fcc15",
        "5726deed5951b619008f80c5",
        "5726534d708984140094c26d",
        "57274f49f1498d1400e8f61e",
        "572843304b864d1900164848",
        "57294279af94a219006aa209",
        "573786b51c4567190057448d",
        "572a13841d0469140077973b",
        "570611c475f01819005e793c",
        "5733a6ac4776f41900660f58",
        "57339902d058e614000b5e70",
        "5725d42a89a1e219009abf58",
        "5730042804bcaa1900d77011",
        "5730131c947a6a140053d052",
        "57309446396df919000961b8"
      ],
      "candidate_sha256": [
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "41e21fa726a42940bf952e16ff6c966f65aef98539bfab9974b8f54fbaa8f53c",
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "85d84a95e51eb97532726aac42704a556a47167854b2dc7d40458c7a19042198",
        "db91c1023a94550f1ffe6e9f87e0b45aaad95f3bef9ada0d0c3872b88286bf7c",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "9cf1d4ada7d1f8dd2a414fdfc4982c71c15f0b82695deb09c470275d7082a80a"
      ],
      "example_id": "5726534d708984140094c26d",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.20240149,
      "neural_order": [
        3,
        8,
        6,
        15,
        0,
        7,
        10,
        9,
        12,
        1,
        2,
        4,
        5,
        11,
        13,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "018b55784145ac743e12906b228b5760fb932bbf95f9d0b86aa94f73f77bc0ae",
      "trial_index": 489
    },
    {
      "answer_sha256": "ee2f4a7062ae4e00abedb87d7a6259389c2909e3915b5839e77e7d0b935dfbeb",
      "candidate_ids": [
        "572ffce5a23a5019007fcc15",
        "572a07fc6aef0514001551dc",
        "5726545f708984140094c2a5",
        "5726e313f1498d1400e8eeb2",
        "572671a5f1498d1400e8dfe4",
        "57096f37200fba1400367fe5",
        "5730088e947a6a140053cfac",
        "5726f48df1498d1400e8f0da",
        "56e1aff7cd28a01900c67a68",
        "56de4bb84396321400ee27a2",
        "572fdd03a23a5019007fca9d",
        "57268a37f1498d1400e8e33c",
        "5726542ff1498d1400e8dc28",
        "57339ad74776f41900660e86",
        "57339dd94776f41900660ecc",
        "5726e179dd62a815002e93ae"
      ],
      "candidate_sha256": [
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "7e56efa7fa5e85dca5e963da3b35db17158b1f5057094c80e889026cfac43e77",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "a28b1741ae11a0ca192dc49f8fe81a5afe9328ea6751781fc77943aa575931d9",
        "060f4efa09fb73e4fb2d8efae87df8f79d0d553cdecfb983e3502b774d0bb037",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "b1883919e17ff672ebc5c8754b9babd683410134d88304f2048788353467a297",
        "2d5e9f304697ac88e70d959415ed2a0826e4e17344d07025e79fe80526e13c31",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "827c9ff031e0d998f3901455949a7349bbd03c9ee89635495a219c5ce6dc0e71",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434"
      ],
      "example_id": "5726542ff1498d1400e8dc28",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        5,
        9,
        0,
        11,
        2,
        1,
        3,
        4,
        6,
        7,
        8,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.44504397,
      "neural_order": [
        12,
        9,
        0,
        6,
        15,
        3,
        11,
        14,
        7,
        1,
        2,
        4,
        5,
        8,
        10,
        13
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "208b1ef9872756a2c768047fe63a1e8e7bf5792558680c45f9f354eabdcec3b7",
      "trial_index": 490
    },
    {
      "answer_sha256": "5b175985edb14218ae811e2395053928a60b0c0cc4598f01b48c099d7af6db7c",
      "candidate_ids": [
        "57115ff82419e314009555c3",
        "57267d52708984140094c7d9",
        "5725bbec271a42140099d0d1",
        "573749741c4567190057445d",
        "5730b7ce069b5314008322c3",
        "5733a560d058e614000b5f77",
        "5733834ed058e614000b5c26",
        "5726f635dd62a815002e9656",
        "570610b275f01819005e792a",
        "5726722bdd62a815002e8528",
        "5727b0892ca10214002d93e8",
        "57302ad804bcaa1900d772af",
        "572663a9f1498d1400e8ddf2",
        "57108073b654c5140001f925",
        "573361404776f4190066093c",
        "572654e2708984140094c2b7"
      ],
      "candidate_sha256": [
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "66cbf5ef7ba18a058b7e88c4c66ba9e11f4ef0efae05c7495c2ee6e756648699",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "ac807a49b5fe00f008d875243af3583cef90d1685b4709f5654c707fa5c1da17",
        "b88f2e98aa5ed39684015bb91e028e3f43f8a9f57f9434564aed6b294987397c",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "d0282348263a9123328cb2baa7c6e8ed31817139c4b7270b227dfb27628e5063",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "9d0852e7a492edfb020138aeed2e89901eaf6cbc0450e6da752320d2c7c024e0"
      ],
      "example_id": "572654e2708984140094c2b7",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.43031642,
      "neural_order": [
        15,
        13,
        5,
        14,
        12,
        7,
        4,
        10,
        6,
        2,
        9,
        0,
        1,
        3,
        8,
        11
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "9defb1adf1729f2f7e8aa290c43444535722358ac7f8216470b37f4fc8ce7010",
      "trial_index": 491
    },
    {
      "answer_sha256": "19e8a1e8899848052443329fbf450b29e9dd5fc0a9fd26d623f901e1a3d444dd",
      "candidate_ids": [
        "5728facd4b864d1900165170",
        "5726eb4b5951b619008f826b",
        "57300200b2c2fd1400568729",
        "57115ac550c2381900b54a77",
        "57265e11708984140094c3bb",
        "5737a7351c456719005744f1",
        "5725c0f289a1e219009abdf2",
        "57286ec63acd2414000df9d1",
        "5726534d708984140094c26d",
        "5728fb002ca10214002dab6a",
        "5726e3c4dd62a815002e9404",
        "572943ab1d04691400779219",
        "5730040f947a6a140053cf4c",
        "57107e6ca58dae1900cd69f2",
        "5726559edd62a815002e81c8",
        "56de17f9cffd8e1900b4b5e0"
      ],
      "candidate_sha256": [
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "8bceec592964d39cac4d75b27e6ea2d64b1139a54fd9ee544bfc96ca1c1c168e",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "f1fd21822238bc914acc770ffbb65f55d8c7f833cb4d0a1c420d323ecdd137dd",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "65f4aeaa7b395a0e2d28e6076ed9cf050be1fd9a7c38882893bab1f0e193b947",
        "1d470bf791028e4f456c100c44c5b3410fac821035b36b868d821eb23e1f2a74",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "d786a2ad8a37fb60494b48d0f306463dea0456e9879866b23371917e8023d131",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "68e815c1eccce28acd02576891be89765c5d2769195c1714b1fb27828cd727a1"
      ],
      "example_id": "5726559edd62a815002e81c8",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.0667914,
      "neural_order": [
        14,
        11,
        13,
        15,
        8,
        7,
        2,
        4,
        12,
        5,
        3,
        0,
        1,
        6,
        9,
        10
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "ec91a0c3c897ed7eb0d013924c8789285e57f1b365fd43dbd68809a786f3ab9b",
      "trial_index": 492
    },
    {
      "answer_sha256": "9925953f1faef050547e5f7b811c3f7d245143926d17b5657e7c480ccc6f8232",
      "candidate_ids": [
        "57265642f1498d1400e8dc68",
        "56e190bce3433e1400422fc8",
        "5725dd7d89a1e219009abfea",
        "5733d2dbd058e614000b6339",
        "572673f5708984140094c69b",
        "5728f50baf94a219006a9e55",
        "56e1a0dccd28a01900c67a2e",
        "5733fb7bd058e614000b66ff",
        "57337ea24776f41900660bd0",
        "570607f575f01819005e78b4",
        "573007fab2c2fd140056876b",
        "573750f51c45671900574467",
        "57294baaaf94a219006aa26b",
        "572f6a0ba23a5019007fc5eb",
        "571161092419e314009555d7",
        "56e1cbe2cd28a01900c67bac"
      ],
      "candidate_sha256": [
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "6ee5d1fa17d81ceddc66e821d2760925c2f8f375ae09c69d361c0fc75ba38ba0",
        "d206c39c98d46bd01f841c82284343c93a112882c53094883d72423bafd6ee01",
        "7de07b845a972b4982551bf842e82fdbdf48395f2413738aec8dad46c04f1ddd",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "ca5603bb3818cc014efbe9976936fcfc69ae6340045d8c80d3e235e1bfbc4163",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95"
      ],
      "example_id": "57265642f1498d1400e8dc68",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        9,
        13,
        14,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.49122844,
      "neural_order": [
        0,
        7,
        10,
        8,
        9,
        3,
        13,
        11,
        4,
        2,
        1,
        5,
        6,
        12,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "f7211912509880eacc81ed4b551dc5f38c391771615ed0383729d1348abb5553",
      "trial_index": 493
    },
    {
      "answer_sha256": "c7abff3c91311ad7f9fcf329b57b3c549c542891fec890ce7fd89434cf06e6bc",
      "candidate_ids": [
        "57092322efce8f15003a7db0",
        "57297a276aef051400154f88",
        "56de3f784396321400ee26fa",
        "572faec7b2c2fd1400568333",
        "56e1c4fce3433e140042314e",
        "573406d1d058e614000b6801",
        "572fc78d04bcaa1900d76d07",
        "5726887e708984140094c917",
        "56de1728cffd8e1900b4b5d7",
        "5725edfe38643c19005ace9f",
        "56e1b169cd28a01900c67a72",
        "57265700dd62a815002e820e",
        "57264c42dd62a815002e80c6",
        "573786b51c4567190057448d",
        "57282dfb4b864d1900164668",
        "572ff890a23a5019007fcbcd"
      ],
      "candidate_sha256": [
        "bc5f1c2743b9de3bbf84078611d839665374c71f9f5ce23ea41ca3d84c8793ba",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "e77cc2842075cf67e0f42d9c55fea1df5857504c655c9e1e596c5f52dc859507",
        "7c68f0f764734062722a3019d2cfcb70711c068ef007e2f096ee98152d82b275",
        "8910fbbf56034ece8e91508b95e4c9b5a93d59dd5f4dddb51a734f71b66480c4",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "9d3029bec84c7f113735fae2aee0eaa8c897d305858ac03459a04545e068759b",
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41"
      ],
      "example_id": "57265700dd62a815002e820e",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        15,
        12,
        10,
        1,
        3,
        0,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        13,
        14
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.28574399,
      "neural_order": [
        11,
        12,
        2,
        7,
        1,
        5,
        14,
        13,
        10,
        15,
        0,
        8,
        6,
        3,
        4,
        9
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "1ff31cc387833cd4a106916c1a51534ea4eabd71bb42d03352e18b6fd065d2b4",
      "trial_index": 494
    },
    {
      "answer_sha256": "2147ac4b4543aebe423f7ded658e7ed4be6f32eb84c993748628a85ac7ac0732",
      "candidate_ids": [
        "57377ec7c3c5551400e51f05",
        "572750df5951b619008f882f",
        "572951f16aef051400154cce",
        "56de1645cffd8e1900b4b5d0",
        "5725d16aec44d21400f3d61b",
        "57286f373acd2414000df9db",
        "5728f2e26aef051400154896",
        "572883153acd2414000dfa6f",
        "5728e8212ca10214002daa6c",
        "5710f2e2a58dae1900cd6b72",
        "572fff45947a6a140053cf26",
        "57282358ff5b5019007d9dde",
        "571c9348dd7acb1400e4c114",
        "57296f293f37b319004783a3",
        "5726577f708984140094c301",
        "572fc043a23a5019007fc95d"
      ],
      "candidate_sha256": [
        "23540e5a8f451870249ba35ad85543793aeca060999d59750185c89849a95c71",
        "cf3314dc1ff1c19a23260f955334192ba867e2b07ef93e9d0d07451dfe6631c6",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "7bebf180af4efdb13d4fe92b74423fe7c97ed3c1f01639bb261882a5ce0ac568",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "e786cb5a8f15ebdbfcd122da827cd2109527d1982c521961d4311026ae8f10bd",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "482cc8356a93089acbdd93a4535c71be03a9b3d51facfa839ee7d6646a0a3bfd",
        "f3e994c58b1e4906937c035c80fb9519764a0f4f42ff12a50c7aeac3190ebd39"
      ],
      "example_id": "5726577f708984140094c301",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.51602439,
      "neural_order": [
        14,
        6,
        10,
        9,
        11,
        7,
        4,
        5,
        13,
        0,
        1,
        2,
        3,
        8,
        12,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "153b83cce80ef10d0f768cf118ac520909454ab51938800df0ec6e9521983af4",
      "trial_index": 495
    },
    {
      "answer_sha256": "ac3f8de1d1aee8d2eb6b8cfd369fad1a5459bad805408a10b81cf2bf46b6b514",
      "candidate_ids": [
        "572f65e9b2c2fd14005680cb",
        "57264b1ddd62a815002e80a0",
        "57264cac708984140094c1b3",
        "57286ab72ca10214002da31e",
        "57339555d058e614000b5df3",
        "57108198b654c5140001f937",
        "56e1a38de3433e140042305c",
        "5725d34aec44d21400f3d639",
        "57283db6ff5b5019007d9fbc",
        "5730a314069b5314008321eb",
        "5705fc3a52bb89140068976a",
        "57115b2850c2381900b54a7f",
        "572657d9dd62a815002e8230",
        "572667e2f1498d1400e8de90",
        "573314e3d058e614000b56f0",
        "572810ec2ca10214002d9d06"
      ],
      "candidate_sha256": [
        "3bafbc5d56c1857ae4a5346827458b7900e775e4ebe79997c59538a3013f8c57",
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "eec1efea8f64191af05fff202291bdbaff719dd86f085904ea04701e00928bc1",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "ca0bb7cbca98ddf47c2746553de4009e05c52aa03b5153c84527be6cb17a88a9",
        "fdca2bfa5203b813cc715877a5fe7c4b90d80bcb6d8b54d88236c9bdeda42cf6",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "5aee5e7fbe75fde0fa4b31457b94e333a03d00b752032439d31a60d62d68616a",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "68cc7e443dab62acd50ead8d7c118537cf11388e2cd488044cf040b21f06c60c",
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87"
      ],
      "example_id": "572657d9dd62a815002e8230",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.18691522,
      "neural_order": [
        12,
        13,
        1,
        14,
        8,
        6,
        4,
        10,
        9,
        0,
        2,
        3,
        5,
        7,
        11,
        15
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "d99f8d3ee50d52af20f15b1216b858b677f8e0a659300d2fc51aac3c0857b05a",
      "trial_index": 496
    },
    {
      "answer_sha256": "8689a381c5096ae08aceba4642aa956f86e52fe868099c41e643eb1c2a1e42b3",
      "candidate_ids": [
        "57264228ec44d21400f3dcf5",
        "5728759cff5b5019007da25a",
        "5729f8516aef05140015516c",
        "57266e72f1498d1400e8df8c",
        "5728804b4b864d1900164a46",
        "573020f7b2c2fd14005688f7",
        "57302a3aa23a5019007fcecf",
        "57300761947a6a140053cf9a",
        "57265d08708984140094c397",
        "5733314e4776f4190066076a",
        "570d391fb3d812140066d575",
        "57377c98c3c5551400e51efb",
        "572a12381d0469140077972b",
        "57332b66d058e614000b5758",
        "57060eaf75f01819005e7910",
        "572855973acd2414000df925"
      ],
      "candidate_sha256": [
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "f9d0ba77dd8dff4446935bfec391a5986633d68d8bdf2319ad27e1fd5890d87c",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "fd5a552211149231e544e6442834ce1d4e83af381af05c00db894e1c2c8cc09b",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "1c326ae15d585661849c0a43f71be232f457f11a2a48964b11b0d9172e7cd31b",
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "3990a188c951ac2ac5a549ad29f598b583da6695cf862bd17be6cdd20e3d149c",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5"
      ],
      "example_id": "57265d08708984140094c397",
      "gated_correct": true,
      "gated_top": 8,
      "gold_index": 8,
      "lexical_correct": true,
      "lexical_order": [
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 8,
      "neural_correct": true,
      "neural_margin": 0.35116042,
      "neural_order": [
        8,
        3,
        7,
        1,
        0,
        10,
        4,
        9,
        13,
        12,
        2,
        5,
        6,
        11,
        14,
        15
      ],
      "neural_top": 8,
      "partition": "test",
      "question_sha256": "d272070264341813f91e0f1a26259e9bcfdd31394c17d920d33c3b4097ad8030",
      "trial_index": 497
    },
    {
      "answer_sha256": "6061e671d770226c3a98492338e10dc818cf8cd92098b10ddb104b8a80d83ac9",
      "candidate_ids": [
        "571a50df4faf5e1900b8a960",
        "57265f605951b619008f70db",
        "57107e6ca58dae1900cd69f2",
        "572726c9708984140094da7b",
        "57340b1bd058e614000b6869",
        "5726e985dd62a815002e94da",
        "5729f3831d0469140077967b",
        "573099ee8ab72b1400f9c5dc",
        "5725b33f6a3fe71400b8952d",
        "572a03a06aef0514001551aa",
        "5705ffde52bb891400689784",
        "5730042804bcaa1900d77011",
        "571144d1a58dae1900cd6d6e",
        "571c91c8dd7acb1400e4c10a",
        "57094d489928a8140047150a",
        "56de3dbacffd8e1900b4b6d2"
      ],
      "candidate_sha256": [
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "c3fae4238688c4590bb4910c491613f34cd2b5871fb6c949bbd22c1c82bf5247",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "af9c49f97fc8aa685320f6a979a809ea599a1ed61f77f300b94cc58c1608c80b",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "96dc015bb2dea5aca473659180a62193857f51811c3f2bcaaa72c8d88f712607"
      ],
      "example_id": "57265f605951b619008f70db",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        12,
        13,
        8,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.24760966,
      "neural_order": [
        1,
        11,
        10,
        8,
        13,
        12,
        2,
        3,
        6,
        15,
        0,
        4,
        5,
        7,
        9,
        14
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "82b918e4a36f78570cc571ff4c69ba19fd508683e45a14ddc7420b4abf73eaa3",
      "trial_index": 498
    },
    {
      "answer_sha256": "70c56ed0da0158660fd14b8ea5a107e3939ec3ac9a0e7f10606b0aee70c0d2d0",
      "candidate_ids": [
        "5705e33f52bb89140068964c",
        "57265642f1498d1400e8dc68",
        "5733dc95d058e614000b644a",
        "57266193dd62a815002e832a",
        "572667e2f1498d1400e8de90",
        "57286951ff5b5019007da20e",
        "57299021af94a219006aa50b",
        "57300ec0947a6a140053d004",
        "5726f0e35951b619008f82f7",
        "57339ad74776f41900660e86",
        "5726f868dd62a815002e9684",
        "572864542ca10214002da2de",
        "570963a5200fba1400367f33",
        "5733a6ac4776f41900660f58",
        "5733f309d058e614000b6648",
        "572f5533a23a5019007fc55c"
      ],
      "candidate_sha256": [
        "8a148fbb97c8eaaa2ffba94677745201a37bfac096e9222f9299c75c2b5afe76",
        "8b14af3080fc770b0bf19c5b30cb00acc2ede39893860404f2c8434840a522be",
        "c60675200199ca4d14c9f4c048d21e6e502a20e47c3999e1ea557a8e627ac9bb",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "4eef5ee2e8e5a99f7d072734f1ac52b08194c79b66eefd1f527eb2304547b585",
        "33e49ae942c39aefd29d877e8334707b666747d16641fd8392ae988a08b47f20",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "40f970a92f69db6815bd67e7934fc725f910e6a6e08c5ff4e55067dcc721b938",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "5b469d64ced1175805ddbb53037730159f46f770f6697c84c8b9e8738d9bbb19",
        "7c28011f4adb5bc9518e20aa534136a5ab7d9e920f0263cb0d482a6ce7859ede"
      ],
      "example_id": "57266193dd62a815002e832a",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        8,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.5753356,
      "neural_order": [
        3,
        4,
        12,
        0,
        6,
        13,
        15,
        9,
        1,
        2,
        5,
        7,
        8,
        10,
        11,
        14
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "780bd0e71ab6207225bc7cfb727601bad5fa2b075ea63389c3ac43365f9c8c95",
      "trial_index": 499
    },
    {
      "answer_sha256": "c4346a1748650cc234303a360aa869466250a5f956b3bdebe49dfa12dd61f7ce",
      "candidate_ids": [
        "5725ce4d38643c19005acd4d",
        "5726a1e5dd62a815002e8b86",
        "5726a34bf1498d1400e8e59e",
        "56de51c64396321400ee27f7",
        "571155ae2419e31400955591",
        "5728f2e26aef051400154896",
        "573755afc3c5551400e51eb3",
        "56e200e4cd28a01900c67c14",
        "573099ee8ab72b1400f9c5dc",
        "57265aaf5951b619008f706b",
        "5728223cff5b5019007d9dc4",
        "57377aac1c45671900574479",
        "570963a5200fba1400367f33",
        "572663a9f1498d1400e8ddf2",
        "5728170d3acd2414000df443",
        "5729f4e46aef051400155156"
      ],
      "candidate_sha256": [
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "82f4578e26c4c6a9e4fef850543ca085cec414d5d25b6d235947535b64ec3b54",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "9236eb060e30e96b6f012a230bcabe83aff286e5d483e37af447fc603b2d39c0",
        "f7140e68e270abe4b07098bebe1f59cce949b799b8237e1c88451a74c65cd4a0",
        "24f1d212bb4caf496bcbfa991a109e97c96130782a118606cc0bdb37bea8c508",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "40f970a92f69db6815bd67e7934fc725f910e6a6e08c5ff4e55067dcc721b938",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "3a95085253039f4bf191f1037c44a2641c44c877382c059105507c459d53eaba",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0"
      ],
      "example_id": "572663a9f1498d1400e8ddf2",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.39141984,
      "neural_order": [
        13,
        7,
        1,
        8,
        4,
        3,
        2,
        15,
        0,
        5,
        6,
        9,
        10,
        11,
        12,
        14
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "a514e78741f63bd0bbb1f91f9b870fd3a76bb97a6e993a382c41915ebf06e9db",
      "trial_index": 500
    },
    {
      "answer_sha256": "0428b1f47627d376e5384810e781e40c4a4f3bee50dfa8778d8b5f8e75e75a87",
      "candidate_ids": [
        "572665ff708984140094c4c3",
        "572872dd2ca10214002da37e",
        "57282dfb4b864d1900164668",
        "572a0bafaf94a219006aa765",
        "572ff626947a6a140053ce8e",
        "57115bf350c2381900b54a93",
        "57273a465951b619008f86ff",
        "5727f2714b864d1900164072",
        "5730126ba23a5019007fcd09",
        "5730bd00069b5314008322d7",
        "572f59b4a23a5019007fc587",
        "571a4b0f10f8ca1400304fd5",
        "5729081d3f37b31900477fab",
        "57097b50200fba140036804d",
        "57264228ec44d21400f3dcf5",
        "57377083c3c5551400e51edf"
      ],
      "candidate_sha256": [
        "e71eab6fb81f391a6f9f6f7ddf6a435221f7159360e4d7a0549e3b969d010395",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "9821d0c94f407e266d045f300baf8a2808a74eea26f04459730cfc74b21f5513",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "cca1ba8c6c31621213a71b564b7859f49097fe27c6f49bfcfa84110b3c7b645c",
        "5d119f76b85606c7733b9d125b3a12aa7bd36bdad4b574ba86ff545ac1f73485",
        "181d9049457f9019a809b04fcc434c04837e13ea97eae479f761258fdd3bca1b",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "db6de70b42aaae5f96618f4be9863dbd2b0618f69ea12eeb5dd9dbfb1a8bcef5",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2"
      ],
      "example_id": "572665ff708984140094c4c3",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        15,
        3,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.52220127,
      "neural_order": [
        0,
        6,
        2,
        7,
        15,
        10,
        9,
        4,
        8,
        1,
        3,
        14,
        12,
        11,
        5,
        13
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "892186e3ad6c9a0aaf15b440813e321b2a3906d7040166453f147eafb454bae6",
      "trial_index": 501
    },
    {
      "answer_sha256": "556ed2717cd8c8d45013ece75ccb8f011fef133fe53a965da45f29c6154f4403",
      "candidate_ids": [
        "57274eca5951b619008f8800",
        "572fdd03a23a5019007fca9d",
        "572667e2f1498d1400e8de90",
        "57267947f1498d1400e8e0ec",
        "572a09abaf94a219006aa75b",
        "5733a32bd058e614000b5f32",
        "573004bf947a6a140053cf56",
        "57332b66d058e614000b5758",
        "57264cc6dd62a815002e80e4",
        "572943ab1d04691400779219",
        "5732b6b5328d981900602021",
        "5733a560d058e614000b5f77",
        "57332442d058e614000b5720",
        "5729727baf94a219006aa437",
        "57115b8b50c2381900b54a89",
        "57115dbe2419e314009555a5"
      ],
      "candidate_sha256": [
        "46ceed087a89c8c5146679d04d65800e1d5b68fadcd272ac5f73357c18ad83be",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "6541bd8b199525a6868c0dfeadabe11905d5afdfa405e13f206f046430934e39",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "fcb9ab642fd54564ba8561d66afa42c7a340302550f69419acc9e102acd011a5",
        "0e4d9d4cd8e24f4d459a071e1e7230fb3bd6653bd073749bee0587e7cfaa743a",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806"
      ],
      "example_id": "572667e2f1498d1400e8de90",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        3,
        0,
        1,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.24470486,
      "neural_order": [
        2,
        3,
        4,
        8,
        11,
        5,
        9,
        13,
        0,
        1,
        6,
        7,
        10,
        12,
        14,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "878f1f6acbc9db1cce76b34503bd99415619349174deada30db9c7c504047cfb",
      "trial_index": 502
    },
    {
      "answer_sha256": "d7ee1b70906a6e5008cda2751584d29a08bec74a54b11ca551b20d306ae89afb",
      "candidate_ids": [
        "572ff932a23a5019007fcbd5",
        "572754dd708984140094dc3c",
        "5729517d6aef051400154cc8",
        "572669a9dd62a815002e8416",
        "5726545f708984140094c2a5",
        "5727d6154b864d1900163e34",
        "57094a79efce8f15003a7dc4",
        "572fc41f947a6a140053cc80",
        "571117d4a58dae1900cd6c0a",
        "57265aaf5951b619008f706b",
        "5727500f708984140094dbfd",
        "5710e9f8a58dae1900cd6b30",
        "570d50a5fed7b91900d45e7d",
        "5726559edd62a815002e81c8",
        "5729f39a6aef05140015514c",
        "56e1b8f3e3433e14004230e6"
      ],
      "candidate_sha256": [
        "cc731d5514919c5d1ecdb71a42a5578264065470737af1622f378cc7b83d1853",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "99b52199ae0d66bb5e7cf03964f9cd3473dd6a91b95100f6ae8b634cbc3a7b60",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "c17ef7adcab459b968e2ef462f3cf4de687b01b702916f09322386dd59bebd28",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa"
      ],
      "example_id": "572669a9dd62a815002e8416",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        2,
        0,
        1,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.52357232,
      "neural_order": [
        3,
        9,
        2,
        15,
        4,
        14,
        7,
        0,
        11,
        1,
        8,
        5,
        6,
        10,
        12,
        13
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "e74d2628cebb941dcc1c66f9a899dd5dca5ffad303ca676cedd4bcf8918f29f8",
      "trial_index": 503
    },
    {
      "answer_sha256": "542c516b1148611e0f495e0f194a29c6b32c9196682226cd6f4c0214c6a54eca",
      "candidate_ids": [
        "57297547af94a219006aa45c",
        "573406d1d058e614000b6801",
        "57287c2bff5b5019007da26c",
        "5711669550c2381900b54adf",
        "57299ec43f37b3190047850d",
        "573011de04bcaa1900d770f9",
        "5733647e4776f419006609ae",
        "57280f974b864d1900164370",
        "572951f16aef051400154cce",
        "57267947f1498d1400e8e0ec",
        "5737804dc3c5551400e51f0f",
        "57266c015951b619008f7237",
        "572fca7eb2c2fd1400568473",
        "5737821cc3c5551400e51f19",
        "5729fd111d046914007796a5",
        "57378b141c4567190057449f"
      ],
      "candidate_sha256": [
        "24e9b742bafa6febc93490d25e9da46c531c66e39939020987e86a838117e341",
        "c2c2de6bee275bf61917835a188643ff9b21ac9910212a50ee3ed5725cb00a5d",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "027d971ab8a5d8d8074c1660827c67aafd4362842508947ab70164e11fcf178f",
        "3d17d88bb8ed6f3e733ef5679710ee5b811949199d088a32462a07f19c6f8ef3",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "c097dba35683dcaca54093ce93a25832a41c04ff0b3c823c8486d4ff18bc8ba0",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "f940260eaf284a909912a1db1a7d78a8a63d57532e47769c4005b748d582708e",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef"
      ],
      "example_id": "57266c015951b619008f7237",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.18469859,
      "neural_order": [
        11,
        9,
        6,
        8,
        0,
        5,
        13,
        10,
        1,
        2,
        3,
        4,
        7,
        12,
        14,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "04fd84cf5c7592c89b9283d8e88a9b97a79289d97a412da24619675c899c7859",
      "trial_index": 504
    },
    {
      "answer_sha256": "de5752d2c4e9fae3e1105f35dbae9635e3755db18911e4a4e2aa34f8620ef16d",
      "candidate_ids": [
        "57266e72f1498d1400e8df8c",
        "57109180a58dae1900cd6a40",
        "572864542ca10214002da2de",
        "573028fa04bcaa1900d77287",
        "57113f83b654c5140001fc29",
        "572996c73f37b319004784b3",
        "5725c69738643c19005accb9",
        "572a0bafaf94a219006aa765",
        "570d529fb3d812140066d6bb",
        "57379a4b1c456719005744cd",
        "57108c95b654c5140001f979",
        "57264228ec44d21400f3dcf5",
        "570d2c20fed7b91900d45ca7",
        "572673f5708984140094c69b",
        "5728151b4b864d1900164428",
        "5726d8bd708984140094d35b"
      ],
      "candidate_sha256": [
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "3cfec35f68f54f81f8310166c199a7f07ec54dcbe09b78ea28c9407184fd9741",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "a6b7ec7938b966c86b4cc72fa2059dcdbd60bc13208210cff0105baccb082b8b",
        "39cb7f2223dff7cbd95e9ec9a183317af550b64886fdb57e13076116c1194343",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "37c7853eb3100481bf8eb03dc20a277af3908ee8f3eb55e8315e451b7ed2d724",
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "6ca143110a7fef484f13b2415ee580abdce99536cc9e23199d442b817d2e41cb",
        "c63d7d17399c240315df794eacc14899f635eabcc983ff0ba9da5fd3b542547c",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1"
      ],
      "example_id": "57266e72f1498d1400e8df8c",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        15,
        5,
        11,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.36565478,
      "neural_order": [
        0,
        13,
        6,
        5,
        4,
        15,
        1,
        2,
        3,
        7,
        8,
        9,
        10,
        11,
        12,
        14
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "fe61d17cfd3fc09767938e378349dc4c4596a83e44446786b3fe5906439ceec2",
      "trial_index": 505
    },
    {
      "answer_sha256": "5c7663df2501576f74d0b061366dd9ee592e621207cd23c1d35c579b8e5a6461",
      "candidate_ids": [
        "5730ac782461fd1900a9cf73",
        "5725e45689a1e219009ac048",
        "56e1d9fee3433e14004231cb",
        "56de4c324396321400ee27ab",
        "57273a465951b619008f86ff",
        "572978f91d046914007794d3",
        "571c83f3dd7acb1400e4c0d8",
        "5726701add62a815002e84c8",
        "57274e975951b619008f87f9",
        "5727cd0f4b864d1900163d72",
        "572879574b864d1900164a14",
        "572649d8f1498d1400e8db37",
        "572855973acd2414000df925",
        "5730b8ca8ab72b1400f9c704",
        "5727d1c93acd2414000ded3f",
        "572864dd4b864d1900164976"
      ],
      "candidate_sha256": [
        "c839b93c89294e489a29502c9894ea7f6f62991cc3c0581f2b9e618fca81f105",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "5069342da076e41ece723574effb0a566b22663edb71937944db3629b8595a42",
        "0500dcf91ad6681a7202730d78e85eed22c2b829b1bfc4d44a2560223db14913",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "b0cf21bf27f200003b49114240dd961527b95fe1ea1f273726d8d630f7506cf9",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac",
        "0d7d1211abfb3097fff9c179fab7dbf9f8d92d35c450072cdc237b4f3d8ea82a",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad"
      ],
      "example_id": "5726701add62a815002e84c8",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.35956676,
      "neural_order": [
        7,
        4,
        6,
        8,
        2,
        1,
        12,
        0,
        3,
        5,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "28619bd01cdbc0643bde9e77c4a03b78370f665517cf3f8440e6ad322e3a8d37",
      "trial_index": 506
    },
    {
      "answer_sha256": "26f1de33979d065ba8d86789de634228e3540fee2f6e5a66eebf93f78d83077d",
      "candidate_ids": [
        "57115f0a50c2381900b54aa7",
        "57097141200fba1400367ff9",
        "570d47b8b3d812140066d62d",
        "5730a951069b531400832213",
        "5726a638dd62a815002e8bf6",
        "57338160d058e614000b5bf9",
        "572671a5f1498d1400e8dfe4",
        "573749741c4567190057445d",
        "57263cfcec44d21400f3dc8d",
        "57297781af94a219006aa4a3",
        "571cd5b1dd7acb1400e4c178",
        "5727cff1ff5b5019007d95c4",
        "57269016708984140094ca41",
        "5725cb33271a42140099d1db",
        "5726a00cf1498d1400e8e550",
        "570d4030fed7b91900d45d9f"
      ],
      "candidate_sha256": [
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "450ad678c6861fc9db13e4beb27c521357425d7920629491fe1bc940fa4ede2b",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "3d7e59aba330ef5a0fd09e9c266887dee9cde9452fe5bebabceb91e21abd669c",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "3a6d439fe6e7862d1ba7ec845fce9f16deb80c798dced2b2a186946b4a7c57db",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b"
      ],
      "example_id": "572671a5f1498d1400e8dfe4",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.35565003,
      "neural_order": [
        6,
        0,
        2,
        9,
        11,
        10,
        4,
        1,
        13,
        7,
        3,
        5,
        8,
        12,
        14,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "436cd3f10c5b9f4f90c574771fdb3d718c971e17f240587fcfbfbba8b1605410",
      "trial_index": 507
    },
    {
      "answer_sha256": "1aa4a107dc2a0437d19329e9fab8fc4cc79ecd0af3bcc93ddeee25dee807c8bb",
      "candidate_ids": [
        "5725d662ec44d21400f3d687",
        "572a005f1d046914007796b7",
        "572753335951b619008f8853",
        "5729fe5c3f37b3190047862f",
        "571ccc00dd7acb1400e4c15a",
        "57332a734776f41900660726",
        "572978f91d046914007794d3",
        "572faf74b2c2fd1400568347",
        "5733f1784776f41900661575",
        "56e17644e3433e1400422f40",
        "572673f5708984140094c69b",
        "572826634b864d19001645be",
        "5727aec03acd2414000de991",
        "5727dc473acd2414000dee43",
        "5727cff1ff5b5019007d95c4",
        "5728facd4b864d1900165170"
      ],
      "candidate_sha256": [
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "b37ba322136b8858601bb5358e7576cd8b84e6f3a8fd1363ee9b99f1196e5836",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "3a40c66bbb21cb9a2998905cc0864a218331098a81b887689e124ee36e4285f4",
        "8390cff73f62c80823b3b17768d83f7ebde3afbf065a93bf7b338250bd3b3770",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "afa714a72ebd441495497201ca8e5b34e7410ce35809dcecc075c5b3d84568f3",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac"
      ],
      "example_id": "572673f5708984140094c69b",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.39989088,
      "neural_order": [
        10,
        9,
        15,
        4,
        6,
        0,
        8,
        2,
        7,
        3,
        12,
        1,
        5,
        11,
        13,
        14
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "85f7c6c2ad40c6a73f22bdf50a5d2ceeeef96355efcac733dbe41344ee39e1cb",
      "trial_index": 508
    },
    {
      "answer_sha256": "31b8d415438c9117b30ed898078e5f8cafbfee6055a851143990027f4ec25b67",
      "candidate_ids": [
        "57378c9b1c456719005744a7",
        "571a4b0f10f8ca1400304fd5",
        "570d32e4fed7b91900d45d01",
        "572fae4b04bcaa1900d76bdf",
        "57297bc9af94a219006aa4c7",
        "57264865dd62a815002e8062",
        "5729f60caf94a219006aa6ef",
        "57300911947a6a140053cfb6",
        "573004bf947a6a140053cf56",
        "5725fb8138643c19005acf3f",
        "572659ea5951b619008f7051",
        "572ffe6fb2c2fd14005686ef",
        "57107e6ca58dae1900cd69f2",
        "5733266d4776f41900660712",
        "573083dc2461fd1900a9ce6d",
        "57267640f1498d1400e8e074"
      ],
      "candidate_sha256": [
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "3655da224fa6e564876c355abd0b1207caca32ca4b67b3fa6aa41fa709a2c652",
        "0823871567c926124407b4674df9aa4e8bf9d7846b67df2c44d1b37fc733a0dd",
        "e154bf723ad4445d0e204018449b16d521e83192271d4e25bbadf5ea7feca6e4",
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "d56f2fcb959ed2126fc7f3de90e5e8ec9a6c058dcfe2d222afbcf158fcc4dc06",
        "808648d31e3c0a580e378e5e38b6e262d80eae03ea2f5f3343f713667cc29b1d",
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "69c99e63810dab9d4dcfccfa3d602efa9e3a5d06f950ccc90287eb4cb0e3afa1"
      ],
      "example_id": "57267640f1498d1400e8e074",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.21290228,
      "neural_order": [
        15,
        14,
        10,
        3,
        9,
        1,
        11,
        0,
        2,
        4,
        5,
        6,
        7,
        8,
        12,
        13
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "af9673a22181ed45f0b85503908f0c2acefad3dffb0aa48f48645a5b9c6afda3",
      "trial_index": 509
    },
    {
      "answer_sha256": "d1dc1f4a9f1a64cadba04041cbc1cf8803e9f80e362d3f9d09c3161a3ef08b81",
      "candidate_ids": [
        "5726534d708984140094c26d",
        "573083dc2461fd1900a9ce6d",
        "57281940ff5b5019007d9d44",
        "572677e7708984140094c723",
        "572998673f37b319004784d5",
        "57294e6b1d04691400779275",
        "5728202c4b864d19001644ec",
        "57060a6e52bb8914006897f8",
        "5730b6592461fd1900a9cfcf",
        "5733a32bd058e614000b5f32",
        "57287f6a3acd2414000dfa51",
        "57264991f1498d1400e8db2e",
        "572754dd708984140094dc3c",
        "572647e2dd62a815002e805c",
        "5727ef664b864d1900164060",
        "572a070c6aef0514001551c8"
      ],
      "candidate_sha256": [
        "8f276d70be8d055bdb87d70110f37f7c7b5d5e15c897613a49a56e3869f11913",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "56c5a58a20c281345ad1b156034653805b4b37502a76613ea7935761d3263ade",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "b483ecccba6a52ccfd793181e241aa17529265c1f2f0924661f7092091552717",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "af1de7092c5c75bf8d6d491ca1d950d37ed13921f9f555316d339ab9aa41092c",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b"
      ],
      "example_id": "572677e7708984140094c723",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.3304844,
      "neural_order": [
        3,
        14,
        5,
        8,
        13,
        6,
        1,
        4,
        0,
        10,
        15,
        7,
        11,
        9,
        2,
        12
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "001485f87f29534ea1fb28d8cf75b07430c792b14bffaf96c9c39d9ba935932b",
      "trial_index": 510
    },
    {
      "answer_sha256": "414a7bf0f84b541654c0228d2f124147302b4ba884d237beb4d0126bfb0f5b99",
      "candidate_ids": [
        "56e1ec83cd28a01900c67c0a",
        "572885c44b864d1900164a78",
        "5727d1c93acd2414000ded3f",
        "5725bcb6271a42140099d0eb",
        "56de3cd0cffd8e1900b4b6be",
        "572ffd9e04bcaa1900d76fc7",
        "5726a3c6f1498d1400e8e5ae",
        "572973f76aef051400154f0b",
        "572a0f073f37b3190047867f",
        "57109180a58dae1900cd6a40",
        "5726545f708984140094c2a5",
        "57267947f1498d1400e8e0ec",
        "5726a09f708984140094cc39",
        "57378b141c4567190057449f",
        "57274f67708984140094dbf5",
        "56de40da4396321400ee2708"
      ],
      "candidate_sha256": [
        "aa681b6c7d1a2367ebf36bb39734e4d4b923b5605ff91c3720031204a2a479a6",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "dd048906ab302fce12c3ce4e44838277e35ed59f6a19cbfff81607b84d7d408a",
        "889592b3e637c0400bdbe22ed06f541a89a8b1c11477b6a55029e9af6bd2131d",
        "f3b1e19e4395205ce512bdbfdf599fe533bc1368c47793208ea4922b7c8559ab",
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "dd04072038d1a3ede59bf62dd3dc9ade4227b046627d4885431216fa66204dee",
        "8b8190fdb07a1af9c1462db72bed5dbe5d3d2445992eda03965818ceb4ef0857",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "c2aa9999cd49b96b950c98756b6cb8ee69d8e051665b4e9b169149c7c2fe349d"
      ],
      "example_id": "57267947f1498d1400e8e0ec",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.33751124,
      "neural_order": [
        11,
        4,
        8,
        10,
        12,
        3,
        2,
        5,
        0,
        14,
        1,
        6,
        7,
        9,
        13,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "f1debc9f1a4cb58ddfeff01fa1afc7b15d3a25cf5125b52950ff8c0363b30f40",
      "trial_index": 511
    },
    {
      "answer_sha256": "c3bec6777732b3c05dd8bb492ea37ddf9d27bd657c4ef77e2130e99d5528faf7",
      "candidate_ids": [
        "572ff760b2c2fd1400568677",
        "572647e2dd62a815002e805c",
        "5725dd7d89a1e219009abfea",
        "573011de04bcaa1900d770f9",
        "57293c246aef051400154bb8",
        "57286ab72ca10214002da31e",
        "57273e50dd62a815002e9a02",
        "57264fe65951b619008f6fa1",
        "57265526708984140094c2bd",
        "57265aaf5951b619008f706b",
        "570d28bdb3d812140066d4a3",
        "57286d7d4b864d19001649e0",
        "572a1a5c6aef051400155284",
        "5727c69d3acd2414000dec13",
        "57267b3c5951b619008f7423",
        "571a4d1a4faf5e1900b8a958"
      ],
      "candidate_sha256": [
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "7e9ee265449fa22688d3f8cc460a08187bf8f5ac34d69544390c50c250eb0448",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "844e8303cc9e09a9a85efd51caddcc2513491415703eebada9fed8f443460d31",
        "df268bb105020f67580468ccdf8fabe37da3958a51a789b0a115440e53bc06ac",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "679435646d97368fa3947fb204446db071897fb9320a01dbbfeaadb616a2c50a",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "1f0fc4793c092f70c224c8914502cf9b4ed7474bba7dcc89bbcfafda36029ad9",
        "ba4b5380e10df31bf28d52a0694e9f4d2d11dda0dc4a8b72368a73abf8c1e23a",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "63083aa84294da2de8401c0b772521bbfe426e2587fa80448b3830f78d5bb8ae",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8"
      ],
      "example_id": "57267b3c5951b619008f7423",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.49547106,
      "neural_order": [
        14,
        15,
        9,
        6,
        0,
        4,
        10,
        8,
        1,
        2,
        3,
        5,
        7,
        11,
        12,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "5f11105f7f3f0453a7597761b4a3b7046d53f7d42c8a43ed81c96317d8c251b3",
      "trial_index": 512
    },
    {
      "answer_sha256": "c9cfc34ee1f512f1c9fd96b344df0e930c368bf1fb425184f927e20cc503828a",
      "candidate_ids": [
        "57264a74708984140094c18b",
        "5737aafd1c456719005744fb",
        "5729efab3f37b319004785cf",
        "571135b8a58dae1900cd6d0e",
        "571cac5d5efbb31900334dd0",
        "57114aceb654c5140001fc47",
        "5728223cff5b5019007d9dc4",
        "5729517d6aef051400154cc8",
        "5733e771d058e614000b6545",
        "57115b8b50c2381900b54a89",
        "572864dd4b864d1900164976",
        "5726f2375951b619008f830f",
        "57267d52708984140094c7d9",
        "570960cf200fba1400367f01",
        "572fdc34a23a5019007fca93",
        "5733f7b9d058e614000b66a9"
      ],
      "candidate_sha256": [
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "11e489536cc7ac71a8ee73d6c7e52469ad3ad8807c404fd6cba2907c8189cab7",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "444f24a8544c7dc22408fef82ea8acb119de07b7303bd5d88d18de6abc73df01",
        "f6b5085bcb5e1d0be2800bcd57e042898b9faa4c93c0198f7f6f15f73eb8f4ad",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "ab6a56d5f9eb9f32fe7cbcd4297ecd8cbcee3a6a500f85ad70bb607c6bc05c7b",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "5d46e9f7c620b7d8f42a0c90216d4bb7625ee95e4d9af99b663434ae1e08a483"
      ],
      "example_id": "57267d52708984140094c7d9",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        11,
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.59137617,
      "neural_order": [
        12,
        4,
        11,
        7,
        1,
        2,
        15,
        0,
        3,
        5,
        6,
        8,
        9,
        10,
        13,
        14
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "68c8b30c5585117617d49fc3a7edb64f32cd0ee68bab414655eada1f26ee9381",
      "trial_index": 513
    },
    {
      "answer_sha256": "7784235e28151b7b23839b44b9c9c5aa563426bfef456672bf1f7da9f5fc340d",
      "candidate_ids": [
        "57286fa83acd2414000df9e5",
        "5733ea04d058e614000b6594",
        "5725ff8238643c19005acf49",
        "5728facd4b864d1900165170",
        "5733fe73d058e614000b673d",
        "572a1a5c6aef051400155284",
        "57267ebfdd62a815002e872c",
        "56e1a38de3433e140042305c",
        "57264d9edd62a815002e80fe",
        "5726a638dd62a815002e8bf6",
        "573098f38ab72b1400f9c5d2",
        "5727f7523acd2414000df10d",
        "57283c464b864d19001647c8",
        "5729fa40af94a219006aa70f",
        "57287ddf3acd2414000dfa3f",
        "571cdcb85efbb31900334e0c"
      ],
      "candidate_sha256": [
        "c4fd59d733b1633198ab8d307fb4d275d55c73f2c3fa5e12deb57198789613e4",
        "255db98fb0b6e32abd3b8317106e8eeb1841b1f3860f985c69bb911aae85aadc",
        "2313f2d61a682bc65173ff0058fcaf0f307a9e3925747c90df3c178cdcf53308",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "54dc2ba078167fe1c832fc71dd808ac1358612b92a5802c139d6ae3350ceacd2",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "6e771f0502b0ffe4b6f4d609fc4c890fa8ed0de3b50ba8cf080f14d1b0e4278f",
        "9819559c80db18970983cbfdf2bcb3f75443deb410b562890bbe13cea74a03dc",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "374416a10bbeb03a4bf4e621397e249423ee2b3129054d9c1c003741b4d6e541",
        "d3edef5595bf65279f954f6a5aa570bfc1f15283c4ba84a74fed072800d92e91",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75"
      ],
      "example_id": "57267ebfdd62a815002e872c",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        1,
        0,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.51257038,
      "neural_order": [
        6,
        11,
        14,
        1,
        2,
        4,
        0,
        9,
        10,
        5,
        3,
        7,
        8,
        12,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "894f543065bc1d178ed3ccab2b24e248164b227ae09a15095b6a93c1ada9cc5d",
      "trial_index": 514
    },
    {
      "answer_sha256": "7a855d14e534fd6fef2033dea862766c543673f6774a4e9644b5936ebe4ee715",
      "candidate_ids": [
        "56e16182e3433e1400422e28",
        "570953a7efce8f15003a7dfe",
        "572863363acd2414000df985",
        "57060a1175f01819005e78d2",
        "56de15104396321400ee25b7",
        "56de0ffd4396321400ee258d",
        "5725ce4d38643c19005acd4d",
        "5725d42a89a1e219009abf58",
        "56e1a0dccd28a01900c67a2e",
        "5728809f2ca10214002da40c",
        "572f65e9b2c2fd14005680cb",
        "5705e4fe75f01819005e7704",
        "5711541350c2381900b54a6f",
        "572fadcbb2c2fd1400568329",
        "57268066708984140094c821",
        "5728f9882ca10214002dab5a"
      ],
      "candidate_sha256": [
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "8082f6ac9145af8d3dd02d6d8f221e3f2ea140fd3ea974adc4ce26b460334eff",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "8d0f136ca3e21898494994833851f5c381fc610d00c5a286083f4b15d65caad4",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "e83796106923fb4c041f48e03aac86a7ce1c0ca388e14d335f5261ec964785b9",
        "3bafbc5d56c1857ae4a5346827458b7900e775e4ebe79997c59538a3013f8c57",
        "dbdab84ab2698351a7891a905e5c427429c23ef61bbd63fa6959258a98080440",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "fbdd4577c5f9e5f4af71320383d43f13f7697f04d5670525f7d260f7ff56759d"
      ],
      "example_id": "57268066708984140094c821",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.53415697,
      "neural_order": [
        14,
        0,
        9,
        3,
        12,
        11,
        8,
        5,
        13,
        2,
        10,
        4,
        1,
        6,
        7,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "521c19985926fe54bb836c761d9ef261d4edf5c05f9385df47c69c1e6ecf1467",
      "trial_index": 515
    },
    {
      "answer_sha256": "a093bc84df3debe38542af2fe823f12819c2468e94b337279e3dfd8dc0af4cad",
      "candidate_ids": [
        "573004bf947a6a140053cf56",
        "57265360dd62a815002e819a",
        "57268220f1498d1400e8e216",
        "5725cfd0271a42140099d225",
        "573093598ab72b1400f9c5ae",
        "5727c0402ca10214002d9564",
        "571cde695efbb31900334e16",
        "57373f80c3c5551400e51e8f",
        "56e19724cd28a01900c679f6",
        "57268527708984140094c8bf",
        "57282dfb4b864d1900164668",
        "572ff673b2c2fd1400568669",
        "573010fab2c2fd14005687d7",
        "572ff83ab2c2fd1400568681",
        "5729fc3d1d0469140077969d",
        "572855973acd2414000df925"
      ],
      "candidate_sha256": [
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "3b25729dedd14dc1875957be2bee80801b54dcf03df63728e1bc796da721e86b",
        "6740991a42bdf84c8cdf4e82a485806a299304b992b7b57dfa26d34e0d97ab8f",
        "ccfb450b288c15536e52e7e1c54aa458c0e2c2936babc78f6cab1b418fa9e1db",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "4ddd2ea4a2c377fbff94602ffde9e3feead6020c3937f8e6feda19f75b6fd619",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "7cc34b2ba0d428283f531c95c3e77a52e090693b4ea5bacd2464f1fa3615352b",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "5b7d91c3c1eef1abc10a08a04fb83add5c8ea276568a385e4d806fe0c301e1b9",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5"
      ],
      "example_id": "57268220f1498d1400e8e216",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.43495243,
      "neural_order": [
        2,
        9,
        5,
        7,
        15,
        6,
        11,
        13,
        8,
        0,
        1,
        3,
        4,
        10,
        12,
        14
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "d0a3abaeb9857145c10ed41c8bd44f0dd924f832dead8f2dda5508c084f300a6",
      "trial_index": 516
    },
    {
      "answer_sha256": "97c257107ed2b6970507e782fa0fb5a381fd514c0d387b031562c2299e66c5cf",
      "candidate_ids": [
        "572815d9ff5b5019007d9cbc",
        "57332b66d058e614000b5758",
        "5733647e4776f419006609ae",
        "57340d124776f419006617bf",
        "5729efab3f37b319004785cf",
        "572a1c943f37b319004786e1",
        "57265746dd62a815002e8218",
        "5725bad5271a42140099d0bd",
        "5733e8ae4776f419006614a6",
        "572ff86004bcaa1900d76f65",
        "5728eff82ca10214002daadc",
        "572683f95951b619008f7525",
        "572867d72ca10214002da2fc",
        "572a0e4b6aef051400155214",
        "572f76d1b2c2fd1400568153",
        "573010fab2c2fd14005687d7"
      ],
      "candidate_sha256": [
        "919235ba6c22297c3a405977fe037b089dff6a386086b7bb0f965682bdfedce3",
        "20e502c5589b046d2ff5af04f1eb121472870964f751ab6cda67e9a705f620b8",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "be1e8ef67e19e07dbf69a2cd82b160a507b81ad4a6ae00fb9fbdeda6ebbed945",
        "835545721185c45eb240b5885c89523078762308bf812877fa20ea8badd1dcce",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "bc680335bd36aca0f4b984d60daa7f4a20ca5645844505b98fab0ce65a3cba6d",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "80a90cffe2844e64f980557e6dcea7aa0941bd898ec4710286e8187b5caa1fb2",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "d36d2c6e10f91eeb155dfafe4984e01adacf0f649660a8b602787ecfda42c36f",
        "4256a08b1b95d9e5abfa0b8d1dac5426d5a98dc261f94a8cf5a819e2533f3023",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb"
      ],
      "example_id": "572683f95951b619008f7525",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.31093302,
      "neural_order": [
        11,
        0,
        7,
        4,
        14,
        8,
        5,
        2,
        15,
        12,
        3,
        6,
        9,
        1,
        10,
        13
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "3dcf280931b5f5cd8a0ebae456183a4fe7b14437f06f156432767c7431602ffb",
      "trial_index": 517
    },
    {
      "answer_sha256": "c50ae32a98a7bdb02faecea090218f93c843c702f0e7ecfeb31b2951160a62a9",
      "candidate_ids": [
        "572673f5708984140094c69b",
        "571a4d1a4faf5e1900b8a958",
        "571cc5c45efbb31900334dda",
        "572759dbdd62a815002e9b94",
        "5725f00938643c19005aced7",
        "57267ebfdd62a815002e872c",
        "57268527708984140094c8bf",
        "5728fb6a1d04691400778ef5",
        "5725d7e438643c19005acdf9",
        "57265e97708984140094c3c3",
        "56e19557e3433e1400422fee",
        "572a058aaf94a219006aa751",
        "5733834ed058e614000b5c26",
        "56e1c9bfe3433e1400423192",
        "5728855d3acd2414000dfa8d",
        "57096b66200fba1400367fa7"
      ],
      "candidate_sha256": [
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "c65899f588f28ae99ef5f3d38d2aa89c81477ceeb9b5361a95c37457b815468d",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086",
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701",
        "492c00fb2c1b273fd0498cf2f1114aeba433fd8ffdf8443cd27726649c6e9bf7",
        "13e9fbf3902511de0c59a6c616fe4bb22971b931abe106864f6f57b7b66316da",
        "9b3485cd6afc7ef2acd1cc33457516a7c270e464012f70c61946d63c7c920549",
        "ff55d7c6237f0a29b1f8caa69651484d351b825239093247421026985d719352"
      ],
      "example_id": "57268527708984140094c8bf",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        7,
        14,
        12,
        2,
        1,
        0,
        3,
        4,
        5,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.35731675,
      "neural_order": [
        6,
        5,
        0,
        1,
        8,
        9,
        2,
        14,
        12,
        10,
        13,
        3,
        4,
        7,
        11,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "3a34e205c41ae860b49b1775d64699fab81cc2cf66f3f0d8d41b7547bf13ce32",
      "trial_index": 518
    },
    {
      "answer_sha256": "57e2e04d1ccd4acf967500bb1b40b9de40573ecd94843729b33b7e6aff6d039a",
      "candidate_ids": [
        "572686ac5951b619008f75a9",
        "5729506d6aef051400154cac",
        "5727aec03acd2414000de991",
        "57300c67947a6a140053cff0",
        "571cc6f85efbb31900334de4",
        "573005b9947a6a140053cf6a",
        "56e1b8f3e3433e14004230e6",
        "571ce5055efbb31900334e2a",
        "572a0c541d046914007796f3",
        "572881704b864d1900164a50",
        "5725c63438643c19005acc9f",
        "5726c5a9f1498d1400e8eac4",
        "572ffc0f947a6a140053cef0",
        "5706155352bb891400689894",
        "572949306aef051400154c68",
        "57309bfb8ab72b1400f9c5e6"
      ],
      "candidate_sha256": [
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "0f1150362aa3102a7829de61ce4354d7911643dfdede6c2e8ce3aaaeba9a89f5",
        "afa714a72ebd441495497201ca8e5b34e7410ce35809dcecc075c5b3d84568f3",
        "9be402bb7a93ae75db4b317f4e0e2c040d9f89ef6aab3066e3a569ea09e6d56e",
        "c456b31b38f7b3d656d571ec7289b0630f5f2d8ce31a3965fddb4291053d73ef",
        "02626edefba54656b37e389ca8d9b91f2b351682bf3930851c24a8232cd02ef4",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "e13a9fb9ad133adb340be1865de5d045a2a0397c5c6a1dce73592e4fef97318a",
        "d7cb5bbd861e7e56919b271b776d170cfdbcdb0f8453e34c21c10b5da74495fa",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "08d567948b98aed939d67c729938fa0fdd9dbd74ffe76809a3f4fd07fdd24bc4",
        "da7e3caa2252604ebed016e003b315efd2a4a47f3b063950ffca6d27e6dd03f2",
        "60063789c84ce9bcd013a4935de8903d015c08487be5955e01c7c8279a0cadf7",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee"
      ],
      "example_id": "572686ac5951b619008f75a9",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        4,
        11,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.5876341,
      "neural_order": [
        0,
        10,
        3,
        12,
        14,
        9,
        6,
        4,
        7,
        2,
        1,
        5,
        8,
        11,
        13,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "9ace1c6c8de2a87b47b3f779d0390d008b52d5c94f1577c708512002ea6eb1e5",
      "trial_index": 519
    },
    {
      "answer_sha256": "724bece71fd9514e85454bd8b6ee672742254ba31d6ff4845b3f6c487144cef1",
      "candidate_ids": [
        "5730a0778ab72b1400f9c60a",
        "573085ea8ab72b1400f9c54c",
        "57268882f1498d1400e8e306",
        "572ff760b2c2fd1400568677",
        "5728fd3c3f37b31900477f2b",
        "5705ec1675f01819005e776e",
        "572fbf21a23a5019007fc937",
        "57340b1bd058e614000b6869",
        "5730876a396df9190009617a",
        "572fd264b2c2fd14005684a9",
        "572fbfa504bcaa1900d76c73",
        "572a12386aef051400155234",
        "5733dd4f4776f419006613aa",
        "57108073b654c5140001f925",
        "5728759cff5b5019007da25a",
        "57283f014b864d19001647e8"
      ],
      "candidate_sha256": [
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "0937633978b30fe3006833a53321968630b8eeb2032caa208942bba9ef8bced1",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "101cbad929a29d849498c9f76cb12e8cd254bed0e11856ba0d2a3ad7e2da7af1",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "58de06c671d2cd29f3b8bde136326194d51c92607fc4a624d1d5a6e90dd5cf33",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "1ace3a4bb317ab9f101e7d2a71bc4fda5a5d07685b9f2adbb37eda80a377af4f",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975",
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55"
      ],
      "example_id": "57268882f1498d1400e8e306",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        15,
        0,
        6,
        7,
        11,
        1,
        3,
        4,
        5,
        8,
        9,
        10,
        12,
        13,
        14
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.2557663,
      "neural_order": [
        2,
        12,
        8,
        15,
        7,
        14,
        13,
        10,
        9,
        1,
        4,
        0,
        3,
        5,
        6,
        11
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "846b28f1c4c96ce845bfa34ff446f16f7031919cce9a7a187d801213a98afd22",
      "trial_index": 520
    },
    {
      "answer_sha256": "98a617f3a2b55139c3f10b99de105e90298087c1f138934e137c6b26b9087f3d",
      "candidate_ids": [
        "5726bc1add62a815002e8ea6",
        "5725c69738643c19005accb9",
        "57273cca708984140094db33",
        "5728661e2ca10214002da2e8",
        "57060df252bb891400689820",
        "572989846aef051400154fc0",
        "5733a6ac4776f41900660f58",
        "57297bc9af94a219006aa4c7",
        "57264845f1498d1400e8db0a",
        "56e1b00ce3433e140042309e",
        "5726d8bd708984140094d35b",
        "5728eef92ca10214002daab0",
        "57269f3ef1498d1400e8e534",
        "57294baaaf94a219006aa26b",
        "572a0bf96aef051400155204",
        "572fc043a23a5019007fc95d"
      ],
      "candidate_sha256": [
        "d59faf758de4ddf104de60dfc826f1e0f46b8d2e1e3ff2417493277fe9fd0d56",
        "7f75accf663d59c998893e7d8881c9407df5f03def38be598706225dc5ae7b6e",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "49268cef0d8b03651b6eaeb5bbc7082e1e05af0e4b27382b395e3fae85fb397c",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "e154bf723ad4445d0e204018449b16d521e83192271d4e25bbadf5ea7feca6e4",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "2fae4c948d1afffe2006da4982e6c6de310d14a0d328a01906062fde6bcaa3a1",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "f3e994c58b1e4906937c035c80fb9519764a0f4f42ff12a50c7aeac3190ebd39"
      ],
      "example_id": "5726d8bd708984140094d35b",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.61737868,
      "neural_order": [
        10,
        8,
        12,
        4,
        0,
        3,
        15,
        1,
        5,
        11,
        2,
        6,
        7,
        9,
        13,
        14
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "e10f32502c6f0d424f67534f85612c5f537fa23a1af33dd22dd8de8312aa8af7",
      "trial_index": 521
    },
    {
      "answer_sha256": "90493f0147f7d8774453374e1b40675a7a2171363db67cea0243f3e0c1bfa9cc",
      "candidate_ids": [
        "57300761947a6a140053cf9a",
        "5737534ec3c5551400e51eab",
        "5726d9935951b619008f7fed",
        "57286192ff5b5019007da1dc",
        "5727d1c93acd2414000ded3f",
        "571ce5055efbb31900334e2a",
        "57265bdfdd62a815002e829e",
        "572872822ca10214002da374",
        "5726f0e35951b619008f82f7",
        "56e1ded7cd28a01900c67bd4",
        "573062662461fd1900a9cdf7",
        "571cdcb85efbb31900334e0c",
        "572a03086aef0514001551a2",
        "57264845f1498d1400e8db0a",
        "5733266d4776f41900660712",
        "572847ff3acd2414000df869"
      ],
      "candidate_sha256": [
        "38f1fc8afb6921f54a832d6421c87a2e55173666a1b767db315cbf17cf3ce9c0",
        "d2c301f3e303e0529f1b5862d9c77adabe2cb026b4683982583601544f992a66",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "e1b10ed004e12df6dceedd14f615055081a60be7a8c36c4491296355382176a6",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75",
        "83c6ede3febbd8b4bf4e518d11f85303444c8ef12433662ff23528b2abb801df",
        "f9f2b1b74b5a275b676569a3c1b48baccbc9f8dcf3798a18c7994b729372aa32",
        "e63a90bf9537e28fc15cdb1fa48c249e27bd148da523998fd18858bb7e471395",
        "c44126e5a71b148722d5f363b3fe85277c50806c0cacb2d5d467d0ee1ef9ca2d"
      ],
      "example_id": "5726d9935951b619008f7fed",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.46911704,
      "neural_order": [
        2,
        8,
        12,
        9,
        4,
        13,
        11,
        6,
        15,
        0,
        1,
        3,
        5,
        7,
        10,
        14
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "7a3973064cc38e5cccc189b878380c427d3ede39b00fc9736def0090fa214073",
      "trial_index": 522
    },
    {
      "answer_sha256": "73e0f751942de4cb4d4a38a204293a2ce37a9107f7197073bec26b4b3144d6db",
      "candidate_ids": [
        "571a4d1a4faf5e1900b8a958",
        "57293ca73f37b3190047815b",
        "572a070c6aef0514001551c8",
        "57274e145951b619008f87e7",
        "5733d3cb4776f419006612ea",
        "5727d6faff5b5019007d969a",
        "572faec7b2c2fd1400568333",
        "57274a8ff1498d1400e8f5bc",
        "572810ec2ca10214002d9d06",
        "57111380a58dae1900cd6bd6",
        "57293d116aef051400154bc8",
        "572a06866aef0514001551be",
        "572fbea404bcaa1900d76c5b",
        "5730208fa23a5019007fcded",
        "5726da89dd62a815002e92b2",
        "570d2f5bfed7b91900d45ccf"
      ],
      "candidate_sha256": [
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "ef18442bdf437b3bbcd356b07c0f74d9141147b11850e926a227a44b0d66bf1c",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "06e886aacc94d738c65efee588174362a18838860a9be95343e59e88cf82c6e1",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "d624d91df9c8008b8e784b1043b9d422a861deabd5189cd8991fbff2eacaca87",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "f012f955e53efdd1ab43f86e284322a35a15ee61e762f9e898a87a0f6e2e19c3",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b"
      ],
      "example_id": "5726da89dd62a815002e92b2",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.62308369,
      "neural_order": [
        14,
        1,
        12,
        11,
        5,
        6,
        10,
        15,
        0,
        3,
        9,
        2,
        4,
        7,
        8,
        13
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "707d4ef803af512d051afae58b5f426c48dd715ac956148f5b00663ff3ec43c3",
      "trial_index": 523
    },
    {
      "answer_sha256": "ba4636f35c181d697df58cee935fbaac8af6d8f6318f8f5ace5e61c166981b62",
      "candidate_ids": [
        "5727dc473acd2414000dee43",
        "56de3ebc4396321400ee26e6",
        "572756265951b619008f886e",
        "57265700dd62a815002e820e",
        "56e1ba41cd28a01900c67ae0",
        "57112686b654c5140001fbd3",
        "5726d9935951b619008f7fed",
        "572a12386aef051400155234",
        "572872822ca10214002da374",
        "5706094b52bb8914006897de",
        "57286951ff5b5019007da20e",
        "5726db5add62a815002e92d4",
        "57338255d058e614000b5c0d",
        "57293d6d1d046914007791b7",
        "5726b718dd62a815002e8dbe",
        "572681c1dd62a815002e8796"
      ],
      "candidate_sha256": [
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "ce72fed7b084941e90079823ae33cc19adb21829e9f5aa99491ba4d96d782570",
        "5df020d955be1c7df32e882ac62108a50dec46bd1b1b064de8825485d8d90aba",
        "a21efd80668b227ef68dd022f77a31b092d02d626d2e188520af3bf6059d57dc",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "a310999bb352084aaea2b5f616253f626dcd988d9dbb861b7e065dfe2b33c1a8",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14",
        "082c3acd69dee4bbe9a4c27102d2530b3abbd5a086538dce498b2070231b63b3",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "96c954021714a1e6c2685d867965c36da19e03a77e71f8ae95eed9cf64ffcccc",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "9a1d3f81d55fe6e9af70adf663cb163faf4656522893b668bd17f7a470d9cad8"
      ],
      "example_id": "5726db5add62a815002e92d4",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        10,
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.37861862,
      "neural_order": [
        11,
        6,
        10,
        15,
        3,
        14,
        1,
        12,
        2,
        4,
        8,
        13,
        0,
        5,
        7,
        9
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "2a6062305ce56a8843e6c1ea8b02d06ceba4705fbcacc19c61db505b8335c3bb",
      "trial_index": 524
    },
    {
      "answer_sha256": "4e6e9781116e81e1b3bda2dcaf03bb29a3420ee84a8874735d5284c129cd3135",
      "candidate_ids": [
        "571094b7a58dae1900cd6a66",
        "5710eca0a58dae1900cd6b3a",
        "56de15104396321400ee25b7",
        "570d29c2b3d812140066d4bd",
        "56e1ddfce3433e14004231d5",
        "57284b904b864d19001648e2",
        "56dddf4066d3e219004dad5f",
        "571ccc00dd7acb1400e4c15a",
        "57106d2fb654c5140001f8ef",
        "571cda1bdd7acb1400e4c196",
        "5725d7e438643c19005acdf9",
        "573735e8c3c5551400e51e71",
        "5726dcbddd62a815002e9320",
        "5726deed5951b619008f80c5",
        "5733e771d058e614000b6545",
        "57111ab8a58dae1900cd6c3c"
      ],
      "candidate_sha256": [
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "00490c1aee74110c3cd1afd78d24a5278d94f9a92455ff33a952663986874447",
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "57ea608fc0f4249a93c9de52fc73db3ff0e90f245a9c336159e74c986a34743d",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "6a4859b84af16a97c8e1dcf1acad6e45d077ea9ef8ecf5e9a80b9bc201a26760",
        "dd790b7fdd62790386db0c2a1e657cb576e6313042d6722d73c517fa827ed359",
        "385d26c2d9938088181ecd6b3120582d075635f9838e132747efaaf577e28109",
        "730ba2104466230111a89907773728154408650beef8a8f777c6ef19249e8d1b",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "80e800db40294efc728c4dcbf06c36437000cef741a99c5f9ed99d6d0756badc",
        "2614908637c6c0562a8695ddb732fc647887bd1b1a4c6cfe9a586d68b00f747c",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "c4f7a9257e453b0a752577b87b26fac2f158793df00e5ae7513ed4aafac440a2"
      ],
      "example_id": "5726dcbddd62a815002e9320",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.39902243,
      "neural_order": [
        12,
        13,
        2,
        15,
        11,
        7,
        0,
        3,
        1,
        4,
        5,
        6,
        8,
        9,
        10,
        14
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "6d8c66477813354b3952dc24996b278d5673383abdb118997a3c2515f02c07b9",
      "trial_index": 525
    },
    {
      "answer_sha256": "96b488508cb3cf75894569a224fc2b5f63e25ffabea17475792436a5ccbc9092",
      "candidate_ids": [
        "5727490bdd62a815002e9a82",
        "56e1ce08e3433e14004231a4",
        "5733db8dd058e614000b6428",
        "572fd47fa23a5019007fca55",
        "5733a5f54776f41900660f44",
        "572671a5f1498d1400e8dfe4",
        "572673f5708984140094c69b",
        "56e1febfe3433e1400423236",
        "5733fc6ed058e614000b670f",
        "5726ddf6f1498d1400e8ee04",
        "57340a094776f4190066177d",
        "5728151b4b864d1900164428",
        "57267b3c5951b619008f7423",
        "573085ea8ab72b1400f9c54c",
        "56e2042ecd28a01900c67c1e",
        "5711475ca58dae1900cd6d8b"
      ],
      "candidate_sha256": [
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "7544f3f342f82852b6fd8a74d46285e0772381cc4cd0b61c47fea00590e8938e",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "80050447c9b41fb5fac1950338fd3ace00106108cadec17c27aa5b41f7aee5cf",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "d0c3193100a2c9cefa5efa046ca711a3a7943ddbb731e1aa29b247fe5eb27c7c",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "48408b13d0ded7ecf339ad91fb7341eec773b90b40878f7f40a4e65b40a92268",
        "0771c41591ce859133397b70f7fe295ca178804c352b68cc34f3f35caa1c92ab",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd"
      ],
      "example_id": "5726ddf6f1498d1400e8ee04",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.51078746,
      "neural_order": [
        9,
        14,
        4,
        15,
        12,
        1,
        7,
        10,
        0,
        2,
        3,
        5,
        6,
        8,
        11,
        13
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "aef20b8bb2e7eeeb09da5a9841befc9b3b21b1e450e6d4609b7398c8995f9d40",
      "trial_index": 526
    },
    {
      "answer_sha256": "845aab778f5f137db62ee9d8f87c24797456ad8f6b148d8f38acf858ce11ef80",
      "candidate_ids": [
        "57300888b2c2fd1400568775",
        "5726deed5951b619008f80c5",
        "5729fd56af94a219006aa72f",
        "57274beff1498d1400e8f5e4",
        "57332e48d058e614000b5762",
        "57340549d058e614000b67dd",
        "5710eb6fb654c5140001fa17",
        "5725ce4d38643c19005acd4d",
        "5726eb4b5951b619008f826b",
        "5705e26d75f01819005e76d4",
        "570966e0200fba1400367f4f",
        "5728ed94ff5b5019007da97c",
        "571cb010dd7acb1400e4c128",
        "570d4a6bfed7b91900d45e13",
        "5728eff82ca10214002daadc",
        "57265aaf5951b619008f706b"
      ],
      "candidate_sha256": [
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "b793fdc89c85940b1b19b51325a87a32c522a99afb18a4e3bed15e221fc6419d",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "83c030609aaae7ee99b65c7e83ec20b9108afd9f06a6c12d3840a14fa398a72f",
        "9867017bbd82b6ead270fd4186419e6e4523be5168798139a5848593a87e9cb4",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e"
      ],
      "example_id": "5726deed5951b619008f80c5",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        15,
        5,
        0,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.26571601,
      "neural_order": [
        1,
        8,
        6,
        0,
        4,
        3,
        13,
        7,
        11,
        5,
        9,
        2,
        10,
        12,
        14,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "fc4c66d9d3de7455698dc24016033f6860aefcb01207fb62eaeea335cde2657e",
      "trial_index": 527
    },
    {
      "answer_sha256": "a01bf6acb3b1b470e15a339ef638d1833bf967d3ac0be32f400343fb377ee19e",
      "candidate_ids": [
        "5728223cff5b5019007d9dc4",
        "572759dbdd62a815002e9b94",
        "57114b1a2419e31400955575",
        "57094d489928a8140047150a",
        "57268882f1498d1400e8e306",
        "56e1c3e1e3433e1400423148",
        "572945b11d0469140077922f",
        "57114667a58dae1900cd6d80",
        "5706149552bb891400689880",
        "57264a0ef1498d1400e8db40",
        "5737a7351c456719005744f1",
        "570966e0200fba1400367f4f",
        "57060a6e52bb8914006897f8",
        "572a005f1d046914007796b7",
        "57274e0d708984140094dbe5",
        "5726e08e5951b619008f810f"
      ],
      "candidate_sha256": [
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "c18330dd0d345a1e5eedb14b423dc1fbbdd345ce4298bcde654d618cf5d24707",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "9fce3973f75ff4020fa9dafdc9f609d40df12aee27d2baae8850bee3e8614117",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "071a5970e18127aa4ea55223be8ce6067227160e20689d0175c1386528e5d93d",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9"
      ],
      "example_id": "5726e08e5951b619008f810f",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.36144897,
      "neural_order": [
        15,
        12,
        10,
        5,
        9,
        1,
        2,
        14,
        13,
        11,
        3,
        4,
        0,
        6,
        7,
        8
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "a1c1f9beae7324d752e13a3c2f1f4c9c99b28f2c87a62e847fff3d4a1fac603c",
      "trial_index": 528
    },
    {
      "answer_sha256": "cb2825eb2546062def53662213819b68295b6162c25e338a0dc981f6d66a195b",
      "candidate_ids": [
        "5705f7c875f01819005e77dc",
        "57339c16d058e614000b5ec5",
        "572a0bafaf94a219006aa765",
        "573784fa1c45671900574483",
        "5726e179dd62a815002e93ae",
        "573362b94776f41900660974",
        "57265d86f1498d1400e8dd50",
        "572ffb02b2c2fd14005686b7",
        "5733a5f54776f41900660f44",
        "5729f4e46aef051400155156",
        "572a142e3f37b319004786b9",
        "572860e03acd2414000df977",
        "5726e5c95951b619008f81cb",
        "5737821cc3c5551400e51f19",
        "57264d58f1498d1400e8db7a",
        "572905ce1d04691400778f83"
      ],
      "candidate_sha256": [
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "29045634dddebb4bde837e85fc0d8d0e2816fa9a90d810be5e7e7db6b4b81661",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "b7991d39b23b7c72fc2a048b728292818e15fadf9f4db417aefd475c9db7b8b2",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "a38345a7ecdd441e34d4cf32d9e5e2d6984d33e6becad500456638aba80cfa88",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "6c330844d7a86f35d0c2ad99b16e53b4a661cde9c2483cf5f3dbbd4be24235f0",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "ecce67f5ebd93f7f51596b468b36d1918c0f3d320dd7e6bd4fa7ac0ca537f812",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1"
      ],
      "example_id": "5726e179dd62a815002e93ae",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        11,
        15,
        10,
        2,
        0,
        1,
        3,
        5,
        6,
        7,
        8,
        9,
        12,
        13,
        14
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.24064267,
      "neural_order": [
        4,
        12,
        5,
        9,
        1,
        6,
        10,
        8,
        0,
        2,
        3,
        7,
        11,
        13,
        14,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "61bff2c7604259f5c12e2e903ef0bb1ffc35677e882d23cc1911a33b082e295d",
      "trial_index": 529
    },
    {
      "answer_sha256": "c3831cbfcd3c1ce1ac76ad87f69e4bdeaed80ebd80b73f3c817a813c478dab59",
      "candidate_ids": [
        "5737a4511c456719005744df",
        "5726e313f1498d1400e8eeb2",
        "5728f2e26aef051400154896",
        "571cc6f85efbb31900334de4",
        "572ff7ab04bcaa1900d76f51",
        "57264a8cdd62a815002e808c",
        "572881022ca10214002da416",
        "5737a0acc3c5551400e51f47",
        "56e1aba0e3433e1400423094",
        "572a0bebaf94a219006aa76f",
        "5733e8ae4776f419006614a6",
        "57107c24a58dae1900cd69e8",
        "572658daf1498d1400e8dcac",
        "572fc8a904bcaa1900d76d1f",
        "572a0d21af94a219006aa783",
        "56e1bc3ae3433e1400423104"
      ],
      "candidate_sha256": [
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "a18c9573d677ad16634bd00da5e9087c5513180b5f0f4dc369059d2355f3d2d6",
        "698730d8005e0b016081ec0d4e6ad22090ccdbccb4e028f155b0f346dc47ea56",
        "c456b31b38f7b3d656d571ec7289b0630f5f2d8ce31a3965fddb4291053d73ef",
        "7fa0e91894a36e57517604cec2da8259e4140eec24fc847af2df39ee2fef8df0",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "edfc01315d6a697f5cca00b265434e36306b27bd91f762a30a96dce4a827bede",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8"
      ],
      "example_id": "5726e313f1498d1400e8eeb2",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        11,
        5,
        0,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.45147186,
      "neural_order": [
        1,
        6,
        10,
        8,
        14,
        3,
        7,
        4,
        15,
        0,
        2,
        5,
        9,
        11,
        12,
        13
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "886afce36b3a78799faaf94a76291de2833a16c30900e010a10228f5a18a3c16",
      "trial_index": 530
    },
    {
      "answer_sha256": "e3b58d65c67e30c7615d8dbd69cb7bde6ffc1096613447d2b7f414a68c98df3c",
      "candidate_ids": [
        "5728759cff5b5019007da25a",
        "57284d484b864d1900164900",
        "57286bf84b864d19001649d2",
        "57264a8cdd62a815002e808c",
        "56e1c720e3433e140042316a",
        "573020f7b2c2fd14005688f7",
        "57263c78ec44d21400f3dc7b",
        "5727cb4b2ca10214002d9676",
        "57379829c3c5551400e51f3d",
        "5728facd4b864d1900165170",
        "5705eee952bb8914006896de",
        "57111713a58dae1900cd6c00",
        "573792ee1c456719005744b9",
        "5725daa8ec44d21400f3d6b3",
        "5726e3c4dd62a815002e9404",
        "57332e48d058e614000b5762"
      ],
      "candidate_sha256": [
        "79df8b2517140f9af8ebfd4d413ff6251d921f90c8297b5519d66514cf768fc5",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "3243e4c3f26f22351bf8c52e8438c396a19b76af187bad37a4ca97532160e33d",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "0580bbeb499ff45e51cfc0808c7015f610f8c224212364146fc8ea5550249474",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "742f9a655ee821050238ea7e2700d2687a845379dd8dd37cddf67a4e742828b3",
        "7c6ea35f0d3bbbf5ecd30b29f3e3ac28df4cafdefaaef9e20c7ebfc53be3bbb2",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "1d470bf791028e4f456c100c44c5b3410fac821035b36b868d821eb23e1f2a74",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708"
      ],
      "example_id": "5726e3c4dd62a815002e9404",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        4,
        3,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.56276087,
      "neural_order": [
        14,
        6,
        0,
        11,
        3,
        12,
        1,
        2,
        4,
        5,
        7,
        8,
        9,
        10,
        13,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "1e5a5e4dcf6f3f36a373c2b6044d53815675baf76e33f6242f75bc6c85e2d5b0",
      "trial_index": 531
    },
    {
      "answer_sha256": "d5eb7d74fb050b99d2ee7d1300910dd8343eb82354e5598d21728e09867d0694",
      "candidate_ids": [
        "57268882f1498d1400e8e306",
        "57269016708984140094ca41",
        "571c9074dd7acb1400e4c100",
        "57287d4a2ca10214002da3e4",
        "5726e5c95951b619008f81cb",
        "57271f125951b619008f8635",
        "572a20816aef0514001552e4",
        "570d30fdfed7b91900d45ce3",
        "572753335951b619008f8853",
        "572f76d1b2c2fd1400568153",
        "572ffd75b2c2fd14005686e5",
        "570d4e6cb3d812140066d66d",
        "572a070c6aef0514001551c8",
        "5727db85ff5b5019007d96fc",
        "56de16ca4396321400ee25c5",
        "572ff935b2c2fd140056869b"
      ],
      "candidate_sha256": [
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "069f593a47b1c7d532212a62aba1e27bdd08d0259cb3e0a96bc73f362f4036e4",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "9bfd34dc5945f3f1cb2977bcaf00d7025ecfd1ae65eff22ed5f0ad4225e7ab2e",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "01606aac86b03063e22ad5a865a75b8fe1d68dd0fbf1e05402546a0aed0d2f52",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "7e7a84f3e53d1a0e66add577d62a5f546843a0fad602c1934af6bce74153ff2b",
        "89800e35b6d4485ba445c61b407eb2b1f9444e20f6df90cb4d849d20b48ccf19",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333"
      ],
      "example_id": "5726e5c95951b619008f81cb",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        15,
        5,
        6,
        2,
        13,
        0,
        1,
        3,
        7,
        8,
        9,
        10,
        11,
        12,
        14
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.53048214,
      "neural_order": [
        4,
        11,
        8,
        3,
        1,
        6,
        0,
        2,
        5,
        7,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "8dbee05a7e11ab2332c6b1d519c0355faa8fb89d95218fbfc46f0c6a88f4660e",
      "trial_index": 532
    },
    {
      "answer_sha256": "2185567cc3cf028ebdd46b51f378b6161422bf15e37eac4caf170a1db565a4be",
      "candidate_ids": [
        "5728f3724b864d1900165116",
        "572978f91d046914007794d3",
        "5710f114a58dae1900cd6b60",
        "56e1b754cd28a01900c67abc",
        "5726e985dd62a815002e94da",
        "5729f4b41d0469140077968b",
        "572fcd86947a6a140053ccda",
        "572fce12a23a5019007fca11",
        "5727ea45ff5b5019007d986e",
        "572a05eb3f37b31900478653",
        "573766251c45671900574471",
        "572a0a391d046914007796df",
        "57339902d058e614000b5e70",
        "57287e512ca10214002da3f8",
        "57265526708984140094c2bd",
        "57273a465951b619008f86ff"
      ],
      "candidate_sha256": [
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "20b577df3799579130e026193eace003121ed48c7a637a90e343d9d5a11b552e",
        "64586c5ebc23364070fb9d83b5dedb5d9c847f19daf128b303edc4490b3d3b89",
        "7b6e0d4b29b58911af0fdd9c0fbe4598a87c0fc055d9f95dcd4143510fedcff3",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "cabcb2e17697c1ca5e1bf56512bbf3851f990033a3f5d302aa90e0f3bfed2236",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "4857e3047000dadc16bfb2d7c84b52b10f8e6992770db9d16c52203a42a603e2",
        "c510ffd948b62d2e66c69cbbad6f3aaf252b777869459bb4bd91bd314d3c61c9",
        "24716e3f0a26cae2e731e20b4c52753dac2a195e6dd4ac82b59c7d61faa7329c",
        "f4a941b8fb7b8c4a1f8010d912eaa080c3e3ddf68b132d4a96fc1da34d4ea0d2",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "0647d147367db34d8108a9fd785f9e4e6909853050082dd01b4b0f7d527e30c8",
        "a828c58f3d12ed3b04a42280e3053c818961fb2b99589a63f4145c2a9ab650b5",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f"
      ],
      "example_id": "5726e985dd62a815002e94da",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.59030324,
      "neural_order": [
        4,
        7,
        6,
        0,
        11,
        14,
        5,
        10,
        13,
        1,
        8,
        2,
        3,
        9,
        12,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "efa46d8d8539599cbe88c7ac44d02bc421bc78847d7041e146054dde30437417",
      "trial_index": 533
    },
    {
      "answer_sha256": "a7c7e753df4e2f3974ec0799ed942cb20cfb3f2d1966ce4822c85c1e6c3b8a70",
      "candidate_ids": [
        "5710f4b8b654c5140001fa45",
        "571117d4a58dae1900cd6c0a",
        "571c879bdd7acb1400e4c0ec",
        "572872822ca10214002da374",
        "572fad30a23a5019007fc86d",
        "5726ea985951b619008f8261",
        "57286ead2ca10214002da346",
        "570960cf200fba1400367f01",
        "57296f293f37b319004783a3",
        "572a019f3f37b31900478643",
        "57339ad74776f41900660e86",
        "573092088ab72b1400f9c594",
        "57300a9a04bcaa1900d77063",
        "5733ffa7d058e614000b674f",
        "5725d34089a1e219009abf50",
        "570d2556fed7b91900d45c47"
      ],
      "candidate_sha256": [
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "8ce2dcdbc9d03cccc09926a2d4518e85b49d3e4f59e03003d552e94ad5a5203c",
        "5aa012bff584bfd7dabfd77b6df3d32254728f8f9aab5f34ddc0f06b7a4f00a1",
        "dee2c52f4da8052dfc4b6450b3f6ed3756a0d92188a28cf15a427acd227772fd",
        "7a964e744589792d6e944294f3245ad4605a6cfd1edb01ff0a55eaa561f86c05",
        "2b7326d0cd8188e7b1ea38f54e115afbbb5e41e5f6886b800cf1fd31ac49dd48",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "6cfe10acbb60c6d8a0089776a1f286752d23ddda6886a1bea5e5b3516e94b48b",
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "6226507e15e79f5a2d61709502ceb92ab2b542f41d5fc2ba605c11e187201b32",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf"
      ],
      "example_id": "5726ea985951b619008f8261",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.64998694,
      "neural_order": [
        5,
        14,
        0,
        4,
        7,
        8,
        1,
        2,
        3,
        6,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "133e15bdd31e091f533518a8a5b270a4aee5465485f1fe2789f7b1a22f98fa36",
      "trial_index": 534
    },
    {
      "answer_sha256": "b0ce448b77d91c61ebc069285fe06e7dd7d9f01bf4f96e3e27a8c2abfde723ec",
      "candidate_ids": [
        "57378862c3c5551400e51f21",
        "572fc8a904bcaa1900d76d1f",
        "573010fab2c2fd14005687d7",
        "572669a9dd62a815002e8416",
        "5726eb4b5951b619008f826b",
        "56de10b44396321400ee2593",
        "57263ea0271a42140099d7c3",
        "5727f2714b864d1900164072",
        "570602fa52bb89140068979e",
        "57283c464b864d19001647c8",
        "57339ad74776f41900660e86",
        "572ff5fcb2c2fd1400568657",
        "571cc8815efbb31900334dee",
        "5727d6154b864d1900163e34",
        "5728fc2eaf94a219006a9ec7",
        "56e19557e3433e1400422fee"
      ],
      "candidate_sha256": [
        "06a36c034cb1681bb677178da6125ae4b9b63cd95527bdc90f5a7af7da958523",
        "1d7b64cc0b222af39964a18ef992b128008ad85619e30c391b33964678add944",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "c018c8eb3d4256f40e35bc0be8827b1ac58191a915621d04874fc9bbf2a895b7",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "63be2650416e20d766eb5aad8897346cb4004a2c3df1e14ea035a76981989e30",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "4a16f2e561ce8db196ec31eb41bcde2f63b1dafe443c9820e3fa2984615e174d",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89",
        "6233f5ff4a063a8401e5bbc7f802133992c089ea6b276dbbe6462d513f33c086"
      ],
      "example_id": "5726eb4b5951b619008f826b",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.46015067,
      "neural_order": [
        4,
        6,
        9,
        2,
        14,
        0,
        11,
        7,
        1,
        5,
        15,
        3,
        8,
        10,
        12,
        13
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "40740620097df7b0b75ca1b56d87f482875062c5c3e61e440eda8c2db792ea2c",
      "trial_index": 535
    },
    {
      "answer_sha256": "7b9244cb714f8e3d73de746755ad2eb41e268088a1279fdc5cd050eeac939f6b",
      "candidate_ids": [
        "5725b7f389a1e219009abd5d",
        "57108c95b654c5140001f979",
        "572ffce5a23a5019007fcc15",
        "572a1f086aef0514001552c0",
        "5725cf3238643c19005acd61",
        "572973f76aef051400154f0b",
        "572758e0f1498d1400e8f6ae",
        "571cd88ddd7acb1400e4c18d",
        "5726f0e35951b619008f82f7",
        "572f64ccb2c2fd14005680b7",
        "56de4b5c4396321400ee2799",
        "572fffb1b2c2fd14005686f9",
        "572878942ca10214002da3a2",
        "5726ed6cf1498d1400e8f00c",
        "57373d0cc3c5551400e51e85",
        "57269e3bf1498d1400e8e516"
      ],
      "candidate_sha256": [
        "0b1769dcd4b05af8e663c9fb941ba1d9c5ce34cd2639f1abfef5b71e0bcc146c",
        "6f87ea6fbf41c9ab83ff8e4d59f5ad75c2e93087643d6fa9220276e0bf7508c7",
        "b5982bf3efbf127bdc0d4129376c8149d17dd872f55180b77b46c56c7b601a03",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "4e29e864fdfd291ba260b61248bc4aded067f95f96277117c78dcb8a7376f655",
        "9eb39676db175f0c94d503b3226389be2c375893fb86b70554e208cebd37f6d7",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "809ab002069708b05d7022073e5692a5d269e21b863617ea0d1573256c8af15f",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "954d9def48563ca27c8bff07d310208952b4bb23f2f669d06ee206180e015c1c",
        "c49d89872cec88c8052a5a2ee0f541f718244647ba1148ede8c4a2a4bfb0282c",
        "bcb1003142eb15184ba0d74747c2dbc7e26be98c4f3660693e8dde9997d2101f"
      ],
      "example_id": "5726ed6cf1498d1400e8f00c",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.27014916,
      "neural_order": [
        13,
        8,
        15,
        7,
        12,
        5,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        9,
        10,
        11
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "dfa87a912790b3d5091121437d5cd50ca50ba6667d08f954a10bed277867bdad",
      "trial_index": 536
    },
    {
      "answer_sha256": "3b05cec378eeca98c87aab49c0b7f35fd7ae0d463930e548ad30f4881b1928e0",
      "candidate_ids": [
        "571cd88ddd7acb1400e4c18d",
        "57332c1e4776f4190066073a",
        "5727cb4b2ca10214002d9676",
        "5726542ff1498d1400e8dc28",
        "572a096e6aef0514001551e4",
        "572980f9af94a219006aa4d1",
        "573361404776f4190066093c",
        "5726edecdd62a815002e957f",
        "57284142ff5b5019007da00a",
        "57263cfcec44d21400f3dc8d",
        "570d2d68fed7b91900d45cbb",
        "5726f2375951b619008f830f",
        "5726462b708984140094c117",
        "57267b3c5951b619008f7423",
        "5726a3c6f1498d1400e8e5ae",
        "57267ebfdd62a815002e872c"
      ],
      "candidate_sha256": [
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "d269a25086da393f4a61cc5b6e49f9b63f3dd265fde5a3da08b68bb335ea4037",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "f1a8665f96b70d2299f62cd87020dd1ad27dde10867e4d88d875b99d36be62ea",
        "32a0dfbb09ed32aec7e5781017443faf0d181dc3baf25713e1ccabb73b1ee77d",
        "280b492999eb9dfb26d1ae2fae6534020b32046a7306f0fef949f217a05c5061",
        "c59e7593c862a0c68cee17ab74270592bac6538a668da2cdfed71c7db8087fec",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "aeb98d5feda0ab3efc8296ff3f73372e13a699fa97064d33bb575deef3451fe3",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "8e1bc438d981c2306ca00117b2d9999aaa7aeffc884e13e6bc7111787865a3ee",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b"
      ],
      "example_id": "5726edecdd62a815002e957f",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.41022242,
      "neural_order": [
        7,
        11,
        0,
        14,
        13,
        15,
        4,
        5,
        8,
        6,
        1,
        2,
        3,
        9,
        10,
        12
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "abebf516b4c34a1b6c78208b16575b79efc93823a9b38a6fd17329ed8731addf",
      "trial_index": 537
    },
    {
      "answer_sha256": "cae8975ed971ee4b90cc03a0a8c3ae39e51adad6292d50bda566fd4d40c3a7f7",
      "candidate_ids": [
        "57337f6ad058e614000b5bcb",
        "5727f16c3acd2414000df05b",
        "5727d1c93acd2414000ded3f",
        "57281ab63acd2414000df493",
        "5733e771d058e614000b6545",
        "5726f0e35951b619008f82f7",
        "571094b7a58dae1900cd6a66",
        "570d2d68fed7b91900d45cbb",
        "56e1ce08e3433e14004231a4",
        "572855973acd2414000df925",
        "572ff935b2c2fd140056869b",
        "5729f5a03f37b31900478605",
        "5727c0402ca10214002d9564",
        "56de1645cffd8e1900b4b5d0",
        "572843304b864d1900164848",
        "5725f00938643c19005aced7"
      ],
      "candidate_sha256": [
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "a8497768ed7455269c8c0e43100892fa8dc1ffff0a532613071c4cb9e3f05a18",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "3d913b06d94a02376db7ad12801739d73c4412833b9f3a3d45390da376256d0d",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "a81ba15411e0a7218bd18d61a8a4ebe10bc50b359b4e54f7cef60996f42c29cf",
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "f7e1321f7259ee2bca9b0347de06ef86a74981b20a2b48975253aa242d09bb09",
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "2eb036f29c316461f4c29577e8a7d145eaf561304aa07090de17f07637b9b838",
        "76356e52b29acc92b0c35dce4f20450bc523d71a1b1beb49d854c21d052f7bf7",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5"
      ],
      "example_id": "5726f0e35951b619008f82f7",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.42901326,
      "neural_order": [
        5,
        6,
        4,
        0,
        7,
        8,
        1,
        2,
        10,
        14,
        9,
        12,
        13,
        11,
        3,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "13ea085a8bc234fb7e649af6646b02fbcfa6b8a549b9f4bc4f9f69a1a8b9b10d",
      "trial_index": 538
    },
    {
      "answer_sha256": "c6898b5e809eaf0a8a096be959f559bc9d68dc6a31a2518de67ff497e76f3aa7",
      "candidate_ids": [
        "572860e03acd2414000df977",
        "57266c015951b619008f7237",
        "5725be0f271a42140099d117",
        "5705fc3a52bb89140068976a",
        "5726f2375951b619008f830f",
        "5730b541396df919000962c2",
        "572759dbdd62a815002e9b94",
        "57114aceb654c5140001fc47",
        "571cc5c45efbb31900334dda",
        "5725d42a89a1e219009abf58",
        "5733d68ed058e614000b637f",
        "57337ea24776f41900660bd0",
        "5726e985dd62a815002e94da",
        "5729f1fcaf94a219006aa6d3",
        "5730bc308ab72b1400f9c73c",
        "572a0ecb1d04691400779717"
      ],
      "candidate_sha256": [
        "ecce67f5ebd93f7f51596b468b36d1918c0f3d320dd7e6bd4fa7ac0ca537f812",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "c573b153ffd8943e12d1903fa463c60e3bbbe8df9681c3b4057a0344790d1b5c",
        "673428db2413933cbe54551ca0f426547227c6870d0b452644c3e43e2faa5e4e",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "18514753d866298f71238efff2e516d2386f7da19255feb3918f789671cbfb12",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "3c0389077487bd149337084460bb68be55b15e60b87e6b03b2daf2236c6d7d48",
        "0fe05bd1833002131a7e69e040cc114a9dfc4409000d5d0674152325ab91cb19",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "99afd74c371b051e0dd2704df0676d8589e7c060ad964a7cae71b5a5bde23c1e",
        "d28ae44a1bd659369983c5d7b048773512040e9150f20bcb6fbae1f618e3266a",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835"
      ],
      "example_id": "5726f2375951b619008f830f",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        13,
        12,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.32226259,
      "neural_order": [
        4,
        12,
        1,
        15,
        14,
        0,
        5,
        13,
        6,
        8,
        2,
        3,
        7,
        9,
        10,
        11
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "ed381f50e8019ea44c950088ab8777def037d42d1fe0f94cbdf1a217e4366075",
      "trial_index": 539
    },
    {
      "answer_sha256": "fdcd81461e414e29a8ffe3d1247f39adc3b0a3db0fb6a8bbd701d2514e826265",
      "candidate_ids": [
        "5730b8ca8ab72b1400f9c704",
        "570614ff52bb89140068988a",
        "571112ada58dae1900cd6bcc",
        "572941273f37b319004781ad",
        "5727aa413acd2414000de921",
        "57114e8d50c2381900b54a5b",
        "572658435951b619008f7025",
        "57115ac550c2381900b54a77",
        "572f76d1b2c2fd1400568153",
        "5726e08e5951b619008f810f",
        "56de3f784396321400ee26fa",
        "5726f36cdd62a815002e95fe",
        "57105da9a58dae1900cd699e",
        "5706149552bb891400689880",
        "5728fc9e1d04691400778f11",
        "5711488ab654c5140001fc3d"
      ],
      "candidate_sha256": [
        "7a9cb3d198d99d90ec117c5b3df6405aac0eb74e382297c46c208b7e76dfd936",
        "5f10859c15b5f6c66012ca3e4f33a4fc38b8221d1aefeaef9c774ef8e6e5949a",
        "b5bef27d4795b8ba1cc2377c83a25e718efed2b9f4d6b689dae0aec1692c424f",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4",
        "0316b036cf4be739a3634997660434772d5dc0c8158e0025882c9dd3afd15f47",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "8bb6cf326072649a03d162b7e149c5ebba76ee9ffeb0e53e8eac685cb78d0b19",
        "dbfbff5b6105a52bac9a00a207444dd2cba3a97e33a1d1a82edb0389f6cac1e9",
        "7250341734ca75ae43d8f7cafbba939053160807e1106cee1d1a0c2b5bab85a2",
        "4cb08df8d00c330db23afb5af129580ac1c4437b39cc7557de63a8b111d7abe4",
        "f20bb42b5e294e6d2d01963844230d83bfdf35592386b5cad0545b14a5462b4c",
        "bfc15b62d8c29f0dc1c0d417aa10a1f77a5bc1848cbd1df2d1f0872ba8f904b2",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16"
      ],
      "example_id": "5726f36cdd62a815002e95fe",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.27182544,
      "neural_order": [
        11,
        9,
        1,
        7,
        13,
        12,
        5,
        15,
        4,
        8,
        6,
        3,
        0,
        2,
        10,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "8301035a9eec6b1ce7f2c061fa02335865d8cd4498b4db8f6607008d8f3a93f6",
      "trial_index": 540
    },
    {
      "answer_sha256": "6fb88f7df9df66e2e10c28027a34e514d612ea7b794d0879028fede75a5dddcd",
      "candidate_ids": [
        "56ddde6b9a695914005b9628",
        "57293bc91d0469140077919b",
        "57335c20d058e614000b58f9",
        "572853e8ff5b5019007da188",
        "571c7d55dd7acb1400e4c0c4",
        "57265e11708984140094c3bb",
        "5730208fa23a5019007fcded",
        "573027d6a23a5019007fce9d",
        "5726a5525951b619008f78dd",
        "5726f48df1498d1400e8f0da",
        "57106644b654c5140001f8e5",
        "572648e8dd62a815002e8076",
        "572ff83ab2c2fd1400568681",
        "5737a7351c456719005744f1",
        "572a1046af94a219006aa78d",
        "570d529fb3d812140066d6bb"
      ],
      "candidate_sha256": [
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "eb9321cdd1b9c157ced5a544ecb2c130392fae1ffc7aca72e06d7fba133c3f52",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "7df6faae258d409de0ae8a4e193fc099c32995566d475ab601795567925e75eb",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "ba71a4259a62247db3f4e9130708c3bd7ff3d84f900927a591bddb688e7ad277",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "3526ee48cec62575d3bca0e00db54328f4551645e442d93a226e7c4e813665ce",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683"
      ],
      "example_id": "5726f48df1498d1400e8f0da",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.59651266,
      "neural_order": [
        9,
        3,
        8,
        5,
        13,
        1,
        14,
        11,
        6,
        10,
        15,
        0,
        2,
        4,
        7,
        12
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "0cb57f62cd5416d3629699d42397ecdaea486a75317ae989c54067098708c854",
      "trial_index": 541
    },
    {
      "answer_sha256": "4f4d3b0291cd842d609ce91b081998ac1a2d8d23a39ca097d2d871232e91f66a",
      "candidate_ids": [
        "57296f293f37b319004783a3",
        "5726f635dd62a815002e9656",
        "5733ef47d058e614000b662b",
        "57373a9fc3c5551400e51e7b",
        "5728848cff5b5019007da298",
        "572879574b864d1900164a14",
        "571c91c8dd7acb1400e4c10a",
        "57274f67708984140094dbf5",
        "572ffe6fb2c2fd14005686ef",
        "572691bedd62a815002e89dc",
        "57106644b654c5140001f8e5",
        "572f5875947a6a140053c89a",
        "56e1c3e1e3433e1400423148",
        "5728742cff5b5019007da246",
        "5727490bdd62a815002e9a82",
        "57114667a58dae1900cd6d80"
      ],
      "candidate_sha256": [
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "baa0a539daf238508db12b50810d17233a6fc9321763a721dc6986e289b1d436",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "2c2d88b5bd133e6e920c44bd056b4ce93bc633f042a94955803d584d4313073c",
        "26e9c23786fcae3ca7788b223db779bc690e26ecd71419f682d8934df95d4628",
        "8f934d4cbb09c9df1a914f649c29dd32db3f498ba2f5303d2ca72b5eceab49e4",
        "fd80faaf43d6d56307f1567cba40b08ce93dcfb538d6704d48b9c014969757ef",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "808648d31e3c0a580e378e5e38b6e262d80eae03ea2f5f3343f713667cc29b1d",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "1b05d25302149995ccf924cfe9d3dd6cbf03a0cc7e58f7a1dc97784ba0be55dc",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "836fc4670df58e0f64317be9b717f3d2d78b471dc16251d4f836c660af6a3216",
        "9f74b3dd8fbafbd78a984fba505bc529aeeb1ecc9aa4c7f2f6efc2eef835a5d3",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "c3575becd21dc14844269008f4c0df552a9d85114cd7cf6ee48a0b11dd086cea"
      ],
      "example_id": "5726f635dd62a815002e9656",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        4,
        5,
        0,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.4909838,
      "neural_order": [
        1,
        8,
        4,
        5,
        10,
        14,
        13,
        12,
        0,
        2,
        3,
        6,
        7,
        9,
        11,
        15
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "302f9ef6ad66ea8be987af8c669a0402677713150671325ceb8c8864b3f4aa83",
      "trial_index": 542
    },
    {
      "answer_sha256": "308b1e7af7dc1b664c3e62318845d10ddfb2075a6f3286aa8b1679e65a20095b",
      "candidate_ids": [
        "572f6a0ba23a5019007fc5eb",
        "57336755d058e614000b5a3c",
        "572629c6271a42140099d6a3",
        "5705e99452bb891400689688",
        "5725fb8138643c19005acf3f",
        "5737821cc3c5551400e51f19",
        "572753335951b619008f8853",
        "5733cff84776f419006612b4",
        "5726ba2c708984140094cf59",
        "5726f7715951b619008f838d",
        "5730876a396df9190009617a",
        "572fc41f947a6a140053cc80",
        "572885c44b864d1900164a78",
        "572663a9f1498d1400e8ddf2",
        "57274971708984140094dbbb",
        "56e17e6ee3433e1400422f7f"
      ],
      "candidate_sha256": [
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "fceaf8d74e889c1852beb63317dd4d8cb5b3cab9ec5ec8491f11a9ca314a73a0",
        "9d6db7d999c0eeca9c2501df4a7fad2034d6cf68c86f755582e7b767a293e353",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "1e5e6f12f794ecb7885b1a3242af3c0eec6a527662235a35ff2642a0af281201",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "2188fc405f0961edead242b989ef4287705c372ff9f0af4d4d548a74cff11667",
        "0b13637d765e595a303e2580975cf2d7d8948022bf44ddefd6f0eff6b8df1684",
        "312a70e3db7cba7ab2f059baf65d323e5e949b04b9c08a807456e9366dcd38b4",
        "0485f70ec1fd26d82a22017be716a779777acc8928fc5c1353ca7f2bb2274ab1",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "00b76f48835bfc2434242a5dea1681e9e4d9c363cc74180aa0b9b9fec6f02bef"
      ],
      "example_id": "5726f7715951b619008f838d",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.35750097,
      "neural_order": [
        9,
        11,
        10,
        8,
        12,
        15,
        3,
        2,
        14,
        13,
        1,
        0,
        5,
        4,
        6,
        7
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "67e1e6371a93b1f778761ccb5a0b351ed0632e98ec771639b3e017b1f3316b7e",
      "trial_index": 543
    },
    {
      "answer_sha256": "ae20f169ee909175ef6d819287eeede5c1c3fd5677d772539cdc0336b8398c6e",
      "candidate_ids": [
        "56e1ce08e3433e14004231a4",
        "5728202c4b864d19001644ec",
        "5733647e4776f419006609ae",
        "5725f00938643c19005aced7",
        "571cd703dd7acb1400e4c182",
        "5725d662ec44d21400f3d687",
        "572758e0f1498d1400e8f6ae",
        "5728fd3c3f37b31900477f2b",
        "5727515f708984140094dc11",
        "571c8539dd7acb1400e4c0e2",
        "57265e455951b619008f70bb",
        "57107d73b654c5140001f91d",
        "57284142ff5b5019007da00a",
        "57337ea24776f41900660bd0",
        "5726f868dd62a815002e9684",
        "571c97e2dd7acb1400e4c11e"
      ],
      "candidate_sha256": [
        "11f3c106363aa4f6fea948825269829f23db795723c77a21a4dee2752cee9822",
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "069e21e2e1d44c53ecd8f15a91dd88f41c8c41ba3608bc69461d8d74313fafc3",
        "5ee9e17954c8db6e08d7e774796ee372c5de505e121d05365dda2f195f5b33f5",
        "8657b03977310d90fc72a13d570f1c07e80c6100a3b2eeb92d2acdff9adb8b16",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "b4ba2bbeeee2a856da55303653574a4e172145e70ab00ddcb48b0d01f5f34139",
        "5e9573dfeb2a0db4f639ca16519759712547a4abf0de1e03015f67d069799f3d",
        "d96efa04967dc1620b0ea8b56914b0773d312093fa35a36584e9e9423e6f42d1",
        "2da67be73df602e23d08efcdf5790b6b7bd46a59a48eae551e9fe4b8ee631c51",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "c206679cc4afbecc5d61f9d8ed6a31b94b426372acc9d19298e47a403405d9de",
        "7b09b48eb2457b9e9876081bdc2b44c1ed9bdd845ee1824d2cae03ccd3b16fbb",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8"
      ],
      "example_id": "5726f868dd62a815002e9684",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.47305538,
      "neural_order": [
        14,
        7,
        9,
        0,
        13,
        12,
        1,
        2,
        3,
        4,
        5,
        6,
        8,
        10,
        11,
        15
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "4455e83f91652b51c7d75cd75edfc195bbcbf72222af503c0643c9352369d736",
      "trial_index": 544
    },
    {
      "answer_sha256": "235b0c4cdc7fa7408f882bb6ebf2150e3411ae05f5f0bf9084f3a0211f7d911c",
      "candidate_ids": [
        "5727c55bff5b5019007d94ce",
        "56de51c64396321400ee27f7",
        "571cc8815efbb31900334dee",
        "56dde2fa66d3e219004dad9b",
        "5726542ff1498d1400e8dc28",
        "57265d86f1498d1400e8dd50",
        "57284b904b864d19001648e2",
        "5730035e04bcaa1900d77001",
        "57373f80c3c5551400e51e8f",
        "5725cb33271a42140099d1db",
        "5733a45d4776f41900660f23",
        "5727d1c93acd2414000ded3f",
        "5726f96ddd62a815002e9698",
        "5727fc37ff5b5019007d9a18",
        "572822233acd2414000df555",
        "5711488ab654c5140001fc3d"
      ],
      "candidate_sha256": [
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "739008a2b11054ac3e7f4ddcaa1d8e35db764247129ae46031be460b212648b4",
        "ab5c862550d9c095acff44d9d96550d8bdd1760386df7e8e7ac8f6e47650d6b8",
        "27749618f729fa62e79b34e0666fb6ce7980b9e0e582584b2a78d0294543512a",
        "1ff281c692c83a32d73d6857c03271c9f747409739b4017f6984d587571d84c0",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "39ba76c733bd48c9aac79e3466a117b35d59d247245dbec816e9246fea04072f",
        "223421a455a8533ec3d82673a2643075bb99a003336eb360498d4d764f30a174",
        "f54cb46616d11638b33e64c6095c152f84bd3dc6c255e97bee06af37ee14f79b",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "bb06356b5b40045b0880d1a1e45376141d30e0cd812388383b320d3271f487e6",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "7c320b94ab7050066a5ef46a73f65b5c8afec9ba966a60947299928c000f866f",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16"
      ],
      "example_id": "5726f96ddd62a815002e9698",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.60205701,
      "neural_order": [
        12,
        11,
        13,
        14,
        7,
        15,
        1,
        0,
        10,
        5,
        2,
        3,
        4,
        6,
        8,
        9
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "839d141d4f427930d15b6857c41370b8f9a45a0406547bc909617e1b4d55a10c",
      "trial_index": 545
    },
    {
      "answer_sha256": "8b9db98c23830283bdd16effc7c3451fb75aca2127052c3ca322f2a0cae6597f",
      "candidate_ids": [
        "5725fb8138643c19005acf3f",
        "5710968aa58dae1900cd6a86",
        "57300911947a6a140053cfb6",
        "57376df3c3c5551400e51ed7",
        "573382d24776f41900660c37",
        "5726fa525951b619008f83f7",
        "572f6ec7a23a5019007fc621",
        "572ff5fcb2c2fd1400568657",
        "5733d4c8d058e614000b6353",
        "5726414e271a42140099d7e5",
        "5727dd2e4b864d1900163eba",
        "57267b3c5951b619008f7423",
        "572659535951b619008f703f",
        "5733e771d058e614000b6545",
        "57378b141c4567190057449f",
        "57340111d058e614000b677d"
      ],
      "candidate_sha256": [
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca",
        "49a9c9283f8bc43e0bd3f16475e09307d23e2f9acd4a96b59b3c418034a28ae3",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "8680c2d1511073dfa6129eaab02ad788c74a59c1cb87f2123d2a4768527f9002",
        "27c39d3595282b73584a3275b9f7366ab4fd3124b8e962814c6b9f1e89a5ba2a",
        "d7b639c11d264c6a9459674a47c65dc31a0d3d9615af12876a8c7d90493d2915",
        "8cd0f21119a00deba08c53802785db26e753a1e07eb7b261f07e1f5d56bdcb26",
        "acd15f4158acfabce9c5519524687c67beb251f8591bbe133adf84f4ff581eeb",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2"
      ],
      "example_id": "5726fa525951b619008f83f7",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        8,
        10,
        14,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        9,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.55346588,
      "neural_order": [
        5,
        1,
        12,
        7,
        15,
        14,
        8,
        0,
        4,
        13,
        11,
        9,
        2,
        3,
        6,
        10
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "9709fe8323c93273331f212afd9173dbd100f52550682089e974fb653c230aa0",
      "trial_index": 546
    },
    {
      "answer_sha256": "71ebeb52261d138ec7928090b2bb08e936923eb7a3d9fca79c58e1340fb0ebb5",
      "candidate_ids": [
        "5726e179dd62a815002e93ae",
        "5728667eff5b5019007da1fc",
        "572fc78d04bcaa1900d76d07",
        "572881d34b864d1900164a5a",
        "572f60f4947a6a140053c8ea",
        "57268066708984140094c821",
        "571126dfa58dae1900cd6cb2",
        "57264d58f1498d1400e8db7a",
        "5734025d4776f419006616c3",
        "572745c6708984140094db99",
        "57269016708984140094ca41",
        "5727478cf1498d1400e8f59a",
        "571142b3a58dae1900cd6d5a",
        "5725f190ec44d21400f3d76f",
        "5726e985dd62a815002e94da",
        "57271c235951b619008f860b"
      ],
      "candidate_sha256": [
        "cb0bbdad5ec4984d2cc4e9738d8ad48ab016642d4d8b1e374bb5d9796d722434",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "f13e50f6e6854915c887789f95aac6c2b8ac4dfc887c56edb77102ecf62c7e08",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "e506629a68c63a7cf4a1fa5216ccd76592f2355601e104882cf833dd45804915",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "a6bf93aaa317a4cbe3dd6095edaf3c9cad24ebd2270fecf0196ce024b0880c94",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "a1b693cf3641547347ecd52bb81710b33d18725de929eb2e5828069b5e055cdd",
        "795ccd56a08606b9fbd7643b1676dc4b2e42a3949da49b759987c4ffe07771e9",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe"
      ],
      "example_id": "57271c235951b619008f860b",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        8,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.28446404,
      "neural_order": [
        15,
        2,
        11,
        9,
        3,
        4,
        12,
        1,
        0,
        5,
        6,
        7,
        8,
        10,
        13,
        14
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "2f691b5b8e3011a39ea7fe7376ca7436909ac4be510030de627abbbfed3cc9cd",
      "trial_index": 547
    },
    {
      "answer_sha256": "e369a7b61ce619dceea1b479c8da1554dd2960f9898d0e174037b4ac4eef05ff",
      "candidate_ids": [
        "57264865dd62a815002e8062",
        "572878942ca10214002da3a2",
        "5725f8f5ec44d21400f3d7b1",
        "5733fe73d058e614000b673d",
        "57271f125951b619008f8635",
        "57115dbe2419e314009555a5",
        "5728eff82ca10214002daadc",
        "570d2556fed7b91900d45c47",
        "57287c2bff5b5019007da26c",
        "57339dd94776f41900660ecc",
        "5733d3cb4776f419006612ea",
        "57299c2c6aef051400155020",
        "57281edd3acd2414000df4eb",
        "57274beff1498d1400e8f5e4",
        "57332e48d058e614000b5762",
        "57261f9f271a42140099d4a9"
      ],
      "candidate_sha256": [
        "1a10ced916ac064f05f770ad0914687cbcf2870959d1012753e81b033904bc15",
        "4f62474ee95129ce6fc26c7def147066b84695bb8a8c7d509cd145479f23b04c",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "a4b78735d9437218b996e1de1bd7c0efa4be31b9c825136ff1bf41597b0c1704",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "1d8012f6b4161807b9a569b52018c2f64f2a26a1db82d82d20559732fa1adc49",
        "114b4c83f090585170508b423750087f82e2cd6538f45a9294aa4265f92637bf",
        "efc06264ba4fcacbf675ac527c22a4897aff1ea1b3ec40b820bf5849e5da9749",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "2271df8c8c1b2d712c122939c9c53f699509a751bc040f6c45e9660ceb9c2177",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "2a2d7de23fc2703c161905cc08266edd06d7bee6136dd40821787bcb87036708",
        "03cb2784f170548bcb7d5bac5f52461bd00a24b18ac95bd094e47c9bdf136670"
      ],
      "example_id": "57271f125951b619008f8635",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        11,
        9,
        13,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        10,
        12,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.43698015,
      "neural_order": [
        4,
        7,
        9,
        8,
        1,
        3,
        14,
        10,
        15,
        5,
        13,
        0,
        11,
        2,
        6,
        12
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "f213df524bcc9b5dd2787e5b6a64dcc679fa303a3c78020c4d8297d38884bdfa",
      "trial_index": 548
    },
    {
      "answer_sha256": "12740bfeac9ed3f0e8507b79e1da2b14e5f08645d12443485f075d5bee35e0bb",
      "candidate_ids": [
        "5706074552bb8914006897d4",
        "570d4a6bfed7b91900d45e13",
        "57299326af94a219006aa515",
        "5727213c708984140094da35",
        "571ccfbadd7acb1400e4c164",
        "57297781af94a219006aa4a3",
        "5733fc6ed058e614000b670f",
        "57340111d058e614000b677d",
        "56e1f10ee3433e1400423223",
        "57339c16d058e614000b5ec5",
        "57094d489928a8140047150a",
        "572a0bebaf94a219006aa76f",
        "572995d46aef051400154fe8",
        "572a10cd6aef051400155222",
        "5725b33f6a3fe71400b8952d",
        "5725e1c4271a42140099d2d7"
      ],
      "candidate_sha256": [
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "73c9e930c6dd292910f185bef250a8ac347a83947ea0900879f5f37b1fd43415",
        "f54da4edb14741599b317e246ffe56becbfd450806ace7773701841b9f5b06c4",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "84f0f44373d22c8c39b22e3d1ba3dcc7bc5d89cbc004c74a3ea1b24dc2e8d40e",
        "30511919f8c0a4872ffe135a14f787cd001899c87463c8a0e386cd86a4a01d29",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "e7faaf2c519b308d025174f8c0530ff4e7bc8f19239b53428fef406411025b93",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "2a8d2aa463245d980e23aefdfe31db650d5d676f415970557b36b6d0e09480b5",
        "7a450d7d1f943d4cd023d7a9a5f3b23b7186f75e0c1eed291a5a400addca2d3a",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012"
      ],
      "example_id": "5727213c708984140094da35",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.38301593,
      "neural_order": [
        3,
        9,
        7,
        6,
        1,
        14,
        8,
        2,
        15,
        0,
        4,
        5,
        10,
        11,
        12,
        13
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "2c5ca5a9c27d1c083196f934173f61e52cefaaa3d8e5e73af8c368e3a81792af",
      "trial_index": 549
    },
    {
      "answer_sha256": "aac00fb596b5452766c56cc522348482f67b4ae9cd7f06ef32291da9dfc46bf7",
      "candidate_ids": [
        "57377862c3c5551400e51ef3",
        "572847ff3acd2414000df869",
        "5730a40f396df91900096234",
        "5730a314069b5314008321eb",
        "5726249538643c19005ad07f",
        "571c83f3dd7acb1400e4c0d8",
        "572726c9708984140094da7b",
        "5726d8bd708984140094d35b",
        "5727cebc2ca10214002d96f0",
        "571ccfbadd7acb1400e4c164",
        "572655e5f1498d1400e8dc5e",
        "572a213e6aef0514001552ee",
        "57269016708984140094ca41",
        "5725cbb289a1e219009abed2",
        "57308f6b8ab72b1400f9c580",
        "572fb059947a6a140053cb80"
      ],
      "candidate_sha256": [
        "69d738412b3b632659539c0a89dbc327c38082560140b90731ca66303090475b",
        "c44126e5a71b148722d5f363b3fe85277c50806c0cacb2d5d467d0ee1ef9ca2d",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "a4008fb587348767c254824b7e5bb46aaec5eaa762404c37c3e9e2f52fc93eac",
        "f831f1f184fd3a9793b6dde3bdbf892925d66cff3ec221ca3f9d0387b41ad687",
        "5f3df575767cc56af56584e2d66c47695434645572896b722eea367b9205fae4",
        "ae439c6bb406e0e21cc46430e9a8878fde998b295e01cc85bc4a1f4f346d6ab1",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "3b7ac2e35093c9f8c2b22f00bf75316eee3b144d65fffac08fff051c11609234",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "040a5627ef7bb5d949592bbcd7381d5c49b141f464054a8ba8c03c8253480ddb",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c"
      ],
      "example_id": "572726c9708984140094da7b",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        8,
        1,
        0,
        2,
        3,
        4,
        5,
        7,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.42764046,
      "neural_order": [
        6,
        1,
        3,
        11,
        14,
        13,
        10,
        0,
        8,
        5,
        4,
        2,
        7,
        9,
        12,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "a8322a4efccccbc46c164d0d953158af3f24ca7b338971fd9f93783e6841cc50",
      "trial_index": 550
    },
    {
      "answer_sha256": "951b63d90e49871ac8955bef50e5995eba50ac58bb4903505567dfc41bf2dc78",
      "candidate_ids": [
        "572807802ca10214002d9bf8",
        "5730131c947a6a140053d052",
        "572a1ba46aef05140015528e",
        "57261dab38643c19005ad037",
        "572882242ca10214002da420",
        "5728667eff5b5019007da1fc",
        "57115f0a50c2381900b54aa7",
        "57264f18f1498d1400e8dbae",
        "5705e63175f01819005e7720",
        "570d30fdfed7b91900d45ce3",
        "5726559edd62a815002e81c8",
        "5710e8c8a58dae1900cd6b26",
        "56e1efa0e3433e140042321a",
        "57265bdfdd62a815002e829e",
        "5733d4c8d058e614000b6353",
        "5729a3716aef05140015506a"
      ],
      "candidate_sha256": [
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "1d4e08e4500d4e7d4178927e436870a00d446eaa35f333647be4a3c5e47d153e",
        "aa95ca2f8cb345761597dedb47b8dc46be2773412d98ec4f473e747ae7e16b2a",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "3bc57095aeacb5ad4f6c76183802e96d3aa2696d626ea2f062c32497a8c98a94",
        "2c8c5bc876d6673ed7189917afadc92211cb5e6bda48d383222cef764603e6c8",
        "9044345e28cea939c1df374c16b6ddb6f833fead030a072c7d784bcf02c24e56",
        "0b363ccd79edf5a3fae3de2a9ab5967af3dba12ea67dec7a1e57170a0dd0ba11",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "72b788bd3798b7b363e83daea211a9068ded0f80fbaed48424cf8ae6d8380aa6",
        "cbcbc6a35007c78d404e0f5abda0e92527d7a8e803cb1c1c312036f7c30c1717",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea"
      ],
      "example_id": "572807802ca10214002d9bf8",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.54260863,
      "neural_order": [
        0,
        11,
        15,
        12,
        10,
        4,
        2,
        1,
        3,
        6,
        14,
        5,
        7,
        8,
        9,
        13
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "b4084da66dd417c0c9744387f758379d34068b6121130b54c9ab8a6e06e5c642",
      "trial_index": 551
    },
    {
      "answer_sha256": "0fdf8ecdcd806fea8bc160548322dfdad74222ec67a29abd215ba98bbfe364cc",
      "candidate_ids": [
        "5726a638dd62a815002e8bf6",
        "572fb059947a6a140053cb80",
        "572a213e6aef0514001552ee",
        "5706074552bb8914006897d4",
        "572881022ca10214002da416",
        "5728284e3acd2414000df5cd",
        "57266193dd62a815002e832a",
        "57280f974b864d1900164370",
        "57273a465951b619008f86ff",
        "57096505ed30961900e84082",
        "57340d124776f419006617bf",
        "5725f6e138643c19005acf19",
        "57264a0ef1498d1400e8db40",
        "57281940ff5b5019007d9d44",
        "5711623e50c2381900b54ab9",
        "57265e97708984140094c3c3"
      ],
      "candidate_sha256": [
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "0577e0dcf4d2d18ab7d9a4c7450fa8e061f67a47b2b7377ac280f6b5c429909c",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "bd0515eb648a6bbe79d8e62d8fbc5ca3a387b3b1796b859c503480732d04dc0b",
        "ec748287f0d6e1ec9c1673e03ffe3bc51a64e8f47db0cbf66fe8f680fbb0e5c6",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "1a2af49f4a71ebbe6894aa6e81cdfdbb693ec13086cd52778cee2f5b92cbccf0",
        "e9cb625a732c3ee55bee95e5cdc73e57cf6cf438d250f0101db0bc232ec11f8a",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "2892ce1ee90d7e758e8bd4b12fd7bebc4276caad2f0d0af29d0444e47b6a4a05",
        "be1e8ef67e19e07dbf69a2cd82b160a507b81ad4a6ae00fb9fbdeda6ebbed945",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "07c74ed461874b9d65a20073628bf44782dd35b57d89d4d5a16b494c99bf9f34",
        "a35e1c42b814138bccc4c2e3471cae108b831e3a97eabed131e090b2af71e6fb",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "936d1474d94981a1e1610f5e29a490816e40c53f6d62829fb5d7ed61491d28b6"
      ],
      "example_id": "57280f974b864d1900164370",
      "gated_correct": true,
      "gated_top": 7,
      "gold_index": 7,
      "lexical_correct": true,
      "lexical_order": [
        7,
        5,
        4,
        1,
        14,
        0,
        2,
        3,
        6,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 7,
      "neural_correct": true,
      "neural_margin": 0.04910211,
      "neural_order": [
        7,
        5,
        0,
        14,
        4,
        15,
        12,
        6,
        9,
        1,
        10,
        8,
        2,
        3,
        11,
        13
      ],
      "neural_top": 7,
      "partition": "test",
      "question_sha256": "6d2fc42aa646973cae4445f5c57d3a568ef605d8fa4448675f2963d233818a3c",
      "trial_index": 552
    },
    {
      "answer_sha256": "a5dd538bd44e7499912abfb460b1ba35d00aa126b4d39de7660716c37a5cf6a7",
      "candidate_ids": [
        "57268f05dd62a815002e8990",
        "57299d1c1d04691400779581",
        "5733a2a9d058e614000b5f28",
        "57274beff1498d1400e8f5e4",
        "572686ac5951b619008f75a9",
        "57293bc91d0469140077919b",
        "572812e74b864d19001643ce",
        "57339dd94776f41900660ecc",
        "5733fd66d058e614000b6735",
        "57286c8cff5b5019007da218",
        "572881d34b864d1900164a5a",
        "57286ec63acd2414000df9d1",
        "5725c57a89a1e219009abe5e",
        "5726581fdd62a815002e823a",
        "5727ee372ca10214002d99ec",
        "5729f60caf94a219006aa6ef"
      ],
      "candidate_sha256": [
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "ee6a40759a8e9342f506db55cd0c40e075a1ce7fd0ccc2da16d91a2be11ea856",
        "96487eda760f5bfa959c6a45ae9719d65ff38f88506a3c48debe04c7539f2c27",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89",
        "f9ec01d3339f2f3a722a9da12e57f12411863a85102e632c5de1f83f576deacb",
        "cb5ae64f4a40d66719685e8c2bb610700b700b4a88b81e771cd6a21e092bd04d",
        "a5fa34951087126c25acb3541e6d73c4b8adce114834d595c12ad94aa258a4b7",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "b3f180d900be2bb7702a2932c1b2d2af31cd2543eca4a0134755f299d0c10cbc",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "a291ae3d8765a5815d6133af33ae3669a134001df26ae0c0825447be74c47d62",
        "53c687b7e79255fc1fb1ae30141e7b477c22a848e5afa6a2f785a636c16c3076",
        "9f31c92a48276f1d6af6a8e3463fab962dda8dd56ba755ba6fbd9673fc3b446a"
      ],
      "example_id": "572812e74b864d19001643ce",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.42526668,
      "neural_order": [
        6,
        8,
        14,
        11,
        15,
        5,
        4,
        10,
        7,
        0,
        1,
        2,
        3,
        9,
        12,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "534e086dcc22fc0cdaec765a95cfe55c2839d99954a9b98e3e17be4753acc490",
      "trial_index": 553
    },
    {
      "answer_sha256": "18a285713d5b1c02ea93c88a7f197baa0357cfae27f7e3d227b54434ed18301f",
      "candidate_ids": [
        "57281bb84b864d190016449a",
        "57274beff1498d1400e8f5e4",
        "57269f3ef1498d1400e8e534",
        "5711648850c2381900b54ac3",
        "5728151b4b864d1900164428",
        "572989846aef051400154fc0",
        "57283f014b864d19001647e8",
        "57293bc91d0469140077919b",
        "56de1728cffd8e1900b4b5d7",
        "572683f95951b619008f7525",
        "571cd5b1dd7acb1400e4c178",
        "572649d8f1498d1400e8db37",
        "5725f8f5ec44d21400f3d7b1",
        "56e1c720e3433e140042316a",
        "572758e0f1498d1400e8f6ae",
        "572a0ce11d046914007796fd"
      ],
      "candidate_sha256": [
        "34b3cf8b2ee4c5e03d253cbd14d037704ed4a2f0fbffba0798b1eb00604b8c7e",
        "7164a06bbe7a859a1701acc5b96841f34611c92ab1541b17f9f20f5e6d21a0bf",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "c9d8fd86f5032d24129329b73bf27e421ad722b5613a8be3e291130697ec25cd",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "9323aa77e16442a4ebc3e47425b81a77c963c9796da7b75cebd9de1a6eb1ed55",
        "8eda53faed808afcd0dea0728c86e0ec04a0c1c440c949ea105f28c501592e89",
        "e77cc2842075cf67e0f42d9c55fea1df5857504c655c9e1e596c5f52dc859507",
        "67afc3f9c17106ee04d8b927f7d7b05326e595ba6c0a85561c5a168c58bf0616",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "697fc195c9b97bc8be1791335b67318ae68f44c890d45b0503189769ebc68eba",
        "2894c924f3b7949239f3846f11a2cdb403c344416a9e245dc179f33fdd345fb9",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869"
      ],
      "example_id": "5728151b4b864d1900164428",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        2,
        0,
        1,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.47135649,
      "neural_order": [
        4,
        2,
        11,
        14,
        13,
        1,
        9,
        8,
        6,
        0,
        7,
        15,
        3,
        5,
        10,
        12
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "e75732c09feef79c27175d52f7b17d043d9e8b792811d1a6dea61d527ea95a08",
      "trial_index": 554
    },
    {
      "answer_sha256": "71ebeb52261d138ec7928090b2bb08e936923eb7a3d9fca79c58e1340fb0ebb5",
      "candidate_ids": [
        "572648d1708984140094c15d",
        "572826634b864d19001645be",
        "573784fa1c45671900574483",
        "5729fd56af94a219006aa72f",
        "570961aa200fba1400367f15",
        "5729f9953f37b3190047861f",
        "56ddde6b9a695914005b9628",
        "5729fd111d046914007796a5",
        "56e1a564cd28a01900c67a48",
        "572673f5708984140094c69b",
        "57293b843f37b31900478133",
        "572818f54b864d190016446c",
        "5737a4511c456719005744df",
        "570953a7efce8f15003a7dfe",
        "572a18a4af94a219006aa7df",
        "57108073b654c5140001f925"
      ],
      "candidate_sha256": [
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "1ee077fd22f6021abe57c6217dbbbf9e6db11baa3d7205f6f4f7249ae297f76c",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad",
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "03498919a733654be7b400ca269d15daacb4e16f0dac95a4236a46b9cdc486d7",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "12614d82ce4ab7673831e5a8b91a94c2835cd9456fa1bf01dc62c16ab070086e",
        "d3683b277766779b663c8154455034cafe15388f870f4b875fba1e113af91660",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "30b0e064c3aca4d389caa6801b1bbf9dd1b6f813a453a0ce92bc1de3ed202a95",
        "e72c427b33209e555b5fc09b718f5d3a1abc94dbdf4f37900c15e3a035c2a954",
        "d0340f7215724f2365361615e8a563799243a08b9728bfbb9a3f113a32d96975"
      ],
      "example_id": "572818f54b864d190016446c",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        12,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.24032233,
      "neural_order": [
        11,
        15,
        9,
        7,
        3,
        14,
        12,
        8,
        10,
        2,
        13,
        6,
        0,
        1,
        4,
        5
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "696075be1a7105b4b464470b9f58344f6986ca18a9dd7c0f8e6ce0bdc7a40bc7",
      "trial_index": 555
    },
    {
      "answer_sha256": "1bf318d39980baeabac8653fbb8018528b33c11d5c166ff4e634b29358f0abb3",
      "candidate_ids": [
        "5729f5a03f37b31900478605",
        "5725fe63ec44d21400f3d7dd",
        "573062662461fd1900a9cdf7",
        "5726fa525951b619008f83f7",
        "571c9348dd7acb1400e4c114",
        "57340111d058e614000b677d",
        "57281c594b864d19001644a2",
        "573011de04bcaa1900d770f9",
        "57293ca73f37b3190047815b",
        "57297d421d046914007794e5",
        "572f6a0ba23a5019007fc5eb",
        "57284b904b864d19001648e2",
        "56de4b5c4396321400ee2799",
        "572872dd2ca10214002da37e",
        "5737a7351c456719005744f1",
        "5727ed2e3acd2414000df001"
      ],
      "candidate_sha256": [
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd",
        "95d3b7479d4d43d7873edd7611bb743b131da7e7a4a5448aa98742e182b2ddf8",
        "48ac719ba3d1968c2d1b7c8c8c71906d06af7e5edeb9f373fa41f4e0690cb488",
        "3b43cbab19f7f3133d1da9d6519cc013ff3e5e23c2c70f574acda48ff54ebc31",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2",
        "f21c4becc50a26d5a0b767f8b5f6b586451094f5b1969f2164fbbb4f46fc814d",
        "669c732e68d4698e85daf523d3d93830928903f8fb4cdd4338411f7b305ecc65",
        "9a562188faaaebd17a115e2b46254c0a1a8bea147240b2909f2847decc87f17c",
        "576db96228f229ebd5188bfb187b0285ce8e7a6eae31bd74cad56712e9e871a5",
        "862c16275df11bd413d934f1001609084d3ec4aeb15df211fd7d395246ec3271",
        "bfad7e54a6646e5123dd89555e482034c8ab13b5be5cf77b34df2ee9913f4b28",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "a0f688a9430021f2f00f50897bc554590fdc0d0de8b0b9fd69b3255997504bd9",
        "b38835b0da15953f81e7fa39f6745a69d2e09f953b85ebc4995b8b8252bb8940"
      ],
      "example_id": "57281c594b864d19001644a2",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.46952078,
      "neural_order": [
        6,
        2,
        7,
        5,
        14,
        12,
        15,
        9,
        8,
        1,
        0,
        3,
        4,
        10,
        11,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "6fe28cc98ffc05680031796432ea2aefe05f7bcd09352d15bb07bfae5aea9735",
      "trial_index": 556
    },
    {
      "answer_sha256": "ec23c82146fe0c1dbd9d88d049c9ae75d1170a59a311e81199a6ad37d8187e2d",
      "candidate_ids": [
        "5728202c4b864d19001644ec",
        "57266e72f1498d1400e8df8c",
        "572facb0a23a5019007fc863",
        "57265aaf5951b619008f706b",
        "5725ec8289a1e219009ac0ae",
        "571099b2b654c5140001f9b3",
        "5726249538643c19005ad07f",
        "57264991f1498d1400e8db2e",
        "5726e3c4dd62a815002e9404",
        "5725cbb289a1e219009abed2",
        "57265a58dd62a815002e8270",
        "570d4c3bfed7b91900d45e31",
        "5727cc15ff5b5019007d9576",
        "57309bfb8ab72b1400f9c5e6",
        "572754dd708984140094dc3c",
        "572f64ccb2c2fd14005680b7"
      ],
      "candidate_sha256": [
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "65a4f29a52c31659b66baa59545297d8af56b2b1f884801f998c7d4c6e8396d8",
        "411996e4acd466c356dbf728b0c0e5a574dd85d6b64ed23507e56eccac9ccb4e",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58",
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "a4008fb587348767c254824b7e5bb46aaec5eaa762404c37c3e9e2f52fc93eac",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "1d470bf791028e4f456c100c44c5b3410fac821035b36b868d821eb23e1f2a74",
        "040a5627ef7bb5d949592bbcd7381d5c49b141f464054a8ba8c03c8253480ddb",
        "8dc2fb949d6dc11eb01624afbc85544a7cddd88de2de7c18d613d591db256f55",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "f396280baabc232b2dbd56220ead1c5d4dcf5e8af7a80222042b7a3a4d987f4a",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb"
      ],
      "example_id": "5728202c4b864d19001644ec",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        6,
        11,
        14,
        2,
        13,
        1,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.10826925,
      "neural_order": [
        0,
        13,
        2,
        11,
        14,
        7,
        5,
        3,
        9,
        4,
        6,
        1,
        8,
        10,
        12,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "f95ef13421e7b38080da51163b7cbf7ad6aa38007ef722418af7cac3b9a13d57",
      "trial_index": 557
    },
    {
      "answer_sha256": "3d7e732336470063f8019e28aaaf0a0e02485dc5a32ee5ed839704ecb6d1a445",
      "candidate_ids": [
        "572822233acd2414000df555",
        "572658daf1498d1400e8dcac",
        "571153422419e3140095557d",
        "572691bedd62a815002e89dc",
        "5725e45689a1e219009ac048",
        "57340d124776f419006617bf",
        "57264684708984140094c123",
        "5727cff1ff5b5019007d95c4",
        "572a0d21af94a219006aa783",
        "572a005f1d046914007796b7",
        "5705edcd52bb8914006896ca",
        "56e1ded7cd28a01900c67bd4",
        "57286f373acd2414000df9db",
        "572989846aef051400154fc0",
        "57115ff82419e314009555c3",
        "5725ce4d38643c19005acd4d"
      ],
      "candidate_sha256": [
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "4b9dbc30727db9ec57f0b7c9ce93419cf17747304d9b72e525e7366146ee1b62",
        "f875007e3f454eb0be5f61d3b23b8218effde8c5456b270f4d2aa3d318d261ca",
        "be1e8ef67e19e07dbf69a2cd82b160a507b81ad4a6ae00fb9fbdeda6ebbed945",
        "0d50de71a8220db34991fd3710e965ad95973a5e898da92d2e09819ee0ff21b8",
        "2d3fc7ca497bbbc2dbcd1b6cdda8ad72ab350d972900ce1725419106f82bdad6",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "22a8092c42faa51f3d6dbe3ed281515ca4a9627a0e1ba161e0cb10299afcec3e",
        "5621a695e0f4cee7162cef3437520925a230303661d56d42374bb5adaf0aaaf9",
        "5c0fd2f8509736ad1d703d7ef3c3bd05e32333b37735101e2f5d067a36cf3aa5",
        "c8747b5ba720574699c69280b74ae88eae87bd9bedd966eb9dd5206f8293a85f",
        "cef2854d822d069c1c644c1a0084b1d935135b00232dbe5c01cfe2dacc77e20f",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "d619b23bf7b93c1267c56dec97767d7a43f0b1d60b5a2d4b27068ccf50232fbf"
      ],
      "example_id": "572822233acd2414000df555",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        11,
        6,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.47351669,
      "neural_order": [
        0,
        3,
        6,
        5,
        12,
        2,
        11,
        8,
        1,
        4,
        7,
        9,
        10,
        13,
        14,
        15
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "c017cb763f69efc3997c4ace05a44f71e9526b4abd5179b5bc3fbcc60c4346ef",
      "trial_index": 558
    },
    {
      "answer_sha256": "4e73ff840ac8644a388365f8b5cff0b016219096686f4e6f552523dc9bb61c0c",
      "candidate_ids": [
        "570d4030fed7b91900d45d9f",
        "57115ac550c2381900b54a77",
        "572825a92ca10214002d9efe",
        "57286c8cff5b5019007da218",
        "57332562d058e614000b5730",
        "5728e8212ca10214002daa6c",
        "5726db5add62a815002e92d4",
        "572f6ec7a23a5019007fc621",
        "5729a03f1d04691400779593",
        "572f5703a23a5019007fc573",
        "57286bb84b864d19001649c8",
        "5705fb7f52bb891400689750",
        "5737534ec3c5551400e51eab",
        "572650325951b619008f6fa9",
        "572a09abaf94a219006aa75b",
        "572a058aaf94a219006aa751"
      ],
      "candidate_sha256": [
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "6246745d994cea4cb5f83c0b426e98ac22cce16f849ce38cae3a8ecb9720f415",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "c24cf42ac4098a341bcd386d785b12b502be6b8c632f94ed19d4ac10b3f225a3",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "03e73a55447d3ce769f7ff263041774683881e82e7f349390d25c89043f01ec6",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "2424c677d91ead7b6884aaa51d2cc04783f884e3c10ae22d2a4a0c7175ed4d8b",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "d2c301f3e303e0529f1b5862d9c77adabe2cb026b4683982583601544f992a66",
        "b0008033f8b97b0cf458d8e505384798d796a2fc78a3fced41f39a2591ef64cd",
        "2c21a41415000684af4848067660868e561f9f1525f219a341976cd6bfae3027",
        "7a4ee6ca6d7934a5657f7c69d91e7ec9cc82611a9a4235417ebc4011313a1701"
      ],
      "example_id": "572825a92ca10214002d9efe",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        5,
        1,
        12,
        14,
        6,
        8,
        10,
        11,
        0,
        3,
        4,
        7,
        9,
        13,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.02318759,
      "neural_order": [
        2,
        5,
        12,
        1,
        11,
        13,
        10,
        6,
        0,
        7,
        8,
        3,
        4,
        9,
        14,
        15
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "0dd0f2f8b1acc3386e009cd0d6fc5baaea06c12630015b5270a7657c61109ec1",
      "trial_index": 559
    },
    {
      "answer_sha256": "5e03d3a580a90e6b9779cfece4d6967be8cc881ff3e6aa8785a1ddd663da7fbc",
      "candidate_ids": [
        "572a0b0b6aef0514001551f6",
        "571cca4add7acb1400e4c150",
        "57283db6ff5b5019007d9fbc",
        "571166352419e314009555f1",
        "572677e7708984140094c723",
        "573010fab2c2fd14005687d7",
        "5725d52f89a1e219009abf78",
        "57302bd0b2c2fd14005689db",
        "56de179dcffd8e1900b4b5da",
        "570d2f5bfed7b91900d45ccf",
        "57287fec4b864d1900164a3c",
        "572a142e3f37b319004786b9",
        "570d391fb3d812140066d575",
        "57286c8cff5b5019007da218",
        "5728284e3acd2414000df5cd",
        "57108ee6a58dae1900cd6a1a"
      ],
      "candidate_sha256": [
        "b4427bebd3b8c72f24147e8acc1aee82ad622448f607f927c8649c8f574382dc",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "61a34f5c534a5f51fb34b62168e5e902cfbf87ce0543e9097da683e835bf4f3f",
        "18421b977c0934cbdee78ce3b3e1e3d999af77d0e67ef9aa7bbed857be3070ec",
        "d93c7d5b00d35e994837759344de468ee6476cff3aa4d621189ac8123a4b67db",
        "9af8cb5926de692d247fc6e89effa7ba163babc4fd5a06f434d38ad148210efb",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "88d1e5596470300feec3b26a2ca2a1caa48fae730f7dfdcf3cfae74056033518",
        "823249b3c179c7bf3d9de7f322b83644f50d2ae221e5d56785b0dce7e235e2e2",
        "1dd13bf14660147737c3329dcbc515e988f9c216e86af57e2f8c27638569229b",
        "99cd537a51fac2706e5aea86d147305bae17a981427e1eb08932fb668f5e95b7",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "a4b301bdb4849b7fdb9725e59b30bf1926329f291c911b9365263d2212e3da78",
        "fd93028701526b473dbe93cb4ded11a23207c124c84b16733c2ee2e9b442c706",
        "faa7e1060dbb7bdc9423952f8889298be2400a4609de6472948393026c30c3f6"
      ],
      "example_id": "5728284e3acd2414000df5cd",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.24972555,
      "neural_order": [
        14,
        15,
        13,
        2,
        1,
        5,
        0,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "b5f151cae82a8cd0c05aa6eec0a1280ebc3ad156ea942d82fc687fe70dfef69f",
      "trial_index": 560
    },
    {
      "answer_sha256": "40a11884ba64223525af5c447d27206abb281dfeb62ff4ab999796ad6ad406cf",
      "candidate_ids": [
        "57282dfb4b864d1900164668",
        "5733f309d058e614000b6648",
        "572fb42904bcaa1900d76c15",
        "5737958b1c456719005744c3",
        "56de3e414396321400ee26d8",
        "57376c50c3c5551400e51ecd",
        "5733140a4776f419006606e0",
        "571cac5d5efbb31900334dd0",
        "57299c2c6aef051400155020",
        "5730285a04bcaa1900d7726d",
        "5727b0892ca10214002d93e8",
        "57309564069b5314008321a5",
        "572881d34b864d1900164a5a",
        "5729a03f1d04691400779593",
        "5710968aa58dae1900cd6a86",
        "5726b718dd62a815002e8dbe"
      ],
      "candidate_sha256": [
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "5b469d64ced1175805ddbb53037730159f46f770f6697c84c8b9e8738d9bbb19",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "9807afc08eed1c309af6758565c9fb73e71562db167a6f2beaae892f76dd712d",
        "196983e68e85513a7fe6690715cedcdf5521e1e6f3f26798c48cc4618989f754",
        "11e489536cc7ac71a8ee73d6c7e52469ad3ad8807c404fd6cba2907c8189cab7",
        "8f85c3d320f0f2f34ba30080ff2604d64f1e338d875da36278cb6bff52a9afa7",
        "5c2f4fc18b23e9b6edb56fe0fbc3d06042a4d34289807c626b06078890ff3065",
        "dbe3b208199caa88b39153756ef6ca8b41ed19f9d4eb07b35c977b49efd4d689",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "1d22fe7376840344d1baeceecf331a1b111373982abd34a9adbd1c7ab732274b",
        "290ff6e5995b6225912bc5c2ba43ba1b4c1fed855787d1836a6727f9810cae27",
        "4aa10fb153da782c2edd4f946166fda375f96bec7cbc6a0f6d1c151025787eca",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4"
      ],
      "example_id": "57282dfb4b864d1900164668",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        5,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.13759156,
      "neural_order": [
        0,
        3,
        15,
        9,
        11,
        5,
        4,
        1,
        2,
        6,
        7,
        8,
        10,
        12,
        13,
        14
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "06eb6b284f87a2160c7a0a45a965e606a74d4a5de9c3559a8301bd5eac50f230",
      "trial_index": 561
    },
    {
      "answer_sha256": "e46653e92497ed11dfe9bca59b28646f0a6b9fd41c5f82e4015f0ddc2181cf19",
      "candidate_ids": [
        "573786b51c4567190057448d",
        "57338160d058e614000b5bf9",
        "571cdcb85efbb31900334e0c",
        "570d3468b3d812140066d543",
        "57263dcd89a1e219009ac5a2",
        "57300a9a04bcaa1900d77063",
        "5728667eff5b5019007da1fc",
        "57340d124776f419006617bf",
        "56e1c4fce3433e140042314e",
        "57265526708984140094c2bd",
        "5733cf61d058e614000b62e9",
        "56e1a38de3433e140042305c",
        "57271f125951b619008f8635",
        "5728e8212ca10214002daa6c",
        "57377aac1c45671900574479",
        "57287f6a3acd2414000dfa51"
      ],
      "candidate_sha256": [
        "38284c73beb7b13e4f83f5564c8f6d7d901bb85c0b530e6ce29f532a5b66465f",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "fa7b1b0d74d9f12de63015fe351bfacee09ceccf626caceb999b7b6413671d75",
        "1d4b2491c909da1e52c580fbfb723ed08bffedb1cbf55d0d87cac70e85a3f3ef",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "37b7c4ca4ba5c4cb091c924dea66f21b686b4d4d2908cb18364f312fe09c8371",
        "362c7d77e45876b8ec90ccd0e364202c6b45142dabbe49bdfe6a843c0886069c",
        "be1e8ef67e19e07dbf69a2cd82b160a507b81ad4a6ae00fb9fbdeda6ebbed945",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "63583ad01af4d3525f39681575e1e7eb014fdf24bbed2a7bca316567f29b4d20",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "661089229cefef25a7ee0f8224a2253cf17a5a8a5d9a6f5890c2586f84f2d5f7",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "b8ebbd89c246d5f748d05db8cc51691a47b9f831e4dbcf49103718b524023e76",
        "bfbe414bdbeab9da323fbcffbe2c73013d4592d5fc040c429776595b323bf4b3"
      ],
      "example_id": "5728e8212ca10214002daa6c",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        12,
        15,
        6,
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        14
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.32580999,
      "neural_order": [
        13,
        12,
        5,
        1,
        6,
        8,
        0,
        11,
        2,
        7,
        9,
        14,
        3,
        4,
        10,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "e66d1bedd21c46af253237eee1b8fde926a7500f261479fe275c052e84bacc2e",
      "trial_index": 562
    },
    {
      "answer_sha256": "1809ee9aae6e2aee7d951bc1edd02f132d4f364c998e0f13c45fb077d0f50de3",
      "candidate_ids": [
        "5728eb1a3acd2414000e01c5",
        "5727cd7dff5b5019007d958e",
        "5726385e271a42140099d797",
        "572871bd3acd2414000dfa03",
        "572a1a5c6aef051400155284",
        "571c7d55dd7acb1400e4c0c4",
        "57287d4a2ca10214002da3e4",
        "572882242ca10214002da420",
        "57340a094776f4190066177d",
        "5726b929f1498d1400e8e8e9",
        "5725d183271a42140099d23d",
        "57293d116aef051400154bc8",
        "572fcb6da23a5019007fc9f1",
        "5733eb34d058e614000b65cc",
        "572fda6fb2c2fd140056850b",
        "57107c24a58dae1900cd69e8"
      ],
      "candidate_sha256": [
        "670c1660953af3287fb138d4ede5ac2f2ee89641d53c09280d41c03ecb38bb8f",
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "d66dc811eec70fc3e4c423de2b401a9c06f15664d2374683267e4dc009eeb470",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "bf6091255163527aa530767e07219aa72506bc6cb126b8e4167dcc9934db0548",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "fc184229a2b15ca7cdfbc4cf9985b525d4188c1b10a28c98f7cb1df638a4e8be",
        "1629b48b9aae07c00bf8bd9ec7d76d9982b5e645f832551c105d868a0f34f81e",
        "b55f93537b0ed64fd665fb8381827e663f16411f4848025713a9aaa3ac0af0d5",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "ecf1e1973489bdea861aef055f7038e7972d136b0a3e0815dad277527a4de447",
        "14e1246d6c57cac98cd47a35fe5c0801832d2bfe7bc8d39af8efc222d99387f9",
        "5b1d7779ff634f8b4049e70e99b5eaadf538e5a3d3a2aa5069b096c68cb79e5d",
        "edb8d00a6415b164c394c0f93d41382a7c573f75814fabfa6e603926be0af604"
      ],
      "example_id": "5728eb1a3acd2414000e01c5",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        6,
        1,
        2,
        9,
        3,
        4,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.33046223,
      "neural_order": [
        0,
        13,
        8,
        15,
        11,
        9,
        10,
        4,
        2,
        6,
        14,
        12,
        1,
        3,
        5,
        7
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "3ae006e45137fb7a63e4490191d3dc1f25ae0331cfd57f4893bf12f2664003f8",
      "trial_index": 563
    },
    {
      "answer_sha256": "173b2b69bc05083b51160996e7bc151d4515caacdb1c20083d4e3970e310f31a",
      "candidate_ids": [
        "572a142e3f37b319004786b9",
        "572856beff5b5019007da190",
        "57308f6b8ab72b1400f9c580",
        "572fbb04a23a5019007fc8f7",
        "5727d6faff5b5019007d969a",
        "5726c3da708984140094d0d9",
        "5725f6e138643c19005acf19",
        "5733d2dbd058e614000b6339",
        "570d4030fed7b91900d45d9f",
        "5727482bf1498d1400e8f5a4",
        "5728ebcb3acd2414000e01d9",
        "5730a0778ab72b1400f9c60a",
        "57286ead2ca10214002da346",
        "5729a3716aef05140015506a",
        "572882242ca10214002da420",
        "57269aa65951b619008f77ab"
      ],
      "candidate_sha256": [
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "5092a26df4dc4e01290d314c8bede184a4ca7700ef46fe5b5ec754d6c56b5796",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "06dabbfdce88f733bf8b7c056c89d87a5ec555147201d86e7703ff7355c1b756",
        "c921148c6781bd32095c95fe8a605f57c9e33119681a07d3d01a7ee290f988dc",
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc",
        "b754709d2be6673ec34c0291a03498bd3136fa91501c6f3047b79318627014c0",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "38ef368895718c227f9e1c396bd2c622788a2307ea4553ac38ea7a2b7896fb5b",
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "56342ec864f7c99a6400cb1b743d2de657d247bc5557e34b707c510c768abb1c",
        "8cecce59146202bdf29d0640494e08c970c9712b7abd814f26d4bbba906ea28e",
        "2b7326d0cd8188e7b1ea38f54e115afbbb5e41e5f6886b800cf1fd31ac49dd48",
        "9bf6a80bb7cef66d730c905dc46cc270db123bc20c72a9ffa0022dddb77436ea",
        "ddc6c9f111ba6093f22cef4e6052499e50be27247fd02709dc8d0dd82041c88c",
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2"
      ],
      "example_id": "5728ebcb3acd2414000e01d9",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.08207376,
      "neural_order": [
        10,
        11,
        2,
        5,
        15,
        9,
        3,
        7,
        12,
        8,
        0,
        1,
        4,
        6,
        13,
        14
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "4b2c3183c677a3fe3fc484b96b0a03de05a385710f77e4e39f380f1c9ba961dd",
      "trial_index": 564
    },
    {
      "answer_sha256": "e7c051c07177804361d53af53427e008c9d0c0f0a192f08a6b464dc71b4e1ab2",
      "candidate_ids": [
        "5725be0f271a42140099d117",
        "57096e1ced30961900e84102",
        "572861cc4b864d190016495e",
        "57268882f1498d1400e8e306",
        "57300f8504bcaa1900d770d1",
        "57293e221d046914007791d5",
        "572ffc99947a6a140053cef6",
        "572fdd03a23a5019007fca9d",
        "5726e5c95951b619008f81cb",
        "5727cc15ff5b5019007d9576",
        "57265526708984140094c2bd",
        "5728ec6a4b864d19001650aa",
        "57264efddd62a815002e8134",
        "56e1aba0e3433e1400423094",
        "5727213c708984140094da35",
        "57340111d058e614000b677d"
      ],
      "candidate_sha256": [
        "24fea2565633e4166a4a302ac559cbe556726d323a6ae3b67d380259c88732be",
        "33e437b3aa69c51c5ea1020e0b70f0916c331f1cf003676c6af11264875e9156",
        "3f83987ebc0c4b6d662d8cc6134fe19bdcd18d63555f50b4c6841486edd3606d",
        "c2c90d7474f65aafbfb0673b1e8914d12583f3a0ae7de9054f81c8cd079800fe",
        "718d81cccb1cfbbb8ab954ba3adc6f9969ebf682d9c40bdd66295995cf5a68aa",
        "a13fb4c358557dc227b4cf451a8338fd695777c87f1452f906d1b60e6dc5cd61",
        "9a0ea788d4b66d4412369821a33e96287b8b91a59dbbe258cb494dfacfb20ad4",
        "0b0c060c48f85197bf26e8fe7a9135f1de267f3c76fb0f2a8b61b88bdb83897d",
        "33b19587e18bf7bf959ece124d8438510f9d20eb4b978252cc77ea7a76e39416",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "e83edd2f6eb1fa0981acc0119542b4a54c226b0e0249686e43082425f6d974b0",
        "df72fdf9177d0ce1999d3245978f17a78cf4721232e3c59b60a2f41a498e9dda",
        "569d9834c490d9ef08715516299c35473302aac1c4aeea04f4647c2e806a18f8",
        "8e240428cc00a7e341c4c98ac2c96623adbfb2366b33b6f3685fcaf86361caa0",
        "652781637aca0d4be073ba7536e6b99b5d346bc3a06e0e0ecbce96a06fde5dd2"
      ],
      "example_id": "5728ec6a4b864d19001650aa",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        3,
        14,
        0,
        1,
        2,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.25471308,
      "neural_order": [
        11,
        14,
        15,
        6,
        4,
        12,
        2,
        10,
        9,
        1,
        0,
        3,
        5,
        7,
        8,
        13
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "71e5dadd0de40b2929ee56911a97c9e22803a7ea2e6f773cefb42bf8b67683da",
      "trial_index": 565
    },
    {
      "answer_sha256": "9d3efafa43abcc64c54955a77e2688a1ca6ed93722dea77be96c7b0cb2641ac8",
      "candidate_ids": [
        "572a12381d0469140077972b",
        "57265f605951b619008f70db",
        "5727e8424b864d1900163fc0",
        "5728ed94ff5b5019007da97c",
        "57271c235951b619008f860b",
        "572885c44b864d1900164a78",
        "5725e1c4271a42140099d2d7",
        "5726462b708984140094c117",
        "5728ec6a4b864d19001650aa",
        "572658daf1498d1400e8dcac",
        "5737477bc3c5551400e51ea3",
        "57287338ff5b5019007da232",
        "57274f67708984140094dbf5",
        "570d35b7b3d812140066d54d",
        "57275409708984140094dc35",
        "572fda6fb2c2fd140056850b"
      ],
      "candidate_sha256": [
        "c4833b3743b47d6540a2e7b1029673e1fbde60f22e9f7a97d46dd8729e18dad6",
        "c8c47f5e5a648817ff11f913f97ee8b366aaf758a4bd7ff32582d54df2a8fa31",
        "3ba76976d0dad7457d2d476110fab5ee215b49fe7e197389981bbb85886e2ec3",
        "c7f395fb45d39675d91269f25c5e193545b2bf5ad8cd9d7f9faffb57b86454a2",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "17204d302a7a5587913391c2cc3ef01540a0e6cd9ac84ee2e0732c35291a5012",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "e83edd2f6eb1fa0981acc0119542b4a54c226b0e0249686e43082425f6d974b0",
        "50dc97ced70e63660fd5569a0f37ced144e103640987962571dfdb558ed06c6b",
        "5b8656a663456a61c1d9692e2b9d3858aa19a6b21b538a75b82941eb5206617c",
        "bb7672c097755bb254dcda07123304b126ddc10e9fc8c157410b39caff2b9954",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "ca0f7b7018127b2736fe74e8ea343545834b32796fc74beb48cde30324c80067",
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "5b1d7779ff634f8b4049e70e99b5eaadf538e5a3d3a2aa5069b096c68cb79e5d"
      ],
      "example_id": "5728ed94ff5b5019007da97c",
      "gated_correct": true,
      "gated_top": 3,
      "gold_index": 3,
      "lexical_correct": true,
      "lexical_order": [
        3,
        4,
        11,
        0,
        1,
        2,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 3,
      "neural_correct": true,
      "neural_margin": 0.18581372,
      "neural_order": [
        3,
        4,
        8,
        14,
        11,
        10,
        2,
        15,
        6,
        9,
        0,
        1,
        5,
        7,
        12,
        13
      ],
      "neural_top": 3,
      "partition": "test",
      "question_sha256": "897d4b6f0bc62f8e20ee11d70fdf495766306a950416ed6fec5cb5e144df598a",
      "trial_index": 566
    },
    {
      "answer_sha256": "f0c11fde4b737ee930f44122c53d7ae60b12a08144a1e606b1888d3c4bad43e4",
      "candidate_ids": [
        "572998673f37b319004784d5",
        "57308cf88ab72b1400f9c576",
        "5730a40f396df91900096234",
        "572a0bebaf94a219006aa76f",
        "572a1f086aef0514001552c0",
        "57339ad74776f41900660e86",
        "57264d58f1498d1400e8db7a",
        "57264cfa708984140094c1c3",
        "572657d9dd62a815002e8230",
        "5728eef92ca10214002daab0",
        "572a064a3f37b3190047865d",
        "57280e1aff5b5019007d9bec",
        "5732b6b5328d981900602021",
        "57261dab38643c19005ad037",
        "572fd47fa23a5019007fca55",
        "573027d6a23a5019007fce9d"
      ],
      "candidate_sha256": [
        "56c5a58a20c281345ad1b156034653805b4b37502a76613ea7935761d3263ade",
        "8e3602480bdea2758de968c9cd5a7a211ace7419d041305ca06ede7a67c543c4",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf",
        "fb7cece550ee7b627e23d5c3f8425c937139da767c728db9729da6ba905465fe",
        "d486c3779d7a4a5926b9156353cda8626ad6409cc3cd5fdb1a098ab7cdad4b93",
        "6cfcaa0f3ee806b7fb2844d8a70b71745e85f01eaf1463c4ffb7adf4d8c86843",
        "32594dd1071ba46226b8f5cd2cfc09b0178c1a9d1d86dc96c20b9dde4371ab1d",
        "db10d196495a161922a1508d574e84c1c57749db05c53e9ad51ef9f511da16ad",
        "a05eabc8f87fdca80834359f2972b2ffc1cf60d28eabf4521b6fbb77c0623996",
        "829e809f79e6cd89a4bf8d645fa85c1b0bf92b9d6362547ad071e708ecc99593",
        "61b4937b9d0514bc8f149849c776fa0f98dcb11b7af677d4c946b355ff06ca52",
        "0606f47852eabec2d5a4ace87ec3d3465bf64d1690c2ed9ed7ea2e30ba0041e8",
        "354de905208b22299c2a39f0c775443c985547505f31c7e5ef9351d30344ba0d",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe"
      ],
      "example_id": "5728eef92ca10214002daab0",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.28340039,
      "neural_order": [
        9,
        10,
        13,
        1,
        15,
        6,
        8,
        0,
        4,
        14,
        2,
        11,
        3,
        5,
        7,
        12
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "f016f49e307808217ed244a0fe3633e125d4486f05729660136df57810d22675",
      "trial_index": 567
    },
    {
      "answer_sha256": "21e1edf6544a2cf3405f374010b0dc56adac4ea3c34fc163f7202b15ebcbd9e6",
      "candidate_ids": [
        "5729081d3f37b31900477fab",
        "5733a6ac4776f41900660f58",
        "572750e8dd62a815002e9af0",
        "572a0ce11d046914007796fd",
        "5728f50baf94a219006a9e55",
        "572905ce1d04691400778f83",
        "57264b3edd62a815002e80aa",
        "57111380a58dae1900cd6bd6",
        "5727c8863acd2414000dec49",
        "57264e66dd62a815002e811a",
        "57284d484b864d1900164900",
        "5726a46cdd62a815002e8bd0",
        "57376c50c3c5551400e51ecd",
        "5726da89dd62a815002e92b2",
        "5725d662ec44d21400f3d687",
        "572fc78d04bcaa1900d76d07"
      ],
      "candidate_sha256": [
        "db6de70b42aaae5f96618f4be9863dbd2b0618f69ea12eeb5dd9dbfb1a8bcef5",
        "501203b730c0f3270043cbb23105201ff5376b0fd2de78b257175fa90c937ad8",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "e45d7b6faadb654909b743c5a61dcd1e6053c8b7ac302e0d99caf5a6e43d1869",
        "7707b43e2d9349ec0fa92f79085391bb5dbe7f2da552f94836a0291fe5c85a21",
        "2cbcea1791f6caa527b6c97411f4641713fe2fc6edb3b45811042048f70202b1",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "b33181c0262419e5c9cfbbf1c8b343becabd6004d529d66cdc65465c54aa31a6",
        "99176151d772b474b85caf9f32312add048f95ea6a74bf0fdc5d1e6eb84a0dda",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "cf83b4012ada6714b8fe507b4b152b6668fd47cd4eac346e991ac37512e34ebf",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "9807afc08eed1c309af6758565c9fb73e71562db167a6f2beaae892f76dd712d",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "325b9415640f2472970fc17f76f5003bc1bbc570f4683d899e36f754823a82a7",
        "8eafeb3409af83fdd03ba65c874ba777f9cdae3c3a1f69a1f857ab52d9dad470"
      ],
      "example_id": "5728f50baf94a219006a9e55",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        14,
        12,
        10,
        0,
        1,
        2,
        3,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.18747496,
      "neural_order": [
        4,
        2,
        14,
        15,
        9,
        10,
        12,
        7,
        11,
        0,
        1,
        3,
        5,
        6,
        8,
        13
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "557478677da5d236b60e7c7d4b16c8cb9f59caf8e11ff2018802a4da8fe3e8a9",
      "trial_index": 568
    },
    {
      "answer_sha256": "c46f49bab62535f7feae0c0f832ed6e65e290e11fd7172adaa53a7fed0fd64bd",
      "candidate_ids": [
        "5727c0402ca10214002d9564",
        "5728facd4b864d1900165170",
        "570d2417fed7b91900d45c3d",
        "5727cebc2ca10214002d96f0",
        "5730a951069b531400832213",
        "57274971708984140094dbbb",
        "572759665951b619008f8883",
        "571cbe35dd7acb1400e4c13c",
        "5729fd56af94a219006aa72f",
        "573028fa04bcaa1900d77287",
        "5729ffda1d046914007796af",
        "5727c3b02ca10214002d95ba",
        "572667e2f1498d1400e8de90",
        "573368044776f41900660a29",
        "57309ef18ab72b1400f9c600",
        "5728fc2eaf94a219006a9ec7"
      ],
      "candidate_sha256": [
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "0452693c2cb8fba37e58cad0335aea50d57c51818ce0841b4e804640f45086ac",
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "3bc8e7da3b9de07cb17a9676516aaabb0ada94d03af66162c7db4c5694dce1fc",
        "36369af6a6ac6372ee4fd5b3a90a276c139484d38876143782e791b973685c35",
        "f6f6d96dd4e446654b7841a39161d16203969d16c05d019d479a1f1657ab0896",
        "0fb23036449013510f8c252304c3eecf5fccaa6ce7e6eda9cdd52fb0feda00d2",
        "6fc646facb48400c43061228449cf9516037f6d3ab2d188ce20799edf2af1a06",
        "1e78d27475aecd3ec0eb73d6278e6a3cbe36d5f520963ba6b65f7dc7602a6dd3",
        "478fe0e0722a2cee8df8d35f5896cb00b61f1afc0fa93dc3410181a8c12ad588",
        "b557caa6a1a9ebdc6ec705acc8c1078fbde1cf27379b5961913015d24cbd8809",
        "cda3f03eaa3903be119a850afe9aeb2ebc58ed24e4dca669ed72f61be9577c8b",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "8ffda8441c55fe91c1d7c352b984785d42dee97084adfcaec49f89486beb85ef",
        "004d13549d731b2a79666053d4a24b060b9c2018ef918f72daf13eb3f515cd89"
      ],
      "example_id": "5728facd4b864d1900165170",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        9,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.28511838,
      "neural_order": [
        1,
        6,
        4,
        11,
        0,
        8,
        10,
        12,
        15,
        5,
        14,
        7,
        2,
        3,
        9,
        13
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "bb5fcf6fa26ff4cbf5a472e354a9729e42881da5894de03497aebe320d1b2765",
      "trial_index": 569
    },
    {
      "answer_sha256": "e5ed2d8cbac6b6f2d4c971cfdc7dda6a42259ba289e5b8ced9f73c06cd61e373",
      "candidate_ids": [
        "571142b3a58dae1900cd6d5a",
        "571163172419e314009555e7",
        "571cc3dedd7acb1400e4c146",
        "56de3e414396321400ee26d8",
        "57263dcd89a1e219009ac5a2",
        "5733d249d058e614000b6331",
        "571cc5c45efbb31900334dda",
        "571135b8a58dae1900cd6d0e",
        "5725c63438643c19005acc9f",
        "57269016708984140094ca41",
        "5728fb6a1d04691400778ef5",
        "57273e50dd62a815002e9a02",
        "572fd264b2c2fd14005684a9",
        "5726bcde708984140094cfbf",
        "57297ed93f37b3190047845f",
        "5725b92e38643c19005acbd3"
      ],
      "candidate_sha256": [
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "90f58333c97a52e6c94616b65018b2aab9e015634bd8b69320412df5a25aebc1",
        "0ed001b44e25271ed198ff145a14f9dc467d1f56b27afead08e9b4bc33f29fd0",
        "8121c40d42d753225c6a469ed73c8181e91e006ed9ad9f72aa74a74b2363b0b8",
        "3d54659f24990c46cee06f5583023244fea05dc6b51b5fc175d27bad04837fea",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "7570c6179b8da3c0ea28959bb249cf7b34072b311b684304210406b0986f7d7a",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "6490f518fe350da9641471f3a1a36ab4a64fb7a97f05b88e0eee6eef8ff7e18f",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "1e39559648bdfe4284447ac60eead4e50ab69917683d26b7c912613f60600260",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "a39e17016b193219ad2d263fa5839b7f520034dd1f60ce7c364d0478c49964ca",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "4db57d18c895cf08141264311300950074a7484693bfac13f09fd3abb15774bf",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d"
      ],
      "example_id": "5728fb6a1d04691400778ef5",
      "gated_correct": true,
      "gated_top": 10,
      "gold_index": 10,
      "lexical_correct": true,
      "lexical_order": [
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 10,
      "neural_correct": true,
      "neural_margin": 0.47868642,
      "neural_order": [
        10,
        13,
        6,
        12,
        8,
        7,
        5,
        3,
        0,
        1,
        2,
        4,
        9,
        11,
        14,
        15
      ],
      "neural_top": 10,
      "partition": "test",
      "question_sha256": "a6eaef83168d93ea335e97f268070f08830d41bd8e81c61fa10af298efbafee2",
      "trial_index": 570
    },
    {
      "answer_sha256": "159ffb347ccc7535b9f4e10ef4d033d208bf74d4102f26b60b554d93cd4a2e2d",
      "candidate_ids": [
        "572649d8f1498d1400e8db37",
        "56e1fc57e3433e140042322c",
        "5728fc9e1d04691400778f11",
        "5726f96ddd62a815002e9698",
        "573027d6a23a5019007fce9d",
        "571154c72419e31400955588",
        "57286192ff5b5019007da1dc",
        "572949306aef051400154c68",
        "572822233acd2414000df555",
        "5725f8f5ec44d21400f3d7b1",
        "57265746dd62a815002e8218",
        "57274f67708984140094dbf5",
        "5711541350c2381900b54a6f",
        "5727f05b4b864d1900164068",
        "5733d2dbd058e614000b6339",
        "5729517d6aef051400154cc8"
      ],
      "candidate_sha256": [
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "ba15f86ae09bc08e0deccb09eabeacc8426433f31366f7bdd5db57801a480ace",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c",
        "a5c5b21972333d3f3f3912faed992876fca81100726aa101c35ba8f3dc2ca6a4",
        "8face3f9e260f534ee98b050e313791fafd75ade587e79dc091494e280786cfe",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "4e8a8ba470a6bf2ac2542b8e6474bad12a5b4aab5322270f08d127b7f9e2986c",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "0078a3aad6d29993d7ba7194bf27bfeb0bcf19d6c79a165e45c45cfd1c239758",
        "46578ca9db66c3a4b1f35c37884c03e7289a6773634dfc580ae01932fe4fdf7d",
        "37330ae11a4883659bf4d70332103d81912cca5a74d7e5fc2ded52e1bd392ce7",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "84f49c43c1af75d4275d5136ba4713a8bdaef62dd919a64b1b24764e92c42856",
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "a3ff0ef7dc3737bd9225e672a62787bcd6362c351ee6a265921f876dedb6b2de",
        "7097fc043d251726d390bf28a141cc5db92ff1ac8c84158fa8666531012e7385"
      ],
      "example_id": "5728fc9e1d04691400778f11",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        4,
        0,
        1,
        3,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.09122415,
      "neural_order": [
        2,
        8,
        0,
        11,
        4,
        13,
        10,
        7,
        15,
        1,
        3,
        5,
        6,
        9,
        12,
        14
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "2c6c7d3138830ad0d00e98b44ed761e117da7885387f33ac01b02b6d86fbbe1c",
      "trial_index": 571
    },
    {
      "answer_sha256": "9c34d9bdea5a3e9f6d3d364ebc7e759c3b052d3301af1d14aa0edca82972ad3a",
      "candidate_ids": [
        "5727cd7dff5b5019007d958e",
        "572820512ca10214002d9e72",
        "5733d4c8d058e614000b6353",
        "57287ccb2ca10214002da3da",
        "5727e9523acd2414000def95",
        "5728fd3c3f37b31900477f2b",
        "57115f0a50c2381900b54aa7",
        "57306797396df919000960ee",
        "572a0ecb1d04691400779717",
        "57264991f1498d1400e8db2e",
        "573081c2069b531400832133",
        "57376828c3c5551400e51eb9",
        "57286bb84b864d19001649c8",
        "571144d1a58dae1900cd6d6e",
        "57293d116aef051400154bc8",
        "572982e66aef051400154f92"
      ],
      "candidate_sha256": [
        "125135dd9d38f51af002ba49b99121835c8d0843192c33251f5a9c21d5b1e374",
        "75cb5a3ffcb3fc0fc634b23eb3e6ae0e8c3365b15ad5892f4ea7a382f0495c6a",
        "848249ede437c8e6985e06665d2ff25a3423bf0cf894fc193dd86cb7166ceb32",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "34aa9a8e3e875665b03b42736977f2f48a2ec8d33658966acc4b94712b217093",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "29f286e9587898e26c90bc809f8b3745ee43be3e41a2975eec8fab07c2d4fe17",
        "cc972e60dbe6da4b4e23470e3f981b6f088a8e6c5eb510d1c0f5e3707e211f2a",
        "9631a5d2abd6b76ca62eadb7dcfcb8cf262295581ef57f199ea042b3aef0c835",
        "798a2ea2ee1414e1c6b9fefa255e2702f5c7435e3ce80f46e2cdc30a22169ec9",
        "0ef4ed4116773ab890cef379f1868f1ba10ce87d84d1fbadb58a9bd614233461",
        "8075afe5649850130d6277b4d42a263b335970200c99b9786462feb745bd73d5",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "973bba514a778416487d8fb41226a28c37601750fe5f3f8ed97fd8cb65c41ff6",
        "b262a50b91142b6190510e526a4ccc399841cf6fa48b978c236895f2f8e035d0",
        "93620703d50b4155fab9038240e64a854d1329e60d76b3887ba89b6b28625e9e"
      ],
      "example_id": "5728fd3c3f37b31900477f2b",
      "gated_correct": true,
      "gated_top": 5,
      "gold_index": 5,
      "lexical_correct": true,
      "lexical_order": [
        5,
        10,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 5,
      "neural_correct": true,
      "neural_margin": 0.3891086,
      "neural_order": [
        5,
        8,
        7,
        13,
        10,
        4,
        11,
        2,
        3,
        6,
        12,
        9,
        14,
        0,
        1,
        15
      ],
      "neural_top": 5,
      "partition": "test",
      "question_sha256": "4baec8a71bb33ad84caa74656628d0c59a9059afa704ec0cf5c3e06a6fc2d695",
      "trial_index": 572
    },
    {
      "answer_sha256": "7a01adea4b8a1084a7aebd7f3256080c31a3013aa5ad7638add2136d6f3e5371",
      "candidate_ids": [
        "571094b7a58dae1900cd6a66",
        "5728151b4b864d1900164428",
        "56e16182e3433e1400422e28",
        "57115ff82419e314009555c3",
        "5729feaf6aef051400155188",
        "571cd5b1dd7acb1400e4c178",
        "572949306aef051400154c68",
        "572ff5fcb2c2fd1400568657",
        "57308f6b8ab72b1400f9c580",
        "5727f2714b864d1900164072",
        "572fc41f947a6a140053cc80",
        "57273a465951b619008f86ff",
        "57115bf350c2381900b54a93",
        "57265526708984140094c2bd",
        "57296f293f37b319004783a3",
        "5733a560d058e614000b5f77"
      ],
      "candidate_sha256": [
        "30002152686f84273913d0dff1b10414c6fe11653db90ebb13ab33320cb98c9c",
        "992197d93539b1235d530bba689513497963a57872bdd0f88de1e347c2d4cdbc",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "45d09dcbde15ec52d7b4cb77166fe10c2c1288f7fd636d21e5bfcf6258d2a1ae",
        "3895e66f384dc9afdd15ffe1c33b4044506755bd22d33e8c94583b00ed06854d",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "e117936f33a37b1d7e89451fa0850df669debc5d22f3a2851f41d29d3030995b",
        "ecf6e2d5bfdccd7ada12e269b544317ad160ef535ef802e816ddb24062f0e875",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "01e35725510ad6491f4169dee36ee61ec70d6931d5fb53003c0e2c6a4bd2a611",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "ea82c9d7a805e960cc889a9692f69c9069b014904492e22cdfb4515383ed6d3f",
        "0db918cf93dd83cdbc4ddb72bc8b078a32ebf1c2dc955b572fbc01d61665d3a7",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "1a3bfa6d8c28e3d95f98010e81aae93407594356ed4a4ca189d76d6b8d7c83f6",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b"
      ],
      "example_id": "57273a465951b619008f86ff",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.44642463,
      "neural_order": [
        11,
        2,
        3,
        10,
        12,
        6,
        7,
        1,
        8,
        5,
        13,
        9,
        14,
        0,
        4,
        15
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "0829319d6592c4aafe665be708900054f906e5ce999ea4c486734c069d234928",
      "trial_index": 573
    },
    {
      "answer_sha256": "2082dab2d8a44c6c75e7869e625947dbd0ffefb93d44465d4e848e59974f99b7",
      "candidate_ids": [
        "56e1b00ce3433e140042309e",
        "572683075951b619008f7513",
        "572fcc43b2c2fd140056847d",
        "571a4b0f10f8ca1400304fd5",
        "573003dd947a6a140053cf42",
        "5725fcbe271a42140099d3ad",
        "57273cca708984140094db33",
        "5737958b1c456719005744c3",
        "57274a8ff1498d1400e8f5bc",
        "57265e11708984140094c3bb",
        "5726baf2dd62a815002e8e74",
        "57376a1bc3c5551400e51ec3",
        "572f57c704bcaa1900d7686d",
        "572673f5708984140094c69b",
        "572807802ca10214002d9bf8",
        "57287ee3ff5b5019007da274"
      ],
      "candidate_sha256": [
        "94a07cc8780eb4400c56a348d92177f5013c97e5647728b384a3a14f2662cc5f",
        "d4b179250b58e123f79fcec3b8e4b94ca1557ab9a7cbe408bc322ced828f1efe",
        "a04348495e1d5c9dbb1605250473569a44febecb13da4593d68c64afff3699c1",
        "d8abf737112efa9397c511991f2e930e293c8a4b7a2251790cb6dc58cc065917",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "299ff1ed613c902a0d3c399c29504dfc2e962c96788b96f369e957d800ff7651",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "e1815c366f56a942832e5364007632abc6e07a59032e51ea5a620e75af47b3de",
        "c4b219e2fa6942ba9f303ff5bec6c539545d31cf8731fd264c58f85069b10c45",
        "4b09e468dc2c2a581560bee6e0748213ac6866c003a02a4f3ce99b88cda3fdf9",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a",
        "77f46222c650ac52f46494875a66ac9bdfd819d463c16ac32e3a05adf3a9528b",
        "76d92e92a6a95722d2402d8f47c5e55073f4f0002468a06046899e2b337d5a02",
        "a0db929a48c83ef36f766c4a7bc7666a9e2de11f6bf7ba38e5249c3c3535034a",
        "cacbb0a60059a48889a14cfd7002a103bbad41b8e8fbc9eefdca1dc45273440a"
      ],
      "example_id": "57273cca708984140094db33",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        10,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.36704321,
      "neural_order": [
        6,
        0,
        10,
        2,
        15,
        13,
        1,
        8,
        12,
        5,
        3,
        4,
        7,
        9,
        11,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "0d7320264c1c10e34061f1bf23fb9191152b7ff24eb62e154083ef6c8f6abb63",
      "trial_index": 574
    },
    {
      "answer_sha256": "67427b038f16b6d8b092fb16c638b53a264582ae12d1c92b4aae4cb1a6f5f00b",
      "candidate_ids": [
        "57269aa65951b619008f77ab",
        "5728501aff5b5019007da166",
        "56de4b074396321400ee2793",
        "5726241189a1e219009ac2de",
        "572fca7eb2c2fd1400568473",
        "57263c78ec44d21400f3dc7b",
        "57378c9b1c456719005744a7",
        "57308f6b8ab72b1400f9c580",
        "570d529fb3d812140066d6bb",
        "57106185b654c5140001f8db",
        "57269016708984140094ca41",
        "5737432bc3c5551400e51e99",
        "5710ed7bb654c5140001fa2b",
        "570d47b8b3d812140066d62d",
        "572648d1708984140094c15d",
        "57273d8bf1498d1400e8f4fe"
      ],
      "candidate_sha256": [
        "dcfb0f7f5bc1204aa3bfd5527705816bbc3708bfd587d9704329ac492eb3e9f2",
        "73270c46e636059094fc099d15918d347d4b6b87326ca7e9d55545b78f5562ab",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "61af1803b0730548c2562a214669d95520c7f20b1ca36c0cd3e8cbc90c590f5b",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "513cf1b85eb2451a79ea1fd226ebdc9790b2e2fb5b807587d641363fd9bd0192",
        "2b614d2ff3c989463a6656eb25ecfde2e55e78a0efe26d54ff94e0766f2be0e6",
        "24e6c613dad838ead406354f2a3968a9b9277f1c58316194ef7bfc596947ee0c",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "76a69efa5b44ac59ec611714b624dd900a527ebd441fa5d4b1077a0b18bdbd5b",
        "8476df131f9ab23532943ab96bb701c1118b000f1bf40e154d173155e3a79f8f",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "dc852fea2c951a6edf6f78960b29556934e7133367dadbb5425bf8800228bcd0",
        "697a7cf1275ff1524ff00842e78c97cc68f1da2ec72cc226223265bdf1d91a77",
        "35a3102ea99632f2f679288422bbe7d74866f848acbb01dc684816ab2d0c09c2",
        "0303b992326c30ab7517bcf7737bdc345dbbbc7eae232baca3af64f05f6565a4"
      ],
      "example_id": "57273d8bf1498d1400e8f4fe",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.56579987,
      "neural_order": [
        15,
        2,
        14,
        13,
        3,
        4,
        0,
        8,
        7,
        5,
        6,
        10,
        1,
        9,
        11,
        12
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "f5f0ab1007e96a4879a73a24b4d47c22d3183e59ef191f27c7385d559ad820de",
      "trial_index": 575
    },
    {
      "answer_sha256": "2521ffb68d96c362594ccc23455656167d24a965223181e5b734572bda01d318",
      "candidate_ids": [
        "573314e3d058e614000b56f0",
        "570d4329fed7b91900d45dd7",
        "57273e50dd62a815002e9a02",
        "572756fe708984140094dc71",
        "572663a9f1498d1400e8ddf2",
        "570d529fb3d812140066d6bb",
        "57298ef11d0469140077952d",
        "5729f5a03f37b31900478605",
        "572872dd2ca10214002da37e",
        "572fc41f947a6a140053cc80",
        "5705ffde52bb891400689784",
        "570d4c3bfed7b91900d45e31",
        "5725f2c838643c19005aceed",
        "5728202c4b864d19001644ec",
        "572826634b864d19001645be",
        "56de16ca4396321400ee25c5"
      ],
      "candidate_sha256": [
        "1eeac96b11db3be66a0a54fbf44c45e97cb5df9cd0c4fbee0e2f1342aab6b8d7",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "2947421ece7b4090c22a0414b5b432c85e51b54b27afe0872df98d7b68e8b683",
        "697b56b27d7b0ea65bf37203f2581443d1622213ae4c8c57bf37844cc487aa68",
        "5bbc964c789f814eb6cd00aa15d9ec35a096030fe54c6d71df18b86c6533397b",
        "db8a6ac9cee9f2cba867592279175e76f2275c9a1698ee9f606bbcd2d12f6ca8",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "63f924475f48b6a4d5e56cf026f3e691dae4f3aac8a27915d395d5fc0e9db228",
        "634104c8914c74e0e747d9d336204833ccb84ac3ecb39557bf5d44d07efc6ed6",
        "ea77756dac40f15e9f8138b99bee439722589f230e2a8715a831bf02ff3b1c0b",
        "660e16581b996f7e18385014c2cc59b2cea9913e155d2df1b672471e5c39aa8e",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "b0cc267c558cbacbae71b948e32e72786e48630b11fbacff7f754248d31c02ee"
      ],
      "example_id": "57273e50dd62a815002e9a02",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.28147225,
      "neural_order": [
        2,
        3,
        10,
        8,
        5,
        0,
        6,
        4,
        9,
        1,
        15,
        7,
        11,
        12,
        13,
        14
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "2921019b3548bc338139a67e907e2c307b35e5e6ed7b6240c14899e30bcd28fd",
      "trial_index": 576
    },
    {
      "answer_sha256": "68eb99612db60dae090e057c8bd984ab320b79f596f370b134bb16633ab6dc17",
      "candidate_ids": [
        "57275409708984140094dc35",
        "572686ac5951b619008f75a9",
        "57265d08708984140094c397",
        "572fc41f947a6a140053cc80",
        "5706111a52bb89140068984c",
        "57094d489928a8140047150a",
        "57271c235951b619008f860b",
        "5710e8c8a58dae1900cd6b26",
        "572671a5f1498d1400e8dfe4",
        "572a213e6aef0514001552ee",
        "56e181d9e3433e1400422fa0",
        "5726baf2dd62a815002e8e74",
        "5730a4d02461fd1900a9cf29",
        "572a20816aef0514001552e4",
        "5705e26d75f01819005e76d4",
        "57273f27dd62a815002e9a0a"
      ],
      "candidate_sha256": [
        "7f5c1af34a119abe54f9f2a403725d6cbdc1db530373505faf8cd53f11cad720",
        "b7ccb0b6320d2cd26aec5b81f0bd352aac0617d45f9be1e73931429e79be605f",
        "550636c40fe4508a2ea69d38f57883943b44dd88402a64050742216683d37a53",
        "d2f59dee20615657cea445b06ca585557b245920adc37cb50a3a3c910ca17965",
        "c2e8ff4717327b79e934dc7433241a41eac7959e0497ae1da52b9acd127d56b5",
        "4dea2ecb9b0134dbb78d5552a81c17867b67cda0141252e26b6e6c0f775dfd57",
        "726bcaf33d9e385e208374dfd2653634a4522742124baaa3999d5c5c292445fe",
        "b4e5bc367ebb9b2d4ccef67770a2f67d1a323f808c17f89bede0f2af3b3a5f3f",
        "23d71dcbdc3d2eb58c4a11a2383b024a7098b0ccb3459c59f68be117a7c2b667",
        "58b533840c840632c3ad539a70592d3f76e2aadfe8acf8597ad2f9f14d34bad0",
        "5ef3de079d69dbfcbf3418f94ba50a6dbe082fa9d93ddac954bac3e8bdfae797",
        "9b37608c96e15deba5be60fd99f3bbf701e5b780225fc1b43915af95af47077c",
        "3344ef93052b85063f75bebf68306da4193c68170ffc88ba5cf12a5bf36faa69",
        "9bfd34dc5945f3f1cb2977bcaf00d7025ecfd1ae65eff22ed5f0ad4225e7ab2e",
        "443784f71b85ab6f58648fe4eb266a7020505d78a8661774accc6a72ddeac883",
        "0cc7e2c8f48dd8eb5da78eae4279e3cb7bc3980e313b32559504f281abc13c99"
      ],
      "example_id": "57273f27dd62a815002e9a0a",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.29792021,
      "neural_order": [
        15,
        10,
        8,
        1,
        11,
        2,
        9,
        4,
        3,
        12,
        14,
        13,
        5,
        0,
        6,
        7
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "e2162adc2c894c2da180515b89057ff7bbc8970a4cc32f8ccda0d184cfc550d7",
      "trial_index": 577
    },
    {
      "answer_sha256": "ca8ba7d4c82b206d78c3be9fb8e219a7cd584fe4136e766fba26afbe56b8819f",
      "candidate_ids": [
        "56dde1d966d3e219004dad8d",
        "571142b3a58dae1900cd6d5a",
        "57336755d058e614000b5a3c",
        "5730b6592461fd1900a9cfcf",
        "5737aafd1c456719005744fb",
        "5726887e708984140094c917",
        "57274046dd62a815002e9a12",
        "5726a7ecf1498d1400e8e654",
        "56e1dc62cd28a01900c67bcb",
        "57300888b2c2fd1400568775",
        "57332c1e4776f4190066073a",
        "572648ed5951b619008f6f03",
        "5733f5f24776f419006615c1",
        "56ddde6b9a695914005b9628",
        "5726a09f708984140094cc39",
        "5725fe63ec44d21400f3d7dd"
      ],
      "candidate_sha256": [
        "92cbfe0adf7c07d3dee4f0e9ccfab845d9dcdc504748be9b51256859cd1a2a6b",
        "be940bee7bb45d696841d22ad78a71a2ca56bbe313476b871dcb65e93cbbc06a",
        "a86d2424a5df24a99c61d4702f1677a30fe23436db4cc91717d6bc2a4e8eb702",
        "34143483711f63c1b1ead895c0a35cfa058958f99f70ac63e451655e59a61e2c",
        "d25aa4629baf5c953b8cea8fa0d95b121ab6eb7da2708a83c56d647a8eb01ae9",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "5caf1cc1422f1d465946defdf660730d9cf842f54b1ae5136e9744b6079a7e92",
        "2b772b90af6e0d587fdbebde9168dcf1328d5b42efb7993c8dd812135bfa1c6f",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "806d9108a9e77885cef62e3c90c279d795143facae239bf4b7e3d230054267bf",
        "2e5e9baaa2d5e643d9202328c0a2e0ee49969312ee9e4915c98092ea7d9b1cee",
        "e26d3a98154d79e4fb93dfbc9b446801d4e38dc1c4681b39878f40025a6b4fce",
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "a9fac3680b78bf804769a812fc8a691e38ce9829b4d7581b8b994de32cd16727",
        "613ea613b64743a4b6c6c5cad6341afdc153a55dba222f368d35cc60e20ef4dd"
      ],
      "example_id": "57274046dd62a815002e9a12",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.56658651,
      "neural_order": [
        6,
        15,
        2,
        4,
        14,
        8,
        7,
        10,
        1,
        0,
        3,
        5,
        9,
        11,
        12,
        13
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "90e52c9767997740e4d938401eb0a9435d6e113e1905e7b1fe85353618e128d0",
      "trial_index": 578
    },
    {
      "answer_sha256": "ee39c872eea1c919474b46215c2dfde76b9df9f245dfb78621f6d5d65dbef843",
      "candidate_ids": [
        "5727f05b4b864d1900164068",
        "5729fe5c3f37b3190047862f",
        "571c97e2dd7acb1400e4c11e",
        "572649d8f1498d1400e8db37",
        "5730b541396df919000962c2",
        "5725c41eec44d21400f3d50b",
        "57274b35f1498d1400e8f5d4",
        "57274f49f1498d1400e8f61e",
        "5728223cff5b5019007d9dc4",
        "5725ca4389a1e219009abeb4",
        "572a0a391d046914007796df",
        "57294209af94a219006aa201",
        "570d391fb3d812140066d575",
        "56e1bc3ae3433e1400423104",
        "572742bd5951b619008f8785",
        "572fbf21a23a5019007fc937"
      ],
      "candidate_sha256": [
        "86b012bdce8785023cf0587f7f63bb8d3f6c6b5c81ea239803399bcb44bd25ce",
        "d33ba8da5a85e17ec64aa5b4e56b91f2eec17d389a054942958b6b7ee645e0b2",
        "b272a915ce32f275f02c333e828edde708d727538b2125a5dfee6f8a3e81aae8",
        "36dcef7286287be9dd8dfc82fd39d18ad3a4563ed1f7d08d30d9ca95667dbce6",
        "2a788c023034a52ad733026fb02cca9fa43db6440104bba1c9c1b811876035e7",
        "f9de1761ee43bd39aac4d0aaaefcd20dbd13070f68926c9fb290571d325156b4",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "85485396c5c90d2472cfcac0f7f0827ceab8a69d637db29a0e6c33e9550349b8",
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "4f4c81ae038012c4fba4f9b9d5af1edba076741a21c3a6d3c06b83ed01143eed",
        "2fe612e2020a1e2f81f43f38b2e3a98266a1753e5167e11315b6eb3c19e82c36",
        "6288ef858c506ff298e5eb7fa2c1d6b45177481e407990e3afb98128c421694a",
        "6ab5cb0cac26721174ae2f62c285e0d0030e849a2f221835b083e9bfe1983155",
        "217d81bcc0dab6b35310da997143cfd073de323168a86a3644e7f8eb39baaaf8",
        "ce8d2359f23498cb006a1bf2f91d5494c463bc986a83749d5f9208b28fa63c51",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad"
      ],
      "example_id": "572742bd5951b619008f8785",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.57255227,
      "neural_order": [
        14,
        11,
        9,
        3,
        8,
        10,
        7,
        6,
        13,
        0,
        15,
        1,
        2,
        4,
        5,
        12
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "5b687bacd223c9768011f05826bbd1a294b15360d29f8e7d2878b82d553810a4",
      "trial_index": 579
    },
    {
      "answer_sha256": "3ec3b9e45d46204398749ab17b2c17c5377031b384465372405b07cb45c0a73b",
      "candidate_ids": [
        "573382d24776f41900660c37",
        "5725d34089a1e219009abf50",
        "571093aba58dae1900cd6a5c",
        "5726b718dd62a815002e8dbe",
        "572744aff1498d1400e8f588",
        "572fbf21a23a5019007fc937",
        "572fbea404bcaa1900d76c5b",
        "572a142e3f37b319004786b9",
        "57115e532419e314009555af",
        "5711623e50c2381900b54ab9",
        "5730a314069b5314008321eb",
        "56e1b8f3e3433e14004230e6",
        "57302700a23a5019007fce89",
        "57285213ff5b5019007da182",
        "56e1dc62cd28a01900c67bcb",
        "5728fc9e1d04691400778f11"
      ],
      "candidate_sha256": [
        "c4689ea65fd78e97df381f5b5fd5ca3b8a882fc1a0c7c748d1a5b87a9217368c",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "dce71b97746bb1141cf617e2046b235127fc073fc2949ebfb609e94f96bb24a4",
        "1b8917ce895b36f178d177be83a5b2be297be41f4e7abc05a66a1e05e65eaf65",
        "695f7892d8dda7e75ff7ac287c45ae74caac840c17b5defa2ce1324a001b8dad",
        "bccd3ed3ef29ad41b91084d85b61d15ae541aba6ff32feae9fd39d09caec0872",
        "ef89fd8649969f09fc4769b5d75714cff1e8abdfe5c2732df28837d2517880d0",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "2fd24747c1f66748170c1c9f051c9e7d266cecb605076c6cae54a1c2917dd5d2",
        "cfdb12034ce378f42908a6860b2e6c3830a6998d27ec7827ea1777a793f97c1a",
        "e3b13d85a858b90fec804e930b4e2453c9d2a925d62565d94d6b00617d3f14fa",
        "62130be8ca9be96234ce2d1001ed5f92b28c71efa655f09e83432829cf934fd9",
        "ce2eeacc6827522ebbd6697ce69e3c55beec4e476763b232db0da53d627e9366",
        "c38ef516e41c288e48baaa51d99898d601d4e886bb759f38af5399abc02dfc4d",
        "95b444dec0e61a10aa63651ec48055a76133f591af32f80988a975441431932c"
      ],
      "example_id": "572744aff1498d1400e8f588",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        13,
        5,
        2,
        1,
        0,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.34534504,
      "neural_order": [
        4,
        8,
        1,
        14,
        11,
        6,
        0,
        9,
        15,
        5,
        10,
        12,
        7,
        13,
        2,
        3
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "7d36a3a7d453b75751032b33799e9aef654da83475e835efb76c75a60308895f",
      "trial_index": 580
    },
    {
      "answer_sha256": "a2e982342ec3b07013553e9e49981889dee225443f6f0b78102ab35ee4938e0a",
      "candidate_ids": [
        "571099b2b654c5140001f9b3",
        "572f64ccb2c2fd14005680b7",
        "572a0e0e1d04691400779707",
        "572fc49d04bcaa1900d76ccb",
        "5725d52f89a1e219009abf78",
        "5727d6154b864d1900163e34",
        "573403394776f419006616dd",
        "57265d86f1498d1400e8dd50",
        "5711658e50c2381900b54ad5",
        "5737432bc3c5551400e51e99",
        "57268066708984140094c821",
        "5728f3724b864d1900165116",
        "57112686b654c5140001fbd3",
        "572745c6708984140094db99",
        "5737a25ac3c5551400e51f51",
        "5727490bdd62a815002e9a82"
      ],
      "candidate_sha256": [
        "465f3a788653969dc5843bdf05150f974f547770065e89d57e06c202b8273776",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "4b184251045ff68cd470a8330d18c3848a5524d5ac96c1ad4b4b92ae5d9d5694",
        "d91531e24877b168a6563e18dbed82fb0db3eb067e565ef509e5a5639905fe98",
        "9ef6e305a45e267b0ca488bc0108dedf4d653b1ec1d52109c09fadd0cce50299",
        "1e4976a9a4b6dfd7616619306dc835cccc38e3ff5e01fc0bb209f1f968e0c22a",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "240c8b1f49cd836bd7a59eadd8b71f086502b482c10f8600748cf4a7146abbca",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "f6d8387f602dd77b227ab8b161d9456f7ad494c5903e27116d9f1009aba9d64b",
        "b0666f44708b16c04b696d03690c4a13c876c6f973ae6384cdcbde973b09ebf2",
        "43d98e0a6026101ae60c61b15713fa0e50a69eebbbe6afa3fabf9e4d3146d377",
        "6b3c0a7178c7d09d6fcfaa669200e662a4c151d4d6093ed06907f82f1b10e5d3",
        "74e88ac6d67515920292029909b403269481a3690e4aeb4db4704db9b3c42403",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8"
      ],
      "example_id": "572745c6708984140094db99",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.23461581,
      "neural_order": [
        13,
        10,
        12,
        1,
        0,
        4,
        14,
        3,
        8,
        9,
        7,
        11,
        6,
        2,
        5,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "6758c87e47a33fbaf2e6d8f0505fa0f8ee8d597bfd411d31b2c2ad0b1b10aa58",
      "trial_index": 581
    },
    {
      "answer_sha256": "471aa8e2532924cfb41adc79d8a4f7163f68f98c0ace853d6ae6ef0c0efdb649",
      "candidate_ids": [
        "570d50a5fed7b91900d45e7d",
        "57097b50200fba140036804d",
        "5737a4511c456719005744df",
        "571090abb654c5140001f995",
        "5733e5a14776f4190066145b",
        "572885c44b864d1900164a78",
        "5725b33f6a3fe71400b8952d",
        "5710f4b8b654c5140001fa45",
        "57286bb84b864d19001649c8",
        "571098f8a58dae1900cd6ab2",
        "572663a9f1498d1400e8ddf2",
        "57264e455951b619008f6f65",
        "5725d34089a1e219009abf50",
        "57060a1175f01819005e78d2",
        "57274e0d708984140094dbe5",
        "56e1cbe2cd28a01900c67bac"
      ],
      "candidate_sha256": [
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "799035149f60a2a0f827c9467bd687e0d62791091aaa7c6f9e5381d13307bd29",
        "78a8c6d2c72573b5448a713a95b9b9a02d3b8e2aaa1cb18364ce9dd02d827451",
        "f684d24fedd06ae1bf5a4b71cb5e509cab0cc5d0fb6eaa14af5d188ebe7e704e",
        "9203327275a06a33cfa7f87016da9684588381b9b63bbd06c7852a26d5e85f92",
        "284309c26e4b936051369f3ca00e547d20c325f01f2b8043bf10e9482b4768c7",
        "1970e37a8bbfd87eb674dceafe71d721c7fa198f09f5c79a0335ff6d3ddce0dc",
        "0464d8e11e00da26823669cbcc75ee1f18730bac83b9161493c0d73e7f5be1d6",
        "a5fefe6a1644e52e96bd506e3743f7ad30207658dcc66d08f4606b777d0aebb6",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "36480f9ea6f7a385ac0f0d884f56e7f8ddc4af7f2b466bc2f55f70e5d2e107ae",
        "78ded4843379125673f81053cbaabe440ae85d1814aaa8296b8d0528e52ec8fd",
        "dfc62fde23936e1ea3cab0f6a04aa5f4eb4ce3a3a1c3c79813ebfd4a6fbee7b8",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "2c447bf3a7af6862d1b0dab5ad502d1e7ff2c612d835aa5688290327662cc657",
        "54711cd280362ef0faf67464e2abcb0c1f7682f248bcaba63afbfda7bda16b95"
      ],
      "example_id": "57274e0d708984140094dbe5",
      "gated_correct": true,
      "gated_top": 14,
      "gold_index": 14,
      "lexical_correct": true,
      "lexical_order": [
        14,
        5,
        12,
        0,
        1,
        2,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        11,
        13,
        15
      ],
      "lexical_top": 14,
      "neural_correct": true,
      "neural_margin": 0.21413814,
      "neural_order": [
        14,
        13,
        12,
        9,
        3,
        15,
        8,
        5,
        10,
        2,
        7,
        1,
        0,
        4,
        6,
        11
      ],
      "neural_top": 14,
      "partition": "test",
      "question_sha256": "d8eca92a089d89e623f9fff04715fda366f89e6dbd620ab46a8c9458d164b471",
      "trial_index": 582
    },
    {
      "answer_sha256": "4dd16141873d48a1b57f716b29fa5eebdab7e2f2c4f873f631f57f867582c6fb",
      "candidate_ids": [
        "571093aba58dae1900cd6a5c",
        "572855973acd2414000df925",
        "573776eec3c5551400e51ee9",
        "5728fd3c3f37b31900477f2b",
        "57287ccb2ca10214002da3da",
        "5733a560d058e614000b5f77",
        "571154c72419e31400955588",
        "573003dd947a6a140053cf42",
        "56de11154396321400ee25aa",
        "56e1c2eee3433e1400423134",
        "57302a3aa23a5019007fcecf",
        "5726887e708984140094c917",
        "572941273f37b319004781ad",
        "5737a0acc3c5551400e51f47",
        "57376df3c3c5551400e51ed7",
        "57274e975951b619008f87f9"
      ],
      "candidate_sha256": [
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "7b34e1e2f91ddbd599fb3d4d15ae6acd8e8d1cac5db50bbd4e9817681d7881c5",
        "f0be3dae6b97bc2b8643d31176d218e779982851746cbd7ec3dcd2eff0a2b865",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "9db4de61ffc7a1c50d0f7117fe6d09882ccd66067b4ab5c5b48d70d0ee79f4d8",
        "0be71852d1cb60f83a2cf3b6dd0bffd363dec529e88d9e07c844c29814e78d9b",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "4af9598890dbade1e5edf1d60b0d84b311495270488650ba5d48027f6102a69b",
        "b1d0c552c7191c039b4fd09d303a727030d84b6cc8c1731a2822fd79722b3862",
        "910b91d2f97cafa3b557bb9be5dd6fdcbaa2991bf2197b4010bd3d78c81969cc",
        "29e683f1e8f85bd1e4bab8f329434f53b74c809aae1934896722f9ba76eff1e3",
        "d0fee370b6584c0602caa46d0b8ef55881343dba8bf6fbc939eb16c852abb7bd",
        "9d0027213b5979657181341596aa1fc2b1e143f3569d40ef327d68c76215135f",
        "06522194b3b62f7819bb2e1d736471092acb6dc38531f9aab4b1e0f29a43ab7b",
        "4df78faf78cb1b5e15488fe4b9288c48672e645c4147f4c979cb59d27e7a4435",
        "db7f043bea8c3ae53b5f17ad1d2466ebcdca95f8a31960816e0c24d9cb2cafac"
      ],
      "example_id": "57274e975951b619008f87f9",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.56253435,
      "neural_order": [
        15,
        1,
        12,
        13,
        6,
        10,
        9,
        14,
        8,
        3,
        0,
        2,
        4,
        5,
        7,
        11
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "dae0b6e94ce5304910eab0097b61cc73bb73a25aec80f41c23b003f6c46d5c46",
      "trial_index": 583
    },
    {
      "answer_sha256": "fab2a993d03ae2f27a51ce64fc8df914a89f7a9bb6f28cd7e889c6796816fbc1",
      "candidate_ids": [
        "56e1b62ecd28a01900c67aa3",
        "571cb010dd7acb1400e4c128",
        "57274f67708984140094dbf5",
        "57376a1bc3c5551400e51ec3",
        "57285841ff5b5019007da19b",
        "572982e66aef051400154f92",
        "57284142ff5b5019007da00a",
        "5733a45d4776f41900660f23",
        "5729f3831d0469140077967b",
        "5726a1e5dd62a815002e8b86",
        "5726db5add62a815002e92d4",
        "57096c95200fba1400367fbb",
        "5710e9f8a58dae1900cd6b30",
        "5705f7c875f01819005e77dc",
        "572a0bf96aef051400155204",
        "572fca7eb2c2fd1400568473"
      ],
      "candidate_sha256": [
        "5944ed72046a98d74e984ac5c604a461fffd2fe1cf76b22760638f1f3d949379",
        "5c55e63733e829bbd22ed99b6f3eace596d8c9e3245bbbf97d42348d58022545",
        "ba3f4fa76f91784474b933b15c21d21e9d89f38989ba7c6375c28cd5befd60ff",
        "ba950b5e5faae63443508d23df7453958b26ef7a41e1c7f7824a2b2050e6685a",
        "61f6d64fb809c0b440be689df0eb78a2fdc048ca335f22d5ac3df24b8dc3b0d7",
        "93620703d50b4155fab9038240e64a854d1329e60d76b3887ba89b6b28625e9e",
        "44b8656c13ef02f0b3c1c8ed5cbcd70007605af64780502e5a0c79d1ccf007e4",
        "5bd9aa5b4017f6cc131de5acee09a6510222c00e1ba418732e5f2ad63fa1fc5e",
        "41361fe7e8b265d1a5fe99de16547bc90fa0d60410f80adeebe9f4dd476de729",
        "c98abf1f37465d91aebd2a9b33f3d0806721cd78c0140059ce0c6d28969bf042",
        "b13b5b0f3b4f559dbf0c505e213127f99a402138a97b37a4e0450454e932a530",
        "f39b9ccf2eadbdb36e5c86e76a1a4054502a28cc4ecc37bdf94e9c39a471c535",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "8c3b7f94a92a31bdb2378e6fe95f3016ac69eadb3ecf88d5947a289faad63343",
        "0f67008f12691a8c9073cd247c444ce0b599ba30111fcdcd2dfc3c73fd6a5730",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b"
      ],
      "example_id": "57274f67708984140094dbf5",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        6,
        0,
        1,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.49745722,
      "neural_order": [
        2,
        0,
        15,
        6,
        13,
        4,
        12,
        8,
        14,
        5,
        3,
        1,
        7,
        9,
        10,
        11
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "e78f78a345e44ef5ca2771fb930c630522d3c4bf70984b098d1fd41f0bae715f",
      "trial_index": 584
    },
    {
      "answer_sha256": "0b3ea5968c926c6d08d257f78dec4512c3f334e1a175aa32521eeba65335bf82",
      "candidate_ids": [
        "57107e6ca58dae1900cd69f2",
        "5727502f708984140094dc07",
        "5728fd3c3f37b31900477f2b",
        "57309bfb8ab72b1400f9c5e6",
        "56de10b44396321400ee2593",
        "572a1c943f37b319004786e1",
        "571a4d1a4faf5e1900b8a958",
        "57335c20d058e614000b58f9",
        "57340b1bd058e614000b6869",
        "5725fabc89a1e219009ac128",
        "5705fb7f52bb891400689750",
        "572a0f073f37b3190047867f",
        "571135b8a58dae1900cd6d0e",
        "572a135daf94a219006aa79f",
        "572ff935b2c2fd140056869b",
        "5726487b5951b619008f6edd"
      ],
      "candidate_sha256": [
        "9736b7ea7cd39c4bc1162efd0078ab91cd174f8fc5b3866084cab54568d1a565",
        "e32237eb13f46727755bd53347a189c0620b3ffff778cecbcd627a94fbebc022",
        "e6c11cd48902e40444e8d32235b4c4325b23c36f593ea48b2607db67b8afa628",
        "e998eef356c2b5074708b8fa85d2d1a12ce0f53d1f90168e87308b61e3d0b6ee",
        "e80ce1ef7c64e3240017398f3333a9fece8bfa318dc05be5457d3ab870d07771",
        "8c238e89447c3cf2ea403544bba8637dfcdd030f059a016732494b6a3046a54c",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "4a4fe99817a1315fcfebc5d539cdae5dbb13833f858ae317984434be400403fc",
        "a6925ffb2204e327f247d6f9af9c4574d99f480697bd611e51b6eae2bd8d30a4",
        "1f02ff6530101359b0083fa972e1e0f5e8a35dc75c5ddb46505577ddf563f4b6",
        "78ec071ddf9b759d3708c4c1ad1235997e73bd747b9e748af29742ebdf7d95d2",
        "777ba60932fa02fb3b72daca6f29f9c6c928a3a46a102597af53de301901e99b",
        "9d6d3d56a911f98076f7ff939a5230eb0bb8caad8bfe5de9bc958e127c7ce475",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333",
        "b8a670602f7e4fb6863032210642e9bad504912ae9c7117d39774190d2c4f534"
      ],
      "example_id": "5727502f708984140094dc07",
      "gated_correct": true,
      "gated_top": 1,
      "gold_index": 1,
      "lexical_correct": true,
      "lexical_order": [
        1,
        14,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        15
      ],
      "lexical_top": 1,
      "neural_correct": true,
      "neural_margin": 0.56707337,
      "neural_order": [
        1,
        5,
        9,
        7,
        10,
        2,
        15,
        8,
        3,
        0,
        4,
        6,
        11,
        12,
        13,
        14
      ],
      "neural_top": 1,
      "partition": "test",
      "question_sha256": "73b0f07bef8ccefc15ad53af77f7294ae4c351299a810d598f2d0f187b1d7575",
      "trial_index": 585
    },
    {
      "answer_sha256": "e75b7ac81b20dfa71c61fdef28b840094672b154c59906a00e99da87403b3497",
      "candidate_ids": [
        "570d29c2b3d812140066d4bd",
        "572654e2708984140094c2b7",
        "57264cc6dd62a815002e80e4",
        "57114aceb654c5140001fc47",
        "5729fc3d1d0469140077969d",
        "571155ae2419e31400955591",
        "56e1a38de3433e140042305c",
        "57265360dd62a815002e819a",
        "56de4b074396321400ee2793",
        "5725daa8ec44d21400f3d6b3",
        "57109180a58dae1900cd6a40",
        "572987e46aef051400154fa2",
        "57297a276aef051400154f88",
        "572750e8dd62a815002e9af0",
        "57378862c3c5551400e51f21",
        "57264a74708984140094c18b"
      ],
      "candidate_sha256": [
        "b8f79857343da03dc1c7c2a5e15e3350bb97060531e8db3a546fadb1276486cb",
        "9d0852e7a492edfb020138aeed2e89901eaf6cbc0450e6da752320d2c7c024e0",
        "5f04960d1ffa118f9b0c1b2ba9cce4e3c0f81c9a07302967bc8bb86716fc8dc8",
        "ffca7827a9c532d19710fdc247a1e643ae84dc2a1b7aeceea5aab0928579ca1d",
        "5b7d91c3c1eef1abc10a08a04fb83add5c8ea276568a385e4d806fe0c301e1b9",
        "babfdcdb4f5e11f89d9b806dc9e84690fc41b249b7a42a6bb189cd52b0be7d24",
        "7da2d7e1a3f7c747e12d14023544c6bd21aa39f41732fff415fa7d8aa83c57fe",
        "0fe75e03f80f9b25361978311951a03c898f6f1a16471cdfdb153d1e2964a156",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "57e770affb89fe4f73b599c4f6316696710741425debe9cafad1dc69f4d289dc",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "90bb1bb50f3f2a7e176f955e343879b294c0bc1743e3521a3db51a9a5f834fab",
        "d09f7c87003b5bdc2d9edcdcff1a2c86df26c7603d224be41276d3fae28c4ac7",
        "083e9d9d0cfbbf2b79a24cc1f179a5db6a5e51a410187c56fa5c8d726cd26992",
        "06a36c034cb1681bb677178da6125ae4b9b63cd95527bdc90f5a7af7da958523",
        "61b66074094351a2b9074e367a8e7de0c9ea38b495749119773fbae59a339d91"
      ],
      "example_id": "572750e8dd62a815002e9af0",
      "gated_correct": true,
      "gated_top": 13,
      "gold_index": 13,
      "lexical_correct": true,
      "lexical_order": [
        13,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        14,
        15
      ],
      "lexical_top": 13,
      "neural_correct": true,
      "neural_margin": 0.26728857,
      "neural_order": [
        13,
        9,
        6,
        8,
        14,
        4,
        10,
        7,
        5,
        3,
        0,
        1,
        2,
        11,
        12,
        15
      ],
      "neural_top": 13,
      "partition": "test",
      "question_sha256": "a183a3fefa1c6382c738e03c0c6f9ff7969cec7e0b02550d9a91bedf4c93ba7f",
      "trial_index": 586
    },
    {
      "answer_sha256": "5242e6d02c58fe5a5b1e08193b12295d893730683ab937ea1d364b3c08236615",
      "candidate_ids": [
        "572751b4708984140094dc1b",
        "572ffa79a23a5019007fcbe9",
        "5728e8212ca10214002daa6c",
        "5725e748ec44d21400f3d733",
        "56ddde6b9a695914005b9628",
        "573083dc2461fd1900a9ce6d",
        "5737432bc3c5551400e51e99",
        "5726472bdd62a815002e8042",
        "5725fb8138643c19005acf3f",
        "572a18a4af94a219006aa7df",
        "56de4b5c4396321400ee2799",
        "5710e9f8a58dae1900cd6b30",
        "572a06af3f37b31900478667",
        "57337f6ad058e614000b5bcb",
        "5726385e271a42140099d797",
        "572ff890a23a5019007fcbcd"
      ],
      "candidate_sha256": [
        "0e1560b33ab1f933fc73de6d686f047c067eb369cc08b6bd463217de1e0437bf",
        "84f4b22501bb2ae58972beaa9397849a79e6fdf259dde0ece6e9c2d26bbb9862",
        "b0b415698a08be3715334c86bcf911274e5b53530b1037d075a0f6e9107df2ea",
        "eb83a38dfdfbd8730a88f6553537813d04b327d6cf7fc0d205c73fe7221e2f5e",
        "dd5c8526091ce0e937a062da23833808b4e54d9ce41cdc101173265b6a718bbd",
        "8f02997e8cb835fed5a4d8bee5503c2ce5eff68d38318aab81769c6d38914c9d",
        "eff5ab41368cf6ec7440b6065cd8d91ef82f7a3ff724d94d70d86cf20e2433bb",
        "d9913b5e5e777efc0d65607a482cfaa5392fd55dcc60d25f59d9c976bb3b26a5",
        "2487c8649dc181a88017e0a080ae60a697f0e6c568e034fc10a65134683cc088",
        "e72c427b33209e555b5fc09b718f5d3a1abc94dbdf4f37900c15e3a035c2a954",
        "00c84baa77993a3ac494e18297fb45693d3de708a3172de621d68d140b0a1bcf",
        "14b88b3e09e2c5d0c0298a21eff363c5aba4a6de26c442f96b4b5da66e67d0f8",
        "651298a168110f388d3688f8765ff1a70797c36a014e4c3a03d0bf31007236d7",
        "c947cb59a7986b7be2fa53ca56328526df98f4da7ba20d1368b007895802474a",
        "415f6ece6468aa11fa79bfdd1e1bc2efe53a2bb9afee147324e9104b7d45596f",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41"
      ],
      "example_id": "572751b4708984140094dc1b",
      "gated_correct": true,
      "gated_top": 0,
      "gold_index": 0,
      "lexical_correct": true,
      "lexical_order": [
        0,
        6,
        2,
        1,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 0,
      "neural_correct": true,
      "neural_margin": 0.19913096,
      "neural_order": [
        0,
        14,
        2,
        6,
        7,
        12,
        5,
        15,
        10,
        13,
        9,
        3,
        8,
        4,
        1,
        11
      ],
      "neural_top": 0,
      "partition": "test",
      "question_sha256": "610c465fcef2658e1e4c35e483eb946823b7415498b05229a0a30dc2274dac49",
      "trial_index": 587
    },
    {
      "answer_sha256": "6517fca535a0ed775aa8dd8f4bad0e50dfeeb9696ca4bcaf4b456060425a5cf3",
      "candidate_ids": [
        "571154c72419e31400955588",
        "5709686c200fba1400367f77",
        "57282358ff5b5019007d9dde",
        "56e1a564cd28a01900c67a48",
        "5710f2e2a58dae1900cd6b72",
        "5726a46cdd62a815002e8bd0",
        "5705eee952bb8914006896de",
        "571117d4a58dae1900cd6c0a",
        "5725f7cd38643c19005acf23",
        "5727e21e4b864d1900163f34",
        "571093aba58dae1900cd6a5c",
        "572754fff1498d1400e8f660",
        "5727526cdd62a815002e9b0e",
        "56e1c0f6cd28a01900c67b2c",
        "5726a5525951b619008f78dd",
        "57114e8d50c2381900b54a5b"
      ],
      "candidate_sha256": [
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "faccce529231f6d9303296f49f7a1b5b32a7ad244c042e4d369db719456ff326",
        "492a42261214e068022116a6c4606540ae02bf33d454186c0c49dc3282a9dd9d",
        "f37ddc9cea4f4e968e127bc1332cd0e7f57964aeceed1b11367c88c09e34c8f4",
        "0a5767102b55172a35d0984dca624f319dfc497b9a90ebbd64386a6b66e2a2ed",
        "fecbe268dcfa78e46c7c2bf6b0bc78e5a507bcaa59c7a1720e47fc2911b8280b",
        "93e67f8281dd40c19fea6d42373c5ae990cf376e9f4f54bf82f9a88dd0c01140",
        "52e533d595a501764514b440ac2ec0f7cebfaac473204708efa685c5a348d032",
        "1e7dacb29402442c93e3c2f418a257081737db545af49703fbf284e8616f229c",
        "874533677a269c5d5fb8bd3a3213307ea1632c2236ea48b8da8182d07905fcc8",
        "4d039cdd6eb2da844ea14281bc883ade8375dda60b3d61f3db090a38fd25e2a4",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "1cf4f15a6611259d8a4188f0b6866ab76852a0cd5f09291751b0ad4f7f3248ba",
        "cfd4f8f5ab4043a0c3cd16823929a42af6d9122d2f0d03670f4f5ea69d7b2b37",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "fc82521fd13994861ed508fa890ed3040da3abc9fa520366cf8003441da1aeb4"
      ],
      "example_id": "5727526cdd62a815002e9b0e",
      "gated_correct": true,
      "gated_top": 12,
      "gold_index": 12,
      "lexical_correct": true,
      "lexical_order": [
        12,
        10,
        1,
        0,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 12,
      "neural_correct": true,
      "neural_margin": 0.43190574,
      "neural_order": [
        12,
        3,
        11,
        13,
        14,
        1,
        0,
        5,
        4,
        10,
        15,
        6,
        7,
        2,
        8,
        9
      ],
      "neural_top": 12,
      "partition": "test",
      "question_sha256": "bc59763800f42dc3dbe02c5cb2156cb008b3fde2b62a9f5017551d38dabc698f",
      "trial_index": 588
    },
    {
      "answer_sha256": "454c0f107fa51b1fae9ab0da8195ce7eac33a3cf42c8a5af5098625275072152",
      "candidate_ids": [
        "572a1dbb3f37b319004786f5",
        "57300888b2c2fd1400568775",
        "572753335951b619008f8853",
        "572ffc0f947a6a140053cef0",
        "57265d86f1498d1400e8dd50",
        "572fca7eb2c2fd1400568473",
        "5727d3843acd2414000ded69",
        "56e190bce3433e1400422fc8",
        "5728683b3acd2414000df9af",
        "57115e532419e314009555af",
        "5726eb4b5951b619008f826b",
        "572849b4ff5b5019007da0f2",
        "571cde695efbb31900334e16",
        "57266c015951b619008f7237",
        "5726a638dd62a815002e8bf6",
        "5730a40f396df91900096234"
      ],
      "candidate_sha256": [
        "093a00ac36f342e74a03659e5c02e73a774dac79d4b9e787da92a64a4b517943",
        "c6bbc021cd7b4c60ac66d6ba3178e8261063b655d79bda24d672b51a2c56ba21",
        "956d35ad3934882f2e9d3120ee8738b6a8ef0bff6d9e54723e71f3457b5592c3",
        "da7e3caa2252604ebed016e003b315efd2a4a47f3b063950ffca6d27e6dd03f2",
        "6a86cf0117278e97e947d4b42f0516d6f55fdec2dc5b293e4b77e48cf4269fde",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "eb6de5520e4ff5dd74226484694777930247235a992261581ce74e58cdb39246",
        "c1eb0d23a5536b9229cc206f583ba22b35e941f0b5315019f5f51be28bbacb20",
        "25c0e585d18699679401cf5f18912437c8e682ace293ae92bc47426596ca51ce",
        "688a529e2d6116cd3c613a951e8d723a25f9e2f22082f3cd72c2f67050b6eb1d",
        "c6305dffc303d36cae10d75429cc053dc20ae8d1309eaab950d2703ef073570e",
        "4b9be09605e42d27002a52c72f3d8886dd1513d21e40716476c7ce97970e08de",
        "bb44cc7d6df50b60e3aa046eb94c63b1c6e157582dd86e876099903c7837fe97",
        "08e7fcfae5bcd655caa1f38fd9fe71bc3ab1492d5cf003ea856a56fd67fbc23d",
        "ca02817db884a9c211c78887f253bd1ad8313827771fe6c139771bc0dd71b434",
        "81827a7fba600de4e9ff3056bd1fcc9f35ada099d3efcb16af4d832b73abdfaf"
      ],
      "example_id": "572753335951b619008f8853",
      "gated_correct": true,
      "gated_top": 2,
      "gold_index": 2,
      "lexical_correct": true,
      "lexical_order": [
        2,
        11,
        9,
        12,
        14,
        0,
        1,
        3,
        4,
        5,
        6,
        7,
        8,
        10,
        13,
        15
      ],
      "lexical_top": 2,
      "neural_correct": true,
      "neural_margin": 0.60087421,
      "neural_order": [
        2,
        11,
        5,
        7,
        8,
        14,
        6,
        15,
        4,
        9,
        13,
        0,
        1,
        3,
        10,
        12
      ],
      "neural_top": 2,
      "partition": "test",
      "question_sha256": "44ead4d9803acef66c5897429828970f2649813d60b4a6c3745040d6ee0f5f99",
      "trial_index": 589
    },
    {
      "answer_sha256": "60d707583c55f4df14e7d263db98045ac6e77183d531a6652e69f14ac8d11f04",
      "candidate_ids": [
        "57284e9fff5b5019007da150",
        "571a4d1a4faf5e1900b8a958",
        "57273cca708984140094db33",
        "572754fff1498d1400e8f660",
        "5726398589a1e219009ac588",
        "5725d7e438643c19005acdf9",
        "5727c0402ca10214002d9564",
        "5729f39a6aef05140015514c",
        "57115ac550c2381900b54a77",
        "57275411dd62a815002e9b30",
        "572fb42904bcaa1900d76c15",
        "5737898f1c45671900574495",
        "57060a1175f01819005e78d2",
        "5733d3cb4776f419006612ea",
        "5711607f2419e314009555cd",
        "5729f9953f37b3190047861f"
      ],
      "candidate_sha256": [
        "21d1013d01a80d08b47b9f871081c86d133bdc66c22a3861450f0acacd5713f2",
        "2eca63750a362ff269957ac209e2ce98797d8646a987f037ecd7aad0aa135bb8",
        "712be0c0b0e3181f3d317747b1a87774a0db59c4052b4f405e2bcd641dc30c59",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "b2e6e0a40bc743d32fc3aac13d5f301658585e90c41cd41c42d54f5cc9e56cf1",
        "757871995f001fd9ee267181bb92c3e00199aa4b7fd30a4fb4cbf24fe6050a52",
        "c2efd8db4a6e03e9a632ded17176ef7b4de416039b6d74b8fc93e8d2420af469",
        "0d39dc7605d4ac279ef93e0dd3825576cdfb4b5add7e7225fed68c1b742cf726",
        "49dc68fb419c7d95e59f7efa07c36ce3082421f1e2da4c5b2eaee57b57b31b14",
        "4bfde5c5f53f69927f5f235510fbbe2969feac3237b78782ea69a8ebe395cbde",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "bb6c3c43811ba3c309a7dadb571d588981ece51bd973eec6dfe9b97e52560379",
        "2016b26388e6290566e2a817031cd701dc81e69574aac7943adc16fffc12937c",
        "f9a034d25c95dbdf3ad3c91d4347b01bc5acdd337851f0f2ed7b23fe601204ad"
      ],
      "example_id": "57275411dd62a815002e9b30",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        2,
        6,
        4,
        0,
        1,
        3,
        5,
        7,
        8,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.13994221,
      "neural_order": [
        9,
        2,
        3,
        4,
        10,
        6,
        12,
        14,
        15,
        1,
        0,
        5,
        7,
        8,
        11,
        13
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "88bedb1135f31498a944d4ec31d4156217d0b1e51f6d71cc7b866a4dd5225c37",
      "trial_index": 590
    },
    {
      "answer_sha256": "f04ee782287cc0c7ced73b507481fcaf8215d72d5e75e1609b42dd8a405e3e9b",
      "candidate_ids": [
        "573060b48ab72b1400f9c4c6",
        "5733a5f54776f41900660f44",
        "572f6c85947a6a140053c940",
        "572863c72ca10214002da2d4",
        "570d50a5fed7b91900d45e7d",
        "57377c98c3c5551400e51efb",
        "5733ef47d058e614000b662b",
        "57274712708984140094dbad",
        "572a0e0e1d04691400779707",
        "572fb42904bcaa1900d76c15",
        "571c7d55dd7acb1400e4c0c4",
        "572754fff1498d1400e8f660",
        "5733db8dd058e614000b6428",
        "572fadcbb2c2fd1400568329",
        "56e16182e3433e1400422e28",
        "572ff890a23a5019007fcbcd"
      ],
      "candidate_sha256": [
        "b7d78d9fd858b52f42115f7144f2b2a6844dffc86f3a087c6833c6f78182e39e",
        "6e7b592cac442aff8533dcb815531150fd8f7ff20eea5c4d508cad74b96bf249",
        "bb3727c243a6cb744686ba6886c54791cd7745eca592eef8baf9c5b06c996b3e",
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "caa4bb3befe82e04c1363d130349629bda110b4f974526ed193a670fd4dc5fed",
        "1c326ae15d585661849c0a43f71be232f457f11a2a48964b11b0d9172e7cd31b",
        "8d62374eba82713af76f705d8f0104ad57b6a0490af0ee67a4f454b213bcc738",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "4b184251045ff68cd470a8330d18c3848a5524d5ac96c1ad4b4b92ae5d9d5694",
        "9e9b2377fcb94d76f2e805157e0273125bd80e22843a3143f64f1a7f052bd4de",
        "e0e9c1b00942a791f21d4d84c2985047c45f3e69d4a43439329f0a03c36f3f75",
        "980a20d86de338dc9049255f4fdc5f4f2b24ff157f6fd588a7bb53dacb45c921",
        "7544f3f342f82852b6fd8a74d46285e0772381cc4cd0b61c47fea00590e8938e",
        "edee229cee4cf080a1cac0953d50a1b6db5b71876e083a25884535844a7e30cc",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "3e999aca4853266bf0a22024b10bedd5cb3fd05120008a3cc31ed492ccf4fd41"
      ],
      "example_id": "572754fff1498d1400e8f660",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.40063228,
      "neural_order": [
        11,
        9,
        13,
        12,
        15,
        2,
        7,
        5,
        10,
        6,
        0,
        1,
        3,
        4,
        8,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "3813cadf1a4da5a1fe562e661546deb936b4bbe0321d8cc8ced8eff21fb35757",
      "trial_index": 591
    },
    {
      "answer_sha256": "be3b2481a3ee3c903d49ec130067db168c15c2dd543f11bf31d55337c304b462",
      "candidate_ids": [
        "5728223cff5b5019007d9dc4",
        "57283c464b864d19001647c8",
        "573407d7d058e614000b6813",
        "571cd88ddd7acb1400e4c18d",
        "57297bc9af94a219006aa4c7",
        "5726c3da708984140094d0d9",
        "572755b7708984140094dc4d",
        "572647d0708984140094c14b",
        "57060a1175f01819005e78d2",
        "57264b3edd62a815002e80aa",
        "572fbfa504bcaa1900d76c73",
        "5706143575f01819005e7950",
        "573380e0d058e614000b5be9",
        "5726da89dd62a815002e92b2",
        "573403394776f419006616dd",
        "57273e50dd62a815002e9a02"
      ],
      "candidate_sha256": [
        "e1d77a1f9e7bd0b456cab2dbd3c49e298456069561cff9836fe9efbb7e0de51b",
        "51afc408b3f8310c28caf84e1cc8788f37c0aa946de19d18f13f7fa3e42f6eea",
        "d03773b18a1be50107069edcf0e896834515c0cc39a5f2a1177372718099eaff",
        "badbd9d07214ab8658b0a88e6e7e4d7d0f49fb5d0523843171461e2db2e5ce51",
        "e154bf723ad4445d0e204018449b16d521e83192271d4e25bbadf5ea7feca6e4",
        "4fa5eaa7154a62b82b2a46016acca4467d3e6ad646c297e4d6e5d6fda0f77dfc",
        "7f004f5ed59315626739ba9c35512c96ceeb6f3af1db7426178150a313595b76",
        "0b5c5e9ff90b17dfedd7b2a52c9f273ef4aa7af6406c551772fe8ff04cdf1858",
        "493072669f8fd14ce95b09d04e07c61e04265ee4d5cd0abca9fe25259df74897",
        "1a671b4180a17632b8ec241e19026597ac1fddc1665774bef0f116b0e55f2038",
        "58de06c671d2cd29f3b8bde136326194d51c92607fc4a624d1d5a6e90dd5cf33",
        "c13e97d5b9e107a95bfa1a5c4501e6f2324d1835f95f811f69b00da962998aa8",
        "65fcd048594e4421741c510ae4270b54b3957f281d73bc8fe4459bfa38f4b774",
        "d65ecd072e715fe2dcda8065e423c90c09a474e3fa4d3330cda31a91cfd6d99f",
        "a28e73d9c272e8bd0f0049546322e1b21da4398070cbbb52ca838d496c2a38c6",
        "c2d62502892e54f510c36e86c2adf3cadf9c533c59c96e28d86c64a63f08853c"
      ],
      "example_id": "572755b7708984140094dc4d",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        15,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.16422535,
      "neural_order": [
        6,
        15,
        8,
        10,
        12,
        11,
        0,
        1,
        13,
        3,
        2,
        4,
        5,
        7,
        9,
        14
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "de9a7c14f50766d94db189381926f0b57ece42518fc3ca2d94ed1ed9d96e65d1",
      "trial_index": 592
    },
    {
      "answer_sha256": "c3a819ce335f24aa7c0391bd062fcdf0da32555aa4171a7aa1dc5199d0f6c0ba",
      "candidate_ids": [
        "572863c72ca10214002da2d4",
        "5727cc15ff5b5019007d9576",
        "57266e72f1498d1400e8df8c",
        "5710eca0a58dae1900cd6b3a",
        "5725f5b1271a42140099d371",
        "571098f8a58dae1900cd6ab2",
        "57338160d058e614000b5bf9",
        "572667e2f1498d1400e8de90",
        "57293d6d1d046914007791b7",
        "57267ebfdd62a815002e872c",
        "571154c72419e31400955588",
        "5734025d4776f419006616c3",
        "57335c20d058e614000b58f9",
        "5728510f4b864d1900164936",
        "5711475ca58dae1900cd6d8b",
        "5727565ef1498d1400e8f674"
      ],
      "candidate_sha256": [
        "33e2cbb8bfbd5c289d039594230be01025af09fb64adfb83f973ad2fbfd72f9a",
        "e9b551e811c9eba193b8d1e1f0eb0fd5e26a5ddaa654c9925633369091e7aeac",
        "62ff2780adc89c5c824605eae7e328e00b1ada3ef2a3670921c1f3f69cdd5d67",
        "4bec549ca300e97a649dd07a6918a6855dc96267742f85f1f1e15aeb21ef05df",
        "9bf726114e7f52735f5cae1281eb0d8f7702706a7351e05dd8ee2290d6aa1685",
        "c56d9ce9f4030093a35de9eecff1f42f4b6a2a04ccb7d561b196b5d1b584c554",
        "1ccdc3ecc98f2ce8281960f061b48c4e568a881f46f589eec7f333b13afb85de",
        "1e0b24e08630dc44ca697627146eff0a852b93e1fc7e4d4a8bb9a1efa7ad0d66",
        "efe5bfaa2f3946825a1254df02525dc6072ac5e90eefa486ff41db2f7edfd51b",
        "ca24626595fcaaf93d7b5eaabdf0a5a0f4193ef886995a0937088d1f4995964b",
        "e2525668eb13ddf3ceee10a8c808caa5ae4a525801b9141bcca4c921cf8a4cfa",
        "09a4a021aa4006e477987f94a8991472b7f607822ed9e8c4a371bc972d212662",
        "bf76765a3bcbb3f449e01da2f548b3b094ea7b414aeef804bd1d9900155a93f6",
        "cef6553c75c9bc99dc2e9bd9be2a7f76921797a86809c6083588e24184b6e391",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "d3a00727a3e30632468c777631dc072465335095156238321d5ff591c1c47875"
      ],
      "example_id": "5727565ef1498d1400e8f674",
      "gated_correct": true,
      "gated_top": 15,
      "gold_index": 15,
      "lexical_correct": true,
      "lexical_order": [
        15,
        4,
        5,
        0,
        1,
        2,
        3,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14
      ],
      "lexical_top": 15,
      "neural_correct": true,
      "neural_margin": 0.60902358,
      "neural_order": [
        15,
        4,
        1,
        12,
        5,
        0,
        9,
        6,
        8,
        14,
        3,
        2,
        7,
        10,
        11,
        13
      ],
      "neural_top": 15,
      "partition": "test",
      "question_sha256": "8f8d1e13d8cd60b3789e0f0d0efc9187343fd1402aa3e42d92a56132e990e634",
      "trial_index": 593
    },
    {
      "answer_sha256": "7a01adea4b8a1084a7aebd7f3256080c31a3013aa5ad7638add2136d6f3e5371",
      "candidate_ids": [
        "572a019f3f37b31900478643",
        "57268f05dd62a815002e8990",
        "573368044776f41900660a29",
        "57265526708984140094c2bd",
        "572a1046af94a219006aa78d",
        "572a12386aef051400155234",
        "57115dbe2419e314009555a5",
        "5726462b708984140094c117",
        "5725b92e38643c19005acbd3",
        "572756fe708984140094dc71",
        "571cd5b1dd7acb1400e4c178",
        "572fcd86947a6a140053ccda",
        "572826634b864d19001645be",
        "5726a5525951b619008f78dd",
        "5706139252bb891400689864",
        "5725ec8289a1e219009ac0ae"
      ],
      "candidate_sha256": [
        "310a0287ae9110b26fba1630c7ed6ae6b2589284cef6bdd0383b565ee8ec5c0b",
        "cabbfd3499fcfd708e407df199eda5fdd9e8bac825431d8e1223fca75483ce2b",
        "01fbee86abe8e1fdb157b2ec40b6bdbb181baa78b7843fc6874859b9c6d9e482",
        "76cdbeb9d90ace35573e2882ad7c0def58ed2e97998e9484cfd8431a288d76df",
        "370a061e5ca65789c1b3de293fa613a451986a288a9add748e5305a00db4ffcb",
        "ad6baa61d0bb46eb7a39342c2879d2b93020cc13598b4a15adfb36b330461023",
        "faf55bac047f0f94f70e9ed7842dc07f9af20019e44018a9b9eedfc802745806",
        "373964db115f2ceafdb8d8e1707645925a2f011a20f774e5520fd95820e1b483",
        "3e8b8158e72efd061d97feacf1b5f6994ba59d3abf59cf06467bd8add6e1929d",
        "1cb7e5de5a224c8dea488ef682fd45db2d8d645f12c7947aa866d443afa6e1b0",
        "1a29c4d900a5861923886c7581703bf0d565d9d2f379ba04c2860be4aa68f14e",
        "e0919cadaf5f25525fab35135b5214829942fcd766dd177767899b1d6fea9300",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "10e098c47481efcd1f8857cdeddc59059f4fb5b1fa8474c25cf60d5b190b9300",
        "4caee0394ea5501c07bbfff7e18699abb2e6bf66f60623a501d3d454f3df8e3b",
        "30b96d00b6abd2b913c23d97aa914ef7cf8a50107f4b66ea802e13359d9c0d58"
      ],
      "example_id": "572756fe708984140094dc71",
      "gated_correct": true,
      "gated_top": 9,
      "gold_index": 9,
      "lexical_correct": true,
      "lexical_order": [
        9,
        11,
        6,
        5,
        3,
        0,
        1,
        2,
        4,
        7,
        8,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 9,
      "neural_correct": true,
      "neural_margin": 0.39914128,
      "neural_order": [
        9,
        4,
        14,
        12,
        2,
        3,
        10,
        13,
        0,
        6,
        7,
        1,
        5,
        8,
        11,
        15
      ],
      "neural_top": 9,
      "partition": "test",
      "question_sha256": "898447f3839fd4ed78f18ba80e75b9ffd53e5c54e42b9cbeb28e5ad523544ab1",
      "trial_index": 594
    },
    {
      "answer_sha256": "0275fca08c216c77b2a122787b6491a6796ab293e4097e5d62162c0ed4910f08",
      "candidate_ids": [
        "57264b1ddd62a815002e80a0",
        "572ffabf04bcaa1900d76f9f",
        "5733d249d058e614000b6331",
        "5711475ca58dae1900cd6d8b",
        "57109180a58dae1900cd6a40",
        "56de4b074396321400ee2793",
        "57309564069b5314008321a5",
        "5727aa413acd2414000de921",
        "572ff83ab2c2fd1400568681",
        "572fdc34a23a5019007fca93",
        "56e1a0dccd28a01900c67a2e",
        "572746addd62a815002e9a5e",
        "57274b35f1498d1400e8f5d4",
        "5705f09e75f01819005e77a4",
        "57377083c3c5551400e51edf",
        "5706094b52bb8914006897de"
      ],
      "candidate_sha256": [
        "37f3d85a43188d708ec87e4ef98f552191843499c9f8f063fbab2fde1677810a",
        "ade6ff401c8944661f8dbe0a4e4be3b05995437ccaed1d24de71c038275f80ef",
        "52f3fc78c5cfc113278260d462256c64e588f9b3bdc83f3c4854fce44d81b74c",
        "24273c4214c280027ec913fc28585ee484edcd0c57967f4677741d4c2c520bdd",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "7b7ff1f8935459f0c21c7e861026fca972eb90d370d58dd89ed28b1f53fa01de",
        "20fa67864c236b5898d308cf6d1075740fb2b80181e8607ef3c3480721033ff2",
        "84ed50acb1a12d2bfe77fc91bcf26a0b90534e1d1e03cd9cc53f99345379e87e",
        "aae646db783930c79488b505a1d5ba5ee0ba743263be085ba544d24103e161dc",
        "174f049052581528d7fbc279f418488ff7a985794153754911495171db5cb243",
        "b6b3b4a40d231c6d0d5783eb7db8789ff44856f59009e6c3b4ee703f6529394d",
        "8a1e73994613f7c76c60df5b5fa0bbf0387c8c4c1fd6cdf6522691e28bc33d2d",
        "aa1a9a4a968dc56da8463c2203affb0194f6839f072ddd7ffb5a7df52eac2259",
        "c6c4ad210ae5106098bad7f1b92af7d378e3f1551cd980396db376b2419de40e",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "b75e5803fbb4f060c696351025c35bd5542acc3855a593521033151e4d0d2c14"
      ],
      "example_id": "572746addd62a815002e9a5e",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        12,
        6,
        0,
        7,
        1,
        2,
        3,
        4,
        5,
        8,
        9,
        10,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.31853088,
      "neural_order": [
        11,
        12,
        7,
        4,
        13,
        15,
        1,
        5,
        9,
        0,
        6,
        2,
        3,
        8,
        10,
        14
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "26652bb721e7fe91a1433897ec77fab1b4642eea889b59f35b48ae937dde1339",
      "trial_index": 595
    },
    {
      "answer_sha256": "7be5908ba4a67f8fc26144ef0f2b6b2fc6907a10f32a299c525132d26daecd91",
      "candidate_ids": [
        "572871bd3acd2414000dfa03",
        "57261dab38643c19005ad037",
        "56e20a27cd28a01900c67c24",
        "57094b4f9928a814004714f6",
        "57281d494b864d19001644be",
        "57308ddc396df919000961a4",
        "57274712708984140094dbad",
        "572655e5f1498d1400e8dc5e",
        "57268da7f1498d1400e8e39c",
        "571cca4add7acb1400e4c150",
        "56e17644e3433e1400422f40",
        "571153422419e3140095557d",
        "57109180a58dae1900cd6a40",
        "56de3ebc4396321400ee26e6",
        "572fca7eb2c2fd1400568473",
        "57378b141c4567190057449f"
      ],
      "candidate_sha256": [
        "598b4e8be0368d958edb3f2b444ad92050591f8638852dbffda8e77de7bf093b",
        "b88785ffc0e7962a8063a65060870a041b78570fe90ccdb19dc2fbfd940bb7cc",
        "86456bb4ed1522c9e166ed46d812534728fcdc279a359f4976015afda6907420",
        "8cd9a361825ad9f2d3cc6576b2de8c81bb02ec9fdbfdaa1559941ae6a93e7500",
        "7df04cb220df5f112c46d2212eeee0bfd81f69d591643d49d0674e8cf59ab2fd",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "cc9d8349d8e4f4705aae8eb7860e4098c56d15c587275bc98e41b37e1702ebfa",
        "e195ac114b1e4e97dcbdf23f18e4937345f46cced929aca6fccd7eb130df1c3c",
        "2cd5effea6825a6ef7ea9db3e15455260ec2dbdcce4dad86a9607b8c1be1318f",
        "8ef143d276f98e3c0b64cb15958e1ff007dda45e722834eed2c95d4919a3429f",
        "c14cb52e0d8ca7b881d0d20570d1ad267d085b79af6bdbfb72715ac665d264d0",
        "e7204a7add4ae4377c10d01ff9587526b8ab126f9d8a405900578a091232eb9d",
        "2630c31b6e02b983acacd9e9cc11de9883a153a58a5fbfc9865686c0c21c8892",
        "d6b793bb23a351b7418793d96d8e3094c74522cf35f731653c539571da8529f6",
        "b4c77dc6b651a68d38c1887f3949da424d9f10940509139effbe1f3c76c00c4b",
        "d3bd27c0e7c73541263df71d3307e6c61ceb6073e5a88dbbf729df847d4e0bef"
      ],
      "example_id": "57274712708984140094dbad",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        4,
        0,
        12,
        1,
        2,
        3,
        5,
        7,
        8,
        9,
        10,
        11,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.37799352,
      "neural_order": [
        6,
        4,
        12,
        11,
        1,
        9,
        3,
        10,
        14,
        0,
        2,
        5,
        7,
        8,
        13,
        15
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "196ff0a97d88e8b2fe806d9fa15e7b272279369eec5400842cb9b1117a07dd31",
      "trial_index": 596
    },
    {
      "answer_sha256": "c5bc3e4cb865869e01f5cb3e842dee76d93b3d97f7bf8f8ce7d8398ab2bc5fa0",
      "candidate_ids": [
        "570d2417fed7b91900d45c3d",
        "571a50df4faf5e1900b8a960",
        "570d4e6cb3d812140066d66d",
        "573004bf947a6a140053cf56",
        "57308ddc396df919000961a4",
        "572647e2dd62a815002e805c",
        "572826634b864d19001645be",
        "56e1ee4de3433e1400423210",
        "57264a8cdd62a815002e808c",
        "57282dfb4b864d1900164668",
        "56dde27d9a695914005b9651",
        "5727478cf1498d1400e8f59a",
        "572a07a86aef0514001551d2",
        "57294e6b1d04691400779275",
        "5727dc473acd2414000dee43",
        "572ff935b2c2fd140056869b"
      ],
      "candidate_sha256": [
        "3548c9026a06bf244b562a301cf08d02a4d7cc064187ea4773284ab64ab44ef3",
        "ad88fe48ac1ab4c46310dbb92ef9eb65cb2d3c8e6a37c2e16ca5b79106cf73e5",
        "a4f39f1d9df3c7a1c8e906ab4740679e3cd0926c0f0ceb7e939b92beda68425d",
        "76dbd11316dabf552d6d379d3ef25104282932d7e715909730b805d8c9b93615",
        "3d22b3f5d1e3403f040c29691b6316b46a56bd72bea2f0052dc88bad8bc76893",
        "4de28cbc201cea5346f39f7dca7b0ba2e34ba42603f55f94d2eb3ee8e023793f",
        "7d5bd174d32d58bb26dd4c85a465c3132509d346fc1fc09b44f99555488ac4a2",
        "d23507599fde3bea64581b405ee4729f02882f17e7602e4f065a2b2a98228ea6",
        "6cbc27f4f6c138ccfcb20b6f35d4cfadf8769e46c293c5a0c77d0edbf311f58d",
        "6ceedf167e6d0db93383ee792e5c7e9aef725362d1eae0cb3ce70d741ffc043d",
        "6cc34106bc6b1deb652e610973c798faa2caaaed6b16c99c39c6f5b49df3a196",
        "a6bf93aaa317a4cbe3dd6095edaf3c9cad24ebd2270fecf0196ce024b0880c94",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "e70fc06ba9ab686a435c00611cdb5be1a2772afabf16c26f218a90a992ce3e77",
        "fe9c5d14f6acc06385618b84fac8d90555fe0e82822b20bcc630fd569cb457b2",
        "037e1b3b1abe4922983061ba833be49ef7e06df5b7d75f2dbfa1e6bef420b333"
      ],
      "example_id": "5727478cf1498d1400e8f59a",
      "gated_correct": true,
      "gated_top": 11,
      "gold_index": 11,
      "lexical_correct": true,
      "lexical_order": [
        11,
        0,
        2,
        5,
        1,
        3,
        4,
        6,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 11,
      "neural_correct": true,
      "neural_margin": 0.33855495,
      "neural_order": [
        11,
        12,
        5,
        4,
        7,
        0,
        2,
        14,
        10,
        3,
        13,
        15,
        8,
        9,
        1,
        6
      ],
      "neural_top": 11,
      "partition": "test",
      "question_sha256": "e6b92f019ed6262229c97438c8bd02ea62a55ee94930ac7036a73ab7144e1332",
      "trial_index": 597
    },
    {
      "answer_sha256": "dd3c6d45a416b60d5da938f148ece6fbc0034e6b9fbae26a4fa3a0defdd8bc9d",
      "candidate_ids": [
        "5727c55bff5b5019007d94ce",
        "5727d9c43acd2414000dee17",
        "57379829c3c5551400e51f3d",
        "572a07a86aef0514001551d2",
        "56e1c4fce3433e140042314e",
        "57269f3ef1498d1400e8e534",
        "5727482bf1498d1400e8f5a4",
        "56e16182e3433e1400422e28",
        "5726bcde708984140094cfbf",
        "572f64ccb2c2fd14005680b7",
        "570d4329fed7b91900d45dd7",
        "570960cf200fba1400367f01",
        "56e1f10ee3433e1400423223",
        "57377083c3c5551400e51edf",
        "57264e66dd62a815002e811a",
        "573784fa1c45671900574483"
      ],
      "candidate_sha256": [
        "cded37032d68b16effd6766161b05dd6133be8b6c51f658dd984a8d9878505c4",
        "ec0145a8460051b10a8900bf9b7695d3054720a4e9802b0ca8152f7b919b72c6",
        "7aa3c41e95ec3f91a2e6241e05c3d435913ef6549395e3a19ca6c397879a305c",
        "8b85b916fece6ff32a3f58eb608b19163304fbead9d97dd4521b33083e3dde75",
        "b12f8869c244fe800cc88e3758ec51e32429412575313c6b2d991e5f60a1d928",
        "47450d0c501c67649dd161f5d7cea2ed48b03534e97e62c2034a4395967f921e",
        "8ca6346e70d1467511f51f9602901b4dd742209975a436a04c0ea22ac490f86d",
        "4d3f905eee7981da9b9da72fa2e46d6de1701497204d573cd2b11c15b077ad99",
        "b425c59663b1f599dfeff34f5eca35cd8787024f5dfa1ce116e21e72f95f0107",
        "d2a2405509f89d50ca530b173f9cfa53b587151a7df14dc0831f4e7b0a132adb",
        "04f203956d3cfbce055adc4cf145999fbeca50dd2e3199c11308d0130f84b6ae",
        "f31033c00bbf282887f5519cd513845f5fcc2323fa573d88ab2ae8db2369c47b",
        "fa4f1279877804a3ee0afc8a8e3f12cffab66a695619e9ab4fb87d466f1278f2",
        "e5cb6b3b41410225b13e886e66c8e3d9f90106c513b4d8727dd4c807d13d84c2",
        "33d8bc85542315bdf838b906552e082e1e24cc97a8cc8ed50f3a5aaf5916d562",
        "6ee156ac8e66c07495287b3eb511c3c813016137102c3784322afc4d9482c0f9"
      ],
      "example_id": "5727482bf1498d1400e8f5a4",
      "gated_correct": true,
      "gated_top": 6,
      "gold_index": 6,
      "lexical_correct": true,
      "lexical_order": [
        6,
        11,
        0,
        1,
        2,
        3,
        4,
        5,
        7,
        8,
        9,
        10,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 6,
      "neural_correct": true,
      "neural_margin": 0.46140644,
      "neural_order": [
        6,
        0,
        1,
        4,
        8,
        14,
        3,
        15,
        12,
        13,
        5,
        2,
        7,
        9,
        10,
        11
      ],
      "neural_top": 6,
      "partition": "test",
      "question_sha256": "9ff0cc2eacc01e6ac276b4aa5466988aaff75ee9929d01aaad66274bc12982d5",
      "trial_index": 598
    },
    {
      "answer_sha256": "359647569b38a9ff0fcf245ec75804686bdd6052999c086fad5e6fad28e80846",
      "candidate_ids": [
        "5705e3f252bb89140068966a",
        "570966e0200fba1400367f4f",
        "572a11663f37b31900478693",
        "5725c57a89a1e219009abe5e",
        "5727490bdd62a815002e9a82",
        "5729f8516aef05140015516c",
        "572a0d21af94a219006aa783",
        "572f5875947a6a140053c89a",
        "57060df252bb891400689820",
        "5725d16aec44d21400f3d61b",
        "5725bbec271a42140099d0d1",
        "572fd47fa23a5019007fca55",
        "5737898f1c45671900574495",
        "5711488ab654c5140001fc3d",
        "5733f5264776f419006615a3",
        "5733fb7bd058e614000b66ff"
      ],
      "candidate_sha256": [
        "1fde3c0825f0073047b85448d0c82e4588739036d0d0e5cc25ef81a0eb7ba822",
        "3a2ba0c4865678a89dbdb5a6f7c74e3eb86122b429459f4c72c70515c24b55c7",
        "68cd564f9e510dca0c3554774995fc8d352aac44126d4a3d512117ab3b855c6e",
        "70150ba659ef2c4bad22a2a83dde1ff6810a8679153106769c21c3b383bdb336",
        "7d40046d35a495a564aa1049e78bbaf0c72f50500a115c58bba2db578b607cf8",
        "bbe67a3db01f54c6447e86ae926f8d1b5a84031edea1bb6431a4e264d4882fed",
        "c1901061039c2bb1ad545a817f00d92dd5d20ffb490cfbe72629441af32a7e7b",
        "68ce7c2699d3d3cb0d5a059ed7e339be4e9e7346b01e5c356a3699ae3d212e8c",
        "be2792db1ce39051057f50003730431c5d1e0c20852368f563eda2c372771da2",
        "e9030e5e203b110c1161f7cb8b77776f171a3a793ec1fafe8d14e2b9bc0dff07",
        "ef012a508c71706fbc3911158abf7b8fd6080d7533d4072dc26dac90ce4584ac",
        "20b1166829f552d99ba27d3ce34d06f8742dc04198b7aedf74db2fee2b569db1",
        "5328904ae022c368d40bd1b0ac886d149846551d037994b11605aab765c6a753",
        "e14ec1d9643c7f64c33f3209672e6a3d0290fe2ccf398853c6741857c7e88d16",
        "9384b2ae8d06f600126ecfff9a80168fbb32d4cda2c1d70e200f7ab4d0578b47",
        "046e69fc2e3832debebd63f0302a7b549b843ea6c086e13ee2f397347e3b527e"
      ],
      "example_id": "5727490bdd62a815002e9a82",
      "gated_correct": true,
      "gated_top": 4,
      "gold_index": 4,
      "lexical_correct": true,
      "lexical_order": [
        4,
        3,
        6,
        2,
        0,
        1,
        5,
        7,
        8,
        9,
        10,
        11,
        12,
        13,
        14,
        15
      ],
      "lexical_top": 4,
      "neural_correct": true,
      "neural_margin": 0.53905812,
      "neural_order": [
        4,
        2,
        11,
        6,
        9,
        1,
        3,
        0,
        10,
        8,
        12,
        5,
        13,
        7,
        14,
        15
      ],
      "neural_top": 4,
      "partition": "test",
      "question_sha256": "9f00b0c5f817e1e9417f60d6d7e49425ef52c552d2b2705363256decac6c9439",
      "trial_index": 599
    }
  ]
}