{ "created_utc": "2026-10-23T13:28:09.536596+00:00", "protocol_sha256": "88754a9c5b4160aed0dc50a981165f8a8b2156218cc1c5b1499992d2e2c5bee3", "prediction_freeze_sha256": "398999cb183ed29775c32784b438032033257075bd7b2a8dc5e75b357422f0cd", "responses_sha256": "cfaf481649d3d2f3cd46f47cd651152c40001462ff341f9d672d41247b465a83", "evaluation_code_sha256": "1fbbcb35ff9261a724de8051f068fa526f08149e3a129b80e38d73def9224a77", "participant_count": 1, "phase_response_counts": { "train": { "right": 14, "left": 8, "tie": 2 }, "test_other": { "right": 3, "left": 4, "tie": 1 }, "test_target": { "right": 2, "left": 2 }, "repeat": { "left": 4 } }, "repeats": { "agree": 4, "usable": 4, "scheduled": 4, "rows": [ { "trial_id": "0f2756272361", "repeat_of": "e60641edcb3b", "first_selected": "natalie_portman", "repeat_selected": "natalie_portman", "usable": true, "agrees": true }, { "trial_id": "dd8c77c145bc", "repeat_of": "c4531359359a", "first_selected": "keira_knightley", "repeat_selected": "keira_knightley", "usable": true, "agrees": true }, { "trial_id": "2333a59ee3bd", "repeat_of": "42ecaecf4d00", "first_selected": "scarlett_johansson", "repeat_selected": "scarlett_johansson", "usable": true, "agrees": true }, { "trial_id": "a33fba8c36a4", "repeat_of": "000340fb28e8", "first_selected": "scarlett_johansson", "repeat_selected": "scarlett_johansson", "usable": true, "agrees": true } ], "rule": "Same selected identity or tie after reversing left/right; any skipped response makes that repeat pair unusable." }, "encoders": [ { "encoder": "openai/clip-vit-base-patch32", "variants": [ { "variant": "portrait", "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 12, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.693368511724823, "brier_score": 0.2292773268869486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 1, "accuracy": 0.09090909090909091, "mean_cross_entropy": 0.6961942687496107, "brier_score": 0.23069019379997246, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 1, "accuracy": 0.14285714285714285, "mean_cross_entropy": 0.6956667452202111, "brier_score": 0.22000977128817517, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.6972493158084097, "brier_score": 0.25205103882356716, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6936387703041482, "brier_score": 0.25024578780231527, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.00811773497719772, "right_score": -0.0067508278673061, "margin": -0.0013669071098916198, "p_left": 0.4996582732757349, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6924639605593622, "brier": 0.249658390052889 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.002939517470945549, "right_score": -0.004391418775907505, "margin": 0.007330936246853054, "p_left": 0.5018327258537623, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6968193664966108, "brier": 0.2518360847378173 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.007748799415882214, "right_score": -0.003597153819498512, "margin": 0.011345953235380726, "p_left": 0.5028364578806538, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6988362484231784, "brier": 0.2528445033739625 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.011449947886593682, "right_score": -0.006741344373869345, "margin": -0.004708603512724337, "p_left": 0.4988228512966933, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.690795650169403, "brier": 0.248824236975763 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.00811773497719772, "right_score": -0.006741344373869345, "margin": -0.001376390603328375, "p_left": 0.49965590240349084, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.692459222064649, "brier": 0.24965602080664676 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.002939517470945549, "right_score": 0.0003852273927488262, "margin": 0.002554290078196723, "p_left": 0.5006385721723576, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6944251411485475, "brier": 0.2506389799467769 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.00811773497719772, "right_score": 0.005164555741378715, "margin": -0.013282290718576435, "p_left": 0.496679476137239, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6865280874443977, "brier": 0.24669050201596218 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.0007515761431032171, "right_score": 0.012649193277060322, "margin": -0.011897617133957105, "p_left": 0.49702563080240764, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6991136831842469, "brier": 0.2529832160697159 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0053332860828347855, "right_score": 0.009102165607821594, "margin": -0.0037688795249868085, "p_left": 0.49905778123406147, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6950333958779971, "brier": 0.25094310654214136 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.003597153819498512, "right_score": -0.01055111858513303, "margin": 0.0069539647656345184, "p_left": 0.5017384841856669, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6896762428681938, "brier": 0.2482645381415969 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.005164555741378715, "right_score": 0.012544297084619325, "margin": -0.00737974134324061, "p_left": 0.4981550730371653, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6968438587889046, "brier": 0.25184833071833285 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.004391418775907505, "right_score": 0.0003852273927488262, "margin": -0.004776646168656331, "p_left": 0.4988058407283646, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6955383556851397, "brier": 0.2511955852880014 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.005164555741378715, "right_score": 0.007748799415882214, "margin": -0.002584243674503499, "p_left": 0.49935393944092377, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6918558935118825, "brier": 0.24935435683516977 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.006741344373869345, "right_score": 0.007748799415882214, "margin": -0.01449014378975156, "p_left": 0.49637752743482333, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7004184977585959, "brier": 0.25363559487266213 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.009102165607821594, "right_score": 0.005338096656073755, "margin": 0.003764068951747839, "p_left": 0.500941016126894, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6912669171099101, "brier": 0.24905986938445707 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.004237671323987392, "right_score": 0.007748799415882214, "margin": -0.003511128091894822, "p_left": 0.4991222188788014, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.691393157515766, "brier": 0.24912298937849817 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.0067508278673061, "right_score": -0.00811773497719772, "margin": 0.0013669071098916198, "p_left": 0.5003417267242651, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6924639605593622, "brier": 0.24965839005288895 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.0067508278673061, "right_score": -0.011449947886593682, "margin": 0.004699120019287582, "p_left": 0.5011747778430622, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931499407735252, "brier": 1.3801029805498548e-06 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.0067508278673061, "right_score": -0.01055111858513303, "margin": 0.0038002907178269303, "p_left": 0.5009500715360293, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6950491311939649, "brier": 0.2509509741719529 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.01055111858513303, "right_score": 0.004237671323987392, "margin": -0.014788789909120421, "p_left": 0.4963028699052296, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6857801238946287, "brier": 0.24631653867616726 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.005164555741378715, "right_score": 0.003729669154608954, "margin": 0.001434886586769761, "p_left": 0.5003587215851448, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.692429994628978, "brier": 0.24964140709603083 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.005338096656073755, "right_score": 0.0007515761431032171, "margin": 0.004586520512970538, "p_left": 0.501146628118188, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6954430703354277, "brier": 0.2511479428742294 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.004237671323987392, "right_score": -0.00811773497719772, "margin": 0.012355406301185111, "p_left": 0.5030888122815689, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.686988559296088, "brier": 0.2469207284797419 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.009965973349332014, "right_score": -0.0003609175344081515, "margin": -0.009605055814923862, "p_left": 0.4975987545072355, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6979612405602283, "brier": 0.25240701147268096 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.003597153819498512, "right_score": 0.007748799415882214, "margin": -0.011345953235380726, "p_left": 0.49716354211934627, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6988362484231784, "brier": 0.2528445033739624 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.006741344373869345, "right_score": 0.012544297084619325, "margin": -0.01928564145848867, "p_left": 0.4951787390678135, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7028364925645137, "brier": 0.2548445054891627 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.012649193277060322, "right_score": 0.0011099537028006288, "margin": 0.011539239574259693, "p_left": 0.5028847778836477, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6989334445109763, "brier": 0.2528930998270857 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": -0.00037095407816814877, "right_score": -0.01055111858513303, "margin": 0.010180164506964882, "p_left": 0.5025450191471986, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6982502172261623, "brier": 0.2525514962696582 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": -0.00037095407816814877, "right_score": -0.011449947886593682, "margin": 0.011078993808425534, "p_left": 0.5027697201215965, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6876230265902397, "brier": 0.2472379512279555 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.004237671323987392, "right_score": 0.003729669154608954, "margin": 0.0005080021693784379, "p_left": 0.5001270005396135, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934012139029098, "brier": 0.25012701666875053 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.0067508278673061, "right_score": 0.012544297084619325, "margin": -0.019295124951925424, "p_left": 0.4951763684150024, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931937175689069, "brier": 2.3267421667786465e-05 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.012544297084619325, "right_score": -0.00037095407816814877, "margin": 0.012915251162787474, "p_left": 0.5032287679099487, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6996256564605016, "brier": 0.25323919285216506 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.006741344373869345, "right_score": -0.011449947886593682, "margin": 0.004708603512724337, "p_left": 0.5011771487033068, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.690795650169403, "brier": 0.24882423697576295 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.011449947886593682, "right_score": -0.003597153819498512, "margin": -0.00785279406709517, "p_left": 0.49803681157177526, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6892284918034244, "brier": 0.24804066568057997 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.012649193277060322, "right_score": -0.004391418775907505, "margin": 0.017040612052967828, "p_left": 0.5042600499267597, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7017037839546532, "brier": 0.25427819795213813 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.003597153819498512, "right_score": 0.003729669154608954, "margin": -0.007326822974107466, "p_left": 0.4981683024506159, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6968173023238514, "brier": 0.2518350526652966 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.0008174589967550785, "right_score": 0.00807681196985207, "margin": -0.008894270966607148, "p_left": 0.49777644691674233, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931570690343547, "brier": 4.944188314064681e-06 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.003729669154608954, "right_score": -0.00037095407816814877, "margin": 0.004100623232777103, "p_left": 0.5010251543716876, "prediction": "left", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6951995940637234, "brier": 0.2510262053131734 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.004391418775907505, "right_score": -0.0008174589967550785, "margin": -0.0035739597791524264, "p_left": 0.49910651100627024, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6949357570972347, "brier": 0.2508942873163117 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.006741344373869345, "right_score": -0.00811773497719772, "margin": 0.001376390603328375, "p_left": 0.5003440975965091, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.692459222064649, "brier": 0.2496560208066468 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 1, "regressions": 2, "both_correct": 0, "both_wrong": 8, "net_fixes": -1, "trial_ids": { "fixes": [ "70f39a6b70be" ], "regressions": [ "0add2ec437a1", "34bec29a8d38" ], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 1, "regressions": 2, "both_correct": 0, "both_wrong": 4, "net_fixes": -1, "trial_ids": { "fixes": [ "70f39a6b70be" ], "regressions": [ "0add2ec437a1", "34bec29a8d38" ], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.6291788311260436, "brier_score": 0.19735566728112044, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6893675850856567, "brier_score": 0.22727952226443535, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6861084443352852, "brier_score": 0.21523471389165968, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.6958858665863998, "brier_score": 0.2513691390099867, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5739784888718696, "brier_score": 0.1908329514139377, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.1265949959823316, "right_score": 0.17132358972696207, "margin": -0.29791858570929364, "p_left": 0.4260663808795703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5552415354817556, "brier": 0.18153256091581504 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.027774114325737596, "right_score": -0.01949319021824058, "margin": -0.008280924107497017, "p_left": 0.4979297808033287, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6890152901947146, "brier": 0.247934066610851 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.16567434654769975, "right_score": 0.2696931958974109, "margin": -0.10401884934971117, "p_left": 0.4740187097315416, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6424896367088768, "brier": 0.2246937371755555 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.047856209842893016, "right_score": 0.22563546795734885, "margin": -0.27349167780024186, "p_left": 0.43205009486248436, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5657220593308394, "brier": 0.18666728447068173 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.1265949959823316, "right_score": 0.22563546795734885, "margin": -0.35223046393968044, "p_left": 0.4128416440885034, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5324607239660066, "brier": 0.1704382230936985 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.027774114325737596, "right_score": -0.07007248855972983, "margin": 0.04229837423399224, "p_left": 0.5105730172105211, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7145199950645631, "brier": 0.26068480590345505 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.1265949959823316, "right_score": 0.048412983520545225, "margin": -0.17500797950287683, "p_left": 0.45635933308353804, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6094667891345371, "brier": 0.20826384089245162 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.007981090840691728, "right_score": 0.03297671911077339, "margin": -0.024995628270081664, "p_left": 0.49375141826224606, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7057230903410653, "brier": 0.2562876265114874 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.08532994512607553, "right_score": 0.07982050614181349, "margin": 0.005509438984262038, "p_left": 0.501377356262054, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6903962553027557, "brier": 0.24862454084821858 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.2696931958974109, "right_score": 0.06260752427213923, "margin": 0.2070856716252717, "p_left": 0.5515871920674121, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5949553529347954, "brier": 0.20107404631798795 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.048412983520545225, "right_score": -0.06723424900656319, "margin": 0.11564723252710842, "p_left": 0.5288796282788033, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6369944188030695, "brier": 0.22195440465071856 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.01949319021824058, "right_score": -0.07007248855972983, "margin": 0.050579298341489255, "p_left": 0.5126421295400436, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6681772804855092, "brier": 0.2375176938992636 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.048412983520545225, "right_score": 0.16567434654769975, "margin": -0.11726136302715452, "p_left": 0.4707182041687297, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.636234293622416, "brier": 0.22157562773583392 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.22563546795734885, "right_score": 0.16567434654769975, "margin": 0.0599611214096491, "p_left": 0.514985790708599, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.663615969556125, "brier": 0.23523878321456299 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.07982050614181349, "right_score": 0.04504033581250766, "margin": 0.03478017032930583, "p_left": 0.5086941661843933, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6759082953056847, "brier": 0.2413814223412485 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.042099847649401986, "right_score": 0.16567434654769975, "margin": -0.12357449889829776, "p_left": 0.4691456291615692, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6332675498964141, "brier": 0.22009762136140462 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.17132358972696207, "right_score": -0.1265949959823316, "margin": 0.29791858570929364, "p_left": 0.5739336191204297, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5552415354817556, "brier": 0.18153256091581504 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.17132358972696207, "right_score": -0.047856209842893016, "margin": 0.2191797995698551, "p_left": 0.554576637136734, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6991401720826234, "brier": 0.0029786093211547323 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.17132358972696207, "right_score": 0.06260752427213923, "margin": 0.10871606545482285, "p_left": 0.5271522784848227, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7489818841529776, "brier": 0.27788952471174005 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.06260752427213923, "right_score": 0.042099847649401986, "margin": 0.02050767662273724, "p_left": 0.5051267394797632, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7034535885501749, "brier": 0.2551530229374565 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.048412983520545225, "right_score": -0.019710662777204516, "margin": 0.06812364629774974, "p_left": 0.5170243281642083, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6596653491703652, "brier": 0.23326549958523438 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.04504033581250766, "right_score": 0.007981090840691728, "margin": 0.03705924497181593, "p_left": 0.5092637510404583, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7118484666775627, "brier": 0.2593495681237979 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.042099847649401986, "right_score": -0.1265949959823316, "margin": 0.1686948436317336, "p_left": 0.5420739799316017, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6123527925182469, "brier": 0.20969623985568314 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.14175601777529534, "right_score": 0.06822206442499257, "margin": 0.07353395335030277, "p_left": 0.5183752091496648, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.65705595694393, "brier": 0.2319624391616291 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.2696931958974109, "right_score": 0.16567434654769975, "margin": 0.10401884934971117, "p_left": 0.5259812902684584, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6424896367088768, "brier": 0.2246937371755555 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.22563546795734885, "right_score": -0.06723424900656319, "margin": 0.29286971696391206, "p_left": 0.572698541552085, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5573958062031952, "brier": 0.18258653639171524 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.03297671911077339, "right_score": 0.1442578851270274, "margin": -0.111281166016254, "p_left": 0.472208382364988, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6390537367460923, "brier": 0.22298075637575873 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1755314487777539, "right_score": 0.06260752427213923, "margin": 0.11292392450561467, "p_left": 0.5282010196501049, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7512022732023029, "brier": 0.27899631715941053 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1755314487777539, "right_score": -0.047856209842893016, "margin": 0.2233876586206469, "p_left": 0.5556158281468653, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.587678180122448, "brier": 0.1974772921935964 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.042099847649401986, "right_score": -0.019710662777204516, "margin": 0.0618105104266065, "p_left": 0.515447709704755, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7245299271689655, "brier": 0.26568634143987735 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.17132358972696207, "right_score": -0.06723424900656319, "margin": 0.23855783873352526, "p_left": 0.5593582207469808, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7002441061996602, "brier": 0.003523398370247305 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.06723424900656319, "right_score": 0.1755314487777539, "margin": -0.24276569778431709, "p_left": 0.43960490096611715, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5791132100359659, "brier": 0.19325246895342968 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.22563546795734885, "right_score": -0.047856209842893016, "margin": 0.27349167780024186, "p_left": 0.5679499051375158, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5657220593308394, "brier": 0.18666728447068165 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.047856209842893016, "right_score": 0.2696931958974109, "margin": -0.31754940574030394, "p_left": 0.4212730922710641, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5469245746301079, "brier": 0.17747101827162445 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.03297671911077339, "right_score": -0.01949319021824058, "margin": 0.05246990932901397, "p_left": 0.513114468692737, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7197262321781948, "brier": 0.2632864579818298 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.2696931958974109, "right_score": -0.019710662777204516, "margin": 0.28940385867461543, "p_left": 0.571850181404948, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5588782425302556, "brier": 0.1833122671629759 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.007282545977960499, "right_score": 0.10587542820936538, "margin": -0.09859288223140489, "p_left": 0.47537172625452406, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6943617583006266, "brier": 0.0006065518676820993 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.019710662777204516, "right_score": 0.1755314487777539, "margin": -0.1952421115549584, "p_left": 0.45134393600006956, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6002835110229207, "brier": 0.2037113485640349 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.01949319021824058, "right_score": 0.007282545977960499, "margin": -0.026775736196201078, "p_left": 0.4933064658513962, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7066246634871807, "brier": 0.2567383375480024 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.22563546795734885, "right_score": -0.1265949959823316, "margin": 0.35223046393968044, "p_left": 0.5871583559114967, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5324607239660066, "brier": 0.17043822309369844 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 5, "regressions": 1, "both_correct": 1, "both_wrong": 4, "net_fixes": 4, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 3, "regressions": 1, "both_correct": 1, "both_wrong": 2, "net_fixes": 2, "trial_ids": { "fixes": [ "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654" ] } }, "target": { "fixes": 2, "regressions": 0, "both_correct": 0, "both_wrong": 2, "net_fixes": 2, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 8, "regressions": 1, "both_correct": 1, "both_wrong": 1, "net_fixes": 7, "trial_ids": { "fixes": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "fe73774e1a9e" ], "regressions": [ "34bec29a8d38" ], "both_correct": [ "0add2ec437a1" ], "both_wrong": [ "c469d02ac7ba" ] } }, "other": { "fixes": 5, "regressions": 1, "both_correct": 1, "both_wrong": 0, "net_fixes": 4, "trial_ids": { "fixes": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [ "34bec29a8d38" ], "both_correct": [ "0add2ec437a1" ], "both_wrong": [] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.6345545847138067, "fold_losses": [ 0.42762889091049494, 0.6910085786197415, 0.24823515112097208, 0.14592212725556453, 0.3552525672729096, 0.4704503124158155, 0.5375262499388462, 0.7620696982640343, 0.824479013474704, 0.618376783068437, 0.7616353412381618, 0.9258052759663866, 1.5857897553378622, 0.7239893279957255, 0.48252141110380975, 0.24876347619024952, 1.1482807264940307, 0.5157850201130045, 1.276175424900226, 0.7804209842486303, 0.48153901104228286, 0.20330385783877247, 0.27320953019818717, 0.741141518122511 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.4378640697857176, "brier_score": 0.1121936655119955, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6957970440678333, "brier_score": 0.23012100909669564, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6778323430125522, "brier_score": 0.21082211726086586, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.7317264461783953, "brier_score": 0.26871879276835514, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2674411950991548, "brier_score": 0.06042183733470195, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.1126605912173593, "right_score": -0.04886667167380007, "margin": -1.0637939195435593, "p_left": 0.2565850995610893, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2965009776839797, "brier": 0.06583591331677412 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3996939996665949, "right_score": -0.45546410888819955, "margin": 0.055770109221604636, "p_left": 0.5139389146362522, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7214209729311118, "brier": 0.2641332079774889 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.4932112789390976, "right_score": 1.0808961869030456, "margin": -0.5876849079639479, "p_left": 0.3571662200447918, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4418690952313347, "brier": 0.12756770874108464 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6144727449462777, "right_score": 0.8761638205574049, "margin": -1.4906365655036826, "p_left": 0.18382620164565772, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20312795851745113, "brier": 0.03379207241147001 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.1126605912173593, "right_score": 0.8761638205574049, "margin": -1.9888244117747642, "p_left": 0.12038128953238127, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.12826674896385343, "brier": 0.014491654869479008 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3996939996665949, "right_score": -0.7373134453142033, "margin": 0.3376194456476084, "p_left": 0.5836121421756691, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8761381025504125, "brier": 0.34060313249487345 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.1126605912173593, "right_score": 0.09476245786682631, "margin": -1.2074230490841855, "p_left": 0.2301573302942574, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.26156911006909306, "brier": 0.05297239668817989 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.46370841103163757, "right_score": 0.027018264025910843, "margin": -0.4907266750575484, "p_left": 0.37972239653066364, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9683148286783754, "brier": 0.3847443053656633 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.12796741890570604, "right_score": -0.06795366631111911, "margin": 0.19592108521682516, "p_left": 0.5488241944437717, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5999771174566345, "brier": 0.20355960751931151 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.0808961869030456, "right_score": 0.18329040304640906, "margin": 0.8976057838566365, "p_left": 0.7104572420863765, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34184651328912175, "brier": 0.08383500866022718 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.09476245786682631, "right_score": -0.6579283915312993, "margin": 0.7526908493981257, "p_left": 0.6797647388797221, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3860085129098889, "brier": 0.10255062246477256 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.45546410888819955, "right_score": -0.7373134453142033, "margin": 0.28184933642600374, "p_left": 0.5699995549682009, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5621196989113884, "brier": 0.1849003827275453 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.09476245786682631, "right_score": 0.4932112789390976, "margin": -0.3984488210722713, "p_left": 0.40168508427930105, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5136380487269225, "brier": 0.1613509069324692 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8761638205574049, "right_score": 0.4932112789390976, "margin": 0.38295254161830733, "p_left": 0.5945850222354289, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5198915583831862, "brier": 0.1643613041958477 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.06795366631111911, "right_score": -0.4611475707927415, "margin": 0.3931939044816224, "p_left": 0.5970513297577363, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5157521897918642, "brier": 0.16236763085000855 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.19022976736821798, "right_score": 0.4932112789390976, "margin": -0.6834410463073156, "p_left": 0.33549372994535115, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4087109651251986, "brier": 0.11255604283264421 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.04886667167380007, "right_score": -1.1126605912173593, "margin": 1.0637939195435593, "p_left": 0.7434149004389107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29650097768397976, "brier": 0.06583591331677412 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.04886667167380007, "right_score": -0.6144727449462777, "margin": 0.5656060732724776, "p_left": 0.6377486825680193, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7326140243484605, "brier": 0.018974699549224948 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.04886667167380007, "right_score": 0.18329040304640906, "margin": -0.23215707472020913, "p_left": 0.44222001205709516, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5837906811929933, "brier": 0.19555853906377738 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.18329040304640906, "right_score": -0.19022976736821798, "margin": 0.37352017041462704, "p_left": 0.5923093010690355, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8972464829296349, "brier": 0.3508303081328893 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.09476245786682631, "right_score": -0.3337226315303794, "margin": 0.4284850893972057, "p_left": 0.6055118638076538, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5016811227744085, "brier": 0.1556208895965111 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.4611475707927415, "right_score": -0.46370841103163757, "margin": 0.0025608402388960783, "p_left": 0.5006402097098547, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6944284204170106, "brier": 0.2506406195783273 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.19022976736821798, "right_score": -1.1126605912173593, "margin": 0.9224308238491413, "p_left": 0.7155371438006068, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33472176826443534, "brier": 0.08091911655711663 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.43348850143172435, "right_score": 0.08811026304444586, "margin": 0.3453782383872785, "p_left": 0.5854963669115558, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.535295301067706, "brier": 0.17181326184351955 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.0808961869030456, "right_score": 0.4932112789390976, "margin": 0.5876849079639479, "p_left": 0.6428337799552082, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44186909523133466, "brier": 0.12756770874108464 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8761638205574049, "right_score": -0.6579283915312993, "margin": 1.5340922120887042, "p_left": 0.8226042649122524, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19528003852595877, "brier": 0.03146924682732234 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.027018264025910843, "right_score": 0.5576568856219835, "margin": -0.5306386215960726, "p_left": 0.37036795200494743, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4626196810221706, "brier": 0.13717241987233905 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22955665095454952, "right_score": 0.18329040304640906, "margin": 0.04626624790814046, "p_left": 0.5115644991697762, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7165478513646625, "brier": 0.26169823681082394 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22955665095454952, "right_score": -0.6144727449462777, "margin": 0.8440293959008272, "p_left": 0.6993131740960845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3576566054735273, "brier": 0.09041256727217158 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.19022976736821798, "right_score": -0.3337226315303794, "margin": 0.14349286416216142, "p_left": 0.5358117894521098, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7674651828191439, "brier": 0.2870942737158721 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.04886667167380007, "right_score": -0.6579283915312993, "margin": 0.6090617198574992, "p_left": 0.6477267379969487, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.738817232121317, "brier": 0.021823189119219124 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.6579283915312993, "right_score": 0.22955665095454952, "margin": -0.8874850424858489, "p_left": 0.29162909890476285, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3447874508577982, "brier": 0.08504753132800395 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8761638205574049, "right_score": -0.6144727449462777, "margin": 1.4906365655036826, "p_left": 0.8161737983543422, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20312795851745125, "brier": 0.03379207241147003 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6144727449462777, "right_score": 1.0808961869030456, "margin": -1.6953689318493232, "p_left": 0.15507107753387475, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1685027705875678, "brier": 0.024047039087516996 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.027018264025910843, "right_score": -0.45546410888819955, "margin": 0.4824823729141104, "p_left": 0.6183338793979324, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9632090823998708, "brier": 0.3823367864112968 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.0808961869030456, "right_score": -0.3337226315303794, "margin": 1.414618818433425, "p_left": 0.8044934305656349, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21754247844303376, "brier": 0.03822281869199421 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.48161740288924254, "right_score": 0.3132747791541187, "margin": -0.7948921820433612, "p_left": 0.31111918953243517, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7701331031162446, "brier": 0.03567596056288415 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.3337226315303794, "right_score": 0.22955665095454952, "margin": -0.5632792824849289, "p_left": 0.36278903707595617, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4506544962542517, "brier": 0.1316158854224995 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.45546410888819955, "right_score": -0.48161740288924254, "margin": 0.026153294001042993, "p_left": 0.506537950844151, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6801560304712102, "brier": 0.24350479395708954 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8761638205574049, "right_score": -1.1126605912173593, "margin": 1.9888244117747642, "p_left": 0.8796187104676187, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1282667489638536, "brier": 0.014491654869479004 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 5, "regressions": 1, "both_correct": 1, "both_wrong": 4, "net_fixes": 4, "trial_ids": { "fixes": [ "974972ffb84a", "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb", "fe73774e1a9e" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "b20ebcae7654", "c469d02ac7ba" ] } }, "other": { "fixes": 3, "regressions": 1, "both_correct": 1, "both_wrong": 2, "net_fixes": 2, "trial_ids": { "fixes": [ "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654" ] } }, "target": { "fixes": 2, "regressions": 0, "both_correct": 0, "both_wrong": 2, "net_fixes": 2, "trial_ids": { "fixes": [ "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba" ] } } } }, { "name": "semantic_bt_tuned", "selected": { "ridge": 1.0, "consistency": 0.0, "mean_cross_entropy": 0.6249194318880952, "fold_losses": [ 0.5202343929073778, 0.5120575682355707, 0.32338185568831723, 0.2864809829012261, 0.6822826365003241, 0.4524468998743053, 0.6702601676935027, 0.7211654276239624, 0.38836507543174503, 0.5842663402756011, 0.7303940609664913, 1.1885497341530216, 0.977318066061178, 0.6801158178887854, 0.9219426925845362, 0.30837435416391634, 0.9219101524249234, 0.6287876635183459, 0.6720886406823104, 0.7563604677594846, 0.5904617777137419, 0.4338493261422586, 0.43127290173524524, 0.6156993623881124 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5468190581224316, "brier_score": 0.15978668441890334, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.7842820808638336, "brier_score": 0.2741208052859679, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.7595746021742544, "brier_score": 0.2515653728845686, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8336970382429919, "brier_score": 0.3192316700887664, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3479511318468654, "brier_score": 0.08807260775920062, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.35907693351647996, "right_score": 0.30660091506639575, "margin": -0.6656778485828757, "p_left": 0.3394653173358272, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4147056466872569, "brier": 0.11523670167391385 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.1722365814281969, "right_score": -0.3185823213545482, "margin": 0.14634573992635133, "p_left": 0.5365212766577938, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7689947993616731, "brier": 0.2878550803065089 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.035000085084426265, "right_score": 0.5854024901085143, "margin": -0.6204025751929406, "p_left": 0.3496898975215688, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43030594912323816, "brier": 0.1222830244286453 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.18862087124804516, "right_score": 0.8859345744035987, "margin": -1.0745554456516437, "p_left": 0.2545377300887782, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2937507565058199, "brier": 0.06478945603874771 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.35907693351647996, "right_score": 0.8859345744035987, "margin": -1.2450115079200788, "p_left": 0.22356486507386505, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2530421750711468, "brier": 0.04998124889549549 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.1722365814281969, "right_score": -0.21217914371743196, "margin": 0.03994256228923507, "p_left": 0.5099843131862746, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7133178744843368, "brier": 0.2600839996960762 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.35907693351647996, "right_score": -0.09251778025355897, "margin": -0.266559153262921, "p_left": 0.4337520121130061, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.568723155591659, "brier": 0.1881408080120814 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.21171238222303662, "right_score": 0.21886883154762604, "margin": -0.43058121377066266, "p_left": 0.3939875516303024, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.931435965030279, "brier": 0.3672510875790354 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.0032179298992850713, "right_score": -0.3467450712284191, "margin": 0.343527141329134, "p_left": 0.5850470524981739, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5360630033639199, "brier": 0.17218594864045325 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5854024901085143, "right_score": -0.16973482480453833, "margin": 0.7551373149130527, "p_left": 0.6802970624661668, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38522571963814145, "brier": 0.10220996826776203 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.09251778025355897, "right_score": -0.30678843796521815, "margin": 0.21427065771165918, "p_left": 0.5533636515354234, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5917398958691927, "brier": 0.19948402776977067 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.3185823213545482, "right_score": -0.21217914371743196, "margin": -0.10640317763711626, "p_left": 0.47342427425237943, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7477633068070857, "brier": 0.2772819949466333 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.09251778025355897, "right_score": -0.035000085084426265, "margin": -0.05751769516913271, "p_left": 0.48562453916630766, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6648018116412311, "brier": 0.2358311930404887 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8859345744035987, "right_score": -0.035000085084426265, "margin": 0.920934659488025, "p_left": 0.7152325107304869, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3351475993180342, "brier": 0.08109252294486227 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.3467450712284191, "right_score": 0.028558731706222817, "margin": -0.3753038029346419, "p_left": 0.40726006016332594, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8983033291327107, "brier": 0.351340636277584 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.4359504958482179, "right_score": -0.035000085084426265, "margin": -0.4009504107637916, "p_left": 0.401084014921335, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.51263394933713, "brier": 0.16086838702541767 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.30660091506639575, "right_score": -0.35907693351647996, "margin": 0.6656778485828757, "p_left": 0.6605346826641728, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41470564668725685, "brier": 0.11523670167391385 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.30660091506639575, "right_score": -0.18862087124804516, "margin": 0.4952217863144409, "p_left": 0.621335777953328, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7234945310954304, "brier": 0.014722371011539306 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.30660091506639575, "right_score": -0.16973482480453833, "margin": 0.4763357398709341, "p_left": 0.6168822405818057, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9594128712155778, "brier": 0.38054369874522886 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.16973482480453833, "right_score": -0.4359504958482179, "margin": 0.26621567104367955, "p_left": 0.566163622886642, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8350878272278885, "brier": 0.3205412478801278 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.09251778025355897, "right_score": -0.25733857318144726, "margin": 0.1648207929278883, "p_left": 0.5411121696078395, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6141286840895162, "brier": 0.21057804088202428 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.028558731706222817, "right_score": -0.21171238222303662, "margin": 0.24027111392925943, "p_left": 0.5597804599511518, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8204817218498413, "brier": 0.31335416334312305 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.4359504958482179, "right_score": -0.35907693351647996, "margin": -0.07687356233173792, "p_left": 0.4807910681554223, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7323224729814852, "brier": 0.26957791490718735 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.1273089101862397, "right_score": 0.142030443462882, "margin": -0.26933935364912165, "p_left": 0.4330692899338704, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8368575408399884, "brier": 0.32141043001608594 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5854024901085143, "right_score": -0.035000085084426265, "margin": 0.6204025751929406, "p_left": 0.6503101024784311, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43030594912323805, "brier": 0.12228302442864535 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8859345744035987, "right_score": -0.30678843796521815, "margin": 1.1927230123688168, "p_left": 0.767227719047785, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2649716259198813, "brier": 0.05418293477969693 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21886883154762604, "right_score": 0.3359946306101782, "margin": -0.11712579906255216, "p_left": 0.4707519790581619, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6362981083377095, "brier": 0.2216074257871761 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.06709002232342283, "right_score": -0.16973482480453833, "margin": 0.23682484712796115, "p_left": 0.5589310350288603, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8185540326073502, "brier": 0.3124039019184331 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.06709002232342283, "right_score": -0.18862087124804516, "margin": 0.255710893571468, "p_left": 0.563581643674796, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5734430692630432, "brier": 0.1904609817375927 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.4359504958482179, "right_score": -0.25733857318144726, "margin": -0.17861192266677062, "p_left": 0.45546535260197224, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6078237070485341, "brier": 0.2074486874208389 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.30660091506639575, "right_score": -0.30678843796521815, "margin": 0.6133893530316139, "p_left": 0.6487135711036861, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7394586750357846, "brier": 0.022115726230411102 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.30678843796521815, "right_score": 0.06709002232342283, "margin": -0.37387846028864097, "p_left": 0.40760418231534024, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5235802565650357, "brier": 0.1661411694409571 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8859345744035987, "right_score": -0.18862087124804516, "margin": 1.0745554456516437, "p_left": 0.7454622699112218, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29375075650581994, "brier": 0.06478945603874771 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.18862087124804516, "right_score": 0.5854024901085143, "margin": -0.7740233613565595, "p_left": 0.3156094157342051, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.37922649470661013, "brier": 0.0996093033000863 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21886883154762604, "right_score": -0.3185823213545482, "margin": 0.5374511529021743, "p_left": 0.6312192919887089, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9975530988325346, "brier": 0.3984377945787269 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5854024901085143, "right_score": -0.25733857318144726, "margin": 0.8427410632899616, "p_left": 0.6990422013681162, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3580441646535676, "brier": 0.0905755965573495 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.07831973556885548, "right_score": -0.37130785868604743, "margin": 0.29298812311719197, "p_left": 0.5727275170552022, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038392743552494, "brier": 0.005289291737014726 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.25733857318144726, "right_score": 0.06709002232342283, "margin": -0.3244285955048701, "p_left": 0.4195968459664853, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5440323237458037, "brier": 0.17606151314502236 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.3185823213545482, "right_score": -0.07831973556885548, "margin": -0.24026258578569276, "p_left": 0.4402216416087557, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8204769479706742, "brier": 0.3133518105231964 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8859345744035987, "right_score": -0.35907693351647996, "margin": 1.2450115079200788, "p_left": 0.7764351349261348, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2530421750711467, "brier": 0.04998124889549555 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 1, "regressions": 1, "both_correct": 1, "both_wrong": 8, "net_fixes": 0, "trial_ids": { "fixes": [ "396ff40ef0fb" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 1, "regressions": 1, "both_correct": 1, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [ "396ff40ef0fb" ], "regressions": [ "0add2ec437a1" ], "both_correct": [ "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "consistent_bt", "selected": { "ridge": 0.1, "consistency": 10.0, "mean_cross_entropy": 0.5425629612400248, "fold_losses": [ 0.35057437747627623, 0.2816562438835754, 0.16253833416938934, 0.06965013858793583, 0.2962009626938364, 0.2505102870149487, 0.6225380312619966, 1.238405843324399, 0.1274589006752498, 0.5340598443771011, 0.7004317586474527, 1.1306489639475439, 0.9689846679941176, 0.5338901152068204, 0.730479448798273, 0.1668541477258192, 1.0852676330630473, 0.6842651368177488, 0.46483212386233974, 0.7066713246945229, 0.6151175547278165, 0.35230487461110915, 0.3459689711514159, 0.60220138504786 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.48670625128028694, "brier_score": 0.1387265931415332, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.0736498033485107, "brier_score": 0.38542478072994574, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8138346766888587, "brier_score": 0.2649716790805806, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.593280056667814, "brier_score": 0.6263309840286759, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.1863870181745217, "brier_score": 0.03312869865168585, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.9749314387304091, "right_score": 0.1077552461143099, "margin": -1.082686684844719, "p_left": 0.25299792328948173, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29168731378555596, "brier": 0.06400794918879048 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.40538457084282187, "right_score": -1.3323992026768514, "margin": 0.9270146318340295, "p_left": 0.7164692267075398, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2604346139363942, "brier": 0.5133281528189 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.382472366465596, "right_score": 0.9242602465189123, "margin": -1.3067326129845083, "p_left": 0.21303410982496818, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23957037308592372, "brier": 0.0453835319489166 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.11388509529872964, "right_score": 1.7165901397493286, "margin": -1.8304752350480582, "p_left": 0.13818166888898473, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1487107833013741, "brier": 0.019094173616945008 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.9749314387304091, "right_score": 1.7165901397493286, "margin": -2.6915215784797377, "p_left": 0.06347550592229449, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06557960252523266, "brier": 0.004029139852091243 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.40538457084282187, "right_score": 0.4057089424960575, "margin": -0.8110935133388794, "p_left": 0.3076575235988513, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3676745376969412, "brier": 0.09465315182697774 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.9749314387304091, "right_score": 0.2400584937594707, "margin": -1.2149899324898799, "p_left": 0.22881933096268683, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2598326020822828, "brier": 0.052358286222211614 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.10732793090370107, "right_score": 0.29028515464572635, "margin": -0.3976130855494274, "p_left": 0.40188595658169807, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9115869207060415, "brier": 0.3577404089341903 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.43614157078330396, "right_score": -1.4259114025710176, "margin": 0.9897698317877136, "p_left": 0.7290424574206401, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3160233080375191, "brier": 0.07341798988064563 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.9242602465189123, "right_score": -0.42870278961384806, "margin": 1.3529630361327603, "p_left": 0.7946136261781384, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2298992872737935, "brier": 0.0421835625516935 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.17682495482529068, "margin": 0.41688344858476134, "p_left": 0.6027372464933641, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5062739210109648, "brier": 0.15781769532367412 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.3323992026768514, "right_score": 0.4057089424960575, "margin": -1.7381081451729088, "p_left": 0.14955339470989384, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.9001017946660241, "brier": 0.7232594284494657 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.382472366465596, "margin": 0.6225308602250668, "p_left": 0.6507939341609257, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0520930844943788, "brier": 0.4235327447406553 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.7165901397493286, "right_score": -0.382472366465596, "margin": 2.0990625062149246, "p_left": 0.8908120259746567, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11562184349398397, "brier": 0.011922013671759045 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.4259114025710176, "right_score": 0.30982651343540885, "margin": -1.7357379160064264, "p_left": 0.14985510793949683, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.8980863987834182, "brier": 0.7227463374965646 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.382472366465596, "margin": -0.4447205148818776, "p_left": 0.3906167416622167, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4953078851778002, "brier": 0.15258143886680695 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1077552461143099, "right_score": -0.9749314387304091, "margin": 1.082686684844719, "p_left": 0.7470020767105182, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.291687313785556, "brier": 0.0640079491887905 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.11388509529872964, "margin": 0.22164034141303954, "p_left": 0.5551843616878755, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.699275207881456, "brier": 0.0030453137748982667 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.42870278961384806, "margin": 0.536458035728158, "p_left": 0.6309880825745242, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9969263389171275, "brier": 0.3981459603510746 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.42870278961384806, "right_score": -0.8271928813474736, "margin": 0.39849009173362554, "p_left": 0.5983248344308972, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.91211156285614, "brier": 0.3579926074967606 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.19950921901220023, "margin": 0.43956771277167095, "p_left": 0.6081560205342458, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49732381721458385, "brier": 0.15354170424355842 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.30982651343540885, "right_score": -0.10732793090370107, "margin": 0.4171544443391099, "p_left": 0.6028021332805981, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.923320717622617, "brier": 0.3633704118876399 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.9749314387304091, "margin": 0.1477385573829355, "p_left": 0.5368676056040683, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6220037593693053, "brier": 0.21449161473890882 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.1378743244233343, "right_score": 0.5997783888589585, "margin": -0.4619040644356242, "p_left": 0.38653422416566907, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9505348659339574, "brier": 0.3763402581200176 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.9242602465189123, "right_score": -0.382472366465596, "margin": 1.3067326129845083, "p_left": 0.7869658901750317, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23957037308592377, "brier": 0.04538353194891665 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.7165901397493286, "right_score": -0.17682495482529068, "margin": 1.8934150945746193, "p_left": 0.8691444273254256, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1402459680724597, "brier": 0.017123180899990816 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.29028515464572635, "right_score": 0.8583237547568487, "margin": -0.5680386001111224, "p_left": 0.3616895307072613, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.44893048501734956, "brier": 0.1308193166232389 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1148546191515259, "right_score": -0.42870278961384806, "margin": 0.543557408765374, "p_left": 0.6326395729467325, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.001411822799104, "brier": 0.400232829258224 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1148546191515259, "right_score": -0.11388509529872964, "margin": 0.22873971445025554, "p_left": 0.5569368911717647, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5853033467813552, "brier": 0.19630491840454067 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.19950921901220023, "margin": -0.6276836623352734, "p_left": 0.34803594568826673, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.42776584999899897, "brier": 0.12112901949112616 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.17682495482529068, "margin": 0.28458020093960057, "p_left": 0.5706687616896483, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.703236440239967, "brier": 0.004994073878748304 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.17682495482529068, "right_score": 0.1148546191515259, "margin": -0.29167957397681654, "p_left": 0.4275927293631001, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5579045293593723, "brier": 0.18283554220418538 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.7165901397493286, "right_score": -0.11388509529872964, "margin": 1.8304752350480582, "p_left": 0.8618183311110152, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1487107833013741, "brier": 0.01909417361694502 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.11388509529872964, "right_score": 0.9242602465189123, "margin": -1.0381453418176418, "p_left": 0.2615080099563315, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.30314502332048027, "brier": 0.06838643927132078 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.29028515464572635, "right_score": -1.3323992026768514, "margin": 1.6226843573225778, "p_left": 0.8351650028592288, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8028103227611456, "brier": 0.6975005820008556 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.9242602465189123, "right_score": -0.19950921901220023, "margin": 1.1237694655311126, "p_left": 0.7546872440012016, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.28145186191286076, "brier": 0.060178348255726016 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.2026023285627484, "right_score": -0.30743086069639713, "margin": 0.10482853213364873, "p_left": 0.5261831601486575, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6945201797130258, "brier": 0.0006855578753702469 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.19950921901220023, "right_score": 0.1148546191515259, "margin": -0.31436383816372615, "p_left": 0.4220499344203131, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5482678057723855, "brier": 0.17812614714419056 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.3323992026768514, "right_score": -0.2026023285627484, "margin": -1.1297968741141031, "p_left": 0.244198589015949, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4097734953076921, "brier": 0.5712357728454823 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.7165901397493286, "right_score": -0.9749314387304091, "margin": 2.6915215784797377, "p_left": 0.9365244940777057, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.06557960252523287, "brier": 0.004029139852091224 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 1, "regressions": 0, "both_correct": 2, "both_wrong": 8, "net_fixes": 1, "trial_ids": { "fixes": [ "396ff40ef0fb" ], "regressions": [], "both_correct": [ "0add2ec437a1", "34bec29a8d38" ], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 1, "regressions": 0, "both_correct": 2, "both_wrong": 4, "net_fixes": 1, "trial_ids": { "fixes": [ "396ff40ef0fb" ], "regressions": [], "both_correct": [ "0add2ec437a1", "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "semantic_bt_matched", "selected": { "ridge": 0.1, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.42649155272262745, "brier_score": 0.11541633393374527, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.2832506495499598, "brier_score": 0.4605957349818586, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.1419260720228168, "brier_score": 0.39682435102055347, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5658998046042458, "brier_score": 0.5881385029044687, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.13509477441572143, "brier_score": 0.019870074633199286, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.2285541156585884, "right_score": 0.33495713709529673, "margin": -1.563511252753885, "p_left": 0.17314338232325338, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1901239754387955, "brier": 0.02997863084233629 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.4231597968171382, "right_score": -1.0803168293030971, "margin": 0.6571570324859589, "p_left": 0.6586214663793394, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0747633481011283, "brier": 0.4337822359756713 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.015638843556511794, "right_score": 1.3428597707562475, "margin": -1.3584986143127593, "p_left": 0.20448442515313583, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22876485270725136, "brier": 0.04181388013020841 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.481674003380385, "right_score": 1.9798172070665478, "margin": -2.461491210446933, "p_left": 0.07860226992643977, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.08186349004580323, "brier": 0.006178316837588899 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.2285541156585884, "right_score": 1.9798172070665478, "margin": -3.208371322725136, "p_left": 0.03885190758075491, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03962677947103538, "brier": 0.0015094707226635206 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.4231597968171382, "right_score": 0.14125176228550596, "margin": -0.5644115591026442, "p_left": 0.36252732580931857, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4502438668828614, "brier": 0.13142606195845583 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.2285541156585884, "right_score": 0.301641146725362, "margin": -1.5301952623839505, "p_left": 0.177965118365578, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19597244974792735, "brier": 0.03167158335487418 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.8101263671981345, "right_score": 1.3827391302369676, "margin": -2.1928654974351023, "p_left": 0.10039300181932888, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.298662777148607, "brier": 0.809292751175638 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.3202782771731662, "right_score": -1.1428344984848768, "margin": 0.8225562213117106, "p_left": 0.6947786853667468, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3641619224508208, "brier": 0.09316005090645134 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.3428597707562475, "right_score": -0.16280707309220271, "margin": 1.5056668438484502, "p_left": 0.8184181452855058, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20038189293177644, "brier": 0.03297196996155567 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.702144872012727, "margin": 1.0037860187380891, "p_left": 0.731802303600279, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3122448785469989, "brier": 0.07193000435411694 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.0803168293030971, "right_score": 0.14125176228550596, "margin": -1.221568591588603, "p_left": 0.22766052539955925, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4798996832622036, "brier": 0.5965082640260848 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.015638843556511794, "margin": 0.3172799902818738, "p_left": 0.5786612222885962, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8643180712435343, "brier": 0.33484881018053214 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9798172070665478, "right_score": -0.015638843556511794, "margin": 1.9954560506230596, "p_left": 0.8803191663218501, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12747074826354, "brier": 0.014323501949896969 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.1428344984848768, "right_score": 0.30849630552081075, "margin": -1.4513308040056876, "p_left": 0.18979683847670328, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.661801050075161, "brier": 0.6564291629423453 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -1.0880003238390579, "right_score": -0.015638843556511794, "margin": -1.072361480282546, "p_left": 0.25495425537755806, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2943096603098397, "brier": 0.06500167233512509 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.33495713709529673, "right_score": -1.2285541156585884, "margin": 1.563511252753885, "p_left": 0.8268566176767466, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19012397543879556, "brier": 0.02997863084233631 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.481674003380385, "margin": 0.8166311404756817, "p_left": 0.6935207571680118, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7742896788848808, "brier": 0.03745028345488057 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.16280707309220271, "margin": 0.49776421018749945, "p_left": 0.6219337686196174, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9726858834189382, "brier": 0.38680161254939976 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.16280707309220271, "right_score": -1.0880003238390579, "margin": 0.9251932507468552, "p_left": 0.7160990835624259, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2591299874789252, "brier": 0.5127978974789462 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.45655246904279134, "margin": 0.7581936157681534, "p_left": 0.6809614198081254, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38424962670155793, "brier": 0.10178561565084719 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.30849630552081075, "right_score": -0.8101263671981345, "margin": 1.1186226727189452, "p_left": 0.7537331470164986, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4013395627600662, "brier": 0.5681136569113947 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -1.0880003238390579, "right_score": -1.2285541156585884, "margin": 0.14055379181953054, "p_left": 0.5350807142585638, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.625337675692512, "brier": 0.21614994225432715 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.3488687732479686, "right_score": 0.562722286861267, "margin": -0.9115910601092356, "p_left": 0.2866743675058912, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2494083153708642, "brier": 0.5088334579731204 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.3428597707562475, "right_score": -0.015638843556511794, "margin": 1.3584986143127593, "p_left": 0.7955155748468642, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22876485270725144, "brier": 0.0418138801302084 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9798172070665478, "right_score": -0.702144872012727, "margin": 2.681962079079275, "p_left": 0.9359538396068241, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.06618912038002733, "brier": 0.004101910661108413 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.3827391302369676, "right_score": 0.8172877098167708, "margin": 0.5654514204201968, "p_left": 0.6377129530852735, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0153184340947154, "brier": 0.40667781053274027 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17609643893880728, "right_score": -0.16280707309220271, "margin": 0.33890351203101, "p_left": 0.5839241483349075, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8768876996086178, "brier": 0.340967411008647 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17609643893880728, "right_score": -0.481674003380385, "margin": 0.6577704423191923, "p_left": 0.6587593715516727, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41739695296335855, "brier": 0.11644516650380933 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -1.0880003238390579, "right_score": -0.45655246904279134, "margin": -0.6314478547962665, "p_left": 0.34718231365152463, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.42645738264092387, "brier": 0.12053555891242562 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.702144872012727, "margin": 1.0371020091080236, "p_left": 0.7382904498617852, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8219689728631221, "brier": 0.056782338495331944 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.702144872012727, "right_score": 0.17609643893880728, "margin": -0.8782413109515343, "p_left": 0.29354235500173537, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3474920291171262, "brier": 0.08616711417996484 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9798172070665478, "right_score": -0.481674003380385, "margin": 2.461491210446933, "p_left": 0.9213977300735602, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08186349004580329, "brier": 0.006178316837588905 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.481674003380385, "right_score": 1.3428597707562475, "margin": -1.8245337741366325, "p_left": 0.13889074482269487, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1495338892512546, "brier": 0.01929063899740294 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.3827391302369676, "right_score": -1.0803168293030971, "margin": 2.463055959540065, "p_left": 0.921510980689984, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.5447965453791594, "brier": 0.8491824875322161 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.3428597707562475, "right_score": -0.45655246904279134, "margin": 1.7994122397990389, "p_left": 0.8580773723757019, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15306100596587147, "brier": 0.020142032231785175 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.29042149107132736, "right_score": -0.3934186779750347, "margin": 0.10299718690370735, "p_left": 0.5257265575676519, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6944726473994376, "brier": 0.0006618557642817048 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.45655246904279134, "right_score": 0.17609643893880728, "margin": -0.6326489079815987, "p_left": 0.34691014883123855, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4260405616694423, "brier": 0.12034665136211209 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.0803168293030971, "right_score": -0.29042149107132736, "margin": -0.7898953382317697, "p_left": 0.312191142770099, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1641396416744914, "brier": 0.4730810240839024 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9798172070665478, "right_score": -1.2285541156585884, "margin": 3.208371322725136, "p_left": 0.961148092419245, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.039626779471035434, "brier": 0.001509470722663525 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 9, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "0add2ec437a1", "34bec29a8d38" ], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 5, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "0add2ec437a1", "34bec29a8d38" ], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.0008174589967550785, "rank": 11.0, "portrait_scores": [ -0.004750221962545225, 0.0031153039690350677 ], "portrait_score_gap": 0.007865525931580292, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.012649193277060322, "rank": 1.0, "portrait_scores": [ 0.01140145279911671, 0.013896933755003935 ], "portrait_score_gap": 0.002495480955887225, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.5625, "single_photo_rank_quantiles": [ 1.0, 1.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.0003609175344081515, "rank": 10.0, "portrait_scores": [ -0.007406036662704777, 0.006684201593888474 ], "portrait_score_gap": 0.014090238256593252, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.5, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.0003852273927488262, "rank": 9.0, "portrait_scores": [ -0.00297437399909557, 0.0037448287845932224 ], "portrait_score_gap": 0.006719202783688792, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.5, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.009965973349332014, "rank": 13.0, "portrait_scores": [ 0.0014172777562307063, -0.021349224454894733 ], "portrait_score_gap": 0.022766502211125438, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 12.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.009102165607821594, "rank": 2.0, "portrait_scores": [ 0.005835977804672911, 0.012368353410970277 ], "portrait_score_gap": 0.0065323756062973665, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0053332860828347855, "rank": 5.0, "portrait_scores": [ -0.0009606743250214975, 0.011627246490691068 ], "portrait_score_gap": 0.012587920815712566, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.5, 12.0 ] }, { "identity": "keira_knightley", "mean_score": 0.0007515761431032171, "rank": 8.0, "portrait_scores": [ 0.000933504428473896, 0.0005696478577325382 ], "portrait_score_gap": 0.0003638565707413579, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "lily_collins", "mean_score": 0.002939517470945549, "rank": 6.0, "portrait_scores": [ 0.0008563177330119655, 0.005022717208879132 ], "portrait_score_gap": 0.004166399475867166, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.00807681196985207, "rank": 3.0, "portrait_scores": [ 0.007919329314875483, 0.008234294624828657 ], "portrait_score_gap": 0.0003149653099531746, "single_photo_rank_min": 2.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": 0.005338096656073755, "rank": 4.0, "portrait_scores": [ 0.0027246622599493243, 0.007951531052198185 ], "portrait_score_gap": 0.00522686879224886, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.0011099537028006288, "rank": 7.0, "portrait_scores": [ 0.012738369637481203, -0.010518462231879945 ], "portrait_score_gap": 0.023256831869361148, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 7.0, 13.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.004391418775907505, "rank": 12.0, "portrait_scores": [ 0.0009465584085515417, -0.009729395960366552 ], "portrait_score_gap": 0.010675954368918094, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "5.0": 32, "6.0": 224, "7.0": 672, "8.0": 1120, "9.0": 1120, "10.0": 672, "11.0": 1248, "12.0": 2080, "13.0": 1024 }, "target_primary_score": -0.004391418775907505, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -0.00760822156320198, 0.0062703371216861915 ], "target_alternate_panel_score": -0.0006689422207578942, "target_alternate_panel_rank": 11.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.0036084596917034433, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.0010541696135067205, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.013318135497818216, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.00014851677599718427, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 3.3288165288700414e-05, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.007282545977960499, "rank": 10.0, "portrait_scores": [ -0.027120692288182632, 0.04168578424410363 ], "portrait_score_gap": 0.06880647653228626, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.03297671911077339, "rank": 8.0, "portrait_scores": [ 0.03776429491436684, 0.028189143307179948 ], "portrait_score_gap": 0.009575151607186895, "single_photo_rank_min": 5.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.06822206442499257, "rank": 6.0, "portrait_scores": [ 0.176900639299852, -0.04045651044986685 ], "portrait_score_gap": 0.21735714974971887, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.07007248855972983, "rank": 13.0, "portrait_scores": [ -0.10675816300111377, -0.033386814118345896 ], "portrait_score_gap": 0.07337134888276786, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.14175601777529534, "rank": 2.0, "portrait_scores": [ 0.15823810325250437, 0.12527393229808628 ], "portrait_score_gap": 0.03296417095441809, "single_photo_rank_min": 1.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.09375, "single_photo_rank_quantiles": [ 1.0, 3.0, 4.0 ] }, { "identity": "emma_watson", "mean_score": 0.07982050614181349, "rank": 5.0, "portrait_scores": [ 0.12377956256252573, 0.035861449721101266 ], "portrait_score_gap": 0.08791811284142446, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.08532994512607553, "rank": 4.0, "portrait_scores": [ 0.07702081350932906, 0.09363907674282201 ], "portrait_score_gap": 0.016618263233492955, "single_photo_rank_min": 2.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": 0.007981090840691728, "rank": 9.0, "portrait_scores": [ -0.11513048080702394, 0.1310926624884074 ], "portrait_score_gap": 0.24622314329543132, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.027774114325737596, "rank": 12.0, "portrait_scores": [ 0.02577292880791591, -0.0813211574593911 ], "portrait_score_gap": 0.10709408626730701, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.10587542820936538, "rank": 3.0, "portrait_scores": [ 0.16501587127327938, 0.04673498514545137 ], "portrait_score_gap": 0.118280886127828, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.5, 7.0 ] }, { "identity": "natalie_portman", "mean_score": 0.04504033581250766, "rank": 7.0, "portrait_scores": [ 0.08244642354636084, 0.007634248078654482 ], "portrait_score_gap": 0.07481217546770635, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.1442578851270274, "rank": 1.0, "portrait_scores": [ 0.08683446877832886, 0.2016813014757259 ], "portrait_score_gap": 0.11484683269739705, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.01949319021824058, "rank": 11.0, "portrait_scores": [ -0.0054916147206259015, -0.03349476571585526 ], "portrait_score_gap": 0.028003150995229358, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 10.5, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 256, "9.0": 1280, "10.0": 2560, "11.0": 2560, "12.0": 1280, "13.0": 256 }, "target_primary_score": -0.01949319021824058, "target_primary_rank": 11.0, "target_alternate_photo_scores": [ -0.029040199171518132, -0.013639893774366613 ], "target_alternate_panel_score": -0.021340046472942374, "target_alternate_panel_rank": 11.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.0064340678527952215, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.04873244208678746, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.054316765583715766, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.028622592450902873, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.0033115784347285955, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.48161740288924254, "rank": 12.0, "portrait_scores": [ -0.6202201131721171, -0.34301469260636797 ], "portrait_score_gap": 0.27720542056574915, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.027018264025910843, "rank": 6.0, "portrait_scores": [ 0.1522292174780946, -0.0981926894262729 ], "portrait_score_gap": 0.2504219069043675, "single_photo_rank_min": 3.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.08811026304444586, "rank": 5.0, "portrait_scores": [ 0.6637920804604099, -0.4875715543715182 ], "portrait_score_gap": 1.1513636348319283, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7373134453142033, "rank": 13.0, "portrait_scores": [ -0.9385736286675956, -0.5360532619608109 ], "portrait_score_gap": 0.4025203667067847, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.43348850143172435, "rank": 2.0, "portrait_scores": [ 0.5894289549475543, 0.2775480479158944 ], "portrait_score_gap": 0.31188090703165994, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 3.0, 4.0 ] }, { "identity": "emma_watson", "mean_score": -0.06795366631111911, "rank": 7.0, "portrait_scores": [ 0.038573250271633434, -0.17448058289387167 ], "portrait_score_gap": 0.2130538331655051, "single_photo_rank_min": 4.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.12796741890570604, "rank": 4.0, "portrait_scores": [ 0.21067359471524438, 0.045261243096167705 ], "portrait_score_gap": 0.1654123516190767, "single_photo_rank_min": 3.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.0, 6.0 ] }, { "identity": "keira_knightley", "mean_score": -0.46370841103163757, "rank": 11.0, "portrait_scores": [ -1.1025624040145803, 0.17514558195130517 ], "portrait_score_gap": 1.2777079859658855, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.3996939996665949, "rank": 8.0, "portrait_scores": [ -0.159017513401933, -0.6403704859312568 ], "portrait_score_gap": 0.48135297252932374, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.5, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.3132747791541187, "rank": 3.0, "portrait_scores": [ 0.6048057840223211, 0.021743774285916298 ], "portrait_score_gap": 0.5830620097364048, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.5, 7.0 ] }, { "identity": "natalie_portman", "mean_score": -0.4611475707927415, "rank": 10.0, "portrait_scores": [ -0.1995555444746426, -0.7227395971108403 ], "portrait_score_gap": 0.5231840526361977, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.5, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5576568856219835, "rank": 1.0, "portrait_scores": [ 0.3863958447196197, 0.7289179265243473 ], "portrait_score_gap": 0.3425220818047276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.5625, "single_photo_rank_quantiles": [ 1.0, 1.0, 4.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.45546410888819955, "rank": 9.0, "portrait_scores": [ -0.34729727148506306, -0.563630946291336 ], "portrait_score_gap": 0.21633367480627297, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 128, "8.0": 768, "9.0": 1920, "10.0": 2560, "11.0": 1920, "12.0": 768, "13.0": 128 }, "target_primary_score": -0.45546410888819955, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ -0.34361414295433534, -0.34234757690131445 ], "target_alternate_panel_score": -0.3429808599278249, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.05671313973877001, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3943325853863784, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.36999912395373574, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.13863654296141764, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.08537078078874814, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.07831973556885548, "rank": 6.0, "portrait_scores": [ -0.26604414041476393, 0.10940466927705297 ], "portrait_score_gap": 0.37544880969181693, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.0, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21886883154762604, "rank": 2.0, "portrait_scores": [ -0.19748162098615438, 0.6352192840814065 ], "portrait_score_gap": 0.8327009050675609, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.142030443462882, "rank": 3.0, "portrait_scores": [ 0.4181667659478982, -0.13410587902213422 ], "portrait_score_gap": 0.5522726449700324, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.5, 9.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.21217914371743196, "rank": 10.0, "portrait_scores": [ -0.678566338513062, 0.2542080510781981 ], "portrait_score_gap": 0.9327743895912601, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 8.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.1273089101862397, "rank": 7.0, "portrait_scores": [ -0.10700086207005174, -0.14761695830242763 ], "portrait_score_gap": 0.04061609623237589, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.5, 10.0 ] }, { "identity": "emma_watson", "mean_score": -0.3467450712284191, "rank": 12.0, "portrait_scores": [ -0.030402196936448994, -0.6630879455203892 ], "portrait_score_gap": 0.6326857485839402, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.0032179298992850713, "rank": 5.0, "portrait_scores": [ 0.08764754976715827, -0.09408340956572842 ], "portrait_score_gap": 0.1817309593328867, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.21171238222303662, "rank": 9.0, "portrait_scores": [ -0.6954301456541114, 0.2720053812080382 ], "portrait_score_gap": 0.9674355268621496, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 8.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.1722365814281969, "rank": 8.0, "portrait_scores": [ -0.12567963766192108, -0.2187935251944727 ], "portrait_score_gap": 0.09311388753255162, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.37130785868604743, "rank": 13.0, "portrait_scores": [ -0.22854520151932314, -0.5140705158527717 ], "portrait_score_gap": 0.28552531433344863, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.028558731706222817, "rank": 4.0, "portrait_scores": [ 0.0965269126288692, -0.039409449216423566 ], "portrait_score_gap": 0.13593636184529276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.009765625, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.3359946306101782, "rank": 1.0, "portrait_scores": [ -0.10520875043095902, 0.7771980116513154 ], "portrait_score_gap": 0.8824067620822744, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.3185823213545482, "rank": 11.0, "portrait_scores": [ -0.36187462374975865, -0.2752900189593378 ], "portrait_score_gap": 0.08658460479042085, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "9.0": 512, "10.0": 2048, "11.0": 3072, "12.0": 2048, "13.0": 512 }, "target_primary_score": -0.3185823213545482, "target_primary_rank": 11.0, "target_alternate_photo_scores": [ -0.5203710741708442, -0.33117850970800367 ], "target_alternate_panel_score": -0.4257747919394239, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.25353821051122705, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.21359564822199195, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.64464362348705, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.34745505637056845, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.08221416325792243, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.2026023285627484, "rank": 8.0, "portrait_scores": [ -0.6993366308094117, 0.29413197368391486 ], "portrait_score_gap": 0.9934686044933265, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 8.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.29028515464572635, "rank": 5.0, "portrait_scores": [ -0.15807422860050335, 0.738644537891956 ], "portrait_score_gap": 0.8967187664924594, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.5997783888589585, "rank": 2.0, "portrait_scores": [ 1.167330185432654, 0.03222659228526287 ], "portrait_score_gap": 1.1351035931473912, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.4057089424960575, "rank": 3.0, "portrait_scores": [ -0.6808915354259366, 1.4923094204180516 ], "portrait_score_gap": 2.1732009558439884, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.1378743244233343, "rank": 6.0, "portrait_scores": [ 0.8006691919416679, -0.5249205430949992 ], "portrait_score_gap": 1.325589735036667, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": -1.4259114025710176, "rank": 13.0, "portrait_scores": [ -0.863196044528949, -1.9886267606130863 ], "portrait_score_gap": 1.1254307160841373, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.43614157078330396, "rank": 11.0, "portrait_scores": [ 0.26649183428691137, -1.1387749758535193 ], "portrait_score_gap": 1.4052668101404306, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.10732793090370107, "rank": 7.0, "portrait_scores": [ -1.1679495782259992, 0.9532937164185971 ], "portrait_score_gap": 2.1212432946445965, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 8.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.40538457084282187, "rank": 10.0, "portrait_scores": [ -0.4520683845428617, -0.3587007571427821 ], "portrait_score_gap": 0.09336762740007959, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.5, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.30743086069639713, "rank": 9.0, "portrait_scores": [ -0.04149806771454024, -0.573363653678254 ], "portrait_score_gap": 0.5318655859637138, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.30982651343540885, "rank": 4.0, "portrait_scores": [ 1.2129519209025243, -0.5932988940317067 ], "portrait_score_gap": 1.8062508149342311, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8583237547568487, "rank": 1.0, "portrait_scores": [ 0.6779713777586348, 1.0386761317550628 ], "portrait_score_gap": 0.360704753996428, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.3323992026768514, "rank": 12.0, "portrait_scores": [ -1.3096253168561975, -1.355173088497505 ], "portrait_score_gap": 0.04554777164130752, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "12.0": 4096, "13.0": 4096 }, "target_primary_score": -1.3323992026768514, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -0.7249942834660589, -1.8712793634017248 ], "target_alternate_panel_score": -1.298136823433892, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.89275225259107, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.7038457659299495, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.5884219780796183, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.0955344948711434, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.4035942381141289, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.29042149107132736, "rank": 6.0, "portrait_scores": [ -0.6552549695820656, 0.07441198743941091 ], "portrait_score_gap": 0.7296669570214765, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.3827391302369676, "rank": 1.0, "portrait_scores": [ -0.20044330871049293, 2.9659215691844283 ], "portrait_score_gap": 3.1663648778949214, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.562722286861267, "rank": 3.0, "portrait_scores": [ 0.21550808892686757, 0.9099364847956665 ], "portrait_score_gap": 0.6944283958687989, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.14125176228550596, "rank": 5.0, "portrait_scores": [ -1.496604584136168, 1.7791081087071798 ], "portrait_score_gap": 3.2757126928433475, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.3488687732479686, "rank": 8.0, "portrait_scores": [ -0.7857104999831721, 0.08797295348723483 ], "portrait_score_gap": 0.873683453470407, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 12.0 ] }, { "identity": "emma_watson", "mean_score": -1.1428344984848768, "rank": 13.0, "portrait_scores": [ -0.2948021482546887, -1.990866848715065 ], "portrait_score_gap": 1.6960647004603762, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.3202782771731662, "rank": 7.0, "portrait_scores": [ 0.034020799721046546, -0.674577354067379 ], "portrait_score_gap": 0.7085981537884255, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "keira_knightley", "mean_score": -0.8101263671981345, "rank": 11.0, "portrait_scores": [ -1.9893039586725187, 0.3690512242762499 ], "portrait_score_gap": 2.3583551829487686, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.4231597968171382, "rank": 10.0, "portrait_scores": [ -0.26328623186390687, -0.5830333617703696 ], "portrait_score_gap": 0.3197471299064627, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.3934186779750347, "rank": 9.0, "portrait_scores": [ -0.22584310694734566, -0.5609942490027238 ], "portrait_score_gap": 0.3351511420553781, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.30849630552081075, "rank": 4.0, "portrait_scores": [ 0.895320732903357, -0.2783281218617355 ], "portrait_score_gap": 1.1736488547650925, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8172877098167708, "rank": 2.0, "portrait_scores": [ 0.2672306349156934, 1.3673447847178481 ], "portrait_score_gap": 1.1001141498021547, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.140625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.0803168293030971, "rank": 12.0, "portrait_scores": [ -1.4616250014360064, -0.6990086571701877 ], "portrait_score_gap": 0.7626163442658187, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "9.0": 256, "10.0": 1536, "11.0": 3072, "12.0": 2560, "13.0": 768 }, "target_primary_score": -1.0803168293030971, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.0525735568445354, -0.7477002970562409 ], "target_alternate_panel_score": -0.900136926950388, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.4769771301332498, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.0413886892358941, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.2828760571873556, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.6097154358790606, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6703570113311726, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, "consistency_ablation": [ { "name": "gamma_0", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.42649155272262745, "brier_score": 0.11541633393374527, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.2832506495499598, "brier_score": 0.4605957349818586, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.1419260720228168, "brier_score": 0.39682435102055347, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5658998046042458, "brier_score": 0.5881385029044687, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.13509477441572143, "brier_score": 0.019870074633199286, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.2285541156585884, "right_score": 0.33495713709529673, "margin": -1.563511252753885, "p_left": 0.17314338232325338, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1901239754387955, "brier": 0.02997863084233629 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.4231597968171382, "right_score": -1.0803168293030971, "margin": 0.6571570324859589, "p_left": 0.6586214663793394, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0747633481011283, "brier": 0.4337822359756713 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.015638843556511794, "right_score": 1.3428597707562475, "margin": -1.3584986143127593, "p_left": 0.20448442515313583, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22876485270725136, "brier": 0.04181388013020841 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.481674003380385, "right_score": 1.9798172070665478, "margin": -2.461491210446933, "p_left": 0.07860226992643977, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.08186349004580323, "brier": 0.006178316837588899 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.2285541156585884, "right_score": 1.9798172070665478, "margin": -3.208371322725136, "p_left": 0.03885190758075491, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03962677947103538, "brier": 0.0015094707226635206 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.4231597968171382, "right_score": 0.14125176228550596, "margin": -0.5644115591026442, "p_left": 0.36252732580931857, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4502438668828614, "brier": 0.13142606195845583 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.2285541156585884, "right_score": 0.301641146725362, "margin": -1.5301952623839505, "p_left": 0.177965118365578, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19597244974792735, "brier": 0.03167158335487418 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.8101263671981345, "right_score": 1.3827391302369676, "margin": -2.1928654974351023, "p_left": 0.10039300181932888, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.298662777148607, "brier": 0.809292751175638 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.3202782771731662, "right_score": -1.1428344984848768, "margin": 0.8225562213117106, "p_left": 0.6947786853667468, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3641619224508208, "brier": 0.09316005090645134 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.3428597707562475, "right_score": -0.16280707309220271, "margin": 1.5056668438484502, "p_left": 0.8184181452855058, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20038189293177644, "brier": 0.03297196996155567 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.702144872012727, "margin": 1.0037860187380891, "p_left": 0.731802303600279, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3122448785469989, "brier": 0.07193000435411694 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.0803168293030971, "right_score": 0.14125176228550596, "margin": -1.221568591588603, "p_left": 0.22766052539955925, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4798996832622036, "brier": 0.5965082640260848 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.015638843556511794, "margin": 0.3172799902818738, "p_left": 0.5786612222885962, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8643180712435343, "brier": 0.33484881018053214 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9798172070665478, "right_score": -0.015638843556511794, "margin": 1.9954560506230596, "p_left": 0.8803191663218501, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12747074826354, "brier": 0.014323501949896969 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.1428344984848768, "right_score": 0.30849630552081075, "margin": -1.4513308040056876, "p_left": 0.18979683847670328, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.661801050075161, "brier": 0.6564291629423453 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -1.0880003238390579, "right_score": -0.015638843556511794, "margin": -1.072361480282546, "p_left": 0.25495425537755806, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2943096603098397, "brier": 0.06500167233512509 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.33495713709529673, "right_score": -1.2285541156585884, "margin": 1.563511252753885, "p_left": 0.8268566176767466, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19012397543879556, "brier": 0.02997863084233631 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.481674003380385, "margin": 0.8166311404756817, "p_left": 0.6935207571680118, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7742896788848808, "brier": 0.03745028345488057 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.16280707309220271, "margin": 0.49776421018749945, "p_left": 0.6219337686196174, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9726858834189382, "brier": 0.38680161254939976 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.16280707309220271, "right_score": -1.0880003238390579, "margin": 0.9251932507468552, "p_left": 0.7160990835624259, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2591299874789252, "brier": 0.5127978974789462 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.301641146725362, "right_score": -0.45655246904279134, "margin": 0.7581936157681534, "p_left": 0.6809614198081254, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38424962670155793, "brier": 0.10178561565084719 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.30849630552081075, "right_score": -0.8101263671981345, "margin": 1.1186226727189452, "p_left": 0.7537331470164986, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4013395627600662, "brier": 0.5681136569113947 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -1.0880003238390579, "right_score": -1.2285541156585884, "margin": 0.14055379181953054, "p_left": 0.5350807142585638, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.625337675692512, "brier": 0.21614994225432715 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.3488687732479686, "right_score": 0.562722286861267, "margin": -0.9115910601092356, "p_left": 0.2866743675058912, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2494083153708642, "brier": 0.5088334579731204 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.3428597707562475, "right_score": -0.015638843556511794, "margin": 1.3584986143127593, "p_left": 0.7955155748468642, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22876485270725144, "brier": 0.0418138801302084 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9798172070665478, "right_score": -0.702144872012727, "margin": 2.681962079079275, "p_left": 0.9359538396068241, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.06618912038002733, "brier": 0.004101910661108413 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.3827391302369676, "right_score": 0.8172877098167708, "margin": 0.5654514204201968, "p_left": 0.6377129530852735, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0153184340947154, "brier": 0.40667781053274027 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17609643893880728, "right_score": -0.16280707309220271, "margin": 0.33890351203101, "p_left": 0.5839241483349075, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8768876996086178, "brier": 0.340967411008647 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17609643893880728, "right_score": -0.481674003380385, "margin": 0.6577704423191923, "p_left": 0.6587593715516727, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41739695296335855, "brier": 0.11644516650380933 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -1.0880003238390579, "right_score": -0.45655246904279134, "margin": -0.6314478547962665, "p_left": 0.34718231365152463, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.42645738264092387, "brier": 0.12053555891242562 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.33495713709529673, "right_score": -0.702144872012727, "margin": 1.0371020091080236, "p_left": 0.7382904498617852, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8219689728631221, "brier": 0.056782338495331944 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.702144872012727, "right_score": 0.17609643893880728, "margin": -0.8782413109515343, "p_left": 0.29354235500173537, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3474920291171262, "brier": 0.08616711417996484 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9798172070665478, "right_score": -0.481674003380385, "margin": 2.461491210446933, "p_left": 0.9213977300735602, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08186349004580329, "brier": 0.006178316837588905 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.481674003380385, "right_score": 1.3428597707562475, "margin": -1.8245337741366325, "p_left": 0.13889074482269487, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1495338892512546, "brier": 0.01929063899740294 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.3827391302369676, "right_score": -1.0803168293030971, "margin": 2.463055959540065, "p_left": 0.921510980689984, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.5447965453791594, "brier": 0.8491824875322161 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.3428597707562475, "right_score": -0.45655246904279134, "margin": 1.7994122397990389, "p_left": 0.8580773723757019, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15306100596587147, "brier": 0.020142032231785175 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.29042149107132736, "right_score": -0.3934186779750347, "margin": 0.10299718690370735, "p_left": 0.5257265575676519, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6944726473994376, "brier": 0.0006618557642817048 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.45655246904279134, "right_score": 0.17609643893880728, "margin": -0.6326489079815987, "p_left": 0.34691014883123855, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4260405616694423, "brier": 0.12034665136211209 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.0803168293030971, "right_score": -0.29042149107132736, "margin": -0.7898953382317697, "p_left": 0.312191142770099, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1641396416744914, "brier": 0.4730810240839024 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9798172070665478, "right_score": -1.2285541156585884, "margin": 3.208371322725136, "p_left": 0.961148092419245, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.039626779471035434, "brier": 0.001509470722663525 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.29042149107132736, "rank": 6.0, "portrait_scores": [ -0.6552549695820656, 0.07441198743941091 ], "portrait_score_gap": 0.7296669570214765, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.3827391302369676, "rank": 1.0, "portrait_scores": [ -0.20044330871049293, 2.9659215691844283 ], "portrait_score_gap": 3.1663648778949214, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.562722286861267, "rank": 3.0, "portrait_scores": [ 0.21550808892686757, 0.9099364847956665 ], "portrait_score_gap": 0.6944283958687989, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.14125176228550596, "rank": 5.0, "portrait_scores": [ -1.496604584136168, 1.7791081087071798 ], "portrait_score_gap": 3.2757126928433475, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.3488687732479686, "rank": 8.0, "portrait_scores": [ -0.7857104999831721, 0.08797295348723483 ], "portrait_score_gap": 0.873683453470407, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 12.0 ] }, { "identity": "emma_watson", "mean_score": -1.1428344984848768, "rank": 13.0, "portrait_scores": [ -0.2948021482546887, -1.990866848715065 ], "portrait_score_gap": 1.6960647004603762, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.3202782771731662, "rank": 7.0, "portrait_scores": [ 0.034020799721046546, -0.674577354067379 ], "portrait_score_gap": 0.7085981537884255, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "keira_knightley", "mean_score": -0.8101263671981345, "rank": 11.0, "portrait_scores": [ -1.9893039586725187, 0.3690512242762499 ], "portrait_score_gap": 2.3583551829487686, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.4231597968171382, "rank": 10.0, "portrait_scores": [ -0.26328623186390687, -0.5830333617703696 ], "portrait_score_gap": 0.3197471299064627, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.3934186779750347, "rank": 9.0, "portrait_scores": [ -0.22584310694734566, -0.5609942490027238 ], "portrait_score_gap": 0.3351511420553781, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.30849630552081075, "rank": 4.0, "portrait_scores": [ 0.895320732903357, -0.2783281218617355 ], "portrait_score_gap": 1.1736488547650925, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8172877098167708, "rank": 2.0, "portrait_scores": [ 0.2672306349156934, 1.3673447847178481 ], "portrait_score_gap": 1.1001141498021547, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.140625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.0803168293030971, "rank": 12.0, "portrait_scores": [ -1.4616250014360064, -0.6990086571701877 ], "portrait_score_gap": 0.7626163442658187, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "9.0": 256, "10.0": 1536, "11.0": 3072, "12.0": 2560, "13.0": 768 }, "target_primary_score": -1.0803168293030971, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.0525735568445354, -0.7477002970562409 ], "target_alternate_panel_score": -0.900136926950388, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.4769771301332498, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.0413886892358941, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.2828760571873556, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.6097154358790606, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6703570113311726, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.01", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.428847557148128, "brier_score": 0.11616054844000374, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.2092600303157262, "brier_score": 0.4362734371357222, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.0912449997896565, "brier_score": 0.37884321553419853, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.4452900913678655, "brier_score": 0.5511338803387695, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.13671916958086677, "brier_score": 0.019900607588821555, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.2326019223468587, "right_score": 0.34735993397774056, "margin": -1.5799618563245992, "p_left": 0.17080088416479228, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18729496470761295, "brier": 0.02917294203147479 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.4862482416546444, "right_score": -0.9287054390133549, "margin": 0.4424571973587105, "p_left": 0.6088443756852105, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9386497820338313, "brier": 0.3706914738035138 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.040658972764937545, "right_score": 1.3261957926021346, "margin": -1.3668547653670722, "p_left": 0.2031284817265449, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22706181987035928, "brier": 0.04126118008853129 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.4461745099840295, "right_score": 1.9123465488421039, "margin": -2.3585210588261334, "p_left": 0.08639085220407455, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.09035242721904306, "brier": 0.007463379344546253 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.2326019223468587, "right_score": 1.9123465488421039, "margin": -3.144948471188963, "p_left": 0.041290784573968564, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.04216746652645177, "brier": 0.0017049288907338804 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.4862482416546444, "right_score": 0.17702501801095227, "margin": -0.6632732596655967, "p_left": 0.3400047029336733, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41552256964413414, "brier": 0.11560319801701542 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.2326019223468587, "right_score": 0.29805253371936186, "margin": -1.5306544560662205, "p_left": 0.17789795123215715, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19589074471203136, "brier": 0.031647681052598964 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.7895039705150808, "right_score": 1.2981514037327988, "margin": -2.0876553742478796, "p_left": 0.11030245599613597, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.2045290864598233, "brier": 0.7915617198065076 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.36728077237218054, "right_score": -1.1750030496258916, "margin": 0.8077222772537112, "p_left": 0.691623923124412, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.36871293478743294, "brier": 0.09509580478917855 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.3261957926021346, "right_score": -0.1695464967511024, "margin": 1.495742289353237, "p_left": 0.8169385949166915, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20219134616742584, "brier": 0.03351147803107516 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.29805253371936186, "right_score": -0.6812056602034133, "margin": 0.9792581939227751, "p_left": 0.7269610010063045, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31888244662022536, "brier": 0.07455029497147928 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.9287054390133549, "right_score": 0.17702501801095227, "margin": -1.1057304570243072, "p_left": 0.2486677199070742, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3916377319042188, "brier": 0.5645001951096348 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.29805253371936186, "right_score": -0.040658972764937545, "margin": 0.3387115064842994, "p_left": 0.5838774985421632, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8767755874117646, "brier": 0.3409129333038538 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9123465488421039, "right_score": -0.040658972764937545, "margin": 1.9530055216070414, "p_left": 0.8757739937105179, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1326472193803212, "brier": 0.015432100638634448 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.1750030496258916, "right_score": 0.19183857995619075, "margin": -1.3668416295820824, "p_left": 0.2031306079889077, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.593906117718468, "brier": 0.6350008279241279 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -1.0673748856596799, "right_score": -0.040658972764937545, "margin": -1.0267159128947423, "p_left": 0.2637212881422778, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.30614654734687713, "brier": 0.06954891781942231 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.34735993397774056, "right_score": -1.2326019223468587, "margin": 1.5799618563245992, "p_left": 0.8291991158352077, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18729496470761298, "brier": 0.02917294203147481 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.34735993397774056, "right_score": -0.4461745099840295, "margin": 0.7935344439617701, "p_left": 0.688589739956908, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7698768500288709, "brier": 0.03556609001701417 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.34735993397774056, "right_score": -0.1695464967511024, "margin": 0.5169064307288429, "p_left": 0.6264241022823777, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9846340883123363, "brier": 0.39240715592028275 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.1695464967511024, "right_score": -1.0673748856596799, "margin": 0.8978283889085774, "p_left": 0.7105030315252123, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2396104536135943, "brier": 0.5048145578065169 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.29805253371936186, "right_score": -0.4510782092051926, "margin": 0.7491307429245544, "p_left": 0.6789892629171469, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38714996463072704, "brier": 0.10304789332247664 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.19183857995619075, "right_score": -0.7895039705150808, "margin": 0.9813425504712716, "p_left": 0.7273745264611062, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2996563175012152, "brier": 0.5290737017445185 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -1.0673748856596799, "right_score": -1.2326019223468587, "margin": 0.16522703668717886, "p_left": 0.5412130422240812, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6139422842617547, "brier": 0.21048547262528272 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.32600252391955875, "right_score": 0.5671980147512745, "margin": -0.8932005386708333, "p_left": 0.29044979044301417, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2363245560345972, "brier": 0.5034614998823626 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.3261957926021346, "right_score": -0.040658972764937545, "margin": 1.3668547653670722, "p_left": 0.7968715182734551, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22706181987035934, "brier": 0.04126118008853127 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9123465488421039, "right_score": -0.6812056602034133, "margin": 2.593552209045517, "p_left": 0.9304454562042321, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.07209182234577138, "brier": 0.00483783456263739 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.2981514037327988, "right_score": 0.8941492862468404, "margin": 0.4040021174859584, "p_left": 0.5996488309376011, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.915413194366849, "brier": 0.35957872044483175 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.20468584777387056, "right_score": -0.1695464967511024, "margin": 0.37423234452497295, "p_left": 0.5924812648437795, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8976683715145906, "brier": 0.35103404919088477 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.20468584777387056, "right_score": -0.4461745099840295, "margin": 0.6508603577579001, "p_left": 0.6572043161293046, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41976032538881747, "brier": 0.11750888088037771 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -1.0673748856596799, "right_score": -0.4510782092051926, "margin": -0.6162966764544873, "p_left": 0.3506241824038166, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43174365807852977, "brier": 0.12293731728634485 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.34735993397774056, "right_score": -0.6812056602034133, "margin": 1.028565594181154, "p_left": 0.7366377118576203, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8199418761721449, "brier": 0.05599740667321012 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.6812056602034133, "right_score": 0.20468584777387056, "margin": -0.8858915079772839, "p_left": 0.29195840303603743, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3452524342403355, "brier": 0.08523970910335327 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9123465488421039, "right_score": -0.4461745099840295, "margin": 2.3585210588261334, "p_left": 0.9136091477959254, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.09035242721904302, "brier": 0.007463379344546263 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.4461745099840295, "right_score": 1.3261957926021346, "margin": -1.7723703025861641, "p_left": 0.14524780626516323, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1569436839438359, "brier": 0.02109692522484239 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.2981514037327988, "right_score": -0.9287054390133549, "margin": 2.2268568427461535, "p_left": 0.9026354730307811, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.329293334165062, "brier": 0.8147507971735019 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.3261957926021346, "right_score": -0.4510782092051926, "margin": 1.7772740018073272, "p_left": 0.8553599338721166, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.156232923341588, "brier": 0.020920748729478477 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.2013022437733312, "right_score": -0.3496412989273202, "margin": 0.148339055153989, "p_left": 0.5370169105281097, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6958952218047311, "brier": 0.0013702516650460814 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.4510782092051926, "right_score": 0.20468584777387056, "margin": -0.6557640569790631, "p_left": 0.34169179832860747, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4180820657200618, "brier": 0.11675328504503775 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.9287054390133549, "right_score": -0.2013022437733312, "margin": -0.7274031952400237, "p_left": 0.32576483681995066, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1215795173683498, "brier": 0.4545930552684278 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9123465488421039, "right_score": -1.2326019223468587, "margin": 3.144948471188963, "p_left": 0.9587092154260314, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.04216746652645176, "brier": 0.001704928890733885 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.01, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.2013022437733312, "rank": 6.0, "portrait_scores": [ -0.6103365777028354, 0.20773209015617305 ], "portrait_score_gap": 0.8180686678590086, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.2981514037327988, "rank": 1.0, "portrait_scores": [ -0.24405619111852653, 2.840358998584124 ], "portrait_score_gap": 3.0844151897026504, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.5671980147512745, "rank": 3.0, "portrait_scores": [ 0.27820832771453885, 0.8561877017880103 ], "portrait_score_gap": 0.5779793740734714, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.17702501801095227, "rank": 5.0, "portrait_scores": [ -1.5068040018607656, 1.8608540378826701 ], "portrait_score_gap": 3.3676580397434357, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.32600252391955875, "rank": 7.0, "portrait_scores": [ -0.6759576162522599, 0.023952568413142417 ], "portrait_score_gap": 0.6999101846654023, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.5, 12.0 ] }, { "identity": "emma_watson", "mean_score": -1.1750030496258916, "rank": 13.0, "portrait_scores": [ -0.3168440836693651, -2.033162015582418 ], "portrait_score_gap": 1.7163179319130528, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.36728077237218054, "rank": 9.0, "portrait_scores": [ -0.07309990392845522, -0.6614616408159059 ], "portrait_score_gap": 0.5883617368874506, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.7895039705150808, "rank": 11.0, "portrait_scores": [ -1.984719266767629, 0.40571132573746727 ], "portrait_score_gap": 2.3904305925050964, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.4862482416546444, "rank": 10.0, "portrait_scores": [ -0.3045758784990878, -0.6679206048102011 ], "portrait_score_gap": 0.3633447263111133, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.3496412989273202, "rank": 8.0, "portrait_scores": [ -0.1546944930004434, -0.5445881048541971 ], "portrait_score_gap": 0.38989361185375365, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.5, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.19183857995619075, "rank": 4.0, "portrait_scores": [ 0.8486975981339737, -0.4650204382215922 ], "portrait_score_gap": 1.3137180363555658, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 5.0, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8941492862468404, "rank": 2.0, "portrait_scores": [ 0.3982165030518708, 1.3900820694418101 ], "portrait_score_gap": 0.9918655663899394, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.140625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.9287054390133549, "rank": 12.0, "portrait_scores": [ -1.2902553659955605, -0.5671555120311493 ], "portrait_score_gap": 0.7230998539644111, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "6.0": 32, "7.0": 224, "8.0": 672, "9.0": 1120, "10.0": 1632, "11.0": 2208, "12.0": 1760, "13.0": 544 }, "target_primary_score": -0.9287054390133549, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.0539313052473351, -0.710079434844682 ], "target_alternate_panel_score": -0.8820053700460085, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.3957571283913641, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.0590303880569607, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.1801567737788075, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.6807031262726774, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6705874210897679, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.440430615306475, "brier_score": 0.12022276454161036, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.03101757416613, "brier_score": 0.3746183058256169, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.9338017833941359, "brier_score": 0.32083131165120826, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2254491557101188, "brier_score": 0.48219229417443393, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.14534579212742982, "brier_score": 0.021272456616367355, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.2132929596009705, "right_score": 0.3778185509307743, "margin": -1.5911115105317448, "p_left": 0.16922757265388427, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18539937560732966, "brier": 0.02863797134632568 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.6234831834640212, "right_score": -0.720744737010752, "margin": 0.09726155354673083, "p_left": 0.5242962382563618, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7429599677684868, "brier": 0.27488654544977176 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.1291451499977905, "right_score": 1.2261032342691252, "margin": -1.3552483842669156, "p_left": 0.20501364969703698, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22943033390543668, "brier": 0.04203059656209939 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.35089339987743123, "right_score": 1.7451261478649611, "margin": -2.0960195477423924, "p_left": 0.10948430247596576, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.11595454864357203, "brier": 0.011986812488648762 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.2132929596009705, "right_score": 1.7451261478649611, "margin": -2.9584191074659314, "p_left": 0.04934010608415409, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.050598910353380794, "brier": 0.0024344460683955794 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.6234831834640212, "right_score": 0.19597256855792522, "margin": -0.8194557520219464, "p_left": 0.3058792008429601, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3651092714364607, "brier": 0.09356208550832792 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.2132929596009705, "right_score": 0.28665754873667537, "margin": -1.4999505083376459, "p_left": 0.1824329054282211, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2014223067078448, "brier": 0.03328176498298226 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.6575762071165093, "right_score": 0.9042154025486885, "margin": -1.5617916096651978, "p_left": 0.17338971299471437, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.752213541685508, "brier": 0.6832845665829607 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.4496322500266329, "right_score": -1.2912894966032384, "margin": 0.8416572465766056, "p_left": 0.698814136444274, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3583704713271457, "brier": 0.09071292440580837 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.2261032342691252, "right_score": -0.2349294325058223, "margin": 1.4610326667749474, "p_left": 0.8116905676671099, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20863608574931103, "brier": 0.035460442305535314 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.28665754873667537, "right_score": -0.5673560817934905, "margin": 0.8540136305301659, "p_left": 0.7014084147186467, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.35466494431545026, "brier": 0.0891569348008317 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.720744737010752, "right_score": 0.19597256855792522, "margin": -0.9167173055686773, "p_left": 0.28562723781651145, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2530676825581928, "brier": 0.5103284433496671 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.28665754873667537, "right_score": -0.1291451499977905, "margin": 0.41580269873446585, "p_left": 0.6024784375713708, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.922506101255493, "brier": 0.3629802677384401 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.7451261478649611, "right_score": -0.1291451499977905, "margin": 1.8742712978627516, "p_left": 0.8669517290865906, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1427719795455631, "brier": 0.01770184239304799 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.2912894966032384, "right_score": -0.05003989701782141, "margin": -1.2412495995854171, "p_left": 0.22421854973755137, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.495134034311006, "brier": 0.601836858571308 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.9732868896358398, "right_score": -0.1291451499977905, "margin": -0.8441417396380493, "p_left": 0.30066320343682856, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.35762282651869987, "brier": 0.09039836190089576 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.3778185509307743, "right_score": -1.2132929596009705, "margin": 1.5911115105317448, "p_left": 0.8307724273461158, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18539937560732977, "brier": 0.02863797134632566 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.3778185509307743, "right_score": -0.35089339987743123, "margin": 0.7287119508082056, "p_left": 0.6745225554363519, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7581061390657373, "brier": 0.03045812235603451 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.3778185509307743, "right_score": -0.2349294325058223, "margin": 0.6127479834365966, "p_left": 0.6485673991223818, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0457373332648532, "brier": 0.4206396712043709 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2349294325058223, "right_score": -0.9732868896358398, "margin": 0.7383574571300174, "p_left": 0.6766365729659621, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1289784271827212, "brier": 0.4578370518751218 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.28665754873667537, "right_score": -0.4274145228214607, "margin": 0.7140720715581361, "p_left": 0.6713003136151191, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3985386808617335, "brier": 0.10804348382951906 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.05003989701782141, "right_score": -0.6575762071165093, "margin": 0.6075363100986879, "p_left": 0.6473785964788682, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0423603088721962, "brier": 0.41909904717894925 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.9732868896358398, "right_score": -1.2132929596009705, "margin": 0.24000606996513074, "p_left": 0.5597151451141379, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5803272941081693, "brier": 0.19385075344186464 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.22589078268166166, "right_score": 0.5288918104955649, "margin": -0.7547825931772265, "p_left": 0.31978009195104506, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1401217320802013, "brier": 0.4626991233061287 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.2261032342691252, "right_score": -0.1291451499977905, "margin": 1.3552483842669156, "p_left": 0.7949863503029629, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22943033390543666, "brier": 0.042030596562099426 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.7451261478649611, "right_score": -0.5673560817934905, "margin": 2.3124822296584515, "p_left": 0.9099055492815395, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.09441447685631177, "brier": 0.008117010050261098 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.9042154025486885, "right_score": 1.0644919070259482, "margin": -0.16027650447725972, "p_left": 0.4600164306099897, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6162165669422658, "brier": 0.21161511643115546 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.2606129544312686, "right_score": -0.2349294325058223, "margin": 0.49554238693709085, "p_left": 0.6214112051728877, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9713046369811011, "brier": 0.3861518859144208 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.2606129544312686, "right_score": -0.35089339987743123, "margin": 0.6115063543086998, "p_left": 0.6482843452390001, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4334258744951771, "brier": 0.12370390180395885 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.9732868896358398, "right_score": -0.4274145228214607, "margin": -0.545872366814379, "p_left": 0.36682258057999156, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4570046126660961, "brier": 0.1345588056233644 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.3778185509307743, "right_score": -0.5673560817934905, "margin": 0.9451746327242648, "p_left": 0.720143723063652, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8008917878884814, "brier": 0.04846325880432592 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.5673560817934905, "right_score": 0.2606129544312686, "margin": -0.8279690362247591, "p_left": 0.3040746779621967, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.36251292032703314, "brier": 0.09246140977781364 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.7451261478649611, "right_score": -0.35089339987743123, "margin": 2.0960195477423924, "p_left": 0.8905156975240343, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11595454864357224, "brier": 0.011986812488648758 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.35089339987743123, "right_score": 1.2261032342691252, "margin": -1.5769966341465564, "p_left": 0.17122125251376907, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18780205031748032, "brier": 0.02931671731238387 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.9042154025486885, "right_score": -0.720744737010752, "margin": 1.6249601395594406, "p_left": 0.8354780581808736, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8047113327529856, "brier": 0.6980235857016832 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.2261032342691252, "right_score": -0.4274145228214607, "margin": 1.653517757090586, "p_left": 0.8393659187432212, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17510853083303957, "brier": 0.025803308061209417 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.023931768565487233, "right_score": -0.2731089060421791, "margin": 0.24917713747669185, "p_left": 0.5619739560220885, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7008883404983042, "brier": 0.0038407712250277657 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.4274145228214607, "right_score": 0.2606129544312686, "margin": -0.6880274772527293, "p_left": 0.3344720143103114, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4071745899053837, "brier": 0.11187152835679717 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.720744737010752, "right_score": -0.023931768565487233, "margin": -0.6968129684452649, "p_left": 0.33251921211422597, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1010576397608096, "brier": 0.44553060219661367 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.7451261478649611, "right_score": -1.2132929596009705, "margin": 2.9584191074659314, "p_left": 0.9506598939158459, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.0505989103533806, "brier": 0.0024344460683955772 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.1, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.023931768565487233, "rank": 5.0, "portrait_scores": [ -0.5540664328919777, 0.5062028957610032 ], "portrait_score_gap": 1.0602693286529807, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.9042154025486885, "rank": 2.0, "portrait_scores": [ -0.3646962774257491, 2.173127082523126 ], "portrait_score_gap": 2.537823359948875, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.5, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.5288918104955649, "rank": 3.0, "portrait_scores": [ 0.565866416605652, 0.4919172043854778 ], "portrait_score_gap": 0.07394921222017414, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.19597256855792522, "rank": 4.0, "portrait_scores": [ -1.460522771869365, 1.8524679089852154 ], "portrait_score_gap": 3.3129906808545804, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.22589078268166166, "rank": 7.0, "portrait_scores": [ -0.24008433338506346, -0.21169723197825985 ], "portrait_score_gap": 0.028387101406803616, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": -1.2912894966032384, "rank": 13.0, "portrait_scores": [ -0.46186761430052403, -2.120711378905953 ], "portrait_score_gap": 1.6588437646054288, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 13.0, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.4496322500266329, "rank": 9.0, "portrait_scores": [ -0.25038422304228913, -0.6488802770109766 ], "portrait_score_gap": 0.3984960539686875, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.5, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.6575762071165093, "rank": 11.0, "portrait_scores": [ -1.868619670818632, 0.5534672565856134 ], "portrait_score_gap": 2.422086927404245, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.6234831834640212, "rank": 10.0, "portrait_scores": [ -0.4273850340574701, -0.8195813328705723 ], "portrait_score_gap": 0.39219629881310225, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.2731089060421791, "rank": 8.0, "portrait_scores": [ 0.008970111267429581, -0.5551879233517877 ], "portrait_score_gap": 0.5641580346192172, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": -0.05003989701782141, "rank": 6.0, "portrait_scores": [ 0.7722840315674324, -0.8723638256030752 ], "portrait_score_gap": 1.6446478571705077, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 6.5, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.0644919070259482, "rank": 1.0, "portrait_scores": [ 0.6868423918060749, 1.4421414222458215 ], "portrait_score_gap": 0.7552990304397467, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.1875, "single_photo_rank_quantiles": [ 1.0, 2.0, 4.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.720744737010752, "rank": 12.0, "portrait_scores": [ -0.9973491766003726, -0.4441402974211316 ], "portrait_score_gap": 0.553208879179241, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "5.0": 32, "6.0": 224, "7.0": 672, "8.0": 1120, "9.0": 1120, "10.0": 1184, "11.0": 1760, "12.0": 1568, "13.0": 512 }, "target_primary_score": -0.720744737010752, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.14183983887682, -0.8409049698725226 ], "target_alternate_panel_score": -0.9913724043746713, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.3678892209106501, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.1873449729325967, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.8955878069233598, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.967440635809184, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6034123525603672, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.46703932242455704, "brier_score": 0.13064177837656554, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 0.9918576959187702, "brier_score": 0.3602126059521777, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8069457643288875, "brier_score": 0.267018570895821, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.3616815590985352, "brier_score": 0.546600676064891, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.16470725938394298, "brier_score": 0.026400177763149183, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.0754412240695697, "right_score": 0.2897366726705205, "margin": -1.3651778967400903, "p_left": 0.20340004705192785, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22740266730053502, "brier": 0.041371579140726464 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.5393115853227693, "right_score": -1.103830192228142, "margin": 0.5645186069053726, "p_left": 0.6374974127091051, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0147236673586857, "brier": 0.40640295121080305 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.2673539236817253, "right_score": 1.034928868058105, "margin": -1.3022827917398303, "p_left": 0.2137810775336416, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24051999802357238, "brier": 0.04570234911144488 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.2402185659665287, "right_score": 1.7207866302908479, "margin": -1.9610051962573767, "p_left": 0.12335830344293162, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.13165692589883618, "brier": 0.015217271028318395 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.0754412240695697, "right_score": 1.7207866302908479, "margin": -2.7962278543604175, "p_left": 0.05752835624374342, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.059249446312828295, "brier": 0.003309511772107052 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.5393115853227693, "right_score": 0.14267119135718243, "margin": -0.6819827766799518, "p_left": 0.3358189111548189, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.40920044252374854, "brier": 0.11277434108920814 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.0754412240695697, "right_score": 0.24390245478390363, "margin": -1.3193436788534734, "p_left": 0.21092750893794696, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23689708521849173, "brier": 0.04449041402676769 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.32553115099832525, "right_score": 0.3096140227888562, "margin": -0.6351451737871814, "p_left": 0.3463448024620742, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0603204612319432, "brier": 0.42726511726834476 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.3941537336662931, "right_score": -1.3932418545689775, "margin": 0.9990881209026845, "p_left": 0.7308792545470914, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31350701133397396, "brier": 0.07242597563312925 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.034928868058105, "right_score": -0.4117328585669646, "margin": 1.4466617266250696, "p_left": 0.8094841387253326, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21135809997199706, "brier": 0.03629629339722829 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.24390245478390363, "right_score": -0.2980771442993998, "margin": 0.5419795990833034, "p_left": 0.6322728026547636, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.45843432819904617, "brier": 0.13522329166738245 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.103830192228142, "right_score": 0.14267119135718243, "margin": -1.2465013835853243, "p_left": 0.2233063535850401, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4992106674058316, "brier": 0.6032530203813666 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.24390245478390363, "right_score": -0.2673539236817253, "margin": 0.5112563784656289, "p_left": 0.6251009526954358, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9810984964423144, "brier": 0.3907512010607414 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.7207866302908479, "right_score": -0.2673539236817253, "margin": 1.9881405539725732, "p_left": 0.8795462782138151, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1283490974124759, "brier": 0.014509099092143624 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.3932418545689775, "right_score": 0.06991775097676695, "margin": -1.4631596055457445, "p_left": 0.1879845473938172, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.6713955142440968, "brier": 0.6593690952712239 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.8542587212749084, "right_score": -0.2673539236817253, "margin": -0.5869047975931831, "p_left": 0.35734535217548447, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.44214779417250144, "brier": 0.12769570072142103 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.2897366726705205, "right_score": -1.0754412240695697, "margin": 1.3651778967400903, "p_left": 0.7965999529480722, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22740266730053493, "brier": 0.04137157914072645 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.2897366726705205, "right_score": -0.2402185659665287, "margin": 0.5299552386370492, "p_left": 0.6294726720296242, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.727850457943466, "brier": 0.016763172802490633 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.2897366726705205, "right_score": -0.4117328585669646, "margin": 0.7014695312374851, "p_left": 0.668513505595247, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1041682110466249, "brier": 0.44691030716324637 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.4117328585669646, "right_score": -0.8542587212749084, "margin": 0.4425258627079438, "p_left": 0.6088607284151651, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9386915891069508, "brier": 0.3707113866062455 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.24390245478390363, "right_score": -0.3295499186081384, "margin": 0.573452373392042, "p_left": 0.6395594100549147, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4469757614887049, "brier": 0.12991741887996114 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.06991775097676695, "right_score": -0.32553115099832525, "margin": 0.3954489019750922, "p_left": 0.5975937204662228, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.910293055115144, "brier": 0.35711825474066206 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.8542587212749084, "right_score": -1.0754412240695697, "margin": 0.22118250279466123, "p_left": 0.5550712934407779, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5886587168355013, "brier": 0.19796155392046239 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.054187027112170466, "right_score": 0.45873812766155697, "margin": -0.5129251547737275, "p_left": 0.3745080517083585, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9821419763671745, "brier": 0.39124017737767347 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.034928868058105, "right_score": -0.2673539236817253, "margin": 1.3022827917398303, "p_left": 0.7862189224663585, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24051999802357238, "brier": 0.04570234911144484 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.7207866302908479, "right_score": -0.2980771442993998, "margin": 2.0188637745902476, "p_left": 0.8827634698719216, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12469798530175868, "brier": 0.013744403996471845 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3096140227888562, "right_score": 0.9929392039338301, "margin": -0.6833251811449739, "p_left": 0.33551956114898845, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4087498386571422, "brier": 0.1125733759136098 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1872777306638577, "right_score": -0.4117328585669646, "margin": 0.5990105892308223, "p_left": 0.6454299120263831, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0368492431763718, "brier": 0.41657977133838464 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1872777306638577, "right_score": -0.2402185659665287, "margin": 0.4274962966303864, "p_left": 0.6052756489770917, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5020713065697728, "brier": 0.1558073132904561 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.8542587212749084, "right_score": -0.3295499186081384, "margin": -0.52470880266677, "p_left": 0.3717518190016196, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46481999790941503, "brier": 0.13819941493101293 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.2897366726705205, "right_score": -0.2980771442993998, "margin": 0.5878138169699203, "p_left": 0.6428633767260797, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7357299636815589, "brier": 0.02040994440957778 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.2980771442993998, "right_score": 0.1872777306638577, "margin": -0.48535487496325747, "p_left": 0.38098844928502024, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4796313461889605, "brier": 0.14515219848860444 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.7207866302908479, "right_score": -0.2402185659665287, "margin": 1.9610051962573767, "p_left": 0.8766416965570684, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.13165692589883626, "brier": 0.015217271028318399 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.2402185659665287, "right_score": 1.034928868058105, "margin": -1.2751474340246338, "p_left": 0.21837736973161975, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24638322490047967, "brier": 0.04768867561090055 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3096140227888562, "right_score": -1.103830192228142, "margin": 1.4134442150169981, "p_left": 0.8043086184561946, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6312164446719548, "brier": 0.6469123537229124 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.034928868058105, "right_score": -0.3295499186081384, "margin": 1.3644787866662433, "p_left": 0.7964866537252447, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2275449059240271, "brier": 0.04141768211194844 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.1198501421852354, "right_score": -0.3535355754105996, "margin": 0.2336854332253642, "p_left": 0.5581569417414336, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6999578151578771, "brier": 0.0033822298727165015 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.3295499186081384, "right_score": 0.1872777306638577, "margin": -0.5168276492719961, "p_left": 0.37359433409719806, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4677570891631785, "brier": 0.13957272646952884 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.103830192228142, "right_score": -0.1198501421852354, "margin": -0.9839800500429066, "p_left": 0.2721027689677052, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3015754569576687, "brier": 0.529834378944482 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.7207866302908479, "right_score": -1.0754412240695697, "margin": 2.7962278543604175, "p_left": 0.9424716437562566, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.05924944631282836, "brier": 0.003309511772107052 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 1.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.1198501421852354, "rank": 7.0, "portrait_scores": [ -0.6754140427393048, 0.435713758368834 ], "portrait_score_gap": 1.111127801108139, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.0, 10.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3096140227888562, "rank": 3.0, "portrait_scores": [ -0.3606422162091868, 0.9798702617868992 ], "portrait_score_gap": 1.340512477996086, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.45873812766155697, "rank": 2.0, "portrait_scores": [ 1.038976955614769, -0.12150070029165502 ], "portrait_score_gap": 1.160477655906424, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 8.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.14267119135718243, "rank": 4.0, "portrait_scores": [ -1.1017262409384023, 1.3870686236527672 ], "portrait_score_gap": 2.4887948645911697, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.054187027112170466, "rank": 6.0, "portrait_scores": [ 0.448571770190578, -0.5569458244149189 ], "portrait_score_gap": 1.0055175946054968, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "emma_watson", "mean_score": -1.3932418545689775, "rank": 13.0, "portrait_scores": [ -0.7456880466149143, -2.040795662523041 ], "portrait_score_gap": 1.2951076159081267, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.3941537336662931, "rank": 10.0, "portrait_scores": [ 0.020041447185989103, -0.8083489145185753 ], "portrait_score_gap": 0.8283903617045644, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.0, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.32553115099832525, "rank": 8.0, "portrait_scores": [ -1.4489154673484113, 0.7978531653517608 ], "portrait_score_gap": 2.246768632700172, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.5393115853227693, "rank": 11.0, "portrait_scores": [ -0.46310417333301895, -0.6155189973125197 ], "portrait_score_gap": 0.15241482397950074, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.3535355754105996, "rank": 9.0, "portrait_scores": [ -0.03059986495685165, -0.6764712858643476 ], "portrait_score_gap": 0.6458714209074959, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.06991775097676695, "rank": 5.0, "portrait_scores": [ 0.9130122468558486, -0.7731767449023147 ], "portrait_score_gap": 1.6861889917581632, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.5, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.9929392039338301, "rank": 1.0, "portrait_scores": [ 0.656075526210804, 1.3298028816568561 ], "portrait_score_gap": 0.6737273554460521, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.265625, "single_photo_rank_quantiles": [ 1.0, 2.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.103830192228142, "rank": 12.0, "portrait_scores": [ -1.1902807906354682, -1.0173795938208157 ], "portrait_score_gap": 0.17290119681465255, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "10.0": 512, "11.0": 2560, "12.0": 3584, "13.0": 1536 }, "target_primary_score": -1.103830192228142, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.14313949636153, -1.5001717184094467 ], "target_alternate_panel_score": -1.3216556073854884, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.782344022062719, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.464326798742671, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.6312696301743446, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.201805465200253, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.4377379436962192, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_10", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.48670625128028694, "brier_score": 0.1387265931415332, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.0736498033485107, "brier_score": 0.38542478072994574, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8138346766888587, "brier_score": 0.2649716790805806, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.593280056667814, "brier_score": 0.6263309840286759, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.1863870181745217, "brier_score": 0.03312869865168585, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.9749314387304091, "right_score": 0.1077552461143099, "margin": -1.082686684844719, "p_left": 0.25299792328948173, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29168731378555596, "brier": 0.06400794918879048 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.40538457084282187, "right_score": -1.3323992026768514, "margin": 0.9270146318340295, "p_left": 0.7164692267075398, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2604346139363942, "brier": 0.5133281528189 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.382472366465596, "right_score": 0.9242602465189123, "margin": -1.3067326129845083, "p_left": 0.21303410982496818, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23957037308592372, "brier": 0.0453835319489166 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.11388509529872964, "right_score": 1.7165901397493286, "margin": -1.8304752350480582, "p_left": 0.13818166888898473, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1487107833013741, "brier": 0.019094173616945008 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.9749314387304091, "right_score": 1.7165901397493286, "margin": -2.6915215784797377, "p_left": 0.06347550592229449, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06557960252523266, "brier": 0.004029139852091243 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.40538457084282187, "right_score": 0.4057089424960575, "margin": -0.8110935133388794, "p_left": 0.3076575235988513, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3676745376969412, "brier": 0.09465315182697774 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.9749314387304091, "right_score": 0.2400584937594707, "margin": -1.2149899324898799, "p_left": 0.22881933096268683, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2598326020822828, "brier": 0.052358286222211614 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.10732793090370107, "right_score": 0.29028515464572635, "margin": -0.3976130855494274, "p_left": 0.40188595658169807, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9115869207060415, "brier": 0.3577404089341903 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.43614157078330396, "right_score": -1.4259114025710176, "margin": 0.9897698317877136, "p_left": 0.7290424574206401, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3160233080375191, "brier": 0.07341798988064563 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.9242602465189123, "right_score": -0.42870278961384806, "margin": 1.3529630361327603, "p_left": 0.7946136261781384, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2298992872737935, "brier": 0.0421835625516935 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.17682495482529068, "margin": 0.41688344858476134, "p_left": 0.6027372464933641, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5062739210109648, "brier": 0.15781769532367412 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.3323992026768514, "right_score": 0.4057089424960575, "margin": -1.7381081451729088, "p_left": 0.14955339470989384, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.9001017946660241, "brier": 0.7232594284494657 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.382472366465596, "margin": 0.6225308602250668, "p_left": 0.6507939341609257, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0520930844943788, "brier": 0.4235327447406553 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.7165901397493286, "right_score": -0.382472366465596, "margin": 2.0990625062149246, "p_left": 0.8908120259746567, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11562184349398397, "brier": 0.011922013671759045 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.4259114025710176, "right_score": 0.30982651343540885, "margin": -1.7357379160064264, "p_left": 0.14985510793949683, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.8980863987834182, "brier": 0.7227463374965646 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.382472366465596, "margin": -0.4447205148818776, "p_left": 0.3906167416622167, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4953078851778002, "brier": 0.15258143886680695 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1077552461143099, "right_score": -0.9749314387304091, "margin": 1.082686684844719, "p_left": 0.7470020767105182, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.291687313785556, "brier": 0.0640079491887905 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.11388509529872964, "margin": 0.22164034141303954, "p_left": 0.5551843616878755, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.699275207881456, "brier": 0.0030453137748982667 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.42870278961384806, "margin": 0.536458035728158, "p_left": 0.6309880825745242, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9969263389171275, "brier": 0.3981459603510746 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.42870278961384806, "right_score": -0.8271928813474736, "margin": 0.39849009173362554, "p_left": 0.5983248344308972, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.91211156285614, "brier": 0.3579926074967606 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.2400584937594707, "right_score": -0.19950921901220023, "margin": 0.43956771277167095, "p_left": 0.6081560205342458, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49732381721458385, "brier": 0.15354170424355842 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.30982651343540885, "right_score": -0.10732793090370107, "margin": 0.4171544443391099, "p_left": 0.6028021332805981, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.923320717622617, "brier": 0.3633704118876399 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.9749314387304091, "margin": 0.1477385573829355, "p_left": 0.5368676056040683, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6220037593693053, "brier": 0.21449161473890882 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.1378743244233343, "right_score": 0.5997783888589585, "margin": -0.4619040644356242, "p_left": 0.38653422416566907, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9505348659339574, "brier": 0.3763402581200176 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.9242602465189123, "right_score": -0.382472366465596, "margin": 1.3067326129845083, "p_left": 0.7869658901750317, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23957037308592377, "brier": 0.04538353194891665 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.7165901397493286, "right_score": -0.17682495482529068, "margin": 1.8934150945746193, "p_left": 0.8691444273254256, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1402459680724597, "brier": 0.017123180899990816 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.29028515464572635, "right_score": 0.8583237547568487, "margin": -0.5680386001111224, "p_left": 0.3616895307072613, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.44893048501734956, "brier": 0.1308193166232389 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1148546191515259, "right_score": -0.42870278961384806, "margin": 0.543557408765374, "p_left": 0.6326395729467325, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.001411822799104, "brier": 0.400232829258224 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1148546191515259, "right_score": -0.11388509529872964, "margin": 0.22873971445025554, "p_left": 0.5569368911717647, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5853033467813552, "brier": 0.19630491840454067 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.8271928813474736, "right_score": -0.19950921901220023, "margin": -0.6276836623352734, "p_left": 0.34803594568826673, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.42776584999899897, "brier": 0.12112901949112616 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1077552461143099, "right_score": -0.17682495482529068, "margin": 0.28458020093960057, "p_left": 0.5706687616896483, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.703236440239967, "brier": 0.004994073878748304 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.17682495482529068, "right_score": 0.1148546191515259, "margin": -0.29167957397681654, "p_left": 0.4275927293631001, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5579045293593723, "brier": 0.18283554220418538 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.7165901397493286, "right_score": -0.11388509529872964, "margin": 1.8304752350480582, "p_left": 0.8618183311110152, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1487107833013741, "brier": 0.01909417361694502 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.11388509529872964, "right_score": 0.9242602465189123, "margin": -1.0381453418176418, "p_left": 0.2615080099563315, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.30314502332048027, "brier": 0.06838643927132078 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.29028515464572635, "right_score": -1.3323992026768514, "margin": 1.6226843573225778, "p_left": 0.8351650028592288, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8028103227611456, "brier": 0.6975005820008556 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.9242602465189123, "right_score": -0.19950921901220023, "margin": 1.1237694655311126, "p_left": 0.7546872440012016, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.28145186191286076, "brier": 0.060178348255726016 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.2026023285627484, "right_score": -0.30743086069639713, "margin": 0.10482853213364873, "p_left": 0.5261831601486575, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6945201797130258, "brier": 0.0006855578753702469 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.19950921901220023, "right_score": 0.1148546191515259, "margin": -0.31436383816372615, "p_left": 0.4220499344203131, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5482678057723855, "brier": 0.17812614714419056 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.3323992026768514, "right_score": -0.2026023285627484, "margin": -1.1297968741141031, "p_left": 0.244198589015949, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4097734953076921, "brier": 0.5712357728454823 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.7165901397493286, "right_score": -0.9749314387304091, "margin": 2.6915215784797377, "p_left": 0.9365244940777057, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.06557960252523287, "brier": 0.004029139852091224 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 10.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.2026023285627484, "rank": 8.0, "portrait_scores": [ -0.6993366308094117, 0.29413197368391486 ], "portrait_score_gap": 0.9934686044933265, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 8.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.29028515464572635, "rank": 5.0, "portrait_scores": [ -0.15807422860050335, 0.738644537891956 ], "portrait_score_gap": 0.8967187664924594, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.5997783888589585, "rank": 2.0, "portrait_scores": [ 1.167330185432654, 0.03222659228526287 ], "portrait_score_gap": 1.1351035931473912, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.4057089424960575, "rank": 3.0, "portrait_scores": [ -0.6808915354259366, 1.4923094204180516 ], "portrait_score_gap": 2.1732009558439884, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.1378743244233343, "rank": 6.0, "portrait_scores": [ 0.8006691919416679, -0.5249205430949992 ], "portrait_score_gap": 1.325589735036667, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": -1.4259114025710176, "rank": 13.0, "portrait_scores": [ -0.863196044528949, -1.9886267606130863 ], "portrait_score_gap": 1.1254307160841373, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.43614157078330396, "rank": 11.0, "portrait_scores": [ 0.26649183428691137, -1.1387749758535193 ], "portrait_score_gap": 1.4052668101404306, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.10732793090370107, "rank": 7.0, "portrait_scores": [ -1.1679495782259992, 0.9532937164185971 ], "portrait_score_gap": 2.1212432946445965, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 8.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.40538457084282187, "rank": 10.0, "portrait_scores": [ -0.4520683845428617, -0.3587007571427821 ], "portrait_score_gap": 0.09336762740007959, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.5, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.30743086069639713, "rank": 9.0, "portrait_scores": [ -0.04149806771454024, -0.573363653678254 ], "portrait_score_gap": 0.5318655859637138, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": 0.30982651343540885, "rank": 4.0, "portrait_scores": [ 1.2129519209025243, -0.5932988940317067 ], "portrait_score_gap": 1.8062508149342311, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8583237547568487, "rank": 1.0, "portrait_scores": [ 0.6779713777586348, 1.0386761317550628 ], "portrait_score_gap": 0.360704753996428, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.3323992026768514, "rank": 12.0, "portrait_scores": [ -1.3096253168561975, -1.355173088497505 ], "portrait_score_gap": 0.04554777164130752, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "12.0": 4096, "13.0": 4096 }, "target_primary_score": -1.3323992026768514, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -0.7249942834660589, -1.8712793634017248 ], "target_alternate_panel_score": -1.298136823433892, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.89275225259107, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.7038457659299495, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.5884219780796183, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.0955344948711434, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.4035942381141289, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_100", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.4934231671191725, "brier_score": 0.1409984769322837, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.1267395198399435, "brier_score": 0.40146310626780995, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8284151524170034, "brier_score": 0.2701298947450476, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.7233882546858235, "brier_score": 0.6641295293133345, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.1974053867545129, "brier_score": 0.0361000323890099, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.0329063322902579, "right_score": 0.08957387639439904, "margin": -1.1224802086846568, "p_left": 0.2455515201683748, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.28176828696038686, "brier": 0.060295549056999784 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3970750170760441, "right_score": -1.4690444664711406, "margin": 1.0719694493950964, "p_left": 0.7449712701892686, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3663790742456026, "brier": 0.5549821934074122 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.3705589699701455, "right_score": 0.8250287179986261, "margin": -1.1955876879687715, "p_left": 0.23226107210039149, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.26430554125707684, "brier": 0.05394520561322325 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.04191206467274794, "right_score": 1.6124473793605056, "margin": -1.6543594440332536, "p_left": 0.16052062836887554, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.17497337497475235, "brier": 0.02576687213193865 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.0329063322902579, "right_score": 1.6124473793605056, "margin": -2.6453537116507633, "p_left": 0.06627595909436518, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06857434382583558, "brier": 0.0043925027538779665 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3970750170760441, "right_score": 0.45646801233628115, "margin": -0.8535430294123252, "p_left": 0.2986901548028823, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3548054850419923, "brier": 0.0892158085761698 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.0329063322902579, "right_score": 0.3148916981397666, "margin": -1.3477980304300243, "p_left": 0.20623060003901456, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23096228817817982, "brier": 0.04253106039245199 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.04421352076003571, "right_score": 0.37824007883957467, "margin": -0.4224535995996104, "p_left": 0.3959297746324042, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9265184202405418, "brier": 0.36490083717565797 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.3641709962803769, "right_score": -1.4132911669873987, "margin": 1.0491201707070217, "p_left": 0.7406059120209388, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3002866277894978, "brier": 0.06728529287848892 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.8250287179986261, "right_score": -0.4176027203719745, "margin": 1.2426314383706005, "p_left": 0.7760217220433947, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25357476686673386, "brier": 0.050166268996406346 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.3148916981397666, "right_score": -0.20629124737222748, "margin": 0.5211829455119941, "p_left": 0.6274243369072044, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4661321938803231, "brier": 0.1388126247290363 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.4690444664711406, "right_score": 0.45646801233628115, "margin": -1.9255124788074218, "p_left": 0.12724811334011146, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.0616164500652565, "brier": 0.7616958556683949 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.3148916981397666, "right_score": -0.3705589699701455, "margin": 0.6854506681099121, "p_left": 0.66495414227607, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0934878677973283, "brier": 0.4421640113301039 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.6124473793605056, "right_score": -0.3705589699701455, "margin": 1.9830063493306511, "p_left": 0.8790012760088828, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1289689296383063, "brier": 0.014640691207478566 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.4132911669873987, "right_score": 0.4049913986305004, "margin": -1.8182825656178991, "p_left": 0.13964007910706344, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.9686870302382764, "brier": 0.7402191934789001 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.7618443357841507, "right_score": -0.3705589699701455, "margin": -0.3912853658140052, "p_left": 0.4034079133831082, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5165216711247386, "brier": 0.16273794458011334 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.08957387639439904, "right_score": -1.0329063322902579, "margin": 1.1224802086846568, "p_left": 0.7544484798316252, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2817682869603868, "brier": 0.06029554905699977 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.08957387639439904, "right_score": -0.04191206467274794, "margin": 0.13148594106714698, "p_left": 0.5328242086324544, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6953066946992688, "brier": 0.001077428672346893 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.08957387639439904, "right_score": -0.4176027203719745, "margin": 0.5071765967663735, "p_left": 0.6241443699692503, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9785501720089644, "brier": 0.38955619456431234 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.4176027203719745, "right_score": -0.7618443357841507, "margin": 0.3442416154121762, "p_left": 0.5852204926807958, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.880008207609938, "brier": 0.3424830250535534 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.3148916981397666, "right_score": -0.09173889843145216, "margin": 0.40663059657121875, "p_left": 0.6002796846025559, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5103595913716985, "brier": 0.1597763305414322 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.4049913986305004, "right_score": -0.04421352076003571, "margin": 0.4492049193905361, "p_left": 0.6104501797898817, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9427635137376102, "brier": 0.3726494220054989 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.7618443357841507, "right_score": -1.0329063322902579, "margin": 0.2710619965061072, "p_left": 0.5673536049594119, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5667725277344592, "brier": 0.18718290314161662 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.22892603288827174, "right_score": 0.6300402384195353, "margin": -0.40111420553126353, "p_left": 0.40104466949598006, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9136824626277377, "brier": 0.35874748793917977 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.8250287179986261, "right_score": -0.3705589699701455, "margin": 1.1955876879687715, "p_left": 0.7677389278996085, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2643055412570767, "brier": 0.05394520561322325 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.6124473793605056, "right_score": -0.20629124737222748, "margin": 1.818738626732733, "p_left": 0.860414703402774, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15034079270299494, "brier": 0.019484055026135542 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.37824007883957467, "right_score": 0.745633958061944, "margin": -0.3673938792223693, "p_left": 0.4091709020040489, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.526228477682716, "brier": 0.16742082704680697 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.08091289699965867, "right_score": -0.4176027203719745, "margin": 0.4985156173716332, "p_left": 0.6221104324163227, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9731532752959859, "brier": 0.387021390121224 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.08091289699965867, "right_score": -0.04191206467274794, "margin": 0.12282496167240661, "p_left": 0.5306676957771493, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6336192619692509, "brier": 0.22027281178713046 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.7618443357841507, "right_score": -0.09173889843145216, "margin": -0.6701054373526985, "p_left": 0.3384732320119004, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41320483065536096, "brier": 0.11456412878858174 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.08957387639439904, "right_score": -0.20629124737222748, "margin": 0.29586512376662655, "p_left": 0.5734314021573987, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.704049524078959, "brier": 0.0053921708228016245 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.20629124737222748, "right_score": 0.08091289699965867, "margin": -0.2872041443718861, "p_left": 0.42868847617634176, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5598206422252281, "brier": 0.18377380960639395 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.6124473793605056, "right_score": -0.04191206467274794, "margin": 1.6543594440332536, "p_left": 0.8394793716311245, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1749733749747524, "brier": 0.025766872131938633 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.04191206467274794, "right_score": 0.8250287179986261, "margin": -0.866940782671374, "p_left": 0.29589125996291465, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3508224744719047, "brier": 0.08755163772244114 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.37824007883957467, "right_score": -1.4690444664711406, "margin": 1.8472845453107154, "p_left": 0.8638079621496482, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.9936893462240515, "brier": 0.746164195473128 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.8250287179986261, "right_score": -0.09173889843145216, "margin": 0.9167676164300782, "p_left": 0.7143830277182143, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33633600709537625, "brier": 0.08157705485541436 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.25788720696212053, "right_score": -0.1598054813089504, "margin": -0.09808172565317014, "p_left": 0.4754992069581723, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6943492019776548, "brier": 0.0006002888596784714 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.09173889843145216, "right_score": 0.08091289699965867, "margin": -0.17265179543111084, "p_left": 0.4569439515737076, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6105427444370892, "brier": 0.20879777487979484 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.4690444664711406, "right_score": -0.25788720696212053, "margin": -1.21115725950902, "p_left": 0.22949635127119417, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4718681482083826, "brier": 0.593675872704403 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.6124473793605056, "right_score": -1.0329063322902579, "margin": 2.6453537116507633, "p_left": 0.9337240409056349, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.06857434382583572, "brier": 0.004392502753877959 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 100.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.25788720696212053, "rank": 9.0, "portrait_scores": [ -0.7578979884774564, 0.2421235745532153 ], "portrait_score_gap": 1.0000215630306717, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.0, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.37824007883957467, "rank": 5.0, "portrait_scores": [ -0.026080758455283792, 0.7825609161344331 ], "portrait_score_gap": 0.8086416745897169, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.6300402384195353, "rank": 2.0, "portrait_scores": [ 1.1216028906666182, 0.13847758617245237 ], "portrait_score_gap": 0.9831253044941659, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.45646801233628115, "rank": 3.0, "portrait_scores": [ -0.6107502282806531, 1.5236862529532154 ], "portrait_score_gap": 2.1344364812338683, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.22892603288827174, "rank": 6.0, "portrait_scores": [ 0.8113553045641932, -0.3535032387876498 ], "portrait_score_gap": 1.164858543351843, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": -1.4132911669873987, "rank": 12.0, "portrait_scores": [ -0.9043408251237784, -1.9222415088510187 ], "portrait_score_gap": 1.0179006837272402, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.3641709962803769, "rank": 10.0, "portrait_scores": [ 0.3769345603810953, -1.1052765529418491 ], "portrait_score_gap": 1.4822111133229443, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.0, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.04421352076003571, "rank": 7.0, "portrait_scores": [ -1.1287062957249137, 1.0402792542048422 ], "portrait_score_gap": 2.168985549929756, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.5, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.3970750170760441, "rank": 11.0, "portrait_scores": [ -0.48417527965987334, -0.30997475449221484 ], "portrait_score_gap": 0.1742005251676585, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.1598054813089504, "rank": 8.0, "portrait_scores": [ 0.03370660705712578, -0.3533175696750266 ], "portrait_score_gap": 0.3870241767321524, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "natalie_portman", "mean_score": 0.4049913986305004, "rank": 4.0, "portrait_scores": [ 1.361478578063302, -0.5514957808023013 ], "portrait_score_gap": 1.9129743588656034, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.745633958061944, "rank": 1.0, "portrait_scores": [ 0.6444436797917006, 0.8468242363321875 ], "portrait_score_gap": 0.20238055654048692, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0390625, "single_photo_rank_quantiles": [ 2.0, 3.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.4690444664711406, "rank": 13.0, "portrait_scores": [ -1.4088169985588075, -1.5292719343834735 ], "portrait_score_gap": 0.12045493582466604, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "12.0": 4096, "13.0": 4096 }, "target_primary_score": -1.4690444664711406, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.557476185167492, -2.064904876428413 ], "target_alternate_panel_score": -1.3111905307979523, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.9141155137219082, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.7676585431342335, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.689430609637527, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.0533033238358318, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.39165069201219116, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } ] }, { "variant": "full", "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 12, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6929321626547033, "brier_score": 0.22905915580080904, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.6949297067316841, "brier_score": 0.2300579251957289, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.694322560091619, "brier_score": 0.21933768743427584, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.696144000011814, "brier_score": 0.25149840071863516, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6932592026587094, "brier_score": 0.2500559951776231, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.009437953141937707, "right_score": 0.00024698642474490443, "margin": -0.00968493956668261, "p_left": 0.4975787840337453, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6883164354875824, "brier": 0.24758464632050053 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.001973474479071992, "right_score": -0.003206817399792488, "margin": 0.0051802918788644795, "p_left": 0.5012950700735712, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6957406809236206, "brier": 0.2512967472800667 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.01074420251303846, "right_score": -0.005245839103148787, "margin": 0.015990041616187247, "p_left": 0.5039974252321262, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7011741612064213, "brier": 0.25401340464061267 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.004070521011260935, "right_score": -0.004002396162004828, "margin": -6.812484925610655e-05, "p_left": 0.49998296878769255, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6931131187154417, "brier": 0.24998296907775475 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.009437953141937707, "right_score": -0.004002396162004828, "margin": -0.005435556979932879, "p_left": 0.49864111410073714, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6904330952253926, "brier": 0.24864296067162436 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.001973474479071992, "right_score": 0.005702293494479819, "margin": -0.003728819015407827, "p_left": 0.49906779632626824, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912845090626407, "brier": 0.24906866532995756 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.009437953141937707, "right_score": 0.0060542952667599955, "margin": -0.015492248408697703, "p_left": 0.49612701536035875, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6854310572756714, "brier": 0.24614201537037764 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.0024525376389379866, "right_score": 0.006366805501861729, "margin": -0.003914267862923742, "p_left": 0.4990214342836928, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6951062296767974, "brier": 0.2509795233071684 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.003528364324779695, "right_score": 0.002713974872993883, "margin": 0.0008143894517858119, "p_left": 0.5002035973516937, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6927400687378225, "brier": 0.2497964441001879 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.005245839103148787, "right_score": -0.00947179196544442, "margin": 0.004225952862295634, "p_left": 0.5010564866432865, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6910364364618358, "brier": 0.24894462952074095 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.007312342515155206, "margin": -0.0012580472483952106, "p_left": 0.4996854882293822, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6937764020194896, "brier": 0.25031461068827165 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.003206817399792488, "right_score": 0.005702293494479819, "margin": -0.008909110894272307, "p_left": 0.4977727370083199, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6976116575063851, "brier": 0.2522322236921142 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.01074420251303846, "margin": -0.004689907246278465, "p_left": 0.49882752533750047, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6908049763380337, "brier": 0.24882890003433467 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.004002396162004828, "right_score": 0.01074420251303846, "margin": -0.014746598675043288, "p_left": 0.4963134171386903, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7005476624227298, "brier": 0.25370017375450304 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.002713974872993883, "right_score": 0.005365045299471259, "margin": -0.002651070426477376, "p_left": 0.4993372327815509, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6944735942947275, "brier": 0.250663206478835 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.0035563270424336956, "right_score": 0.01074420251303846, "margin": -0.007187875470604765, "p_left": 0.4982030388690914, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6895597010049629, "brier": 0.2482062679383974 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.00024698642474490443, "right_score": -0.009437953141937707, "margin": 0.00968493956668261, "p_left": 0.5024212159662547, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6883164354875824, "brier": 0.2475846463205006 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.00024698642474490443, "right_score": -0.004070521011260935, "margin": 0.004317507436005839, "p_left": 0.5010793751822942, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.693149510666943, "brier": 1.16505078415258e-06 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.00024698642474490443, "right_score": -0.00947179196544442, "margin": 0.009718778390189326, "p_left": 0.5024296754730645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6980183765402478, "brier": 0.2524355787959689 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.00947179196544442, "right_score": 0.0035563270424336956, "margin": -0.013028119007878116, "p_left": 0.4967430163157318, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6866543373915719, "brier": 0.2467536242584514 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.0007174158272800026, "margin": 0.005336879439479993, "p_left": 0.5013342166930768, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6904823011212491, "brier": 0.24866756344110733 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.005365045299471259, "right_score": 0.0024525376389379866, "margin": 0.0029125076605332726, "p_left": 0.5007281264004269, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6946044947274462, "brier": 0.25072865656848187 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.0035563270424336956, "right_score": -0.009437953141937707, "margin": 0.012994280184371402, "p_left": 0.5032485243364203, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6866711467339565, "brier": 0.246762028573944 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.009751167958215307, "right_score": -0.000818008680712527, "margin": -0.00893315927750278, "p_left": 0.49776672503213154, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6976237353323637, "brier": 0.2522382624849505 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.005245839103148787, "right_score": 0.01074420251303846, "margin": -0.015990041616187247, "p_left": 0.49600257476787385, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7011741612064213, "brier": 0.25401340464061267 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.004002396162004828, "right_score": 0.007312342515155206, "margin": -0.011314738677160034, "p_left": 0.4971713455084551, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6988205527270782, "brier": 0.2528366557777774 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.006366805501861729, "right_score": 0.001514875938342089, "margin": 0.00485192956351964, "p_left": 0.501212980011295, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6955760879913799, "brier": 0.2512144513318028 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.00454757471227478, "right_score": -0.00947179196544442, "margin": 0.014019366677719202, "p_left": 0.5035047842663213, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7001814315278704, "brier": 0.25351706777907473 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.00454757471227478, "right_score": -0.004070521011260935, "margin": 0.008618095723535715, "p_left": 0.5021545105959927, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6888474166161845, "brier": 0.24785013131991554 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.0035563270424336956, "right_score": 0.0007174158272800026, "margin": 0.002838911215153693, "p_left": 0.5007097273271228, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6945676435942948, "brier": 0.2507102310400017 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.00024698642474490443, "right_score": 0.007312342515155206, "margin": -0.007065356090410302, "p_left": 0.4982336683252216, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931534204540521, "brier": 3.1199275853255016e-06 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.007312342515155206, "right_score": 0.00454757471227478, "margin": 0.0027647678028804258, "p_left": 0.5006911915104346, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6945305199537066, "brier": 0.2506916692561387 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.004002396162004828, "right_score": -0.004070521011260935, "margin": 6.812484925610655e-05, "p_left": 0.5000170312123075, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6931131187154417, "brier": 0.24998296907775475 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.004070521011260935, "right_score": -0.005245839103148787, "margin": 0.0011753180918878522, "p_left": 0.5002938294891479, "prediction": "left", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6937350122774564, "brier": 0.2502939158249166 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.006366805501861729, "right_score": -0.003206817399792488, "margin": 0.009573622901654217, "p_left": 0.5023933874450961, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6979454487489529, "brier": 0.25239911574855844 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.005245839103148787, "right_score": 0.0007174158272800026, "margin": -0.005963254930428789, "p_left": 0.498509190685206, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6961332530697442, "brier": 0.2514930318272071 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.0029447699503040324, "right_score": 0.011078225800300573, "margin": -0.014022995750604606, "p_left": 0.49649430851004667, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931717609097745, "brier": 1.228987282273122e-05 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.0007174158272800026, "right_score": 0.00454757471227478, "margin": -0.0038301588849947777, "p_left": 0.49904246144935127, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912339348809625, "brier": 0.24904337832942725 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.003206817399792488, "right_score": -0.0029447699503040324, "margin": -0.00026204744948845563, "p_left": 0.4999344881380028, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6932782128682977, "brier": 0.2500655161538013 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.004002396162004828, "right_score": -0.009437953141937707, "margin": 0.005435556979932879, "p_left": 0.5013588858992628, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6904330952253926, "brier": 0.24864296067162442 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 2, "regressions": 3, "both_correct": 0, "both_wrong": 6, "net_fixes": -1, "trial_ids": { "fixes": [ "0add2ec437a1", "34bec29a8d38" ], "regressions": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 2, "regressions": 2, "both_correct": 0, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [ "0add2ec437a1", "34bec29a8d38" ], "regressions": [ "e5aa020b210d", "396ff40ef0fb" ], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654" ] } }, "target": { "fixes": 0, "regressions": 1, "both_correct": 0, "both_wrong": 3, "net_fixes": -1, "trial_ids": { "fixes": [], "regressions": [ "974972ffb84a" ], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.6235438969947976, "brier_score": 0.19458252571635956, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 7, "accuracy": 0.6363636363636364, "mean_cross_entropy": 0.6894572536309592, "brier_score": 0.22732437613600961, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6780861588695775, "brier_score": 0.21122479643990671, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.7121994431537227, "brier_score": 0.25952353552821544, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5760940563411876, "brier_score": 0.19192120629951392, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.13902191780764347, "right_score": 0.1280228076121007, "margin": -0.26704472541974417, "p_left": 0.43363275398955076, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5685125666461338, "brier": 0.18803736533256224 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.007578623552076993, "right_score": -0.03571019972460085, "margin": 0.043288823276677844, "p_left": 0.5108205161381384, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7150258141886511, "brier": 0.2609375997076341 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.18499326645481873, "right_score": 0.2546305696703304, "margin": -0.06963730321551165, "p_left": 0.4825977061180897, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6589345757612762, "brier": 0.2329005459504421 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.06872604284636673, "right_score": 0.23423594357815292, "margin": -0.30296198642451966, "p_left": 0.4248335611401853, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5530958211832935, "brier": 0.18048355467105154 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.13902191780764347, "right_score": 0.23423594357815292, "margin": -0.3732578613857964, "p_left": 0.407754042584497, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5238332617740468, "brier": 0.1662633592439998 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.007578623552076993, "right_score": -0.038691975560557396, "margin": 0.04627059911263439, "p_left": 0.5115655863889251, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7165500772887756, "brier": 0.2616993491774448 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.13902191780764347, "right_score": 0.10397774370054166, "margin": -0.24299966150818514, "p_left": 0.43954726424842444, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5790103651787876, "brier": 0.19320179750827426 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.06737305060665771, "right_score": 0.034205489187603175, "margin": 0.03316756141905454, "p_left": 0.5082911302882218, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6767009044391094, "brier": 0.24177761255323446 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.10078551633204691, "right_score": 0.06701979805511567, "margin": 0.03376571827693124, "p_left": 0.5084406276378151, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6764068301181194, "brier": 0.2416306165571052 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.2546305696703304, "right_score": 0.0016934492166865989, "margin": 0.2529371204536438, "p_left": 0.5628992938185841, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5746545410544531, "brier": 0.1910570273442925 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.10397774370054166, "right_score": -0.0697482679735836, "margin": 0.17372601167412527, "p_left": 0.5433225987259541, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6100520309907064, "brier": 0.20855424883441592 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.03571019972460085, "right_score": -0.038691975560557396, "margin": 0.002981775835956546, "p_left": 0.5007454434066786, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6916574040149475, "brier": 0.24925511227919397 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.10397774370054166, "right_score": 0.18499326645481873, "margin": -0.08101552275427706, "p_left": 0.4797571900991788, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6534596342732457, "brier": 0.23016696145185955 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.23423594357815292, "right_score": 0.18499326645481873, "margin": 0.04924267712333419, "p_left": 0.5123081822656693, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6688289165352668, "brier": 0.2378433090850156 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.06701979805511567, "right_score": 0.05006197741303451, "margin": 0.016957820642081158, "p_left": 0.504239353569251, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6847042157683251, "brier": 0.24577861854943414 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.06522470974429768, "right_score": 0.18499326645481873, "margin": -0.11976855671052104, "p_left": 0.4700936016573049, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6350548949333799, "brier": 0.22098799431913685 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1280228076121007, "right_score": -0.13902191780764347, "margin": 0.26704472541974417, "p_left": 0.5663672460104492, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5685125666461339, "brier": 0.1880373653325623 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1280228076121007, "right_score": -0.06872604284636673, "margin": 0.1967488504584674, "p_left": 0.5490291542306274, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6979781598665978, "brier": 0.0024038579645706457 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1280228076121007, "right_score": 0.0016934492166865989, "margin": 0.1263293583954141, "p_left": 0.531540404361607, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7583054229898565, "brier": 0.28253520146890065 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.0016934492166865989, "right_score": 0.06522470974429768, "margin": -0.06353126052761109, "p_left": 0.4841225249254919, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6618859931026151, "brier": 0.23437461914023353 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.10397774370054166, "right_score": -0.09226287730568865, "margin": 0.1962406210062303, "p_left": 0.548903315446035, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5998329632805074, "brier": 0.2034882188155794 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.05006197741303451, "right_score": 0.06737305060665771, "margin": -0.017311073193623204, "p_left": 0.4956723397747233, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6845291026523029, "brier": 0.24569106841774874 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.06522470974429768, "right_score": -0.13902191780764347, "margin": 0.20424662755194117, "p_left": 0.5508848840350232, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5962294135529516, "brier": 0.20170438738823454 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.10570304544684982, "right_score": -0.01863321051351382, "margin": 0.12433625596036363, "p_left": 0.531044080449146, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6329102471571486, "brier": 0.21991965448178707 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.2546305696703304, "right_score": 0.18499326645481873, "margin": 0.06963730321551165, "p_left": 0.5174022938819104, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6589345757612762, "brier": 0.232900545950442 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.23423594357815292, "right_score": -0.0697482679735836, "margin": 0.3039842115517365, "p_left": 0.5754162003635839, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5526616733017284, "brier": 0.18027140291369637 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.034205489187603175, "right_score": 0.10749168954980078, "margin": -0.0732862003621976, "p_left": 0.48168664572526215, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6571752885873621, "brier": 0.2320220246700542 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.15998482842316958, "right_score": 0.0016934492166865989, "margin": 0.15829137920648298, "p_left": 0.5394904226836511, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7754216258562491, "brier": 0.2910499161673845 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.15998482842316958, "right_score": -0.06872604284636673, "margin": 0.22871087126953632, "p_left": 0.5569297738689966, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.585316126233316, "brier": 0.19631122528377853 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.06522470974429768, "right_score": -0.09226287730568865, "margin": 0.15748758704998633, "p_left": 0.5392907218249862, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7749880679441359, "brier": 0.2908344826465146 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1280228076121007, "right_score": -0.0697482679735836, "margin": 0.1977710755856843, "p_left": 0.5492822405219899, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6980284080577683, "brier": 0.0024287392308672663 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.0697482679735836, "right_score": 0.15998482842316958, "margin": -0.22973309639675318, "p_left": 0.44281799758038615, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5848633376347306, "brier": 0.19608777898110288 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.23423594357815292, "right_score": -0.06872604284636673, "margin": 0.30296198642451966, "p_left": 0.5751664388598147, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5530958211832935, "brier": 0.1804835546710516 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.06872604284636673, "right_score": 0.2546305696703304, "margin": -0.3233566125166971, "p_left": 0.4198579341786919, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5444822643636177, "brier": 0.17628068489279877 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.034205489187603175, "right_score": -0.03571019972460085, "margin": 0.06991568891220402, "p_left": 0.5174718056626035, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7287159260503988, "brier": 0.2677770696557153 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.2546305696703304, "right_score": -0.09226287730568865, "margin": 0.34689344697601904, "p_left": 0.5858640457452341, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5346675201699038, "brier": 0.17150858860650556 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.004390702226837337, "right_score": 0.1477118503259845, "margin": -0.14332114809914714, "p_left": 0.4642309196030277, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6957126049454769, "brier": 0.0012794271124450679 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.09226287730568865, "right_score": 0.15998482842316958, "margin": -0.25224770572885824, "p_left": 0.4372703396657634, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5749559431905756, "brier": 0.19120534995141208 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.03571019972460085, "right_score": 0.004390702226837337, "margin": -0.040100901951438186, "p_left": 0.4899761177451408, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7133986283608938, "brier": 0.26012436047031845 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.23423594357815292, "right_score": -0.13902191780764347, "margin": 0.3732578613857964, "p_left": 0.5922459574155029, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5238332617740468, "brier": 0.16626335924399988 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 4, "regressions": 0, "both_correct": 3, "both_wrong": 4, "net_fixes": 4, "trial_ids": { "fixes": [ "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ], "regressions": [], "both_correct": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 4, "regressions": 0, "both_correct": 2, "both_wrong": 1, "net_fixes": 4, "trial_ids": { "fixes": [ "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ], "regressions": [], "both_correct": [ "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 1, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 6, "regressions": 0, "both_correct": 3, "both_wrong": 2, "net_fixes": 6, "trial_ids": { "fixes": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "34bec29a8d38", "c469d02ac7ba" ] } }, "other": { "fixes": 4, "regressions": 0, "both_correct": 2, "both_wrong": 1, "net_fixes": 4, "trial_ids": { "fixes": [ "0add2ec437a1", "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654" ], "regressions": [], "both_correct": [ "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "34bec29a8d38" ] } }, "target": { "fixes": 2, "regressions": 0, "both_correct": 1, "both_wrong": 1, "net_fixes": 2, "trial_ids": { "fixes": [ "1b6d9789fab8", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.610687905859317, "fold_losses": [ 0.49633817997366453, 0.726597380822996, 0.217076853769756, 0.13676752909132028, 0.28766175595700433, 0.41191351053828007, 0.42399517746702575, 0.8098549557860905, 0.893140921972505, 0.5653061647538569, 0.7387903622773608, 0.8655632973018383, 1.3941801741502198, 0.5853845307959373, 0.43496210447562955, 0.2185733406713597, 1.290020362638054, 0.4583533220171834, 1.440358267925417, 0.75589213774868, 0.4364509420151334, 0.1761822386186455, 0.2388508230625177, 0.6542954067931313 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.42288535889461065, "brier_score": 0.10659448896934315, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6977254133832548, "brier_score": 0.23150143537172355, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.7142857142857143, "mean_cross_entropy": 0.657609039028456, "brier_score": 0.20127730493865265, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.777958162092852, "brier_score": 0.2919496962378654, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.276704148750473, "brier_score": 0.06534191890730323, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.1359646514411044, "right_score": -0.21647874626864227, "margin": -0.9194859051724621, "p_left": 0.2850626562430001, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.33556037123705046, "brier": 0.08126071798431483 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.25384977883024473, "right_score": -0.44590745548620714, "margin": 0.1920576766559624, "p_left": 0.5478673726113807, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7937797187639914, "brier": 0.3001586579720975 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.6109806013273638, "right_score": 1.1315212389015756, "margin": -0.5205406375742119, "p_left": 0.3727258232326904, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4663715504093621, "brier": 0.13892453930448675 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6888596594742573, "right_score": 0.9127144861406629, "margin": -1.6015741456149202, "p_left": 0.1677617212776335, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.183636486468336, "brier": 0.02814399512603439 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.1359646514411044, "right_score": 0.9127144861406629, "margin": -2.0486791375817672, "p_left": 0.1141859151313191, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.12124818688714376, "brier": 0.013038423214376807 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.25384977883024473, "right_score": -0.7235027291274588, "margin": 0.4696529502972141, "p_left": 0.6153016110923056, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9552956571513098, "brier": 0.37859607261278694 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.1359646514411044, "right_score": 0.2672866331110632, "margin": -1.4032512845521676, "p_left": 0.19730068853525742, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21977509161905803, "brier": 0.03892756169648666 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.2955598329326959, "right_score": -0.0350447489045001, "margin": -0.2605150840281958, "p_left": 0.43523709335204414, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8318643541863893, "brier": 0.3189571407254477 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.05430339704767575, "right_score": -0.06888642818872334, "margin": 0.1231898252363991, "p_left": 0.5307585674933862, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6334480362899864, "brier": 0.22018752198085897 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.1315212389015756, "right_score": 0.11420429905384884, "margin": 1.0173169398477269, "p_left": 0.734449642492042, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30863384597446597, "brier": 0.0705169923726043 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.2672866331110632, "right_score": -0.730960577325219, "margin": 0.9982472104362823, "p_left": 0.7307138197472471, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3137333873389787, "brier": 0.0725150468751181 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.44590745548620714, "right_score": -0.7235027291274588, "margin": 0.2775952736412517, "p_left": 0.5689565749063046, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5639511660347932, "brier": 0.18579843431650422 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.2672866331110632, "right_score": 0.6109806013273638, "margin": -0.34369396821630055, "p_left": 0.41491244801380534, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5359937814335852, "brier": 0.17215233951680872 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.9127144861406629, "right_score": 0.6109806013273638, "margin": 0.3017338848132991, "p_left": 0.5748663245567294, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5536177442440006, "brier": 0.18073864199590417 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.06888642818872334, "right_score": -0.4142341673736004, "margin": 0.34534773918487705, "p_left": 0.585488965029502, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5353079432107833, "brier": 0.17181939811231342 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.13514720821224746, "right_score": 0.6109806013273638, "margin": -0.7461278095396112, "p_left": 0.3216656164839225, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.38811492159380023, "brier": 0.10346876882798191 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.21647874626864227, "right_score": -1.1359646514411044, "margin": 0.9194859051724621, "p_left": 0.7149373437569998, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3355603712370504, "brier": 0.0812607179843149 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.21647874626864227, "right_score": -0.6888596594742573, "margin": 0.47238091320561504, "p_left": 0.6159471315801146, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7207846010915884, "brier": 0.013443737321656409 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.21647874626864227, "right_score": 0.11420429905384884, "margin": -0.3306830453224911, "p_left": 0.4180744369965549, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5414127380499352, "brier": 0.17478623486998635 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.11420429905384884, "right_score": -0.13514720821224746, "margin": 0.24935150726609628, "p_left": 0.5620168782910266, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8255749042591993, "brier": 0.3158629714839907 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.2672866331110632, "right_score": -0.39955450259734715, "margin": 0.6668411357084103, "p_left": 0.6607954761938873, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4143109027523626, "brier": 0.11505970897053165 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.4142341673736004, "right_score": -0.2955598329326959, "margin": -0.11867433444090453, "p_left": 0.4703661874533079, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6355694309542224, "brier": 0.22124435029936038 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.13514720821224746, "right_score": -1.1359646514411044, "margin": 1.0008174432288568, "p_left": 0.7312192673641482, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3130419088554601, "brier": 0.07224308223626524 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.4989111317645456, "right_score": -0.2065038183030578, "margin": 0.7054149500676035, "p_left": 0.6693872409852216, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40139255076119373, "brier": 0.10930479642336392 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.1315212389015756, "right_score": 0.6109806013273638, "margin": 0.5205406375742119, "p_left": 0.6272741767673096, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4663715504093622, "brier": 0.13892453930448678 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.9127144861406629, "right_score": -0.730960577325219, "margin": 1.6436750634658819, "p_left": 0.8380343838086896, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17669614854383542, "brier": 0.026232860828230883 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.0350447489045001, "right_score": 0.366444041330194, "margin": -0.4014887902346941, "p_left": 0.4009546946486747, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5124180487496977, "brier": 0.16076466716081197 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.27025808678430313, "right_score": 0.11420429905384884, "margin": 0.1560537877304543, "p_left": 0.5389344654296431, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7742150886694675, "brier": 0.2904503580279351 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.27025808678430313, "right_score": -0.6888596594742573, "margin": 0.9591177462585605, "p_left": 0.7229451284595877, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3244219539502533, "brier": 0.07675940184427436 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.13514720821224746, "right_score": -0.39955450259734715, "margin": 0.2644072943850997, "p_left": 0.565719392086256, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8340643917996625, "brier": 0.320038430582443 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.21647874626864227, "right_score": -0.730960577325219, "margin": 0.5144818310565767, "p_left": 0.6258565311726264, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7258750331071717, "brier": 0.015839866438806284 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.730960577325219, "right_score": 0.27025808678430313, "margin": -1.0012186641095222, "p_left": 0.2687018849424327, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3129340842313657, "brier": 0.07220070297161633 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.9127144861406629, "right_score": -0.6888596594742573, "margin": 1.6015741456149202, "p_left": 0.8322382787223664, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18363648646833597, "brier": 0.028143995126034407 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6888596594742573, "right_score": 1.1315212389015756, "margin": -1.820380898375833, "p_left": 0.13938817445456578, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.15011171762459657, "brier": 0.019429063177776464 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.0350447489045001, "right_score": -0.44590745548620714, "margin": 0.41086270658170704, "p_left": 0.601294721651515, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9195327857848312, "brier": 0.36155534228597286 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.1315212389015756, "right_score": -0.39955450259734715, "margin": 1.5310757414989227, "p_left": 0.8221636535157137, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19581581187384822, "brier": 0.03162576613087911 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.18060839689622893, "right_score": 0.5334728648023845, "margin": -0.7140812616986134, "p_left": 0.3286976585265746, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7555762909240655, "brier": 0.02934449219427803 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.39955450259734715, "right_score": 0.27025808678430313, "margin": -0.6698125893816502, "p_left": 0.33853880643986306, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41330396145612747, "brier": 0.11460852346572707 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.44590745548620714, "right_score": -0.18060839689622893, "margin": -0.2652990585899782, "p_left": 0.43406153128022235, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8345689777877923, "brier": 0.3202863503768868 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.9127144861406629, "right_score": -1.1359646514411044, "margin": 2.0486791375817672, "p_left": 0.8858140848686809, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12124818688714356, "brier": 0.013038423214376819 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 3, "regressions": 0, "both_correct": 3, "both_wrong": 5, "net_fixes": 3, "trial_ids": { "fixes": [ "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ], "regressions": [], "both_correct": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 3, "regressions": 0, "both_correct": 2, "both_wrong": 2, "net_fixes": 3, "trial_ids": { "fixes": [ "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ], "regressions": [], "both_correct": [ "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 1, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "semantic_bt_tuned", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.5892915917843412, "fold_losses": [ 0.30532803824504245, 0.3061114327672558, 0.053797643627939526, 0.028246939675317623, 0.28732993840780285, 0.3526751695062367, 0.6098677229515012, 1.0042176619457917, 0.2387001593607665, 0.7215327305444785, 0.8840366503217462, 1.054168538344539, 2.185400095361611, 1.0540839515655303, 0.43859662353075834, 0.06232213833023437, 1.47129167932906, 0.21570541325684522, 0.8324421648333369, 0.900868589267606, 0.23760158910738402, 0.07123355436506629, 0.20224885674858162, 0.625190921429756 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.3775599321032988, "brier_score": 0.09520198673505154, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.1693106187546625, "brier_score": 0.38489281167894, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.34813811995763, "brier_score": 0.4259344363717877, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.8116556163487267, "brier_score": 0.3028095622932446, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.12120189309786672, "brier_score": 0.017939099503156124, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.11831325875144155, "margin": -1.520259690306019, "p_left": 0.17942328187303547, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19774787114479916, "brier": 0.03219271407809074 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.08973321215964836, "right_score": -0.03886648085526412, "margin": 0.12859969301491248, "p_left": 0.5321056887728856, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7595128392793792, "brier": 0.28313646402446696 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.1646357360289511, "right_score": 1.6038835186277196, "margin": -1.4392477825987684, "p_left": 0.19166186076299857, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21277481884850044, "brier": 0.03673426887113505 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.9459779677115197, "margin": -3.0358951170605737, "p_left": 0.045830342712650934, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.046913785517712504, "brier": 0.0021004203131590364 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.6385729490574605, "right_score": 1.9459779677115197, "margin": -3.58455091676898, "p_left": 0.026999902782041108, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027371096880454665, "brier": 0.0007289947502396711 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.08973321215964836, "right_score": -0.8607083987427133, "margin": 0.9504416109023617, "p_left": 0.7212039808699184, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2772748787842165, "brier": 0.5201351820226177 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.1618516026326928, "margin": -1.4767213464247677, "p_left": 0.18592315138486037, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20570050881770224, "brier": 0.03456741822087771 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.8453450229784956, "right_score": 1.0519678982652456, "margin": -2.897312921243741, "p_left": 0.052286554933550826, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.9510160168012463, "brier": 0.8981607739597176 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.24782753137386723, "right_score": 0.4893105018242937, "margin": -0.24148297045042647, "p_left": 0.4399209284038839, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8211602763922697, "brier": 0.3136885664399674 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.6038835186277196, "right_score": 0.24242247087959748, "margin": 1.3614610477481222, "p_left": 0.7959970536479521, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22815979459190072, "brier": 0.04161720212031651 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.925602308842444, "margin": 0.7637507062097512, "p_left": 0.6821674982216793, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3824800527275535, "brier": 0.10101749918666622 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": -0.8607083987427133, "margin": 0.8218419178874492, "p_left": 0.6946271882083916, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.364379997185982, "brier": 0.09325255418151306 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.1618516026326928, "right_score": 0.1646357360289511, "margin": -0.3264873386616439, "p_left": 0.41909555242250546, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5431689976564537, "brier": 0.17564108206032503 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9459779677115197, "right_score": 0.1646357360289511, "margin": 1.7813422316825687, "p_left": 0.855862525212909, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15564551712496733, "brier": 0.020775611637999306 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.4893105018242937, "right_score": 0.9498276872991533, "margin": -0.46051718547485965, "p_left": 0.38686314026384855, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9496842912277652, "brier": 0.3759368087671091 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.43814011720228163, "right_score": 0.1646357360289511, "margin": -0.6027758532312327, "p_left": 0.353708879377193, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43650522559335764, "brier": 0.12510997135026966 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.11831325875144155, "right_score": -1.6385729490574605, "margin": 1.520259690306019, "p_left": 0.8205767181269645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19774787114479908, "brier": 0.03219271407809074 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.11831325875144155, "right_score": -1.0899171493490538, "margin": 0.9716038905976123, "p_left": 0.7254390717510635, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.806780136529394, "brier": 0.05082277507198115 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.11831325875144155, "right_score": 0.24242247087959748, "margin": -0.36073572963103906, "p_left": 0.41078147821892863, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5289581594141014, "brier": 0.16874142284772814 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.24242247087959748, "right_score": -0.43814011720228163, "margin": 0.6805625880818791, "p_left": 0.6638642496663729, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0902401817624918, "brier": 0.44071574198509633 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.18096098415179185, "margin": 0.01910938151909905, "p_left": 0.5047772000070945, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6836381351636471, "brier": 0.2452456216328133 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.9498276872991533, "right_score": -1.8453450229784956, "margin": 2.795172710277649, "p_left": 0.9424144083470759, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.854482887486262, "brier": 0.8881449170601692 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.43814011720228163, "right_score": -1.6385729490574605, "margin": 1.200432831855179, "p_left": 0.768601772930407, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.26318229415302996, "brier": 0.053545139490950924 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.048839619073031366, "right_score": -0.9682958119567393, "margin": 1.0171354310297707, "p_left": 0.73441424071006, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30868204891878737, "brier": 0.07053579553761395 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.6038835186277196, "right_score": 0.1646357360289511, "margin": 1.4392477825987684, "p_left": 0.8083381392370015, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21277481884850036, "brier": 0.03673426887113503 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9459779677115197, "right_score": -0.925602308842444, "margin": 2.8715802765539635, "p_left": 0.9464235340927628, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.055065099665216, "brier": 0.002870437699109349 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.0519678982652456, "right_score": 1.0868302136944603, "margin": -0.034862315429214696, "p_left": 0.49128530376453017, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.675867937782102, "brier": 0.2413612496950067 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.596438676739378, "right_score": 0.24242247087959748, "margin": 0.35401620585978055, "p_left": 0.5875911583845447, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8857400876524333, "brier": 0.34526336941169106 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.596438676739378, "right_score": -1.0899171493490538, "margin": 1.6863558260884317, "p_left": 0.8437443141637132, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16990577558703546, "brier": 0.024415839356168357 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.43814011720228163, "right_score": -0.18096098415179185, "margin": -0.2571791330504898, "p_left": 0.4360572657714607, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5728025676948922, "brier": 0.19014593903208232 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.11831325875144155, "right_score": -0.925602308842444, "margin": 0.8072890500910024, "p_left": 0.6915315166474398, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7724910767417104, "brier": 0.036684321869268524 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.925602308842444, "right_score": 0.596438676739378, "margin": -1.522040985581822, "p_left": 0.17916117050402172, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1974284987937559, "brier": 0.03209872501637114 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.0899171493490538, "margin": 3.0358951170605737, "p_left": 0.954169657287349, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.046913785517712636, "brier": 0.002100420313159039 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.6038835186277196, "margin": -2.6938006679767734, "p_left": 0.06334015704800336, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06543509045362172, "brier": 0.004011975494865729 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.0519678982652456, "right_score": -0.03886648085526412, "margin": 1.0908343791205097, "p_left": 0.7485388080714952, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3804666077961052, "brier": 0.5603103471890947 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.6038835186277196, "right_score": -0.18096098415179185, "margin": 1.7848445027795115, "p_left": 0.8562940336589682, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15514146455692446, "brier": 0.020651404762009773 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.057065352799532953, "right_score": -1.4287911366492683, "margin": 1.485856489448801, "p_left": 0.8154555396440202, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9469366222683924, "brier": 0.09951219749209998 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.18096098415179185, "right_score": 0.596438676739378, "margin": -0.7773996608911699, "p_left": 0.3148805890842144, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3781621334075176, "brier": 0.09914978538202188 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": 0.057065352799532953, "margin": -0.09593183365479707, "p_left": 0.47603541743939987, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7422630211334409, "brier": 0.2745388837779039 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.6385729490574605, "margin": 3.58455091676898, "p_left": 0.9730000972179589, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.027371096880454804, "brier": 0.0007289947502396718 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 3, "both_wrong": 8, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 5, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 1, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "consistent_bt", "selected": { "ridge": 0.1, "consistency": 10.0, "mean_cross_entropy": 0.5945351124527156, "fold_losses": [ 0.3042420109702049, 0.37806157572421906, 0.5364343902262632, 0.2719688885531836, 0.2676223677375381, 0.6397034419794829, 0.5250736159899754, 1.2094767357501512, 0.5739865686320723, 0.5827679293244228, 0.7511100060793736, 0.7590441557094081, 1.898585062440634, 0.3736935730874533, 0.7034849561254136, 0.5350554422886682, 0.8817632125351166, 0.35088004715929944, 0.660770406976741, 0.7712052844521501, 0.3636441232186086, 0.3265569733237463, 0.29709565077121103, 0.30661627980983835 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5120509681518876, "brier_score": 0.14354190222763652, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.45454545454545453, "mean_cross_entropy": 0.8431625737099367, "brier_score": 0.2941468934016725, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.008598956528902, "brier_score": 0.3576766614462926, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.5122898080720065, "brier_score": 0.16708735731243224, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3182347318953053, "brier_score": 0.07610247730271964, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.322054370337053, "margin": -1.2654258040726614, "p_left": 0.22004128446547566, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24851428950116994, "brier": 0.04841816686921638 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.18238416788019518, "right_score": 0.8582962750030222, "margin": -0.675912107122827, "p_left": 0.33717428776286595, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4112432008249994, "brier": 0.11368650032839593 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.20826033514763365, "right_score": 0.6989493118224618, "margin": -0.9072096469700954, "p_left": 0.2875711689829475, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3390752581201555, "brier": 0.08269717723021894 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.30503695862105157, "right_score": 0.3065408150995909, "margin": -0.6115777737206425, "p_left": 0.3516993704681515, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4334007557514525, "brier": 0.12369244718769405 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.3065408150995909, "margin": -1.2499122488351992, "p_left": 0.222715329341627, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.25194862420844344, "brier": 0.04960211792374938 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.18238416788019518, "right_score": -0.5401699237521967, "margin": 0.7225540916323919, "p_left": 0.6731691973130416, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.118312664970008, "brier": 0.4531567682110847 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.4082665152447368, "margin": -1.3516379489803452, "p_left": 0.20560271619628323, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2301715854366309, "brier": 0.042272476907289386 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.4957564569813912, "right_score": 0.9481002769431781, "margin": -1.4438567339245694, "p_left": 0.19094882160614932, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.655749836546468, "brier": 0.6545638092604784 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.5945867768665508, "right_score": 0.44392116584665775, "margin": -1.0385079427132085, "p_left": 0.26143798995776313, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3415581556901037, "brier": 0.5454738426776292 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.6989493118224618, "right_score": 0.4705486828399298, "margin": 0.228400628982532, "p_left": 0.5568532174407075, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5854535972290251, "brier": 0.19637907089265288 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.29128126586411507, "margin": 0.6995477811088519, "p_left": 0.6680875017938281, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4033361233147016, "brier": 0.11016590646546204 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.8582962750030222, "right_score": -0.5401699237521967, "margin": 1.398466198755219, "p_left": 0.801940384651381, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22072100723087584, "brier": 0.039227611232042936 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.20826033514763365, "margin": 0.6165268503923704, "p_left": 0.6494282233764629, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0481898099534108, "brier": 0.421757017317909 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.3065408150995909, "right_score": -0.20826033514763365, "margin": 0.5148011502472245, "p_left": 0.6259312999914822, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4685146583269506, "brier": 0.13992739232606247 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.44392116584665775, "right_score": -0.10779785226845706, "margin": 0.5517190181151148, "p_left": 0.6345343243136319, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4548638966578751, "brier": 0.1335651601048936 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.20826033514763365, "margin": -0.48828114562703007, "p_left": 0.38029856933274947, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47851748037183056, "brier": 0.14462700183653607 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.322054370337053, "right_score": -0.9433714337356084, "margin": 1.2654258040726614, "p_left": 0.7799587155345244, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24851428950116983, "brier": 0.04841816686921633 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.322054370337053, "right_score": -0.30503695862105157, "margin": 0.6270913289581046, "p_left": 0.6518296376704695, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7415177075936744, "brier": 0.02305223887514606 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.322054370337053, "right_score": 0.4705486828399298, "margin": -0.1484943125028768, "p_left": 0.46294448809823446, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6216538156972643, "brier": 0.21431759906053635 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.4705486828399298, "right_score": -0.6965414807746637, "margin": 1.1670901636145934, "p_left": 0.7626186464476711, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4380873443212985, "brier": 0.5815871999096779 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.43690806903301493, "margin": 0.8451745842777517, "p_left": 0.6995539227555715, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3573124002771858, "brier": 0.09026784533156511 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.10779785226845706, "right_score": -0.4957564569813912, "margin": 0.38795860471293414, "p_left": 0.5957911785022925, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9058236496646546, "brier": 0.3549671283811505 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.9433714337356084, "margin": 0.24682995296094468, "p_left": 0.5613960910984787, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5773285778529474, "brier": 0.19237338890369404 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.332305192832029, "right_score": -0.41599073173726603, "margin": 0.08368553890523706, "p_left": 0.5209091834299748, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6521795644574757, "brier": 0.2295280105217335 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.6989493118224618, "right_score": -0.20826033514763365, "margin": 0.9072096469700954, "p_left": 0.7124288310170525, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3390752581201555, "brier": 0.08269717723021897 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.3065408150995909, "right_score": -0.29128126586411507, "margin": 0.597822080963706, "p_left": 0.6451578747540792, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43826022507634654, "brier": 0.12591293384904179 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.9481002769431781, "right_score": 0.46987707022675773, "margin": 0.4782232067164204, "p_left": 0.6173282232225964, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9605776369119794, "brier": 0.3810941351871678 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.6750398478323165, "right_score": 0.4705486828399298, "margin": 0.20449116499238673, "p_left": 0.5509453844682042, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8006107604817561, "brier": 0.3035408166668173 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.6750398478323165, "right_score": -0.30503695862105157, "margin": 0.980076806453368, "p_left": 0.7271234561544773, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31865899997233793, "brier": 0.07446160818107748 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.43690806903301493, "margin": -0.2596334117416488, "p_left": 0.4354538258450435, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5717331021831802, "brier": 0.18962003444308548 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.322054370337053, "right_score": -0.29128126586411507, "margin": 0.613335636201168, "p_left": 0.6487013297850049, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7394506869428821, "brier": 0.0221120854798288 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.29128126586411507, "right_score": 0.6750398478323165, "margin": -0.9663211136964316, "p_left": 0.27561438925545617, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.32243141685798343, "brier": 0.07596329156465811 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.3065408150995909, "right_score": -0.30503695862105157, "margin": 0.6115777737206425, "p_left": 0.6483006295318486, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43340075575145254, "brier": 0.12369244718769398 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.30503695862105157, "right_score": 0.6989493118224618, "margin": -1.0039862704435134, "p_left": 0.2681583952841115, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.31219117543602254, "brier": 0.07190892496134979 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.9481002769431781, "right_score": 0.8582962750030222, "margin": 0.08980400194015592, "p_left": 0.5224359241517104, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7390569378058612, "brier": 0.27293929484425167 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.6989493118224618, "right_score": -0.43690806903301493, "margin": 1.1358573808554766, "p_left": 0.7569182374591771, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2785000400184947, "brier": 0.05908874327995302 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.8280493767402581, "right_score": -0.7593440000306045, "margin": 1.5873933767708626, "p_left": 0.8302490533312744, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.979726247332652, "brier": 0.10906443722620293 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.43690806903301493, "right_score": 0.6750398478323165, "margin": -1.1119479168653315, "p_left": 0.24750791502506425, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.28436480072015724, "brier": 0.061260168000054425 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.8582962750030222, "right_score": 0.8280493767402581, "margin": 0.030246898262764055, "p_left": 0.5075611481157905, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6781380864262897, "brier": 0.24249602284503846 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.3065408150995909, "right_score": -0.9433714337356084, "margin": 1.2499122488351992, "p_left": 0.7772846706583731, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25194862420844344, "brier": 0.049602117923749314 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 3, "regressions": 1, "both_correct": 2, "both_wrong": 5, "net_fixes": 2, "trial_ids": { "fixes": [ "1b6d9789fab8", "70f39a6b70be", "fe73774e1a9e" ], "regressions": [ "396ff40ef0fb" ], "both_correct": [ "974972ffb84a", "e5aa020b210d" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "b20ebcae7654", "c469d02ac7ba" ] } }, "other": { "fixes": 1, "regressions": 1, "both_correct": 1, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [ "70f39a6b70be" ], "regressions": [ "396ff40ef0fb" ], "both_correct": [ "e5aa020b210d" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "b20ebcae7654" ] } }, "target": { "fixes": 2, "regressions": 0, "both_correct": 1, "both_wrong": 1, "net_fixes": 2, "trial_ids": { "fixes": [ "1b6d9789fab8", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "semantic_bt_matched", "selected": { "ridge": 0.1, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.3775599321032988, "brier_score": 0.09520198673505154, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.1693106187546625, "brier_score": 0.38489281167894, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.34813811995763, "brier_score": 0.4259344363717877, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.8116556163487267, "brier_score": 0.3028095622932446, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.12120189309786672, "brier_score": 0.017939099503156124, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.11831325875144155, "margin": -1.520259690306019, "p_left": 0.17942328187303547, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19774787114479916, "brier": 0.03219271407809074 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.08973321215964836, "right_score": -0.03886648085526412, "margin": 0.12859969301491248, "p_left": 0.5321056887728856, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7595128392793792, "brier": 0.28313646402446696 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.1646357360289511, "right_score": 1.6038835186277196, "margin": -1.4392477825987684, "p_left": 0.19166186076299857, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21277481884850044, "brier": 0.03673426887113505 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.9459779677115197, "margin": -3.0358951170605737, "p_left": 0.045830342712650934, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.046913785517712504, "brier": 0.0021004203131590364 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.6385729490574605, "right_score": 1.9459779677115197, "margin": -3.58455091676898, "p_left": 0.026999902782041108, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027371096880454665, "brier": 0.0007289947502396711 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.08973321215964836, "right_score": -0.8607083987427133, "margin": 0.9504416109023617, "p_left": 0.7212039808699184, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2772748787842165, "brier": 0.5201351820226177 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.1618516026326928, "margin": -1.4767213464247677, "p_left": 0.18592315138486037, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20570050881770224, "brier": 0.03456741822087771 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.8453450229784956, "right_score": 1.0519678982652456, "margin": -2.897312921243741, "p_left": 0.052286554933550826, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.9510160168012463, "brier": 0.8981607739597176 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.24782753137386723, "right_score": 0.4893105018242937, "margin": -0.24148297045042647, "p_left": 0.4399209284038839, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8211602763922697, "brier": 0.3136885664399674 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.6038835186277196, "right_score": 0.24242247087959748, "margin": 1.3614610477481222, "p_left": 0.7959970536479521, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22815979459190072, "brier": 0.04161720212031651 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.925602308842444, "margin": 0.7637507062097512, "p_left": 0.6821674982216793, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3824800527275535, "brier": 0.10101749918666622 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": -0.8607083987427133, "margin": 0.8218419178874492, "p_left": 0.6946271882083916, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.364379997185982, "brier": 0.09325255418151306 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.1618516026326928, "right_score": 0.1646357360289511, "margin": -0.3264873386616439, "p_left": 0.41909555242250546, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5431689976564537, "brier": 0.17564108206032503 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9459779677115197, "right_score": 0.1646357360289511, "margin": 1.7813422316825687, "p_left": 0.855862525212909, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15564551712496733, "brier": 0.020775611637999306 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.4893105018242937, "right_score": 0.9498276872991533, "margin": -0.46051718547485965, "p_left": 0.38686314026384855, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9496842912277652, "brier": 0.3759368087671091 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.43814011720228163, "right_score": 0.1646357360289511, "margin": -0.6027758532312327, "p_left": 0.353708879377193, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43650522559335764, "brier": 0.12510997135026966 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.11831325875144155, "right_score": -1.6385729490574605, "margin": 1.520259690306019, "p_left": 0.8205767181269645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19774787114479908, "brier": 0.03219271407809074 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.11831325875144155, "right_score": -1.0899171493490538, "margin": 0.9716038905976123, "p_left": 0.7254390717510635, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.806780136529394, "brier": 0.05082277507198115 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.11831325875144155, "right_score": 0.24242247087959748, "margin": -0.36073572963103906, "p_left": 0.41078147821892863, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5289581594141014, "brier": 0.16874142284772814 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.24242247087959748, "right_score": -0.43814011720228163, "margin": 0.6805625880818791, "p_left": 0.6638642496663729, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0902401817624918, "brier": 0.44071574198509633 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.18096098415179185, "margin": 0.01910938151909905, "p_left": 0.5047772000070945, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6836381351636471, "brier": 0.2452456216328133 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.9498276872991533, "right_score": -1.8453450229784956, "margin": 2.795172710277649, "p_left": 0.9424144083470759, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.854482887486262, "brier": 0.8881449170601692 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.43814011720228163, "right_score": -1.6385729490574605, "margin": 1.200432831855179, "p_left": 0.768601772930407, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.26318229415302996, "brier": 0.053545139490950924 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.048839619073031366, "right_score": -0.9682958119567393, "margin": 1.0171354310297707, "p_left": 0.73441424071006, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30868204891878737, "brier": 0.07053579553761395 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.6038835186277196, "right_score": 0.1646357360289511, "margin": 1.4392477825987684, "p_left": 0.8083381392370015, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21277481884850036, "brier": 0.03673426887113503 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9459779677115197, "right_score": -0.925602308842444, "margin": 2.8715802765539635, "p_left": 0.9464235340927628, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.055065099665216, "brier": 0.002870437699109349 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.0519678982652456, "right_score": 1.0868302136944603, "margin": -0.034862315429214696, "p_left": 0.49128530376453017, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.675867937782102, "brier": 0.2413612496950067 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.596438676739378, "right_score": 0.24242247087959748, "margin": 0.35401620585978055, "p_left": 0.5875911583845447, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8857400876524333, "brier": 0.34526336941169106 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.596438676739378, "right_score": -1.0899171493490538, "margin": 1.6863558260884317, "p_left": 0.8437443141637132, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16990577558703546, "brier": 0.024415839356168357 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.43814011720228163, "right_score": -0.18096098415179185, "margin": -0.2571791330504898, "p_left": 0.4360572657714607, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5728025676948922, "brier": 0.19014593903208232 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.11831325875144155, "right_score": -0.925602308842444, "margin": 0.8072890500910024, "p_left": 0.6915315166474398, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7724910767417104, "brier": 0.036684321869268524 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.925602308842444, "right_score": 0.596438676739378, "margin": -1.522040985581822, "p_left": 0.17916117050402172, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1974284987937559, "brier": 0.03209872501637114 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.0899171493490538, "margin": 3.0358951170605737, "p_left": 0.954169657287349, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.046913785517712636, "brier": 0.002100420313159039 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.6038835186277196, "margin": -2.6938006679767734, "p_left": 0.06334015704800336, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06543509045362172, "brier": 0.004011975494865729 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.0519678982652456, "right_score": -0.03886648085526412, "margin": 1.0908343791205097, "p_left": 0.7485388080714952, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3804666077961052, "brier": 0.5603103471890947 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.6038835186277196, "right_score": -0.18096098415179185, "margin": 1.7848445027795115, "p_left": 0.8562940336589682, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15514146455692446, "brier": 0.020651404762009773 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.057065352799532953, "right_score": -1.4287911366492683, "margin": 1.485856489448801, "p_left": 0.8154555396440202, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9469366222683924, "brier": 0.09951219749209998 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.18096098415179185, "right_score": 0.596438676739378, "margin": -0.7773996608911699, "p_left": 0.3148805890842144, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3781621334075176, "brier": 0.09914978538202188 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": 0.057065352799532953, "margin": -0.09593183365479707, "p_left": 0.47603541743939987, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7422630211334409, "brier": 0.2745388837779039 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.6385729490574605, "margin": 3.58455091676898, "p_left": 0.9730000972179589, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.027371096880454804, "brier": 0.0007289947502396718 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 3, "both_wrong": 8, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a", "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 5, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "e5aa020b210d", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 1, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "974972ffb84a" ], "both_wrong": [ "1b6d9789fab8", "c469d02ac7ba", "fe73774e1a9e" ] } } } } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.0029447699503040324, "rank": 11.0, "portrait_scores": [ -0.009538983211527217, 0.003649443310919152 ], "portrait_score_gap": 0.013188426522446369, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.006366805501861729, "rank": 2.0, "portrait_scores": [ 0.002772053205905692, 0.009961557797817766 ], "portrait_score_gap": 0.007189504591912074, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.5, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.000818008680712527, "rank": 10.0, "portrait_scores": [ 0.0026960506350724557, -0.00433206799649751 ], "portrait_score_gap": 0.0070281186315699655, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.005702293494479819, "rank": 3.0, "portrait_scores": [ 0.0064449784748520835, 0.004959608514107554 ], "portrait_score_gap": 0.0014853699607445293, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.0, 6.0 ] }, { "identity": "emma_stone", "mean_score": -0.009751167958215307, "rank": 13.0, "portrait_scores": [ 0.0025443112275344657, -0.022046647143965082 ], "portrait_score_gap": 0.02459095837149955, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.002713974872993883, "rank": 6.0, "portrait_scores": [ -0.0033791187613289695, 0.008807068507316735 ], "portrait_score_gap": 0.012186187268645704, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": 0.003528364324779695, "rank": 5.0, "portrait_scores": [ 0.0030453779791719425, 0.004011350670387447 ], "portrait_score_gap": 0.0009659726912155048, "single_photo_rank_min": 3.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": 0.0024525376389379866, "rank": 7.0, "portrait_scores": [ -0.0019279379021871915, 0.006833013180063165 ], "portrait_score_gap": 0.008760951082250355, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 6.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": 0.001973474479071992, "rank": 8.0, "portrait_scores": [ 0.003943926375389773, 3.0225827542113004e-06 ], "portrait_score_gap": 0.003940903792635561, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.011078225800300573, "rank": 1.0, "portrait_scores": [ 0.008251297296183728, 0.01390515430441742 ], "portrait_score_gap": 0.0056538570082336915, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": 0.005365045299471259, "rank": 4.0, "portrait_scores": [ 0.0038640235490429893, 0.006866067049899528 ], "portrait_score_gap": 0.0030020435008565388, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.001514875938342089, "rank": 9.0, "portrait_scores": [ 0.005655011872031997, -0.0026252599953478185 ], "portrait_score_gap": 0.008280271867379816, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.5, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.003206817399792488, "rank": 12.0, "portrait_scores": [ 0.0002987519566857791, -0.006712386756270755 ], "portrait_score_gap": 0.007011138712956534, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "6.0": 32, "7.0": 224, "8.0": 672, "9.0": 1120, "10.0": 1120, "11.0": 1696, "12.0": 2272, "13.0": 1056 }, "target_primary_score": -0.003206817399792488, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ 0.0010974737458220116, 0.005048462047064254 ], "target_alternate_panel_score": 0.0030729678964431326, "target_alternate_panel_rank": 6.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.0010994934173711407, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.002629325598036686, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.0032938376054185963, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.006017737846747165, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 2.4660733883063767e-05, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.004390702226837337, "rank": 10.0, "portrait_scores": [ -0.017760228204966984, 0.026541632658641658 ], "portrait_score_gap": 0.04430186086360864, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.034205489187603175, "rank": 8.0, "portrait_scores": [ -0.007828359509347238, 0.07623933788455359 ], "portrait_score_gap": 0.08406769739390083, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.01863321051351382, "rank": 11.0, "portrait_scores": [ 0.06376529039194849, -0.10103171141897613 ], "portrait_score_gap": 0.1647970018109246, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.038691975560557396, "rank": 13.0, "portrait_scores": [ -0.05259200545647568, -0.024791945664639103 ], "portrait_score_gap": 0.027800059791836577, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 12.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.10570304544684982, "rank": 3.0, "portrait_scores": [ 0.10448274238665109, 0.10692334850704854 ], "portrait_score_gap": 0.002440606120397454, "single_photo_rank_min": 2.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 3.5, 5.0 ] }, { "identity": "emma_watson", "mean_score": 0.06701979805511567, "rank": 6.0, "portrait_scores": [ 0.09909417587885525, 0.03494542023137608 ], "portrait_score_gap": 0.06414875564747916, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "gal_gadot", "mean_score": 0.10078551633204691, "rank": 4.0, "portrait_scores": [ 0.07410114116427254, 0.12746989149982127 ], "portrait_score_gap": 0.053368750335548734, "single_photo_rank_min": 2.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 3.5, 7.0 ] }, { "identity": "keira_knightley", "mean_score": 0.06737305060665771, "rank": 5.0, "portrait_scores": [ -0.002918702903504217, 0.13766480411681964 ], "portrait_score_gap": 0.14058350702032385, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "lily_collins", "mean_score": 0.007578623552076993, "rank": 9.0, "portrait_scores": [ 0.026252840075390343, -0.011095592971236357 ], "portrait_score_gap": 0.0373484330466267, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.1477118503259845, "rank": 1.0, "portrait_scores": [ 0.16199305167643008, 0.13343064897553894 ], "portrait_score_gap": 0.028562402700891137, "single_photo_rank_min": 1.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.75, "single_photo_rank_quantiles": [ 1.0, 1.0, 2.0 ] }, { "identity": "natalie_portman", "mean_score": 0.05006197741303451, "rank": 7.0, "portrait_scores": [ 0.10084720279335108, -0.0007232479672820563 ], "portrait_score_gap": 0.10157045076063313, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.10749168954980078, "rank": 2.0, "portrait_scores": [ 0.1204276029670659, 0.09455577613253566 ], "portrait_score_gap": 0.025871826834530237, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.03571019972460085, "rank": 12.0, "portrait_scores": [ -0.06734687943448237, -0.004073520014719327 ], "portrait_score_gap": 0.06327335941976303, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 256, "9.0": 1024, "10.0": 1536, "11.0": 1024, "12.0": 2304, "13.0": 2048 }, "target_primary_score": -0.03571019972460085, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ 0.007792452926562662, -0.012296227243939212 ], "target_alternate_panel_score": -0.0022518871586882754, "target_alternate_panel_rank": 11.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.009830510710765268, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.03644008840186912, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.036457376346291454, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.006642589385525612, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.0015556545254246145, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.18060839689622893, "rank": 7.0, "portrait_scores": [ -0.22418288349032084, -0.13703391030213702 ], "portrait_score_gap": 0.08714897318818382, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.0350447489045001, "rank": 5.0, "portrait_scores": [ -0.22288717307253325, 0.15279767526353305 ], "portrait_score_gap": 0.3756848483360663, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 5.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.2065038183030578, "rank": 8.0, "portrait_scores": [ 0.22380810183933564, -0.6368157384454513 ], "portrait_score_gap": 0.8606238402847869, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.5, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7235027291274588, "rank": 13.0, "portrait_scores": [ -0.8861573045807204, -0.5608481536741973 ], "portrait_score_gap": 0.3253091509065231, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.4989111317645456, "rank": 2.0, "portrait_scores": [ 0.5603683762917054, 0.4374538872373857 ], "portrait_score_gap": 0.12291448905431973, "single_photo_rank_min": 1.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 2.0 ] }, { "identity": "emma_watson", "mean_score": -0.06888642818872334, "rank": 6.0, "portrait_scores": [ 0.1257377927878689, -0.2635106491653156 ], "portrait_score_gap": 0.3892484419531845, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.5, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.05430339704767575, "rank": 4.0, "portrait_scores": [ -0.012339578868379377, 0.12094637296373087 ], "portrait_score_gap": 0.13328595183211026, "single_photo_rank_min": 4.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.2955598329326959, "rank": 10.0, "portrait_scores": [ -0.6631597483238459, 0.07204008245845414 ], "portrait_score_gap": 0.7351998307823001, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.25384977883024473, "rank": 9.0, "portrait_scores": [ -0.07141152615023622, -0.4362880315102532 ], "portrait_score_gap": 0.364876505360017, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.5, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.5334728648023845, "rank": 1.0, "portrait_scores": [ 0.5698765244191858, 0.4970692051855834 ], "portrait_score_gap": 0.0728073192336024, "single_photo_rank_min": 1.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.75, "single_photo_rank_quantiles": [ 1.0, 1.0, 2.0 ] }, { "identity": "natalie_portman", "mean_score": -0.4142341673736004, "rank": 11.0, "portrait_scores": [ -0.15545278314564875, -0.6730155516015521 ], "portrait_score_gap": 0.5175627684559034, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.5, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.366444041330194, "rank": 3.0, "portrait_scores": [ 0.42079746356873915, 0.3120906190916488 ], "portrait_score_gap": 0.10870684447709034, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.44590745548620714, "rank": 12.0, "portrait_scores": [ -0.6380555609840962, -0.253759349988318 ], "portrait_score_gap": 0.3842962109957782, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 128, "8.0": 640, "9.0": 1280, "10.0": 1792, "11.0": 2176, "12.0": 1664, "13.0": 512 }, "target_primary_score": -0.44590745548620714, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -0.15764965000163375, -0.2593780446597911 ], "target_alternate_panel_score": -0.20851384733071243, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.0453359314995323, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.5149888817967464, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.17346909842621233, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.027905450434483503, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.043960528639596624, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.057065352799532953, "rank": 7.0, "portrait_scores": [ -0.9771077384798927, 1.0912384440789586 ], "portrait_score_gap": 2.0683461825588516, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.0519678982652456, "rank": 2.0, "portrait_scores": [ 0.6740879239201923, 1.4298478726102988 ], "portrait_score_gap": 0.7557599486901064, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.9682958119567393, "rank": 11.0, "portrait_scores": [ -1.6766400415866847, -0.25995158232679383 ], "portrait_score_gap": 1.4166884592598907, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.8607083987427133, "rank": 10.0, "portrait_scores": [ -1.5812953017353384, -0.14012149575008825 ], "portrait_score_gap": 1.4411738059852501, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.048839619073031366, "rank": 8.0, "portrait_scores": [ -0.07778574455110553, 0.17546498269716826 ], "portrait_score_gap": 0.2532507272482738, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.4893105018242937, "rank": 4.0, "portrait_scores": [ 1.521128612569642, -0.5425076089210545 ], "portrait_score_gap": 2.0636362214906967, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.24782753137386723, "rank": 5.0, "portrait_scores": [ 0.6884649859811793, -0.19280992323344487 ], "portrait_score_gap": 0.8812749092146241, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -1.8453450229784956, "rank": 13.0, "portrait_scores": [ -2.7560882879980473, -0.9346017579589441 ], "portrait_score_gap": 1.8214865300391032, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.08973321215964836, "rank": 6.0, "portrait_scores": [ 0.9670142958211776, -0.7875478715018809 ], "portrait_score_gap": 1.7545621673230585, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -1.4287911366492683, "rank": 12.0, "portrait_scores": [ -0.13096143825103956, -2.726620835047497 ], "portrait_score_gap": 2.5956593967964574, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.9498276872991533, "rank": 3.0, "portrait_scores": [ 2.0263269912254755, -0.12667161662716891 ], "portrait_score_gap": 2.1529986078526444, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.0, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.0868302136944603, "rank": 1.0, "portrait_scores": [ 0.4591683136751561, 1.7144921137137643 ], "portrait_score_gap": 1.2553238000386082, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.03886648085526412, "rank": 9.0, "portrait_scores": [ -0.11842536620618582, 0.040692404495657586 ], "portrait_score_gap": 0.1591177707018434, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 64, "4.0": 512, "5.0": 1600, "6.0": 2560, "7.0": 2240, "8.0": 1024, "9.0": 192 }, "target_primary_score": -0.03886648085526412, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ -0.7131870836860958, -0.013387833808842231 ], "target_alternate_panel_score": -0.36328745874746904, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.4530206709071174, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.4974209399952443, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.4152553570127147, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.420352811547002, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6423634924485461, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.8280493767402581, "rank": 3.0, "portrait_scores": [ 1.1023359023533006, 0.5537628511272158 ], "portrait_score_gap": 0.5485730512260848, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.9481002769431781, "rank": 1.0, "portrait_scores": [ 0.15314572894819922, 1.743054824938157 ], "portrait_score_gap": 1.5899090959899578, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.5, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.41599073173726603, "rank": 9.0, "portrait_scores": [ -0.8699223169974357, 0.03794085352290366 ], "portrait_score_gap": 0.9078631705203394, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.5401699237521967, "rank": 11.0, "portrait_scores": [ -0.8725280969349682, -0.20781175056942525 ], "portrait_score_gap": 0.664716346365543, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.332305192832029, "rank": 8.0, "portrait_scores": [ -0.14380151743312272, -0.5208088682309352 ], "portrait_score_gap": 0.37700735079781245, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "emma_watson", "mean_score": 0.44392116584665775, "rank": 5.0, "portrait_scores": [ 1.3763184702853273, -0.4884761385920118 ], "portrait_score_gap": 1.864794608877339, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.5, 11.0 ] }, { "identity": "gal_gadot", "mean_score": -0.5945867768665508, "rank": 12.0, "portrait_scores": [ -0.7105041416263659, -0.4786694121067357 ], "portrait_score_gap": 0.2318347295196302, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.4957564569813912, "rank": 10.0, "portrait_scores": [ -0.992767409892572, 0.0012544959297895453 ], "portrait_score_gap": 0.9940219058223615, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.18238416788019518, "rank": 6.0, "portrait_scores": [ 1.1437250622453399, -0.7789567264849495 ], "portrait_score_gap": 1.9226817887302894, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.7593440000306045, "rank": 13.0, "portrait_scores": [ -0.3209388549477432, -1.1977491451134659 ], "portrait_score_gap": 0.8768102901657227, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "natalie_portman", "mean_score": -0.10779785226845706, "rank": 7.0, "portrait_scores": [ 0.5547282377635694, -0.7703239423004835 ], "portrait_score_gap": 1.325052180064053, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.5, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.46987707022675773, "rank": 4.0, "portrait_scores": [ 0.6126307940809954, 0.32712334637252005 ], "portrait_score_gap": 0.28550744770847536, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.8582962750030222, "rank": 2.0, "portrait_scores": [ 1.1207890563202303, 0.5958034936858141 ], "portrait_score_gap": 0.5249855626344162, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.078125, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "1.0": 640, "2.0": 2176, "3.0": 2816, "4.0": 1792, "5.0": 640, "6.0": 128 }, "target_primary_score": 0.8582962750030222, "target_primary_rank": 2.0, "target_alternate_photo_scores": [ 0.0953131147490398, -0.18709787751734763 ], "target_alternate_panel_score": -0.04589238138415391, "target_alternate_panel_rank": 6.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.2282765492643491, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.49427754236804283, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.993992658327332, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.873941758124412, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.29489646576337153, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.057065352799532953, "rank": 7.0, "portrait_scores": [ -0.9771077384798927, 1.0912384440789586 ], "portrait_score_gap": 2.0683461825588516, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.0519678982652456, "rank": 2.0, "portrait_scores": [ 0.6740879239201923, 1.4298478726102988 ], "portrait_score_gap": 0.7557599486901064, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.9682958119567393, "rank": 11.0, "portrait_scores": [ -1.6766400415866847, -0.25995158232679383 ], "portrait_score_gap": 1.4166884592598907, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.8607083987427133, "rank": 10.0, "portrait_scores": [ -1.5812953017353384, -0.14012149575008825 ], "portrait_score_gap": 1.4411738059852501, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.048839619073031366, "rank": 8.0, "portrait_scores": [ -0.07778574455110553, 0.17546498269716826 ], "portrait_score_gap": 0.2532507272482738, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.4893105018242937, "rank": 4.0, "portrait_scores": [ 1.521128612569642, -0.5425076089210545 ], "portrait_score_gap": 2.0636362214906967, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.24782753137386723, "rank": 5.0, "portrait_scores": [ 0.6884649859811793, -0.19280992323344487 ], "portrait_score_gap": 0.8812749092146241, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -1.8453450229784956, "rank": 13.0, "portrait_scores": [ -2.7560882879980473, -0.9346017579589441 ], "portrait_score_gap": 1.8214865300391032, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.08973321215964836, "rank": 6.0, "portrait_scores": [ 0.9670142958211776, -0.7875478715018809 ], "portrait_score_gap": 1.7545621673230585, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -1.4287911366492683, "rank": 12.0, "portrait_scores": [ -0.13096143825103956, -2.726620835047497 ], "portrait_score_gap": 2.5956593967964574, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.9498276872991533, "rank": 3.0, "portrait_scores": [ 2.0263269912254755, -0.12667161662716891 ], "portrait_score_gap": 2.1529986078526444, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.0, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.0868302136944603, "rank": 1.0, "portrait_scores": [ 0.4591683136751561, 1.7144921137137643 ], "portrait_score_gap": 1.2553238000386082, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.03886648085526412, "rank": 9.0, "portrait_scores": [ -0.11842536620618582, 0.040692404495657586 ], "portrait_score_gap": 0.1591177707018434, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 64, "4.0": 512, "5.0": 1600, "6.0": 2560, "7.0": 2240, "8.0": 1024, "9.0": 192 }, "target_primary_score": -0.03886648085526412, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ -0.7131870836860958, -0.013387833808842231 ], "target_alternate_panel_score": -0.36328745874746904, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.4530206709071174, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.4974209399952443, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.4152553570127147, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.420352811547002, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6423634924485461, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, "consistency_ablation": [ { "name": "gamma_0", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.3775599321032988, "brier_score": 0.09520198673505154, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.1693106187546625, "brier_score": 0.38489281167894, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.34813811995763, "brier_score": 0.4259344363717877, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.8116556163487267, "brier_score": 0.3028095622932446, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.12120189309786672, "brier_score": 0.017939099503156124, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.11831325875144155, "margin": -1.520259690306019, "p_left": 0.17942328187303547, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19774787114479916, "brier": 0.03219271407809074 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.08973321215964836, "right_score": -0.03886648085526412, "margin": 0.12859969301491248, "p_left": 0.5321056887728856, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7595128392793792, "brier": 0.28313646402446696 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.1646357360289511, "right_score": 1.6038835186277196, "margin": -1.4392477825987684, "p_left": 0.19166186076299857, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21277481884850044, "brier": 0.03673426887113505 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.9459779677115197, "margin": -3.0358951170605737, "p_left": 0.045830342712650934, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.046913785517712504, "brier": 0.0021004203131590364 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.6385729490574605, "right_score": 1.9459779677115197, "margin": -3.58455091676898, "p_left": 0.026999902782041108, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027371096880454665, "brier": 0.0007289947502396711 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.08973321215964836, "right_score": -0.8607083987427133, "margin": 0.9504416109023617, "p_left": 0.7212039808699184, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2772748787842165, "brier": 0.5201351820226177 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.6385729490574605, "right_score": -0.1618516026326928, "margin": -1.4767213464247677, "p_left": 0.18592315138486037, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20570050881770224, "brier": 0.03456741822087771 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.8453450229784956, "right_score": 1.0519678982652456, "margin": -2.897312921243741, "p_left": 0.052286554933550826, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.9510160168012463, "brier": 0.8981607739597176 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.24782753137386723, "right_score": 0.4893105018242937, "margin": -0.24148297045042647, "p_left": 0.4399209284038839, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8211602763922697, "brier": 0.3136885664399674 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.6038835186277196, "right_score": 0.24242247087959748, "margin": 1.3614610477481222, "p_left": 0.7959970536479521, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22815979459190072, "brier": 0.04161720212031651 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.925602308842444, "margin": 0.7637507062097512, "p_left": 0.6821674982216793, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3824800527275535, "brier": 0.10101749918666622 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": -0.8607083987427133, "margin": 0.8218419178874492, "p_left": 0.6946271882083916, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.364379997185982, "brier": 0.09325255418151306 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.1618516026326928, "right_score": 0.1646357360289511, "margin": -0.3264873386616439, "p_left": 0.41909555242250546, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5431689976564537, "brier": 0.17564108206032503 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.9459779677115197, "right_score": 0.1646357360289511, "margin": 1.7813422316825687, "p_left": 0.855862525212909, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15564551712496733, "brier": 0.020775611637999306 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.4893105018242937, "right_score": 0.9498276872991533, "margin": -0.46051718547485965, "p_left": 0.38686314026384855, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9496842912277652, "brier": 0.3759368087671091 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.43814011720228163, "right_score": 0.1646357360289511, "margin": -0.6027758532312327, "p_left": 0.353708879377193, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43650522559335764, "brier": 0.12510997135026966 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.11831325875144155, "right_score": -1.6385729490574605, "margin": 1.520259690306019, "p_left": 0.8205767181269645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19774787114479908, "brier": 0.03219271407809074 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.11831325875144155, "right_score": -1.0899171493490538, "margin": 0.9716038905976123, "p_left": 0.7254390717510635, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.806780136529394, "brier": 0.05082277507198115 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.11831325875144155, "right_score": 0.24242247087959748, "margin": -0.36073572963103906, "p_left": 0.41078147821892863, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5289581594141014, "brier": 0.16874142284772814 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.24242247087959748, "right_score": -0.43814011720228163, "margin": 0.6805625880818791, "p_left": 0.6638642496663729, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0902401817624918, "brier": 0.44071574198509633 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.1618516026326928, "right_score": -0.18096098415179185, "margin": 0.01910938151909905, "p_left": 0.5047772000070945, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6836381351636471, "brier": 0.2452456216328133 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.9498276872991533, "right_score": -1.8453450229784956, "margin": 2.795172710277649, "p_left": 0.9424144083470759, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.854482887486262, "brier": 0.8881449170601692 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.43814011720228163, "right_score": -1.6385729490574605, "margin": 1.200432831855179, "p_left": 0.768601772930407, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.26318229415302996, "brier": 0.053545139490950924 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.048839619073031366, "right_score": -0.9682958119567393, "margin": 1.0171354310297707, "p_left": 0.73441424071006, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30868204891878737, "brier": 0.07053579553761395 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.6038835186277196, "right_score": 0.1646357360289511, "margin": 1.4392477825987684, "p_left": 0.8083381392370015, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21277481884850036, "brier": 0.03673426887113503 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.9459779677115197, "right_score": -0.925602308842444, "margin": 2.8715802765539635, "p_left": 0.9464235340927628, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.055065099665216, "brier": 0.002870437699109349 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.0519678982652456, "right_score": 1.0868302136944603, "margin": -0.034862315429214696, "p_left": 0.49128530376453017, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.675867937782102, "brier": 0.2413612496950067 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.596438676739378, "right_score": 0.24242247087959748, "margin": 0.35401620585978055, "p_left": 0.5875911583845447, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8857400876524333, "brier": 0.34526336941169106 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.596438676739378, "right_score": -1.0899171493490538, "margin": 1.6863558260884317, "p_left": 0.8437443141637132, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16990577558703546, "brier": 0.024415839356168357 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.43814011720228163, "right_score": -0.18096098415179185, "margin": -0.2571791330504898, "p_left": 0.4360572657714607, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5728025676948922, "brier": 0.19014593903208232 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.11831325875144155, "right_score": -0.925602308842444, "margin": 0.8072890500910024, "p_left": 0.6915315166474398, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7724910767417104, "brier": 0.036684321869268524 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.925602308842444, "right_score": 0.596438676739378, "margin": -1.522040985581822, "p_left": 0.17916117050402172, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1974284987937559, "brier": 0.03209872501637114 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.0899171493490538, "margin": 3.0358951170605737, "p_left": 0.954169657287349, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.046913785517712636, "brier": 0.002100420313159039 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.0899171493490538, "right_score": 1.6038835186277196, "margin": -2.6938006679767734, "p_left": 0.06334015704800336, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06543509045362172, "brier": 0.004011975494865729 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.0519678982652456, "right_score": -0.03886648085526412, "margin": 1.0908343791205097, "p_left": 0.7485388080714952, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3804666077961052, "brier": 0.5603103471890947 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.6038835186277196, "right_score": -0.18096098415179185, "margin": 1.7848445027795115, "p_left": 0.8562940336589682, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15514146455692446, "brier": 0.020651404762009773 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.057065352799532953, "right_score": -1.4287911366492683, "margin": 1.485856489448801, "p_left": 0.8154555396440202, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9469366222683924, "brier": 0.09951219749209998 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.18096098415179185, "right_score": 0.596438676739378, "margin": -0.7773996608911699, "p_left": 0.3148805890842144, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3781621334075176, "brier": 0.09914978538202188 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.03886648085526412, "right_score": 0.057065352799532953, "margin": -0.09593183365479707, "p_left": 0.47603541743939987, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7422630211334409, "brier": 0.2745388837779039 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.9459779677115197, "right_score": -1.6385729490574605, "margin": 3.58455091676898, "p_left": 0.9730000972179589, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.027371096880454804, "brier": 0.0007289947502396718 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.057065352799532953, "rank": 7.0, "portrait_scores": [ -0.9771077384798927, 1.0912384440789586 ], "portrait_score_gap": 2.0683461825588516, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.0519678982652456, "rank": 2.0, "portrait_scores": [ 0.6740879239201923, 1.4298478726102988 ], "portrait_score_gap": 0.7557599486901064, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.9682958119567393, "rank": 11.0, "portrait_scores": [ -1.6766400415866847, -0.25995158232679383 ], "portrait_score_gap": 1.4166884592598907, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.8607083987427133, "rank": 10.0, "portrait_scores": [ -1.5812953017353384, -0.14012149575008825 ], "portrait_score_gap": 1.4411738059852501, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.048839619073031366, "rank": 8.0, "portrait_scores": [ -0.07778574455110553, 0.17546498269716826 ], "portrait_score_gap": 0.2532507272482738, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.4893105018242937, "rank": 4.0, "portrait_scores": [ 1.521128612569642, -0.5425076089210545 ], "portrait_score_gap": 2.0636362214906967, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.24782753137386723, "rank": 5.0, "portrait_scores": [ 0.6884649859811793, -0.19280992323344487 ], "portrait_score_gap": 0.8812749092146241, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -1.8453450229784956, "rank": 13.0, "portrait_scores": [ -2.7560882879980473, -0.9346017579589441 ], "portrait_score_gap": 1.8214865300391032, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.08973321215964836, "rank": 6.0, "portrait_scores": [ 0.9670142958211776, -0.7875478715018809 ], "portrait_score_gap": 1.7545621673230585, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -1.4287911366492683, "rank": 12.0, "portrait_scores": [ -0.13096143825103956, -2.726620835047497 ], "portrait_score_gap": 2.5956593967964574, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.9498276872991533, "rank": 3.0, "portrait_scores": [ 2.0263269912254755, -0.12667161662716891 ], "portrait_score_gap": 2.1529986078526444, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.0, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.0868302136944603, "rank": 1.0, "portrait_scores": [ 0.4591683136751561, 1.7144921137137643 ], "portrait_score_gap": 1.2553238000386082, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.03886648085526412, "rank": 9.0, "portrait_scores": [ -0.11842536620618582, 0.040692404495657586 ], "portrait_score_gap": 0.1591177707018434, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 64, "4.0": 512, "5.0": 1600, "6.0": 2560, "7.0": 2240, "8.0": 1024, "9.0": 192 }, "target_primary_score": -0.03886648085526412, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ -0.7131870836860958, -0.013387833808842231 ], "target_alternate_panel_score": -0.36328745874746904, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.4530206709071174, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.4974209399952443, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.4152553570127147, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.420352811547002, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6423634924485461, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.01", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.3839636996456311, "brier_score": 0.09729194763602023, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 1.1184950876161046, "brier_score": 0.3730009706539194, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.2935134368677963, "brier_score": 0.41740770072834754, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.768458389112721, "brier_score": 0.284187510505063, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.12263203446002935, "brier_score": 0.017664358683043346, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.6188889609740995, "right_score": -0.03577850436373278, "margin": -1.5831104566103666, "p_left": 0.17035541648830319, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18675788253721598, "brier": 0.02902096792690324 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.08922150657520933, "right_score": 0.1227494283976911, "margin": -0.033527921822481765, "p_left": 0.4916188046531899, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.676523728260408, "brier": 0.2416890490886313 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.11939147171796385, "right_score": 1.5439163152583668, "margin": -1.424524843540403, "p_left": 0.1939532107791827, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2156134870179579, "brier": 0.037617847971554076 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.993203943766993, "right_score": 1.8397908645368173, "margin": -2.8329948083038103, "p_left": 0.05556702305138407, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.057170555968070615, "brier": 0.003087694050793049 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.6188889609740995, "right_score": 1.8397908645368173, "margin": -3.4586798255109166, "p_left": 0.03051106000982377, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.030986212316872833, "brier": 0.0009309247829230673 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.08922150657520933, "right_score": -0.7994963118529237, "margin": 0.888717818428133, "p_left": 0.7086255044484778, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2331459128966216, "brier": 0.5021501055548596 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.6188889609740995, "right_score": -0.14825024238447987, "margin": -1.4706387185896197, "p_left": 0.18684555130087915, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20683421367740704, "brier": 0.03491126004092946 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.6712726873456718, "right_score": 1.1008920999342244, "margin": -2.7721647872798965, "p_left": 0.058847004249521155, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.832814351422505, "brier": 0.8857689614101009 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.17358081443801035, "right_score": 0.46421984032460684, "margin": -0.2906390258865965, "p_left": 0.4278474301537693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8489886185547881, "brier": 0.32735856318164586 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.5439163152583668, "right_score": 0.2567534173631103, "margin": 1.2871628978952565, "p_left": 0.7836665930992887, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24377161298603633, "brier": 0.04680014294126874 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.14825024238447987, "right_score": -0.8856042115874215, "margin": 0.7373539692029416, "p_left": 0.6764169713815588, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3909455715250536, "brier": 0.10470597640988297 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.1227494283976911, "right_score": -0.7994963118529237, "margin": 0.9222457402506148, "p_left": 0.7154994696900134, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3347744211598913, "brier": 0.08094055174666358 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.14825024238447987, "right_score": 0.11939147171796385, "margin": -0.26764171410244375, "p_left": 0.4334861421351879, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5682537365629416, "brier": 0.1879102354232483 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.8397908645368173, "right_score": 0.11939147171796385, "margin": 1.7203993928188535, "p_left": 0.8481802735073831, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16466207910034614, "brier": 0.023049229352292994 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.46421984032460684, "right_score": 0.7950703527484774, "margin": -0.33085051242387054, "p_left": 0.41803369478561664, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8721932401711867, "brier": 0.33868478040488076 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.4663344511290638, "right_score": 0.11939147171796385, "margin": -0.5857259228470276, "p_left": 0.3576161258552291, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4425692191790251, "brier": 0.12788929347170308 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.03577850436373278, "right_score": -1.6188889609740995, "margin": 1.5831104566103666, "p_left": 0.8296445835116969, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18675788253721604, "brier": 0.029020967926903212 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.03577850436373278, "right_score": -0.993203943766993, "margin": 0.9574254394032602, "p_left": 0.7226060392085462, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8036038223957607, "brier": 0.04955344869211679 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.03577850436373278, "right_score": 0.2567534173631103, "margin": -0.29253192172684306, "p_left": 0.4273841240122758, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5575401605625543, "brier": 0.18265718945774037 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.2567534173631103, "right_score": -0.4663344511290638, "margin": 0.7230878684921741, "p_left": 0.6732866239995884, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1186720184510417, "brier": 0.4533148780567631 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.14825024238447987, "right_score": -0.2089271523340162, "margin": 0.06067690994953634, "p_left": 0.5151645751707372, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6632688659295966, "brier": 0.23506538916937172 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.7950703527484774, "right_score": -1.6712726873456718, "margin": 2.466343040094149, "p_left": 0.9217484011039634, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.5478260165944415, "brier": 0.849620114937713 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.4663344511290638, "right_score": -1.6188889609740995, "margin": 1.1525545098450358, "p_left": 0.7599771992419775, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2744668471491971, "brier": 0.057610944883725375 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.020763036422905792, "right_score": -0.9004209183927793, "margin": 0.921183954815685, "p_left": 0.7152832832234178, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33507661444227566, "brier": 0.0810636088120365 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.5439163152583668, "right_score": 0.11939147171796385, "margin": 1.424524843540403, "p_left": 0.8060467892208174, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21561348701795802, "brier": 0.037617847971554035 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.8397908645368173, "right_score": -0.8856042115874215, "margin": 2.7253950761242387, "p_left": 0.9385086221732234, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.0634632358126681, "brier": 0.0037811895470353935 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.1008920999342244, "right_score": 1.153365185448032, "margin": -0.05247308551380758, "p_left": 0.48688473780736735, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6672547764120201, "brier": 0.23705674790974884 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.5971353976635491, "right_score": 0.2567534173631103, "margin": 0.34038198030043876, "p_left": 0.5842833075322831, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8777512784474274, "brier": 0.3413869834608645 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.5971353976635491, "right_score": -0.993203943766993, "margin": 1.590339341430542, "p_left": 0.8306638406710641, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18553008983007357, "brier": 0.028674734856274753 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.4663344511290638, "right_score": -0.2089271523340162, "margin": -0.2574072987950476, "p_left": 0.4360011580493294, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.572703080765139, "brier": 0.1900970098203563 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.03577850436373278, "right_score": -0.8856042115874215, "margin": 0.8498257072236888, "p_left": 0.7005305793084827, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.780830114224408, "brier": 0.04021251323779568 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.8856042115874215, "right_score": 0.5971353976635491, "margin": -1.4827396092509706, "p_left": 0.18501397407083858, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20458431198745883, "brier": 0.03423017060148493 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.8397908645368173, "right_score": -0.993203943766993, "margin": 2.8329948083038103, "p_left": 0.9444329769486158, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.05717055596807041, "brier": 0.0030876940507930596 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.993203943766993, "right_score": 1.5439163152583668, "margin": -2.5371202590253596, "p_left": 0.07329653719708797, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.07612165369321172, "brier": 0.0053723823650840994 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.1008920999342244, "right_score": 0.1227494283976911, "margin": 0.9781426715365333, "p_left": 0.7267395263635511, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.297329822791994, "brier": 0.5281503391791186 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.5439163152583668, "right_score": -0.2089271523340162, "margin": 1.752843467592383, "p_left": 0.8523110875310291, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15980369258504634, "brier": 0.02181201486626734 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.26202100660231553, "right_score": -1.4177059819732245, "margin": 1.6797269885755401, "p_left": 0.8428683765175716, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.0108079644485324, "brier": 0.11755872361579527 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.2089271523340162, "right_score": 0.5971353976635491, "margin": -0.8060625499975653, "p_left": 0.30873017649414536, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3692250487906368, "brier": 0.09531432187810614 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.1227494283976911, "right_score": 0.26202100660231553, "margin": -0.13927157820462444, "p_left": 0.46523827548539104, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7652055842385906, "brier": 0.2859701020058386 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.8397908645368173, "right_score": -1.6188889609740995, "margin": 3.4586798255109166, "p_left": 0.9694889399901762, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.03098621231687293, "brier": 0.0009309247829230717 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.01, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.26202100660231553, "rank": 5.0, "portrait_scores": [ -0.649415819323123, 1.173457832527754 ], "portrait_score_gap": 1.822873651850877, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.5, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.1008920999342244, "rank": 2.0, "portrait_scores": [ 0.6217777114501877, 1.5800064884182614 ], "portrait_score_gap": 0.9582287769680737, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.140625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.9004209183927793, "rank": 11.0, "portrait_scores": [ -1.5744928687108046, -0.2263489680747539 ], "portrait_score_gap": 1.3481439006360507, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7994963118529237, "rank": 10.0, "portrait_scores": [ -1.5251116661646118, -0.07388095754123558 ], "portrait_score_gap": 1.4512307086233762, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.020763036422905792, "rank": 9.0, "portrait_scores": [ -0.046323252020089106, 0.08784932486590069 ], "portrait_score_gap": 0.1341725768859898, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.46421984032460684, "rank": 4.0, "portrait_scores": [ 1.535338736195447, -0.6068990555462332 ], "portrait_score_gap": 2.14223779174168, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.17358081443801035, "rank": 6.0, "portrait_scores": [ 0.548902795864753, -0.2017411669887323 ], "portrait_score_gap": 0.7506439628534853, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -1.6712726873456718, "rank": 13.0, "portrait_scores": [ -2.591020598519956, -0.7515247761713876 ], "portrait_score_gap": 1.8394958223485687, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.08922150657520933, "rank": 8.0, "portrait_scores": [ 1.0334222392184906, -0.854979226068072 ], "portrait_score_gap": 1.8884014652865626, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -1.4177059819732245, "rank": 12.0, "portrait_scores": [ -0.200784775635074, -2.634627188311375 ], "portrait_score_gap": 2.433842412676301, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 12.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.7950703527484774, "rank": 3.0, "portrait_scores": [ 1.8991501035319096, -0.30900939803495486 ], "portrait_score_gap": 2.2081595015668647, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.153365185448032, "rank": 1.0, "portrait_scores": [ 0.6164037430137507, 1.6903266278823135 ], "portrait_score_gap": 1.073922884868563, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.1227494283976911, "rank": 7.0, "portrait_scores": [ 0.07167840247830015, 0.17382045431708204 ], "portrait_score_gap": 0.10214205183878189, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 192, "4.0": 1024, "5.0": 2240, "6.0": 2560, "7.0": 1600, "8.0": 512, "9.0": 64 }, "target_primary_score": 0.1227494283976911, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ -0.6535439296481365, 0.008195234912451221 ], "target_alternate_panel_score": -0.3226743473678426, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.41189585394305195, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.4768219644850811, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.423566447302067, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.5846953539701581, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.6201613517664039, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.41311116560853306, "brier_score": 0.10696187755399167, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.45454545454545453, "mean_cross_entropy": 0.9634089532776962, "brier_score": 0.3300727083316121, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 1.1301252198805163, "brier_score": 0.38329006620093586, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.6299764200720555, "brier_score": 0.22363799259296469, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.14388128494331504, "brier_score": 0.021016700478302636, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.4727913904202603, "right_score": 0.22459515884366582, "margin": -1.6973865492639262, "p_left": 0.15480690508853473, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.16819016304234302, "brier": 0.0239651778630906 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.0959988485247284, "right_score": 0.5832000715889002, "margin": -0.4872012230641718, "p_left": 0.38055310925511193, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47892831081326237, "brier": 0.14482066896373316 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.03199914517375774, "right_score": 1.2632432449741138, "margin": -1.2952423901478716, "p_left": 0.21496680373535038, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24202927385862835, "brier": 0.04621072670819265 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7101960219944543, "right_score": 1.4257077663578144, "margin": -2.1359037883522687, "p_left": 0.1056558267513445, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.11166459663241701, "brier": 0.011163153726510123 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.4727913904202603, "right_score": 1.4257077663578144, "margin": -2.8984991567780747, "p_left": 0.052227805002864135, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.05364110623987186, "brier": 0.0027277436154171998 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.0959988485247284, "right_score": -0.7498878996760154, "margin": 0.8458867482007438, "p_left": 0.6997035828344549, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2029852349146914, "brier": 0.4895851038313729 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.4727913904202603, "right_score": -0.029863585759309516, "margin": -1.4429278046609508, "p_left": 0.1910923706448917, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21207054723721472, "brier": 0.03651629411868467 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.1095388841248073, "right_score": 1.1278518208853783, "margin": -2.2373907050101858, "p_left": 0.09644268025734351, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.338806434087879, "brier": 0.8164158300605332 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.08027096067701908, "right_score": 0.36587366541291166, "margin": -0.4461446260899308, "p_left": 0.3902778057742495, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9408964709416261, "brier": 0.37176115413146377 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.2632432449741138, "right_score": 0.3109892450011393, "margin": 0.9522539999729744, "p_left": 0.7215682496255388, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3263283111663582, "brier": 0.07752423961658629 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.029863585759309516, "right_score": -0.710930639411955, "margin": 0.6810670536526455, "p_left": 0.6639768110985151, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4095080531599318, "brier": 0.11291158347952297 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.5832000715889002, "right_score": -0.7498878996760154, "margin": 1.3330879712649155, "p_left": 0.7913509627062353, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23401371466698895, "brier": 0.04353442076361482 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.029863585759309516, "right_score": -0.03199914517375774, "margin": 0.002135559414448221, "p_left": 0.5005338896507067, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6942155303438127, "brier": 0.2505341746888658 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.4257077663578144, "right_score": -0.03199914517375774, "margin": 1.4577069115315722, "p_left": 0.8111817023129595, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20926320271993215, "brier": 0.035652349541431844 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.36587366541291166, "right_score": 0.29734375787218353, "margin": 0.06852990754072813, "p_left": 0.5171257750261364, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6594691554806087, "brier": 0.2331675171441094 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.5719413191029455, "right_score": -0.03199914517375774, "margin": -0.5399421739291879, "p_left": 0.36820103418428457, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.45918402750103254, "brier": 0.13557200157437668 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.22459515884366582, "right_score": -1.4727913904202603, "margin": 1.6973865492639262, "p_left": 0.8451930949114653, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16819016304234302, "brier": 0.023965177863090582 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.22459515884366582, "right_score": -0.7101960219944543, "margin": 0.9347911808381202, "p_left": 0.7180463004556257, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7986168171178196, "brier": 0.047544189142384986 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.22459515884366582, "right_score": 0.3109892450011393, "margin": -0.08639408615747346, "p_left": 0.4784149026100451, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6508828397326925, "brier": 0.22888081903937893 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.3109892450011393, "right_score": -0.5719413191029455, "margin": 0.8829305641040848, "p_left": 0.7074291376865453, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2290483773319838, "brier": 0.500455984847929 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.029863585759309516, "right_score": -0.3379728832749368, "margin": 0.30810929751562727, "p_left": 0.576423695143531, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5509123068475676, "brier": 0.1794168860358604 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.29734375787218353, "right_score": -1.1095388841248073, "margin": 1.4068826419969909, "p_left": 0.8032737887957762, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6259423077406796, "brier": 0.6452487797663212 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.5719413191029455, "right_score": -1.4727913904202603, "margin": 0.9008500713173148, "p_left": 0.7111241612113091, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34090823543558024, "brier": 0.08344925023586976 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.1532724266629415, "right_score": -0.776899144617125, "margin": 0.6236267179541835, "p_left": 0.6510429389192488, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4291796805474237, "brier": 0.12177103047811512 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.2632432449741138, "right_score": -0.03199914517375774, "margin": 1.2952423901478716, "p_left": 0.7850331962646496, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24202927385862827, "brier": 0.04621072670819265 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.4257077663578144, "right_score": -0.710930639411955, "margin": 2.1366384057697694, "p_left": 0.8944135691055336, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11158700551398937, "brier": 0.011148494389031928 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.1278518208853783, "right_score": 1.2041829996426583, "margin": -0.07633117875728002, "p_left": 0.48092646532601535, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6557097205463346, "brier": 0.23129026505097505 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.6411595699608874, "right_score": 0.3109892450011393, "margin": 0.33017032495974813, "p_left": 0.5818008189510704, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8717974502656343, "brier": 0.33849219293213617 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.6411595699608874, "right_score": -0.7101960219944543, "margin": 1.3513555919553417, "p_left": 0.7943511625297272, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2302296453190611, "brier": 0.0422914443528747 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.5719413191029455, "right_score": -0.3379728832749368, "margin": -0.23396843582800875, "p_left": 0.44177326593690935, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5829900656855059, "brier": 0.19516361849656325 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.22459515884366582, "right_score": -0.710930639411955, "margin": 0.9355257982556209, "p_left": 0.7181950041944128, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7987770523510513, "brier": 0.047609059855399836 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.710930639411955, "right_score": 0.6411595699608874, "margin": -1.3520902093728424, "p_left": 0.20552885823317632, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2300786161737625, "brier": 0.042242111566633087 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.4257077663578144, "right_score": -0.7101960219944543, "margin": 2.1359037883522687, "p_left": 0.8943441732486556, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11166459663241701, "brier": 0.011163153726510111 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7101960219944543, "right_score": 1.2632432449741138, "margin": -1.973439266968568, "p_left": 0.12201995518064429, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.13013141360148323, "brier": 0.01488886946228644 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.1278518208853783, "right_score": 0.5832000715889002, "margin": 0.5446517492964781, "p_left": 0.6328938681290013, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0021042850748612, "brier": 0.40055464831528964 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.2632432449741138, "right_score": -0.3379728832749368, "margin": 1.6012161282490505, "p_left": 0.8321882872060105, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.18369655702644216, "brier": 0.028160770950852433 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.7953894231003993, "right_score": -1.361752341226468, "margin": 2.1571417643268673, "p_left": 0.8963342644089664, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1880127547848869, "brier": 0.15708084914459644 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.3379728832749368, "right_score": 0.6411595699608874, "margin": -0.9791324532358242, "p_left": 0.273063957811939, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3189167803006777, "brier": 0.0745639250559204 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.5832000715889002, "right_score": 0.7953894231003993, "margin": -0.2121893515114991, "p_left": 0.44715080507499955, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8048593697331095, "brier": 0.3056422323292212 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.4257077663578144, "right_score": -1.4727913904202603, "margin": 2.8984991567780747, "p_left": 0.9477721949971358, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.05364110623987184, "brier": 0.0027277436154172037 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.1, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.7953894231003993, "rank": 3.0, "portrait_scores": [ 0.3440383912486761, 1.2467404549521226 ], "portrait_score_gap": 0.9027020637034465, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.1278518208853783, "rank": 2.0, "portrait_scores": [ 0.39076448447391504, 1.8649391572968415 ], "portrait_score_gap": 1.4741746728229264, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.776899144617125, "rank": 11.0, "portrait_scores": [ -1.2316259504368858, -0.32217233879736396 ], "portrait_score_gap": 0.9094536116395219, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7498878996760154, "rank": 10.0, "portrait_scores": [ -1.3906399514249888, -0.10913584792704212 ], "portrait_score_gap": 1.2815041034979466, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.1532724266629415, "rank": 9.0, "portrait_scores": [ -0.016373428173684312, -0.2901714251521987 ], "portrait_score_gap": 0.27379799697851437, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": 0.36587366541291166, "rank": 5.0, "portrait_scores": [ 1.5631662608088073, -0.831418929982984 ], "portrait_score_gap": 2.3945851907917914, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": -0.08027096067701908, "rank": 8.0, "portrait_scores": [ 0.05666658411462999, -0.21720850546866816 ], "portrait_score_gap": 0.27387508958329815, "single_photo_rank_min": 5.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -1.1095388841248073, "rank": 12.0, "portrait_scores": [ -1.972223295446124, -0.2468544728034905 ], "portrait_score_gap": 1.7253688226426336, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.0959988485247284, "rank": 7.0, "portrait_scores": [ 1.2149034906596687, -1.022905793610212 ], "portrait_score_gap": 2.237809284269881, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 7.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -1.361752341226468, "rank": 13.0, "portrait_scores": [ -0.3872699756726743, -2.336234706780262 ], "portrait_score_gap": 1.9489647311075877, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 13.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.29734375787218353, "rank": 6.0, "portrait_scores": [ 1.406301493293818, -0.811613977549451 ], "portrait_score_gap": 2.217915470843269, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.5, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.2041829996426583, "rank": 1.0, "portrait_scores": [ 0.9101097104563594, 1.4982562888289574 ], "portrait_score_gap": 0.5881465783725981, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.140625, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.5832000715889002, "rank": 4.0, "portrait_scores": [ 0.6052620189752109, 0.5611381242025895 ], "portrait_score_gap": 0.044123894772621464, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.5, 6.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "2.0": 256, "3.0": 1280, "4.0": 2560, "5.0": 2560, "6.0": 1280, "7.0": 256 }, "target_primary_score": 0.5832000715889002, "target_primary_rank": 4.0, "target_alternate_photo_scores": [ -0.43239016019446463, -0.019424037616514678 ], "target_alternate_panel_score": -0.22590709890548966, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.32190594743021805, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.5239808007705258, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.353758919790868, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.021296522005889, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.545992223195703, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.46962501372944726, "brier_score": 0.12724302775197485, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.8420760542101964, "brier_score": 0.2908150018685924, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.0046582922666778, "brier_score": 0.3502316055176246, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.5169115780972335, "brier_score": 0.17198179457052792, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.22546526812160284, "brier_score": 0.042827596797444835, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.1159263788574267, "right_score": 0.4129444375640396, "margin": -1.5288708164214664, "p_left": 0.17815895889546104, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19620828327781867, "brier": 0.03174061463471458 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.11650601429750823, "right_score": 0.8188534994081573, "margin": -0.702347485110649, "p_left": 0.33129196581081777, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4024077353552554, "brier": 0.10975436661079604 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.14450451534229042, "right_score": 0.8270615919638344, "margin": -0.9715661073061248, "p_left": 0.27456845388418494, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.320988565188342, "brier": 0.0753878358683518 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.5024218410160748, "right_score": 0.7801514464130612, "margin": -1.282573287429136, "p_left": 0.21711251150693145, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24476628617301444, "brier": 0.04713784265284744 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.1159263788574267, "right_score": 0.7801514464130612, "margin": -1.896077825270488, "p_left": 0.130553031500098, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.13989793784723642, "brier": 0.017044094033865583 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.11650601429750823, "right_score": -0.8482236320652671, "margin": 0.9647296463627753, "p_left": 0.724067759105954, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2875999474131088, "brier": 0.5242741197767178 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.1159263788574267, "right_score": 0.290692186935258, "margin": -1.4066185657926846, "p_left": 0.19676794519091736, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21911162196516037, "brier": 0.03871762425465586 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.6172192814030267, "right_score": 0.909490050094982, "margin": -1.5267093314980087, "p_left": 0.17847566013045316, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.7233030448726023, "brier": 0.6749022409980947 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.3869783236410703, "right_score": 0.2346247977071022, "margin": -0.6216031213481725, "p_left": 0.34941693436454163, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.051489415471717, "brier": 0.42325832529163104 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.8270615919638344, "right_score": 0.38696660809291783, "margin": 0.4400949838709166, "p_left": 0.6082816632688475, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49711724233331944, "brier": 0.15344325533142059 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.290692186935258, "right_score": -0.4239861857681613, "margin": 0.7146783727034194, "p_left": 0.6714340838260271, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3983394304193446, "brier": 0.10795556127124219 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.8188534994081573, "right_score": -0.8482236320652671, "margin": 1.6670771314734245, "p_left": 0.8411857377332053, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17294278996541124, "brier": 0.02522196989934626 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.290692186935258, "right_score": -0.14450451534229042, "margin": 0.4351967022775484, "p_left": 0.6071139077877217, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9342355508241365, "brier": 0.3685872970292782 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.7801514464130612, "right_score": -0.14450451534229042, "margin": 0.9246559617553516, "p_left": 0.7159898393917269, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33408930291585937, "brier": 0.08066177132873709 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.2346247977071022, "right_score": -0.05043054881853015, "margin": 0.28505534652563236, "p_left": 0.5707851712630114, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5607423725833953, "brier": 0.18422536920772242 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.7048403720093357, "right_score": -0.14450451534229042, "margin": -0.5603358566670453, "p_left": 0.36346975252821945, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4517233405556109, "brier": 0.13211026100292508 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.4129444375640396, "right_score": -1.1159263788574267, "margin": 1.5288708164214664, "p_left": 0.821841041104539, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1962082832778187, "brier": 0.031740614634714545 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.4129444375640396, "right_score": -0.5024218410160748, "margin": 0.9153662785801144, "p_left": 0.7140970129804061, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7944195926408678, "brier": 0.04583753096713216 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.4129444375640396, "right_score": 0.38696660809291783, "margin": 0.02597782947112176, "p_left": 0.5064940921616659, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.706220448876641, "brier": 0.25653626539467006 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.38696660809291783, "right_score": -0.7048403720093357, "margin": 1.0918069801022536, "p_left": 0.748721834999981, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3811947263889683, "brier": 0.5605843862057388 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.290692186935258, "right_score": -0.5154270432111455, "margin": 0.8061192301464035, "p_left": 0.691281919817117, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.36920755026109386, "brier": 0.095306853031805 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.05043054881853015, "right_score": -0.6172192814030267, "margin": 0.5667887325844966, "p_left": 0.6380218622554306, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0161714619508975, "brier": 0.4070718967158876 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.7048403720093357, "right_score": -1.1159263788574267, "margin": 0.41108600684809105, "p_left": 0.601348254307705, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5085810541852335, "brier": 0.15892321434351425 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.4448924174083337, "right_score": -0.7657776953123261, "margin": 0.3208852779039924, "p_left": 0.5795399859940709, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5455206177301988, "brier": 0.17678662337786613 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.8270615919638344, "right_score": -0.14450451534229042, "margin": 0.9715661073061248, "p_left": 0.7254315461158151, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.32098856518834196, "brier": 0.0753878358683518 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.7801514464130612, "right_score": -0.4239861857681613, "margin": 1.2041376321812225, "p_left": 0.7692600273149202, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2623262296814266, "brier": 0.05324093499471139 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.909490050094982, "right_score": 0.8447635321566144, "margin": 0.06472651793836759, "p_left": 0.5161759824092069, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7260340384030425, "brier": 0.2664376448161099 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.7092900652353251, "right_score": 0.38696660809291783, "margin": 0.3223234571424073, "p_left": 0.5798903918611362, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8672396299927295, "brier": 0.33627286657286215 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.7092900652353251, "right_score": -0.5024218410160748, "margin": 1.2117119062514, "p_left": 0.77060171096356, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2605836264920516, "brier": 0.05262357501284606 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.7048403720093357, "right_score": -0.5154270432111455, "margin": -0.18941332879819017, "p_left": 0.45278773793835747, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6029185041864866, "brier": 0.2050167356273347 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.4129444375640396, "right_score": -0.4239861857681613, "margin": 0.8369306233322009, "p_left": 0.6978183762763573, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7782617276530935, "brier": 0.039132109992614496 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.4239861857681613, "right_score": 0.7092900652353251, "margin": -1.1332762510034864, "p_left": 0.24355698729178094, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2791280787852144, "brier": 0.05932000605864874 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.7801514464130612, "right_score": -0.5024218410160748, "margin": 1.282573287429136, "p_left": 0.7828874884930686, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24476628617301444, "brier": 0.04713784265284744 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.5024218410160748, "right_score": 0.8270615919638344, "margin": -1.3294834329799092, "p_left": 0.20924482429237454, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23476687150306585, "brier": 0.04378339649314669 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.909490050094982, "right_score": 0.8188534994081573, "margin": 0.09063655068682464, "p_left": 0.5226436383658465, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7394919776463735, "brier": 0.2731563727242897 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.8270615919638344, "right_score": -0.5154270432111455, "margin": 1.34248863517498, "p_left": 0.7928988986930781, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23205955766542674, "brier": 0.042890866162539946 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.9348073165965829, "right_score": -1.0638201143888062, "margin": 1.998627430985389, "p_left": 0.8806528916965787, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1264054397084602, "brier": 0.14489662395696726 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.5154270432111455, "right_score": 0.7092900652353251, "margin": -1.2247171084464705, "p_left": 0.22710739261810417, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2576151696945916, "brier": 0.05157776778179372 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.8188534994081573, "right_score": 0.9348073165965829, "margin": -0.11595381718842557, "p_left": 0.4710439819345282, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7528038094218943, "brier": 0.2797944690476797 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.7801514464130612, "right_score": -1.1159263788574267, "margin": 1.896077825270488, "p_left": 0.8694469684999021, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.13989793784723625, "brier": 0.017044094033865552 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 1.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.9348073165965829, "rank": 1.0, "portrait_scores": [ 0.9901310831505203, 0.8794835500426454 ], "portrait_score_gap": 0.1106475331078749, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.078125, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.909490050094982, "rank": 2.0, "portrait_scores": [ 0.19156861017292953, 1.6274114900170344 ], "portrait_score_gap": 1.435842879844105, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.5, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.7657776953123261, "rank": 11.0, "portrait_scores": [ -0.9096932935472553, -0.6218620970773968 ], "portrait_score_gap": 0.28783119646985844, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.8482236320652671, "rank": 12.0, "portrait_scores": [ -1.236118574867569, -0.4603286892629652 ], "portrait_score_gap": 0.7757898856046037, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.4448924174083337, "rank": 9.0, "portrait_scores": [ -0.12723279090143028, -0.7625520439152371 ], "portrait_score_gap": 0.6353192530138069, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "emma_watson", "mean_score": 0.2346247977071022, "rank": 5.0, "portrait_scores": [ 1.3884489288887436, -0.9191993334745392 ], "portrait_score_gap": 2.3076482623632826, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": -0.3869783236410703, "rank": 8.0, "portrait_scores": [ -0.47504944245050706, -0.2989072048316336 ], "portrait_score_gap": 0.1761422376188735, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -0.6172192814030267, "rank": 10.0, "portrait_scores": [ -1.2638954035346444, 0.029456840728590894 ], "portrait_score_gap": 1.2933522442632353, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.11650601429750823, "rank": 6.0, "portrait_scores": [ 1.181219881307391, -0.9482078527123746 ], "portrait_score_gap": 2.1294277340197656, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -1.0638201143888062, "rank": 13.0, "portrait_scores": [ -0.3093891996203736, -1.818251029157239 ], "portrait_score_gap": 1.5088618295368654, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 12.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": -0.05043054881853015, "rank": 7.0, "portrait_scores": [ 0.8502419919310527, -0.951103089568113 ], "portrait_score_gap": 1.8013450814991656, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 8.5, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.8447635321566144, "rank": 3.0, "portrait_scores": [ 0.7528954410118671, 0.9366316233013616 ], "portrait_score_gap": 0.1837361822894945, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.8188534994081573, "rank": 4.0, "portrait_scores": [ 0.9140356708017717, 0.723671328014543 ], "portrait_score_gap": 0.1903643427872287, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "1.0": 128, "2.0": 640, "3.0": 1536, "4.0": 2304, "5.0": 2176, "6.0": 1152, "7.0": 256 }, "target_primary_score": 0.8188534994081573, "target_primary_rank": 4.0, "target_alternate_photo_scores": [ -0.18983852871785956, -0.29776603010841307 ], "target_alternate_panel_score": -0.2438022794131363, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.36030829371064454, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.6044213526521307, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.1532923295081183, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.1786095960097192, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.39071557179761984, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_10", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5120509681518876, "brier_score": 0.14354190222763652, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.45454545454545453, "mean_cross_entropy": 0.8431625737099367, "brier_score": 0.2941468934016725, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.008598956528902, "brier_score": 0.3576766614462926, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.5122898080720065, "brier_score": 0.16708735731243224, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3182347318953053, "brier_score": 0.07610247730271964, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.322054370337053, "margin": -1.2654258040726614, "p_left": 0.22004128446547566, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24851428950116994, "brier": 0.04841816686921638 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.18238416788019518, "right_score": 0.8582962750030222, "margin": -0.675912107122827, "p_left": 0.33717428776286595, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4112432008249994, "brier": 0.11368650032839593 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.20826033514763365, "right_score": 0.6989493118224618, "margin": -0.9072096469700954, "p_left": 0.2875711689829475, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3390752581201555, "brier": 0.08269717723021894 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.30503695862105157, "right_score": 0.3065408150995909, "margin": -0.6115777737206425, "p_left": 0.3516993704681515, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4334007557514525, "brier": 0.12369244718769405 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.3065408150995909, "margin": -1.2499122488351992, "p_left": 0.222715329341627, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.25194862420844344, "brier": 0.04960211792374938 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.18238416788019518, "right_score": -0.5401699237521967, "margin": 0.7225540916323919, "p_left": 0.6731691973130416, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.118312664970008, "brier": 0.4531567682110847 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.9433714337356084, "right_score": 0.4082665152447368, "margin": -1.3516379489803452, "p_left": 0.20560271619628323, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2301715854366309, "brier": 0.042272476907289386 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.4957564569813912, "right_score": 0.9481002769431781, "margin": -1.4438567339245694, "p_left": 0.19094882160614932, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.655749836546468, "brier": 0.6545638092604784 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.5945867768665508, "right_score": 0.44392116584665775, "margin": -1.0385079427132085, "p_left": 0.26143798995776313, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3415581556901037, "brier": 0.5454738426776292 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.6989493118224618, "right_score": 0.4705486828399298, "margin": 0.228400628982532, "p_left": 0.5568532174407075, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5854535972290251, "brier": 0.19637907089265288 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.29128126586411507, "margin": 0.6995477811088519, "p_left": 0.6680875017938281, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4033361233147016, "brier": 0.11016590646546204 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.8582962750030222, "right_score": -0.5401699237521967, "margin": 1.398466198755219, "p_left": 0.801940384651381, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22072100723087584, "brier": 0.039227611232042936 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.20826033514763365, "margin": 0.6165268503923704, "p_left": 0.6494282233764629, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0481898099534108, "brier": 0.421757017317909 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.3065408150995909, "right_score": -0.20826033514763365, "margin": 0.5148011502472245, "p_left": 0.6259312999914822, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4685146583269506, "brier": 0.13992739232606247 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.44392116584665775, "right_score": -0.10779785226845706, "margin": 0.5517190181151148, "p_left": 0.6345343243136319, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4548638966578751, "brier": 0.1335651601048936 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.20826033514763365, "margin": -0.48828114562703007, "p_left": 0.38029856933274947, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47851748037183056, "brier": 0.14462700183653607 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.322054370337053, "right_score": -0.9433714337356084, "margin": 1.2654258040726614, "p_left": 0.7799587155345244, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24851428950116983, "brier": 0.04841816686921633 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.322054370337053, "right_score": -0.30503695862105157, "margin": 0.6270913289581046, "p_left": 0.6518296376704695, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7415177075936744, "brier": 0.02305223887514606 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.322054370337053, "right_score": 0.4705486828399298, "margin": -0.1484943125028768, "p_left": 0.46294448809823446, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6216538156972643, "brier": 0.21431759906053635 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.4705486828399298, "right_score": -0.6965414807746637, "margin": 1.1670901636145934, "p_left": 0.7626186464476711, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4380873443212985, "brier": 0.5815871999096779 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.4082665152447368, "right_score": -0.43690806903301493, "margin": 0.8451745842777517, "p_left": 0.6995539227555715, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3573124002771858, "brier": 0.09026784533156511 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.10779785226845706, "right_score": -0.4957564569813912, "margin": 0.38795860471293414, "p_left": 0.5957911785022925, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9058236496646546, "brier": 0.3549671283811505 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.9433714337356084, "margin": 0.24682995296094468, "p_left": 0.5613960910984787, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5773285778529474, "brier": 0.19237338890369404 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.332305192832029, "right_score": -0.41599073173726603, "margin": 0.08368553890523706, "p_left": 0.5209091834299748, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6521795644574757, "brier": 0.2295280105217335 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.6989493118224618, "right_score": -0.20826033514763365, "margin": 0.9072096469700954, "p_left": 0.7124288310170525, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3390752581201555, "brier": 0.08269717723021897 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.3065408150995909, "right_score": -0.29128126586411507, "margin": 0.597822080963706, "p_left": 0.6451578747540792, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43826022507634654, "brier": 0.12591293384904179 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.9481002769431781, "right_score": 0.46987707022675773, "margin": 0.4782232067164204, "p_left": 0.6173282232225964, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9605776369119794, "brier": 0.3810941351871678 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.6750398478323165, "right_score": 0.4705486828399298, "margin": 0.20449116499238673, "p_left": 0.5509453844682042, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8006107604817561, "brier": 0.3035408166668173 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.6750398478323165, "right_score": -0.30503695862105157, "margin": 0.980076806453368, "p_left": 0.7271234561544773, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31865899997233793, "brier": 0.07446160818107748 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.6965414807746637, "right_score": -0.43690806903301493, "margin": -0.2596334117416488, "p_left": 0.4354538258450435, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5717331021831802, "brier": 0.18962003444308548 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.322054370337053, "right_score": -0.29128126586411507, "margin": 0.613335636201168, "p_left": 0.6487013297850049, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7394506869428821, "brier": 0.0221120854798288 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.29128126586411507, "right_score": 0.6750398478323165, "margin": -0.9663211136964316, "p_left": 0.27561438925545617, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.32243141685798343, "brier": 0.07596329156465811 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.3065408150995909, "right_score": -0.30503695862105157, "margin": 0.6115777737206425, "p_left": 0.6483006295318486, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43340075575145254, "brier": 0.12369244718769398 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.30503695862105157, "right_score": 0.6989493118224618, "margin": -1.0039862704435134, "p_left": 0.2681583952841115, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.31219117543602254, "brier": 0.07190892496134979 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.9481002769431781, "right_score": 0.8582962750030222, "margin": 0.08980400194015592, "p_left": 0.5224359241517104, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7390569378058612, "brier": 0.27293929484425167 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.6989493118224618, "right_score": -0.43690806903301493, "margin": 1.1358573808554766, "p_left": 0.7569182374591771, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2785000400184947, "brier": 0.05908874327995302 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.8280493767402581, "right_score": -0.7593440000306045, "margin": 1.5873933767708626, "p_left": 0.8302490533312744, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.979726247332652, "brier": 0.10906443722620293 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.43690806903301493, "right_score": 0.6750398478323165, "margin": -1.1119479168653315, "p_left": 0.24750791502506425, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.28436480072015724, "brier": 0.061260168000054425 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.8582962750030222, "right_score": 0.8280493767402581, "margin": 0.030246898262764055, "p_left": 0.5075611481157905, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6781380864262897, "brier": 0.24249602284503846 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.3065408150995909, "right_score": -0.9433714337356084, "margin": 1.2499122488351992, "p_left": 0.7772846706583731, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25194862420844344, "brier": 0.049602117923749314 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 10.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.8280493767402581, "rank": 3.0, "portrait_scores": [ 1.1023359023533006, 0.5537628511272158 ], "portrait_score_gap": 0.5485730512260848, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.9481002769431781, "rank": 1.0, "portrait_scores": [ 0.15314572894819922, 1.743054824938157 ], "portrait_score_gap": 1.5899090959899578, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.5, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.41599073173726603, "rank": 9.0, "portrait_scores": [ -0.8699223169974357, 0.03794085352290366 ], "portrait_score_gap": 0.9078631705203394, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.5401699237521967, "rank": 11.0, "portrait_scores": [ -0.8725280969349682, -0.20781175056942525 ], "portrait_score_gap": 0.664716346365543, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.332305192832029, "rank": 8.0, "portrait_scores": [ -0.14380151743312272, -0.5208088682309352 ], "portrait_score_gap": 0.37700735079781245, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "emma_watson", "mean_score": 0.44392116584665775, "rank": 5.0, "portrait_scores": [ 1.3763184702853273, -0.4884761385920118 ], "portrait_score_gap": 1.864794608877339, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.5, 11.0 ] }, { "identity": "gal_gadot", "mean_score": -0.5945867768665508, "rank": 12.0, "portrait_scores": [ -0.7105041416263659, -0.4786694121067357 ], "portrait_score_gap": 0.2318347295196302, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.4957564569813912, "rank": 10.0, "portrait_scores": [ -0.992767409892572, 0.0012544959297895453 ], "portrait_score_gap": 0.9940219058223615, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.18238416788019518, "rank": 6.0, "portrait_scores": [ 1.1437250622453399, -0.7789567264849495 ], "portrait_score_gap": 1.9226817887302894, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.7593440000306045, "rank": 13.0, "portrait_scores": [ -0.3209388549477432, -1.1977491451134659 ], "portrait_score_gap": 0.8768102901657227, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "natalie_portman", "mean_score": -0.10779785226845706, "rank": 7.0, "portrait_scores": [ 0.5547282377635694, -0.7703239423004835 ], "portrait_score_gap": 1.325052180064053, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.5, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.46987707022675773, "rank": 4.0, "portrait_scores": [ 0.6126307940809954, 0.32712334637252005 ], "portrait_score_gap": 0.28550744770847536, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.8582962750030222, "rank": 2.0, "portrait_scores": [ 1.1207890563202303, 0.5958034936858141 ], "portrait_score_gap": 0.5249855626344162, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.078125, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "1.0": 640, "2.0": 2176, "3.0": 2816, "4.0": 1792, "5.0": 640, "6.0": 128 }, "target_primary_score": 0.8582962750030222, "target_primary_rank": 2.0, "target_alternate_photo_scores": [ 0.0953131147490398, -0.18709787751734763 ], "target_alternate_panel_score": -0.04589238138415391, "target_alternate_panel_rank": 6.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.2282765492643491, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.49427754236804283, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.993992658327332, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.873941758124412, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.29489646576337153, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_100", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5371919086428779, "brier_score": 0.1542580957050011, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.8626074279751891, "brier_score": 0.3007241105986929, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 1, "accuracy": 0.14285714285714285, "mean_cross_entropy": 1.033786572847141, "brier_score": 0.3666271478822193, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.5202491382312846, "brier_score": 0.16891803603164, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3891341890825566, "brier_score": 0.10720799431549079, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.8617429677110443, "right_score": 0.16596378989509433, "margin": -1.0277067576061387, "p_left": 0.2635289385238105, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3058853358049795, "brier": 0.06944750143948629 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.261601632393135, "right_score": 0.9194441317490241, "margin": -0.6578424993558891, "p_left": 0.34122443053878343, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4173723647584609, "brier": 0.11643411199651704 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.3050034902757184, "right_score": 0.747685864879143, "margin": -1.0526893551548615, "p_left": 0.25870900530887825, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29936202538849976, "brier": 0.0669303494279092 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.11511511170402645, "right_score": 0.05064127431529053, "margin": -0.16575638601931697, "p_left": 0.458655522421061, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6136994604799243, "brier": 0.2103648882473364 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.8617429677110443, "right_score": 0.05064127431529053, "margin": -0.9123842420263348, "p_left": 0.28651219545986395, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3375899346568231, "brier": 0.08208923814723128 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.261601632393135, "right_score": -0.23921598089243568, "margin": 0.5008176132855706, "p_left": 0.6226514541122198, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9745859937427701, "brier": 0.38769483330806176 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.8617429677110443, "right_score": 0.4011574715259357, "margin": -1.26290043923698, "p_left": 0.22047500185842583, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.24907052154172174, "brier": 0.048609226444472874 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.42620650634561197, "right_score": 1.084321020071715, "margin": -1.510527526417327, "p_left": 0.1808606269236523, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.7100285613280706, "brier": 0.6709893125239119 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.6949809073180091, "right_score": 0.6597613507697451, "margin": -1.3547422580877542, "p_left": 0.20509615200284725, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.5842763756456375, "brier": 0.6318721275606806 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.747685864879143, "right_score": 0.5012205484393386, "margin": 0.2464653164398044, "p_left": 0.5613063044491862, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5774885252260714, "brier": 0.19245215851603015 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.4011574715259357, "right_score": -0.26368667522824385, "margin": 0.6648441467541796, "p_left": 0.6603477178450704, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41498873747607956, "brier": 0.1153636727730519 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.9194441317490241, "right_score": -0.23921598089243568, "margin": 1.1586601126414597, "p_left": 0.7610891650820953, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.27300475967650595, "brier": 0.05707838704117029 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.4011574715259357, "right_score": -0.3050034902757184, "margin": 0.7061609618016541, "p_left": 0.6695523184545048, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1073069331426555, "brier": 0.4483003071478026 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.05064127431529053, "right_score": -0.3050034902757184, "margin": 0.3556447645910089, "p_left": 0.5879857470468596, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5310525710935219, "brier": 0.1697557446365344 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.6597613507697451, "right_score": -0.1323271628801605, "margin": 0.7920885136499056, "p_left": 0.6882795989489928, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.37356012978318587, "brier": 0.09716960843140075 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.6621852264154655, "right_score": -0.3050034902757184, "margin": -0.35718173613974713, "p_left": 0.41164195893761824, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5304196030246372, "brier": 0.1694491023579998 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.16596378989509433, "right_score": -0.8617429677110443, "margin": 1.0277067576061387, "p_left": 0.7364710614761896, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30588533580497956, "brier": 0.06944750143948626 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.16596378989509433, "right_score": -0.11511511170402645, "margin": 0.2810789015991208, "p_left": 0.5698107111691074, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7029905097739902, "brier": 0.00487353539393654 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.16596378989509433, "right_score": 0.5012205484393386, "margin": -0.3352567585442443, "p_left": 0.4169621252565075, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5395031294870914, "brier": 0.17385741389842344 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.5012205484393386, "right_score": -0.6621852264154655, "margin": 1.1634057748548041, "p_left": 0.7619510109792635, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.435278790270449, "brier": 0.5805693431323218 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.4011574715259357, "right_score": -0.32304786034958294, "margin": 0.7242053318755186, "p_left": 0.6735323867556174, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3952191970486485, "brier": 0.10658110249748375 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.1323271628801605, "right_score": -0.42620650634561197, "margin": 0.29387934346545147, "p_left": 0.572945594068476, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8508438595104917, "brier": 0.3282666537624789 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.6621852264154655, "right_score": -0.8617429677110443, "margin": 0.1995577412955788, "p_left": 0.5497245285428528, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5983379834212953, "brier": 0.20274800019595618 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.1671950247196146, "right_score": -0.12062811515324734, "margin": -0.04656690956636725, "p_left": 0.48836037588722353, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7167016704888809, "brier": 0.26177510496226325 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.747685864879143, "right_score": -0.3050034902757184, "margin": 1.0526893551548615, "p_left": 0.7412909946911217, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29936202538849965, "brier": 0.06693034942790922 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.05064127431529053, "right_score": -0.26368667522824385, "margin": 0.3143279495435344, "p_left": 0.5779413114670408, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5482829527192702, "brier": 0.17813353656616152 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.084321020071715, "right_score": 0.31242345161196705, "margin": 0.771897568459748, "p_left": 0.6839312328792476, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1517954715997363, "brier": 0.46776193130772764 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.6641123826292802, "right_score": 0.5012205484393386, "margin": 0.16289183418994158, "p_left": 0.5406331524303523, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7779061559596735, "brier": 0.2922842055067805 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.6641123826292802, "right_score": -0.11511511170402645, "margin": 0.7792274943333066, "p_left": 0.6855135974036675, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3775869444308082, "brier": 0.09890169741798255 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.6621852264154655, "right_score": -0.32304786034958294, "margin": -0.3391373660658826, "p_left": 0.41601903636708526, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5378868932041697, "brier": 0.17307183861979822 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.16596378989509433, "right_score": -0.26368667522824385, "margin": 0.42965046512333815, "p_left": 0.6057901996426612, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7160467906274809, "brier": 0.01119156634043412 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.26368667522824385, "right_score": 0.6641123826292802, "margin": -0.927799057857524, "p_left": 0.28337145120937296, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.33319763567706473, "brier": 0.08029937936050605 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.05064127431529053, "right_score": -0.11511511170402645, "margin": 0.16575638601931697, "p_left": 0.541344477578939, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6136994604799243, "brier": 0.2103648882473364 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.11511511170402645, "right_score": 0.747685864879143, "margin": -0.8628009765831695, "p_left": 0.296754473756949, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.35204919317801386, "brier": 0.08806321769476375 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.084321020071715, "right_score": 0.9194441317490241, "margin": 0.16487688832269098, "p_left": 0.5411260986113886, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7789798313088679, "brier": 0.29281745459838227 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.747685864879143, "right_score": -0.32304786034958294, "margin": 1.070733725228726, "p_left": 0.7447364250340254, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2947249151023341, "brier": 0.06515949270440975 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.7491911835219076, "right_score": -0.6109524905310836, "margin": 1.360143674052991, "p_left": 0.7957830475217095, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9085005206783547, "brier": 0.08748761120122985 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.32304786034958294, "right_score": 0.6641123826292802, "margin": -0.9871602429788631, "p_left": 0.2714733475927486, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.31673106869386475, "brier": 0.0736977784532133 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.9194441317490241, "right_score": 0.7491911835219076, "margin": 0.17025294822711645, "p_left": 0.5424607224614587, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6116395971813037, "brier": 0.20934219049049033 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.05064127431529053, "right_score": -0.8617429677110443, "margin": 0.9123842420263348, "p_left": 0.713487804540136, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33758993465682297, "brier": 0.08208923814723128 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 100.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.7491911835219076, "rank": 3.0, "portrait_scores": [ 1.161650836101114, 0.33673153094270125 ], "portrait_score_gap": 0.8249193051584127, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.084321020071715, "rank": 1.0, "portrait_scores": [ 0.07126738989785779, 2.0973746502455723 ], "portrait_score_gap": 2.0261072603477146, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.12062811515324734, "rank": 7.0, "portrait_scores": [ -0.9232039705708701, 0.6819477402643754 ], "portrait_score_gap": 1.6051517108352455, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.5, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.23921598089243568, "rank": 10.0, "portrait_scores": [ -0.5502010199482814, 0.07176905816341006 ], "portrait_score_gap": 0.6219700781116915, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": -0.1671950247196146, "rank": 9.0, "portrait_scores": [ -0.16491517828904867, -0.1694748711501805 ], "portrait_score_gap": 0.004559692861131837, "single_photo_rank_min": 6.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 8.5, 10.0 ] }, { "identity": "emma_watson", "mean_score": 0.6597613507697451, "rank": 4.0, "portrait_scores": [ 1.421831767927134, -0.10230906638764392 ], "portrait_score_gap": 1.5241408343147778, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.5, 9.0 ] }, { "identity": "gal_gadot", "mean_score": -0.6949809073180091, "rank": 13.0, "portrait_scores": [ -0.786345322534699, -0.6036164921013193 ], "portrait_score_gap": 0.18272883043337973, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "keira_knightley", "mean_score": -0.42620650634561197, "rank": 11.0, "portrait_scores": [ -0.7998199625594106, -0.05259305013181337 ], "portrait_score_gap": 0.7472269124275972, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.261601632393135, "rank": 6.0, "portrait_scores": [ 1.1475413669935748, -0.6243381022073048 ], "portrait_score_gap": 1.7718794692008797, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.6109524905310836, "rank": 12.0, "portrait_scores": [ -0.4360855060999995, -0.7858194749621676 ], "portrait_score_gap": 0.3497339688621681, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": -0.1323271628801605, "rank": 8.0, "portrait_scores": [ 0.386110963658314, -0.650765289418635 ], "portrait_score_gap": 1.036876253076949, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 8.5, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.31242345161196705, "rank": 5.0, "portrait_scores": [ 0.5861114512359971, 0.0387354519879371 ], "portrait_score_gap": 0.54737599924806, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.9194441317490241, "rank": 2.0, "portrait_scores": [ 1.2793826480705386, 0.5595056154275097 ], "portrait_score_gap": 0.7198770326430289, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.1328125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "1.0": 1088, "2.0": 2432, "3.0": 1984, "4.0": 1280, "5.0": 960, "6.0": 384, "7.0": 64 }, "target_primary_score": 0.9194441317490241, "target_primary_rank": 2.0, "target_alternate_photo_scores": [ 0.33356673417559074, 0.20162514565449396 ], "target_alternate_panel_score": 0.2675959399150424, "target_alternate_panel_rank": 5.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.005994307521907383, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.5068119208074781, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.8167250801566727, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.48159524360686523, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.3042029179672889, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } ] } ], "bootstrap": { "iterations": 1000, "interpretation": "Resampling the observed training comparisons with fixed features and selected penalties measures conditional fit sensitivity; these are not population confidence intervals or new human judgments.", "identities": [ { "identity": "alexandra_daddario", "rank_quantiles": [ 6.0, 8.0, 11.0 ], "score_quantiles": [ -0.5723172138961694, -0.20170397723534306, 0.13331596639017013 ], "top_share": 0.0 }, { "identity": "ana_de_armas", "rank_quantiles": [ 2.0, 4.0, 8.0 ], "score_quantiles": [ -0.24377183609608352, 0.2737849079452706, 0.7574629285495843 ], "top_share": 0.005 }, { "identity": "anne_hathaway", "rank_quantiles": [ 1.0, 2.0, 4.0 ], "score_quantiles": [ 0.24608999919770447, 0.5893125312976593, 0.9184604463879461 ], "top_share": 0.177 }, { "identity": "elizabeth_olsen", "rank_quantiles": [ 2.0, 3.0, 6.0 ], "score_quantiles": [ 0.022627347551845332, 0.4040189604643988, 0.7591663548819694 ], "top_share": 0.001 }, { "identity": "emma_stone", "rank_quantiles": [ 3.0, 6.0, 8.0 ], "score_quantiles": [ -0.1310713358392001, 0.14061985097559643, 0.3816225144613701 ], "top_share": 0.01 }, { "identity": "emma_watson", "rank_quantiles": [ 12.0, 13.0, 13.0 ], "score_quantiles": [ -1.8873399009554939, -1.416533420928193, -0.9165309101220868 ], "top_share": 0.0 }, { "identity": "gal_gadot", "rank_quantiles": [ 7.0, 11.0, 11.0 ], "score_quantiles": [ -0.7211733461012964, -0.4416740194456829, -0.12203908159401684 ], "top_share": 0.0 }, { "identity": "keira_knightley", "rank_quantiles": [ 5.0, 7.0, 11.0 ], "score_quantiles": [ -0.6227411815323641, -0.09990396438256641, 0.3283059565239656 ], "top_share": 0.001 }, { "identity": "lily_collins", "rank_quantiles": [ 9.0, 10.0, 11.0 ], "score_quantiles": [ -0.616233414062617, -0.40000338083319487, -0.17568556515297712 ], "top_share": 0.0 }, { "identity": "margot_robbie", "rank_quantiles": [ 7.0, 9.0, 11.0 ], "score_quantiles": [ -0.5892635098108622, -0.31308533502959324, -0.0144352616211573 ], "top_share": 0.0 }, { "identity": "natalie_portman", "rank_quantiles": [ 2.0, 4.0, 6.0 ], "score_quantiles": [ 0.06571645189185937, 0.3052633232202351, 0.5146806655172952 ], "top_share": 0.013 }, { "identity": "scarlett_johansson", "rank_quantiles": [ 1.0, 1.0, 3.0 ], "score_quantiles": [ 0.37146443557080794, 0.8610355073197047, 1.2690539970475534 ], "top_share": 0.793 }, { "identity": "tongdak_cheonsa", "rank_quantiles": [ 10.0, 12.0, 13.0 ], "score_quantiles": [ -2.087240681991438, -1.332346054488473, -0.37855820847516186 ], "top_share": 0.0 } ] }, "fixed_head_full": { "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 12, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6929321626547033, "brier_score": 0.22905915580080904, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.6949297067316841, "brier_score": 0.2300579251957289, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.694322560091619, "brier_score": 0.21933768743427584, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.696144000011814, "brier_score": 0.25149840071863516, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6932592026587094, "brier_score": 0.2500559951776231, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.009437953141937707, "right_score": 0.00024698642474490443, "margin": -0.00968493956668261, "p_left": 0.4975787840337453, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6883164354875824, "brier": 0.24758464632050053 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.001973474479071992, "right_score": -0.003206817399792488, "margin": 0.0051802918788644795, "p_left": 0.5012950700735712, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6957406809236206, "brier": 0.2512967472800667 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.01074420251303846, "right_score": -0.005245839103148787, "margin": 0.015990041616187247, "p_left": 0.5039974252321262, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7011741612064213, "brier": 0.25401340464061267 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.004070521011260935, "right_score": -0.004002396162004828, "margin": -6.812484925610655e-05, "p_left": 0.49998296878769255, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6931131187154417, "brier": 0.24998296907775475 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.009437953141937707, "right_score": -0.004002396162004828, "margin": -0.005435556979932879, "p_left": 0.49864111410073714, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6904330952253926, "brier": 0.24864296067162436 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.001973474479071992, "right_score": 0.005702293494479819, "margin": -0.003728819015407827, "p_left": 0.49906779632626824, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912845090626407, "brier": 0.24906866532995756 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.009437953141937707, "right_score": 0.0060542952667599955, "margin": -0.015492248408697703, "p_left": 0.49612701536035875, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6854310572756714, "brier": 0.24614201537037764 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.0024525376389379866, "right_score": 0.006366805501861729, "margin": -0.003914267862923742, "p_left": 0.4990214342836928, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6951062296767974, "brier": 0.2509795233071684 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.003528364324779695, "right_score": 0.002713974872993883, "margin": 0.0008143894517858119, "p_left": 0.5002035973516937, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6927400687378225, "brier": 0.2497964441001879 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.005245839103148787, "right_score": -0.00947179196544442, "margin": 0.004225952862295634, "p_left": 0.5010564866432865, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6910364364618358, "brier": 0.24894462952074095 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.007312342515155206, "margin": -0.0012580472483952106, "p_left": 0.4996854882293822, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6937764020194896, "brier": 0.25031461068827165 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.003206817399792488, "right_score": 0.005702293494479819, "margin": -0.008909110894272307, "p_left": 0.4977727370083199, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6976116575063851, "brier": 0.2522322236921142 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.01074420251303846, "margin": -0.004689907246278465, "p_left": 0.49882752533750047, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6908049763380337, "brier": 0.24882890003433467 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.004002396162004828, "right_score": 0.01074420251303846, "margin": -0.014746598675043288, "p_left": 0.4963134171386903, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7005476624227298, "brier": 0.25370017375450304 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.002713974872993883, "right_score": 0.005365045299471259, "margin": -0.002651070426477376, "p_left": 0.4993372327815509, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6944735942947275, "brier": 0.250663206478835 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.0035563270424336956, "right_score": 0.01074420251303846, "margin": -0.007187875470604765, "p_left": 0.4982030388690914, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6895597010049629, "brier": 0.2482062679383974 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.00024698642474490443, "right_score": -0.009437953141937707, "margin": 0.00968493956668261, "p_left": 0.5024212159662547, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6883164354875824, "brier": 0.2475846463205006 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.00024698642474490443, "right_score": -0.004070521011260935, "margin": 0.004317507436005839, "p_left": 0.5010793751822942, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.693149510666943, "brier": 1.16505078415258e-06 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.00024698642474490443, "right_score": -0.00947179196544442, "margin": 0.009718778390189326, "p_left": 0.5024296754730645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6980183765402478, "brier": 0.2524355787959689 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.00947179196544442, "right_score": 0.0035563270424336956, "margin": -0.013028119007878116, "p_left": 0.4967430163157318, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6866543373915719, "brier": 0.2467536242584514 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0060542952667599955, "right_score": 0.0007174158272800026, "margin": 0.005336879439479993, "p_left": 0.5013342166930768, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6904823011212491, "brier": 0.24866756344110733 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.005365045299471259, "right_score": 0.0024525376389379866, "margin": 0.0029125076605332726, "p_left": 0.5007281264004269, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6946044947274462, "brier": 0.25072865656848187 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.0035563270424336956, "right_score": -0.009437953141937707, "margin": 0.012994280184371402, "p_left": 0.5032485243364203, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6866711467339565, "brier": 0.246762028573944 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.009751167958215307, "right_score": -0.000818008680712527, "margin": -0.00893315927750278, "p_left": 0.49776672503213154, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6976237353323637, "brier": 0.2522382624849505 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.005245839103148787, "right_score": 0.01074420251303846, "margin": -0.015990041616187247, "p_left": 0.49600257476787385, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7011741612064213, "brier": 0.25401340464061267 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.004002396162004828, "right_score": 0.007312342515155206, "margin": -0.011314738677160034, "p_left": 0.4971713455084551, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6988205527270782, "brier": 0.2528366557777774 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.006366805501861729, "right_score": 0.001514875938342089, "margin": 0.00485192956351964, "p_left": 0.501212980011295, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6955760879913799, "brier": 0.2512144513318028 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.00454757471227478, "right_score": -0.00947179196544442, "margin": 0.014019366677719202, "p_left": 0.5035047842663213, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7001814315278704, "brier": 0.25351706777907473 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.00454757471227478, "right_score": -0.004070521011260935, "margin": 0.008618095723535715, "p_left": 0.5021545105959927, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6888474166161845, "brier": 0.24785013131991554 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.0035563270424336956, "right_score": 0.0007174158272800026, "margin": 0.002838911215153693, "p_left": 0.5007097273271228, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6945676435942948, "brier": 0.2507102310400017 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.00024698642474490443, "right_score": 0.007312342515155206, "margin": -0.007065356090410302, "p_left": 0.4982336683252216, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931534204540521, "brier": 3.1199275853255016e-06 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.007312342515155206, "right_score": 0.00454757471227478, "margin": 0.0027647678028804258, "p_left": 0.5006911915104346, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6945305199537066, "brier": 0.2506916692561387 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.004002396162004828, "right_score": -0.004070521011260935, "margin": 6.812484925610655e-05, "p_left": 0.5000170312123075, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6931131187154417, "brier": 0.24998296907775475 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.004070521011260935, "right_score": -0.005245839103148787, "margin": 0.0011753180918878522, "p_left": 0.5002938294891479, "prediction": "left", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6937350122774564, "brier": 0.2502939158249166 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.006366805501861729, "right_score": -0.003206817399792488, "margin": 0.009573622901654217, "p_left": 0.5023933874450961, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6979454487489529, "brier": 0.25239911574855844 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.005245839103148787, "right_score": 0.0007174158272800026, "margin": -0.005963254930428789, "p_left": 0.498509190685206, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6961332530697442, "brier": 0.2514930318272071 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.0029447699503040324, "right_score": 0.011078225800300573, "margin": -0.014022995750604606, "p_left": 0.49649430851004667, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931717609097745, "brier": 1.228987282273122e-05 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.0007174158272800026, "right_score": 0.00454757471227478, "margin": -0.0038301588849947777, "p_left": 0.49904246144935127, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912339348809625, "brier": 0.24904337832942725 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.003206817399792488, "right_score": -0.0029447699503040324, "margin": -0.00026204744948845563, "p_left": 0.4999344881380028, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6932782128682977, "brier": 0.2500655161538013 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.004002396162004828, "right_score": -0.009437953141937707, "margin": 0.005435556979932879, "p_left": 0.5013588858992628, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6904330952253926, "brier": 0.24864296067162442 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated" }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.6346665037810238, "brier_score": 0.2000496325344503, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.7011981208677063, "brier_score": 0.23318987766321952, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.6928292093988221, "brier_score": 0.21859312743320045, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.7179359438054744, "brier_score": 0.26238337812325774, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5915393000890928, "brier_score": 0.19947521423534884, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.10680135786997269, "right_score": 0.1216719188817411, "margin": -0.2284732767517138, "p_left": 0.4431288555095851, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5854214042550331, "brier": 0.19636318258523475 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.001123901643209961, "right_score": -0.02638904300485292, "margin": 0.02526514136164296, "p_left": 0.5063159493737066, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.705859540039663, "brier": 0.25635584059019784 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.1815549361914133, "right_score": 0.26313412092953903, "margin": -0.08157918473812573, "p_left": 0.4796165071846814, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6531892530326654, "brier": 0.23003199396403354 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": 3.110240007205868e-05, "right_score": 0.2254347509806327, "margin": -0.22540364858056064, "p_left": 0.44388646603452253, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5867828077974478, "brier": 0.19703519472861733 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.10680135786997269, "right_score": 0.2254347509806327, "margin": -0.3322361088506054, "p_left": 0.41769664310778204, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5407637352712248, "brier": 0.17447048566350984 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.001123901643209961, "right_score": -0.05835078900834853, "margin": 0.05722688736513857, "p_left": 0.5143028186760848, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7221699329746087, "brier": 0.2645073892981658 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.10680135786997269, "right_score": 0.0864710953279705, "margin": -0.19327245319794317, "p_left": 0.45183173418621964, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6011729847609687, "brier": 0.20415191601772664 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.02438246627547637, "right_score": 0.0838032179321319, "margin": -0.05942075165665553, "p_left": 0.4851491814663966, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7232988446885962, "brier": 0.26507136534472137 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0834474602740379, "right_score": 0.08989254179224036, "margin": -0.0064450815182024584, "p_left": 0.49838873519798305, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6963749136945316, "brier": 0.25161386097627925 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.26313412092953903, "right_score": 0.055643111092437844, "margin": 0.20749100983710117, "p_left": 0.5516874458210564, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5947736144075471, "brier": 0.2009841462344482 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0864710953279705, "right_score": -0.02638469442216072, "margin": 0.11285578975013122, "p_left": 0.5281840401160163, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6383104951832995, "brier": 0.22261030000124493 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.02638904300485292, "right_score": -0.05835078900834853, "margin": 0.031961746003495606, "p_left": 0.5079897563490184, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6772939962742512, "brier": 0.24207407985749824 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0864710953279705, "right_score": 0.1815549361914133, "margin": -0.0950838408634428, "p_left": 0.4762469329196451, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6467349517622581, "brier": 0.22681114111536893 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.2254347509806327, "right_score": 0.1815549361914133, "margin": 0.0438798147892194, "p_left": 0.5109681938722052, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6714479336271388, "brier": 0.23915210740461312 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.08989254179224036, "right_score": 0.065203931482052, "margin": 0.02468861031018836, "p_left": 0.506171839088605, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6808790644047913, "brier": 0.24386625250913066 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.055071009764953036, "right_score": 0.1815549361914133, "margin": -0.12648392642646028, "p_left": 0.4684211075821728, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6319036586916362, "brier": 0.21941833402850952 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1216719188817411, "right_score": -0.10680135786997269, "margin": 0.2284732767517138, "p_left": 0.5568711444904149, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5854214042550331, "brier": 0.19636318258523477 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1216719188817411, "right_score": 3.110240007205868e-05, "margin": 0.12164081648166904, "p_left": 0.5303727625009338, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6949956024207953, "brier": 0.0009225047019381299 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1216719188817411, "right_score": 0.055643111092437844, "margin": 0.06602880778930326, "p_left": 0.5165012072145535, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7267064609162144, "brier": 0.26677349705409115 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.055643111092437844, "right_score": 0.055071009764953036, "margin": 0.0005721013274848086, "p_left": 0.5001430253279702, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934332721361782, "brier": 0.2501430457842146 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0864710953279705, "right_score": -0.025303244357637613, "margin": 0.11177433968560811, "p_left": 0.5279145284507071, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6388208863137894, "brier": 0.22286469244791826 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.065203931482052, "right_score": 0.02438246627547637, "margin": 0.04082146520657563, "p_left": 0.5102039493593463, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7137661977047162, "brier": 0.26030806994187444 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.055071009764953036, "right_score": -0.10680135786997269, "margin": 0.16187236763492574, "p_left": 0.5403799585185792, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6154827599668982, "brier": 0.21125058253138296 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.1463967414385155, "right_score": 0.03839825682645058, "margin": 0.10799848461206492, "p_left": 0.526973408830489, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6406051893409591, "brier": 0.22375415595344766 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.26313412092953903, "right_score": 0.1815549361914133, "margin": 0.08157918473812573, "p_left": 0.5203834928153186, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6531892530326654, "brier": 0.23003199396403354 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.2254347509806327, "right_score": -0.02638469442216072, "margin": 0.25181944540279344, "p_left": 0.5626242776320524, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5751432312942327, "brier": 0.191297522516884 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.0838032179321319, "right_score": 0.12751449899715323, "margin": -0.043711281065021326, "p_left": 0.4890739193617692, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6715303555274248, "brier": 0.2391932985998823 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.15377849044029013, "right_score": 0.055643111092437844, "margin": 0.09813537934785228, "p_left": 0.5245141742401895, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.743418206568626, "brier": 0.2751151189788679 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.15377849044029013, "right_score": 3.110240007205868e-05, "margin": 0.15374738804021806, "p_left": 0.5383613105469631, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6192253632876081, "brier": 0.2131102795999174 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.055071009764953036, "right_score": -0.025303244357637613, "margin": 0.08037425412259065, "p_left": 0.5200827534441937, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7341415929520444, "brier": 0.27048607043009404 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1216719188817411, "right_score": -0.02638469442216072, "margin": 0.14805661330390182, "p_left": 0.5369466863482676, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.695884776597126, "brier": 0.0013650576321172626 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.02638469442216072, "right_score": 0.15377849044029013, "margin": -0.18016318486245084, "p_left": 0.45508064038033735, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6071174592672645, "brier": 0.20709838924897794 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.2254347509806327, "right_score": 3.110240007205868e-05, "margin": 0.22540364858056064, "p_left": 0.5561135339654775, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5867828077974478, "brier": 0.19703519472861733 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": 3.110240007205868e-05, "right_score": 0.26313412092953903, "margin": -0.263103018529467, "p_left": 0.4346010710834831, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5702237281062652, "brier": 0.18887809098691075 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.0838032179321319, "right_score": -0.02638904300485292, "margin": 0.11019226093698482, "p_left": 0.5275202242209692, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7497603355484581, "brier": 0.27827758696214167 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.26313412092953903, "right_score": -0.025303244357637613, "margin": 0.2884373652871766, "p_left": 0.5716135311152726, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5592921608565882, "brier": 0.18351496672352555 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.06298034485226085, "right_score": 0.14603425518392582, "margin": -0.08305391033166497, "p_left": 0.47924844964581986, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6940091768549489, "brier": 0.00043062684210207376 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.025303244357637613, "right_score": 0.15377849044029013, "margin": -0.17908173479792774, "p_left": 0.4553488338039974, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6076097512717235, "brier": 0.2073425604466604 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.02638904300485292, "right_score": 0.06298034485226085, "margin": -0.08936938785711378, "p_left": 0.4776725116527054, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7388299033595251, "brier": 0.27282600508319316 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.2254347509806327, "right_score": -0.10680135786997269, "margin": 0.3322361088506054, "p_left": 0.582303356892218, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5407637352712248, "brier": 0.17447048566350978 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated" }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.6345545847138067, "fold_losses": [ 0.42762889091049494, 0.6910085786197415, 0.24823515112097208, 0.14592212725556453, 0.3552525672729096, 0.4704503124158155, 0.5375262499388462, 0.7620696982640343, 0.824479013474704, 0.618376783068437, 0.7616353412381618, 0.9258052759663866, 1.5857897553378622, 0.7239893279957255, 0.48252141110380975, 0.24876347619024952, 1.1482807264940307, 0.5157850201130045, 1.276175424900226, 0.7804209842486303, 0.48153901104228286, 0.20330385783877247, 0.27320953019818717, 0.741141518122511 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.4493025288992314, "brier_score": 0.11647698220261526, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.45454545454545453, "mean_cross_entropy": 0.7496120894387125, "brier_score": 0.2560067092164533, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.7130655309864051, "brier_score": 0.2276081930335847, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.8227052063433271, "brier_score": 0.3128037415821906, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.29601020680960244, "brier_score": 0.07141873863748732, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.1058198932836754, "right_score": -0.17107806126309294, "margin": -0.9347418320205825, "p_left": 0.2819636906067488, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.331235141026955, "brier": 0.07950352282057836 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.2838028716048929, "right_score": -0.43717394747375726, "margin": 0.15337107586886434, "p_left": 0.5382677849323922, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7727701770114104, "brier": 0.2897322082960241 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.4942681837116253, "right_score": 0.976092106421734, "margin": -0.48182392271010877, "p_left": 0.3818215250391162, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4809780687824451, "brier": 0.14578767698319645 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.49480359576344846, "right_score": 0.8261935865857203, "margin": -1.3209971823491689, "p_left": 0.2106524361623088, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23654854329827354, "brier": 0.044374448861115585 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.1058198932836754, "right_score": 0.8261935865857203, "margin": -1.9320134798693958, "p_left": 0.1265278858001621, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1352790741307364, "brier": 0.016009305885058864 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.2838028716048929, "right_score": -0.7031027436036137, "margin": 0.41929987199872076, "p_left": 0.6033157032510833, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9246145369468429, "brier": 0.3639898377893492 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.1058198932836754, "right_score": 0.250890212907369, "margin": -1.3567101061910445, "p_left": 0.2047755165175815, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22913083502619855, "brier": 0.04193301216504229 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.47382464316093187, "right_score": 0.23116385294480976, "margin": -0.7049884961057417, "p_left": 0.3307071434887303, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1065220581126913, "brier": 0.447952927777015 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.05161806404118777, "right_score": -0.1876063713639991, "margin": 0.23922443540518687, "p_left": 0.5595225147224032, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5806715112490842, "brier": 0.1940204150364755 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.976092106421734, "right_score": 0.06127312526600459, "margin": 0.9148189811557295, "p_left": 0.7139852622953798, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3368929578984674, "brier": 0.08180443018424267 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.250890212907369, "right_score": -0.5014701645589219, "margin": 0.7523603774662909, "p_left": 0.6796927959433812, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38611435356259827, "brier": 0.10259670497056846 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.43717394747375726, "right_score": -0.7031027436036137, "margin": 0.2659287961298564, "p_left": 0.5660931586520029, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5689966230646534, "brier": 0.1882751469685959 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.250890212907369, "right_score": 0.4942681837116253, "margin": -0.24337797080425627, "p_left": 0.4394540716007232, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5788440979906168, "brier": 0.19311988104645356 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8261935865857203, "right_score": 0.4942681837116253, "margin": 0.331925402874095, "p_left": 0.5822277831394814, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5408935278551151, "brier": 0.17453362518055215 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.1876063713639991, "right_score": -0.44013637296898883, "margin": 0.25253000160498973, "p_left": 0.5627991222355782, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5748325133813954, "brier": 0.19114460751798093 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.11653249923330297, "right_score": 0.4942681837116253, "margin": -0.6108006829449283, "p_left": 0.35187657295746994, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43367412693663443, "brier": 0.12381712259629367 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.17107806126309294, "right_score": -1.1058198932836754, "margin": 0.9347418320205825, "p_left": 0.7180363093932512, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33123514102695495, "brier": 0.07950352282057833 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.17107806126309294, "right_score": -0.49480359576344846, "margin": 0.3237255345003555, "p_left": 0.5802319241655025, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7061901533673864, "brier": 0.006437161655298946 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.17107806126309294, "right_score": 0.06127312526600459, "margin": -0.23235118652909753, "p_left": 0.4421721326895441, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5837048457135189, "brier": 0.19551619492721978 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.06127312526600459, "right_score": -0.11653249923330297, "margin": 0.17780562449930756, "p_left": 0.5443346648452118, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7859966530488076, "brier": 0.296300227352149 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.250890212907369, "right_score": -0.2957079486916785, "margin": 0.5465981615990475, "p_left": 0.6333459789562634, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4567384359220007, "brier": 0.1344351711475408 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.44013637296898883, "right_score": -0.47382464316093187, "margin": 0.03368827019194304, "p_left": 0.5084212711216505, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7101331713916723, "brier": 0.2584921889289548 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.11653249923330297, "right_score": -1.1058198932836754, "margin": 0.9892873940503724, "p_left": 0.7289471463558983, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3161540511712573, "brier": 0.07346964946861081 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.42284865366754787, "right_score": -0.07549702705151831, "margin": 0.49834568071906615, "p_left": 0.6220704813424899, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.47470187860706037, "brier": 0.1428307210726973 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.976092106421734, "right_score": 0.4942681837116253, "margin": 0.48182392271010877, "p_left": 0.6181784749608837, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4809780687824451, "brier": 0.14578767698319647 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8261935865857203, "right_score": -0.5014701645589219, "margin": 1.3276637511446423, "p_left": 0.7904539292520962, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23514790454699286, "brier": 0.04390955576588549 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.23116385294480976, "right_score": 0.4457972104519444, "margin": -0.21463335750713466, "p_left": 0.44654670810437985, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5915779172133279, "brier": 0.19940396251885822 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17286336154994641, "right_score": 0.06127312526600459, "margin": 0.11159023628394182, "p_left": 0.52786864582184, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7504980393624874, "brier": 0.27864530724178316 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17286336154994641, "right_score": -0.49480359576344846, "margin": 0.6676669573133949, "p_left": 0.6609805552377593, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4140308567526163, "brier": 0.11493418392689794 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.11653249923330297, "right_score": -0.2957079486916785, "margin": 0.17917544945837552, "p_left": 0.5446744079224279, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7867425289578004, "brier": 0.2966702106456474 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.17107806126309294, "right_score": -0.5014701645589219, "margin": 0.33039210329582896, "p_left": 0.5818547785541656, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7067304354188488, "brier": 0.006700204772151483 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.5014701645589219, "right_score": 0.17286336154994641, "margin": -0.6743335261088683, "p_left": 0.3375271719707723, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4117757362583744, "brier": 0.1139245918185873 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8261935865857203, "right_score": -0.49480359576344846, "margin": 1.3209971823491689, "p_left": 0.7893475638376912, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.23654854329827346, "brier": 0.04437444886111557 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.49480359576344846, "right_score": 0.976092106421734, "margin": -1.4708957021851825, "p_left": 0.18680650982252142, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20678620245246312, "brier": 0.03489667211207179 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.23116385294480976, "right_score": -0.43717394747375726, "margin": 0.668337800418567, "p_left": 0.6611308650519625, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0821412787331122, "brier": 0.43709402072435627 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.976092106421734, "right_score": -0.2957079486916785, "margin": 1.2718000551134125, "p_left": 0.7810507319400919, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24711517358233714, "brier": 0.047938781983969496 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.11541425574116164, "right_score": 0.5113671369889095, "margin": -0.626781392730071, "p_left": 0.3482407049721603, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7414706609891669, "brier": 0.023030883627346888 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.2957079486916785, "right_score": 0.17286336154994641, "margin": -0.4685713102416249, "p_left": 0.38495445100927295, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48605895051862114, "brier": 0.1481899293518507 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.43717394747375726, "right_score": -0.11541425574116164, "margin": -0.3217596917325956, "p_left": 0.4202469574553437, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8669127465641322, "brier": 0.33611359033978605 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8261935865857203, "right_score": -1.1058198932836754, "margin": 1.9320134798693958, "p_left": 0.8734721141998378, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.13527907413073637, "brier": 0.01600930588505888 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "semantic_bt_tuned", "selected": { "ridge": 1.0, "consistency": 0.0, "mean_cross_entropy": 0.6249194318880952, "fold_losses": [ 0.5202343929073778, 0.5120575682355707, 0.32338185568831723, 0.2864809829012261, 0.6822826365003241, 0.4524468998743053, 0.6702601676935027, 0.7211654276239624, 0.38836507543174503, 0.5842663402756011, 0.7303940609664913, 1.1885497341530216, 0.977318066061178, 0.6801158178887854, 0.9219426925845362, 0.30837435416391634, 0.9219101524249234, 0.6287876635183459, 0.6720886406823104, 0.7563604677594846, 0.5904617777137419, 0.4338493261422586, 0.43127290173524524, 0.6156993623881124 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5522510421649448, "brier_score": 0.1614027017965844, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.8445200976183438, "brier_score": 0.30287607627942575, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8273989839885947, "brier_score": 0.28385662160708824, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.8787623248778422, "brier_score": 0.3409149856241007, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3864268464612882, "brier_score": 0.10341417183311888, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.21600974900570155, "right_score": 0.3113371318493887, "margin": -0.5273468808550903, "p_left": 0.3711358982395059, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4638401000596447, "brier": 0.13774185496204486 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.1288031561002862, "right_score": -0.27857651977989095, "margin": 0.40737967588017715, "p_left": 0.600459408168143, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9174399123495303, "brier": 0.3605515008576366 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.03307778902170853, "right_score": 0.7256399746707074, "margin": -0.6925621856489989, "p_left": 0.3334633448747399, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4056601444386228, "brier": 0.11119780237504971 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.11867954860382637, "right_score": 0.7440923712513091, "margin": -0.8627719198551355, "p_left": 0.2967605376778305, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.35205781598014935, "brier": 0.08806681672283506 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.21600974900570155, "right_score": 0.7440923712513091, "margin": -0.9601021202570107, "p_left": 0.27685774916470374, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.324149325366736, "brier": 0.07665021327254601 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.1288031561002862, "right_score": -0.3538361436996229, "margin": 0.4826392997999091, "p_left": 0.6183709129990109, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9633061185157504, "brier": 0.38238258604323033 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.21600974900570155, "right_score": -0.0006033427753804749, "margin": -0.2154064062303211, "p_left": 0.44635566262435544, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5912327886955359, "brier": 0.1992333775568274 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.3248739075837648, "right_score": 0.26403162025733123, "margin": -0.588905527841096, "p_left": 0.35688601642523127, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0303388299058, "brier": 0.413595595869408 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.17577796812345442, "right_score": 0.011184049166976379, "margin": 0.16459391895647804, "p_left": 0.5410558340545373, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6142328001845445, "brier": 0.21062974745537644 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.7256399746707074, "right_score": 0.18762670474921184, "margin": 0.5380132699214956, "p_left": 0.6313501327758463, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.45989468479281104, "brier": 0.13590272460438615 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": -0.0006033427753804749, "right_score": -0.2967766495245895, "margin": 0.296173306749209, "p_left": 0.5735067844212369, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5559855126287396, "brier": 0.18189646293471326 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.27857651977989095, "right_score": -0.3538361436996229, "margin": 0.07525962391973195, "p_left": 0.5188060303542055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.656225202948564, "brier": 0.23154763642347784 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": -0.0006033427753804749, "right_score": 0.03307778902170853, "margin": -0.033681131797089, "p_left": 0.49158051297093086, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6764484102891656, "brier": 0.24165140073276353 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.7440923712513091, "right_score": 0.03307778902170853, "margin": 0.7110145822296006, "p_left": 0.6706253066222178, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3995447083786874, "brier": 0.10848768863770807 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.011184049166976379, "right_score": 0.32119847632417015, "margin": -0.31001442715719374, "p_left": 0.4231112173664416, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8601202092784622, "brier": 0.3328006675284291 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.2300535482793345, "right_score": 0.03307778902170853, "margin": -0.26313133730104304, "p_left": 0.4345941125234266, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5702114208363355, "brier": 0.1888720426400248 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.3113371318493887, "right_score": -0.21600974900570155, "margin": 0.5273468808550903, "p_left": 0.6288641017604941, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4638401000596447, "brier": 0.13774185496204486 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.3113371318493887, "right_score": -0.11867954860382637, "margin": 0.43001668045321506, "p_left": 0.6058776515629322, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7160855486336645, "brier": 0.011210077100481672 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.3113371318493887, "right_score": 0.18762670474921184, "margin": 0.12371042710017688, "p_left": 0.5308882233442839, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.756914209175558, "brier": 0.2818423056856502 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.18762670474921184, "right_score": -0.2300535482793345, "margin": 0.41768025302854633, "p_left": 0.6029280217525252, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9236377093195298, "brier": 0.36352219941441355 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": -0.0006033427753804749, "right_score": -0.1090412830812981, "margin": 0.10843794030591762, "p_left": 0.5270829517212228, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6403973391818447, "brier": 0.2236505345527113 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.32119847632417015, "right_score": -0.3248739075837648, "margin": 0.646072383907935, "p_left": 0.6561248383577779, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0674765896765215, "brier": 0.4304998035100202 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.2300535482793345, "right_score": -0.21600974900570155, "margin": -0.014043799273632945, "p_left": 0.49648910788534173, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7001937335314203, "brier": 0.25352321847809905 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.05799079655174194, "right_score": -0.07159177730084082, "margin": 0.013600980749098879, "p_left": 0.5034001927715727, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6863698133418357, "brier": 0.2466113685393112 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.7256399746707074, "right_score": 0.03307778902170853, "margin": 0.6925621856489989, "p_left": 0.6665366551252602, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40566014443862286, "brier": 0.11119780237504967 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.7440923712513091, "right_score": -0.2967766495245895, "margin": 1.0408690207758986, "p_left": 0.7390176494216514, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3024334754752134, "brier": 0.06811178731340006 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.26403162025733123, "right_score": 0.34324344896886627, "margin": -0.07921182871153504, "p_left": 0.4802073908233178, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6543253754662859, "brier": 0.2305991382013387 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1302918664408319, "right_score": 0.18762670474921184, "margin": -0.05733483830837993, "p_left": 0.4856702157133576, "prediction": "right", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6648906155960989, "brier": 0.2358755584310593 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1302918664408319, "right_score": -0.11867954860382637, "margin": 0.24897141504465828, "p_left": 0.5619233149014105, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5763898887520045, "brier": 0.1919111820269688 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.2300535482793345, "right_score": -0.1090412830812981, "margin": -0.1210122651980364, "p_left": 0.46978379846218543, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6344704281859047, "brier": 0.22069681729755927 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.3113371318493887, "right_score": -0.2967766495245895, "margin": 0.6081137813739782, "p_left": 0.6475104101394092, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7386772987892929, "brier": 0.02175932109949673 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.2967766495245895, "right_score": 0.1302918664408319, "margin": -0.4270685159654214, "p_left": 0.39482655971357916, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5022401838762539, "brier": 0.1558880122552605 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.7440923712513091, "right_score": -0.11867954860382637, "margin": 0.8627719198551355, "p_left": 0.7032394623221695, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3520578159801494, "brier": 0.08806681672283506 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.11867954860382637, "right_score": 0.7256399746707074, "margin": -0.8443195232745337, "p_left": 0.30062582311348846, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.35756937684387996, "brier": 0.09037588552266244 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.26403162025733123, "right_score": -0.27857651977989095, "margin": 0.5426081400372222, "p_left": 0.6324189287278604, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.000811382557251, "brier": 0.3999537014132946 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.7256399746707074, "right_score": -0.1090412830812981, "margin": 0.8346812577520055, "p_left": 0.6973438463530766, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3604766665469388, "brier": 0.09160074734035013 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.1670372041623521, "right_score": -0.46989054246715634, "margin": 0.6369277466295085, "p_left": 0.6540586437354035, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7430221355395572, "brier": 0.023734065709591967 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.1090412830812981, "right_score": 0.1302918664408319, "margin": -0.23933314952213, "p_left": 0.44045069208824456, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5806236265846411, "brier": 0.1939968121610136 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.27857651977989095, "right_score": 0.1670372041623521, "margin": -0.4456137239422431, "p_left": 0.39040414715814054, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9405728016560239, "brier": 0.37160710380199397 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.7440923712513091, "right_score": -0.21600974900570155, "margin": 0.9601021202570107, "p_left": 0.7231422508352964, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3241493253667359, "brier": 0.07665021327254595 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "consistent_bt", "selected": { "ridge": 0.1, "consistency": 10.0, "mean_cross_entropy": 0.5425629612400248, "fold_losses": [ 0.35057437747627623, 0.2816562438835754, 0.16253833416938934, 0.06965013858793583, 0.2962009626938364, 0.2505102870149487, 0.6225380312619966, 1.238405843324399, 0.1274589006752498, 0.5340598443771011, 0.7004317586474527, 1.1306489639475439, 0.9689846679941176, 0.5338901152068204, 0.730479448798273, 0.1668541477258192, 1.0852676330630473, 0.6842651368177488, 0.46483212386233974, 0.7066713246945229, 0.6151175547278165, 0.35230487461110915, 0.3459689711514159, 0.60220138504786 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.48597453370176313, "brier_score": 0.13750590589750142, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.1759566035194309, "brier_score": 0.4242522288468818, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8789114297662777, "brier_score": 0.2956877096021123, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.7700469510257375, "brier_score": 0.6813812673364206, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.20273813323566692, "brier_score": 0.04148603877442693, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.0454943112613788, "right_score": 0.023790764324913255, "margin": -1.0692850755862922, "p_left": 0.25553906680073024, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2950949021109402, "brier": 0.06530021466138808 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.028898460614838417, "right_score": -1.081246875833081, "margin": 1.0523484152182427, "p_left": 0.7412256043095864, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3517986559854058, "brier": 0.5494153964841115 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.20894135326215107, "right_score": 0.6538060693704881, "margin": -0.8627474226326393, "p_left": 0.2967656501197999, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.35206508585168916, "brier": 0.08806985109102748 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7492710043729757, "right_score": 1.5737691449038709, "margin": -2.3230401492768467, "p_left": 0.08923267703540379, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.09346782272579748, "brier": 0.007962470650904678 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.0454943112613788, "right_score": 1.5737691449038709, "margin": -2.6192634561652497, "p_left": 0.06790889996449255, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.07032472225424091, "brier": 0.004611618694387456 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.028898460614838417, "right_score": 0.5848113826263373, "margin": -0.6137098432411756, "p_left": 0.3512133976990404, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.43265142636184234, "brier": 0.12335085072330433 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.0454943112613788, "right_score": 0.4462504448362998, "margin": -1.4917447560976786, "p_left": 0.18365999348802398, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20292433615577482, "brier": 0.03373099320802101 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.1847842097376341, "right_score": 0.702240568434428, "margin": -0.887024778172062, "p_left": 0.291724190146317, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.231946477379801, "brier": 0.5016546228238905 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.21534275018090138, "right_score": -1.0120986343378737, "margin": 0.7967558841569724, "p_left": 0.6892801059582592, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3721075507310785, "brier": 0.09654685255331065 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.6538060693704881, "right_score": -0.13353577070836503, "margin": 0.7873418400788532, "p_left": 0.687260287325222, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3750421832234634, "brier": 0.09780612788390268 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.4462504448362998, "right_score": 0.1692109931331432, "margin": 0.2770394517031566, "p_left": 0.5688202571390024, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5641907873104514, "brier": 0.185915970653676 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.081246875833081, "right_score": 0.5848113826263373, "margin": -1.6660582584594184, "p_left": 0.15895042316951377, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.839162929348191, "brier": 0.70736439068674 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.4462504448362998, "right_score": -0.20894135326215107, "margin": 0.6551917980984509, "p_left": 0.6581794669505876, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0734694368182203, "brier": 0.43320021071535963 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.5737691449038709, "right_score": -0.20894135326215107, "margin": 1.782710498166022, "p_left": 0.856031234942013, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15544841408787802, "brier": 0.02072700531232184 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -1.0120986343378737, "right_score": 0.5466764741467869, "margin": -1.5587751084846606, "p_left": 0.17382248124348407, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.7497207232876624, "brier": 0.6825692924986732 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.36484039467420637, "right_score": -0.20894135326215107, "margin": -0.1558990414120553, "p_left": 0.4611039868027546, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6182326521081934, "brier": 0.21261688664539488 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.023790764324913255, "right_score": -1.0454943112613788, "margin": 1.0692850755862922, "p_left": 0.7444609331992698, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29509490211094014, "brier": 0.06530021466138808 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.023790764324913255, "right_score": -0.7492710043729757, "margin": 0.7730617686978889, "p_left": 0.6841828433251517, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7660609666279138, "brier": 0.033923319775337364 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.023790764324913255, "right_score": -0.13353577070836503, "margin": 0.1573265350332783, "p_left": 0.5392507071934828, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7749012173080058, "brier": 0.2907913252086713 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.13353577070836503, "right_score": -0.36484039467420637, "margin": 0.23130462396584134, "p_left": 0.5575697108397982, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8154723655656779, "brier": 0.3108839824459762 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.4462504448362998, "right_score": 0.15903273972255014, "margin": 0.28721770511374967, "p_left": 0.5713148450450184, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5598148289139819, "brier": 0.18377096207877658 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.5466764741467869, "right_score": -0.1847842097376341, "margin": 0.731460683884421, "p_left": 0.675125727725922, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1243170260412203, "brier": 0.45579474823745586 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.36484039467420637, "right_score": -1.0454943112613788, "margin": 0.6806539165871724, "p_left": 0.6638846291810345, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4096468958355791, "brier": 0.11297354250077067 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.2791139626448253, "right_score": 0.3022286793885628, "margin": -0.023114716743737496, "p_left": 0.4942215780912575, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7047713237113331, "brier": 0.2558118120684979 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.6538060693704881, "right_score": -0.20894135326215107, "margin": 0.8627474226326393, "p_left": 0.7032343498802001, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.35206508585168916, "brier": 0.08806985109102751 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.5737691449038709, "right_score": 0.1692109931331432, "margin": 1.4045581517707277, "p_left": 0.8029062022395952, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21951738102423013, "brier": 0.03884596511561935 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.702240568434428, "right_score": 1.0503504289650538, "margin": -0.34810986053062587, "p_left": 0.41384084989738623, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5341639390724281, "brier": 0.17126424904379095 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.34905949479209497, "right_score": -0.13353577070836503, "margin": 0.48259526550046, "p_left": 0.6183605213641918, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9632788892145906, "brier": 0.38236973438179517 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.34905949479209497, "right_score": -0.7492710043729757, "margin": 1.0983304991650706, "p_left": 0.7499471607461431, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2877525272721393, "brier": 0.0625264224189152 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.36484039467420637, "right_score": 0.15903273972255014, "margin": -0.5238731343967565, "p_left": 0.3719470122431601, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46513074066436233, "brier": 0.1383445799166135 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.023790764324913255, "right_score": 0.1692109931331432, "margin": -0.14542022880822994, "p_left": 0.4637088744345038, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6957882350534038, "brier": 0.0013170457948106108 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.1692109931331432, "right_score": 0.34905949479209497, "margin": -0.17984850165895178, "p_left": 0.45515867733197124, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6072606777794309, "brier": 0.2071694215505895 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.5737691449038709, "right_score": -0.7492710043729757, "margin": 2.3230401492768467, "p_left": 0.9107673229645962, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.09346782272579768, "brier": 0.007962470650904685 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7492710043729757, "right_score": 0.6538060693704881, "margin": -1.403077073743464, "p_left": 0.1973282803007013, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.21980946593491524, "brier": 0.038938450206432135 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.702240568434428, "right_score": -1.081246875833081, "margin": 1.783487444267509, "p_left": 0.8561269606626031, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.9388240395749314, "brier": 0.7329533727733863 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.6538060693704881, "right_score": 0.15903273972255014, "margin": 0.49477332964793796, "p_left": 0.6212302603922819, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.476053476092551, "brier": 0.14346651564249857 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.7157513373508678, "right_score": -0.5510518742526286, "margin": 1.2668032116034964, "p_left": 0.7801950193489806, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8816129715448553, "brier": 0.07850924886797563 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.15903273972255014, "right_score": 0.34905949479209497, "margin": -0.19002675506954483, "p_left": 0.45263575310269555, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6026407989088843, "brier": 0.20487912498684435 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.081246875833081, "right_score": 0.7157513373508678, "margin": -1.796998213183949, "p_left": 0.1422168634197542, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.9504021791944213, "brier": 0.7357919094014446 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.5737691449038709, "right_score": -1.0454943112613788, "margin": 2.6192634561652497, "p_left": 0.9320911000355074, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.07032472225424069, "brier": 0.004611618694387458 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "semantic_bt_matched", "selected": { "ridge": 0.1, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.42440977019226217, "brier_score": 0.11035307477500644, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 1, "accuracy": 0.09090909090909091, "mean_cross_entropy": 1.259578839049563, "brier_score": 0.455985772392965, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 1, "accuracy": 0.14285714285714285, "mean_cross_entropy": 1.2417858891770985, "brier_score": 0.42521488370222765, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2951647387944925, "brier_score": 0.5175275497744398, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.17341551706132868, "brier_score": 0.029763590143008983, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.9991092874508453, "right_score": 0.008486257192471314, "margin": -1.0075955446433165, "p_left": 0.2674506701126508, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3112245958023493, "brier": 0.07152986094370595 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.143374810069331, "right_score": -0.831055721527555, "margin": 0.974430531596886, "p_left": 0.7260017153501688, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.294633433015739, "brier": 0.5270784906913875 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": -0.16129440660609978, "right_score": 1.38784043475062, "margin": -1.5491348413567196, "p_left": 0.1752112590772856, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1926279970482197, "brier": 0.030698985307447697 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6197621210901691, "right_score": 1.5139054633913145, "margin": -2.1336675844814836, "p_left": 0.10586731804459742, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.11190110100050356, "brier": 0.011207889029955944 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.9991092874508453, "right_score": 1.5139054633913145, "margin": -2.5130147508421596, "p_left": 0.07495081914780109, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.07790837439424217, "brier": 0.0056176252909263865 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.143374810069331, "right_score": -0.36135995335827986, "margin": 0.5047347634276109, "p_left": 0.6235713713276492, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9770268149912382, "brier": 0.38884125513944495 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.9991092874508453, "right_score": 0.08172087330639824, "margin": -1.0808301607572435, "p_left": 0.2533489485958951, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.292157336317921, "brier": 0.06418568975464549 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.4563715539775914, "right_score": 0.7136596860895247, "margin": -2.1700312400671162, "p_left": 0.10247416015926612, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.278144608192374, "brier": 0.8055526331818147 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.019581740371938994, "right_score": -0.2288548054050908, "margin": 0.2092730650331518, "p_left": 0.5521281579849882, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5939750893897267, "brier": 0.20058918686991972 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.38784043475062, "right_score": 0.39148030855775595, "margin": 0.996360126192864, "p_left": 0.7303423344161803, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31424190350793413, "brier": 0.0727152566081151 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.08172087330639824, "right_score": -0.8340757134080307, "margin": 0.9157965867144289, "p_left": 0.7141848576572702, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3366134458705914, "brier": 0.08169029559239487 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.831055721527555, "right_score": -0.36135995335827986, "margin": -0.46969576816927516, "p_left": 0.3846882537302701, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9553220032739562, "brier": 0.3786085450975045 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.08172087330639824, "right_score": -0.16129440660609978, "margin": 0.24301527991249802, "p_left": 0.560456583271043, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.822018780094444, "brier": 0.31411158173185155 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.5139054633913145, "right_score": -0.16129440660609978, "margin": 1.6751998699974142, "p_left": 0.842267868067653, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.171657182236439, "brier": 0.02487942544392333 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.2288548054050908, "right_score": 0.8262534308478345, "margin": -1.0551082362529254, "p_left": 0.2582453863623655, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3538450361473313, "brier": 0.5501999068527165 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.6186983779240002, "right_score": -0.16129440660609978, "margin": -0.4574039713179004, "p_left": 0.38760185441327677, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49037264331109115, "brier": 0.150235197544611 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.008486257192471314, "right_score": -0.9991092874508453, "margin": 1.0075955446433165, "p_left": 0.7325493298873493, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3112245958023494, "brier": 0.07152986094370589 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.008486257192471314, "right_score": -0.6197621210901691, "margin": 0.6282483782826405, "p_left": 0.6520921812727092, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7416935338702364, "brier": 0.02313203160429063 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.008486257192471314, "right_score": 0.39148030855775595, "margin": -0.38299405136528464, "p_left": 0.40540497172686046, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5198747299177128, "brier": 0.16435319110085653 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.39148030855775595, "right_score": -0.6186983779240002, "margin": 1.0101786864817561, "p_left": 0.7330551170149656, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3207130726593126, "brier": 0.5373698045818249 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.08172087330639824, "right_score": -0.26089561501613157, "margin": 0.3426164883225298, "p_left": 0.5848259588927, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5364409821803444, "brier": 0.17236948440936609 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.8262534308478345, "right_score": -1.4563715539775914, "margin": 2.282624984825426, "p_left": 0.9074277883803956, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.379766272867201, "brier": 0.8234251911249361 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.6186983779240002, "right_score": -0.9991092874508453, "margin": 0.3804109095268451, "p_left": 0.5939722054914698, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5209227528161794, "brier": 0.16485856991346123 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.14138894321062326, "right_score": 0.03059651948247402, "margin": -0.17198546269309728, "p_left": 0.45710930423284424, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7828327389339766, "brier": 0.29473030755054647 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.38784043475062, "right_score": -0.16129440660609978, "margin": 1.5491348413567196, "p_left": 0.8247887409227144, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19262799704821965, "brier": 0.030698985307447708 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.5139054633913145, "right_score": -0.8340757134080307, "margin": 2.3479811767993453, "p_left": 0.9127736269153444, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.09126737336334134, "brier": 0.007608440161503538 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.7136596860895247, "right_score": 0.5079456226299373, "margin": 0.20571406345958743, "p_left": 0.5512479162492021, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8012846957346121, "brier": 0.30387426516908733 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.28023464161929446, "right_score": 0.39148030855775595, "margin": -0.1112456669384615, "p_left": 0.47221722972459873, "prediction": "right", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6390704998652287, "brier": 0.22298911204877445 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.28023464161929446, "right_score": -0.6197621210901691, "margin": 0.8999967627094636, "p_left": 0.7109488373603494, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3411548104736044, "brier": 0.08355057462333373 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.6186983779240002, "right_score": -0.26089561501613157, "margin": -0.3578027629078686, "p_left": 0.41149155894832307, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5301640090513196, "brier": 0.16932530308572125 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.008486257192471314, "right_score": -0.8340757134080307, "margin": 0.842561970600502, "p_left": 0.6990045221106804, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7793790526693891, "brier": 0.039602799820500294 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.8340757134080307, "right_score": 0.28023464161929446, "margin": -1.114310355027325, "p_left": 0.24706817874576895, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.28378059810455863, "brier": 0.06104268494875124 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.5139054633913145, "right_score": -0.6197621210901691, "margin": 2.1336675844814836, "p_left": 0.8941326819554026, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.11190110100050354, "brier": 0.011207889029955932 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6197621210901691, "right_score": 1.38784043475062, "margin": -2.007602555840789, "p_left": 0.11840701044844221, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.12602479257567822, "brier": 0.014020220123337502 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.7136596860895247, "right_score": -0.831055721527555, "margin": 1.5447154076170797, "p_left": 0.8241491615042239, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.738119151829123, "brier": 0.6792218404081153 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.38784043475062, "right_score": -0.26089561501613157, "margin": 1.6487360497667516, "p_left": 0.8387201507589682, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17587817909824643, "brier": 0.02601118977120994 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -6.704217334108087e-05, "right_score": -0.7804320601515438, "margin": 0.7803650179782027, "p_left": 0.685758778337271, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7674118571603281, "brier": 0.03450632372935536 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.26089561501613157, "right_score": 0.28023464161929446, "margin": -0.541130256635426, "p_left": 0.36792469495760394, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4587467383854047, "brier": 0.1353685811596459 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.831055721527555, "right_score": -6.704217334108087e-05, "margin": -0.8309886793542139, "p_left": 0.3034360597183115, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1925843670591516, "brier": 0.4852013229007518 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.5139054633913145, "right_score": -0.9991092874508453, "margin": 2.5130147508421596, "p_left": 0.9250491808521989, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.07790837439424214, "brier": 0.005617625290926389 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.0029447699503040324, "rank": 11.0, "portrait_scores": [ -0.009538983211527217, 0.003649443310919152 ], "portrait_score_gap": 0.013188426522446369, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.006366805501861729, "rank": 2.0, "portrait_scores": [ 0.002772053205905692, 0.009961557797817766 ], "portrait_score_gap": 0.007189504591912074, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.5, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.000818008680712527, "rank": 10.0, "portrait_scores": [ 0.0026960506350724557, -0.00433206799649751 ], "portrait_score_gap": 0.0070281186315699655, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.005702293494479819, "rank": 3.0, "portrait_scores": [ 0.0064449784748520835, 0.004959608514107554 ], "portrait_score_gap": 0.0014853699607445293, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.0, 6.0 ] }, { "identity": "emma_stone", "mean_score": -0.009751167958215307, "rank": 13.0, "portrait_scores": [ 0.0025443112275344657, -0.022046647143965082 ], "portrait_score_gap": 0.02459095837149955, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.002713974872993883, "rank": 6.0, "portrait_scores": [ -0.0033791187613289695, 0.008807068507316735 ], "portrait_score_gap": 0.012186187268645704, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": 0.003528364324779695, "rank": 5.0, "portrait_scores": [ 0.0030453779791719425, 0.004011350670387447 ], "portrait_score_gap": 0.0009659726912155048, "single_photo_rank_min": 3.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": 0.0024525376389379866, "rank": 7.0, "portrait_scores": [ -0.0019279379021871915, 0.006833013180063165 ], "portrait_score_gap": 0.008760951082250355, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 6.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": 0.001973474479071992, "rank": 8.0, "portrait_scores": [ 0.003943926375389773, 3.0225827542113004e-06 ], "portrait_score_gap": 0.003940903792635561, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.011078225800300573, "rank": 1.0, "portrait_scores": [ 0.008251297296183728, 0.01390515430441742 ], "portrait_score_gap": 0.0056538570082336915, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": 0.005365045299471259, "rank": 4.0, "portrait_scores": [ 0.0038640235490429893, 0.006866067049899528 ], "portrait_score_gap": 0.0030020435008565388, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.001514875938342089, "rank": 9.0, "portrait_scores": [ 0.005655011872031997, -0.0026252599953478185 ], "portrait_score_gap": 0.008280271867379816, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.5, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.003206817399792488, "rank": 12.0, "portrait_scores": [ 0.0002987519566857791, -0.006712386756270755 ], "portrait_score_gap": 0.007011138712956534, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "6.0": 32, "7.0": 224, "8.0": 672, "9.0": 1120, "10.0": 1120, "11.0": 1696, "12.0": 2272, "13.0": 1056 }, "target_primary_score": -0.003206817399792488, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ 0.0010974737458220116, 0.005048462047064254 ], "target_alternate_panel_score": 0.0030729678964431326, "target_alternate_panel_rank": 6.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.0010994934173711407, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.002629325598036686, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.0032938376054185963, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.006017737846747165, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 2.4660733883063767e-05, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.06298034485226085, "rank": 8.0, "portrait_scores": [ 0.04441598801605031, 0.08154470168847139 ], "portrait_score_gap": 0.03712871367242108, "single_photo_rank_min": 5.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.0838032179321319, "rank": 5.0, "portrait_scores": [ 0.08304462651260677, 0.08456180935165702 ], "portrait_score_gap": 0.0015171828390502518, "single_photo_rank_min": 4.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 6.5, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.03839825682645058, "rank": 9.0, "portrait_scores": [ 0.10275362517953611, -0.025957111526634945 ], "portrait_score_gap": 0.12871073670617106, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.05835078900834853, "rank": 13.0, "portrait_scores": [ -0.05708399235758552, -0.059617585659111544 ], "portrait_score_gap": 0.002533593301526027, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.1463967414385155, "rank": 1.0, "portrait_scores": [ 0.15955486650289008, 0.13323861637414092 ], "portrait_score_gap": 0.026316250128749163, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.375, "single_photo_rank_quantiles": [ 1.0, 2.0, 3.0 ] }, { "identity": "emma_watson", "mean_score": 0.08989254179224036, "rank": 4.0, "portrait_scores": [ 0.1302019368686892, 0.04958314671579153 ], "portrait_score_gap": 0.08061879015289766, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 5.5, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0834474602740379, "rank": 6.0, "portrait_scores": [ 0.06381186948694066, 0.10308305106113513 ], "portrait_score_gap": 0.03927118157419447, "single_photo_rank_min": 4.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": 0.02438246627547637, "rank": 10.0, "portrait_scores": [ -0.06396568713691145, 0.11273061968786419 ], "portrait_score_gap": 0.17669630682477563, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.001123901643209961, "rank": 11.0, "portrait_scores": [ 0.01015290400490012, -0.012400707291320041 ], "portrait_score_gap": 0.02255361129622016, "single_photo_rank_min": 8.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.14603425518392582, "rank": 2.0, "portrait_scores": [ 0.16782850620051337, 0.12424000416733827 ], "portrait_score_gap": 0.0435885020331751, "single_photo_rank_min": 1.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 5.0 ] }, { "identity": "natalie_portman", "mean_score": 0.065203931482052, "rank": 7.0, "portrait_scores": [ 0.13143039098077783, -0.0010225280166738376 ], "portrait_score_gap": 0.13245291899745165, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.12751449899715323, "rank": 3.0, "portrait_scores": [ 0.10310442071561424, 0.1519245772786922 ], "portrait_score_gap": 0.04882015656307798, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.02638904300485292, "rank": 12.0, "portrait_scores": [ -0.0285095745139299, -0.02426851149577594 ], "portrait_score_gap": 0.004241063018153961, "single_photo_rank_min": 10.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "10.0": 1024, "11.0": 4096, "12.0": 3072 }, "target_primary_score": -0.02638904300485292, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -0.016179316670446163, 0.004259568950509621 ], "target_alternate_panel_score": -0.005959873859968271, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.00483597221675831, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.05239091514838026, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.08976309179210017, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.06894021871222912, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.0015435241134771729, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.11541425574116164, "rank": 7.0, "portrait_scores": [ -0.11113812271639456, -0.11969038876592872 ], "portrait_score_gap": 0.008552266049534157, "single_photo_rank_min": 6.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.23116385294480976, "rank": 4.0, "portrait_scores": [ 0.24065229285620837, 0.22167541303341115 ], "portrait_score_gap": 0.01897687982279722, "single_photo_rank_min": 4.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.5, 5.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.07549702705151831, "rank": 6.0, "portrait_scores": [ 0.32640820873488435, -0.477402262837921 ], "portrait_score_gap": 0.8038104715728054, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7031027436036137, "rank": 13.0, "portrait_scores": [ -0.7609246232948674, -0.64528086391236 ], "portrait_score_gap": 0.11564375938250748, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 12.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.42284865366754787, "rank": 3.0, "portrait_scores": [ 0.536377772016885, 0.30931953531821077 ], "portrait_score_gap": 0.2270582366986742, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 4.0 ] }, { "identity": "emma_watson", "mean_score": -0.1876063713639991, "rank": 8.0, "portrait_scores": [ 0.062070910242186125, -0.43728365297018434 ], "portrait_score_gap": 0.4993545632123705, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.5, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.05161806404118777, "rank": 5.0, "portrait_scores": [ 0.08826543840918818, 0.014970689673187352 ], "portrait_score_gap": 0.07329474873600084, "single_photo_rank_min": 5.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 6.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.47382464316093187, "rank": 12.0, "portrait_scores": [ -0.9098375363098774, -0.03781175001198635 ], "portrait_score_gap": 0.8720257862978911, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.2838028716048929, "rank": 9.0, "portrait_scores": [ -0.1677932241604984, -0.3998125190492874 ], "portrait_score_gap": 0.23201929488878897, "single_photo_rank_min": 7.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": 0.5113671369889095, "rank": 1.0, "portrait_scores": [ 0.6201018176570584, 0.4026324563207604 ], "portrait_score_gap": 0.21746936133629802, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": -0.44013637296898883, "rank": 11.0, "portrait_scores": [ -0.09753808748755108, -0.7827346584504266 ], "portrait_score_gap": 0.6851965709628755, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.4457972104519444, "rank": 2.0, "portrait_scores": [ 0.3466334909002033, 0.5449609300036855 ], "portrait_score_gap": 0.19832743910348222, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 3.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.43717394747375726, "rank": 10.0, "portrait_scores": [ -0.4484380089358183, -0.4259098860116962 ], "portrait_score_gap": 0.022528122924122096, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 10.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 256, "9.0": 1536, "10.0": 3072, "11.0": 2560, "12.0": 768 }, "target_primary_score": -0.43717394747375726, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ -0.3403641077808458, -0.2056204490056054 ], "target_alternate_panel_score": -0.2729922783932256, "target_alternate_panel_rank": 9.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.01081059321166733, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.4301104652103881, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.5041561313380354, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.15757802265206394, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.04494405842182469, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.1670372041623521, "rank": 5.0, "portrait_scores": [ -0.08083947733974756, 0.41491388566445175 ], "portrait_score_gap": 0.4957533630041993, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.26403162025733123, "rank": 3.0, "portrait_scores": [ 0.10654421150280911, 0.4215190290118534 ], "portrait_score_gap": 0.31497481750904427, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.07159177730084082, "rank": 9.0, "portrait_scores": [ 0.0674604265571016, -0.21064398115878324 ], "portrait_score_gap": 0.27810440771588485, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3538361436996229, "rank": 12.0, "portrait_scores": [ -0.49739550119969766, -0.21027678619954815 ], "portrait_score_gap": 0.2871187150001495, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.05799079655174194, "rank": 8.0, "portrait_scores": [ -0.00973311147713185, -0.10624848162635203 ], "portrait_score_gap": 0.09651537014922018, "single_photo_rank_min": 6.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 8.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": 0.011184049166976379, "rank": 7.0, "portrait_scores": [ 0.35293306240942157, -0.3305649640754688 ], "portrait_score_gap": 0.6834980264848904, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": 0.17577796812345442, "rank": 4.0, "portrait_scores": [ 0.0786327907137504, 0.27292314553315844 ], "portrait_score_gap": 0.19429035481940804, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.3248739075837648, "rank": 11.0, "portrait_scores": [ -0.6550683883905855, 0.00532057322305593 ], "portrait_score_gap": 0.6603889616136414, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.1288031561002862, "rank": 6.0, "portrait_scores": [ 0.2611476734119429, -0.003541361211370525 ], "portrait_score_gap": 0.2646890346233134, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 6.0, 8.0 ] }, { "identity": "margot_robbie", "mean_score": -0.46989054246715634, "rank": 13.0, "portrait_scores": [ -0.21264612223168902, -0.7271349627026237 ], "portrait_score_gap": 0.5144888404709347, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.5, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.32119847632417015, "rank": 2.0, "portrait_scores": [ 0.4935680069365803, 0.14882894571175997 ], "portrait_score_gap": 0.34473906122482034, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.2578125, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.34324344896886627, "rank": 1.0, "portrait_scores": [ 0.1205412289553165, 0.565945668982416 ], "portrait_score_gap": 0.44540444002709956, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.27857651977989095, "rank": 10.0, "portrait_scores": [ -0.25626675030011714, -0.30088628925966476 ], "portrait_score_gap": 0.04461953895954762, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "9.0": 512, "10.0": 2048, "11.0": 3072, "12.0": 2048, "13.0": 512 }, "target_primary_score": -0.27857651977989095, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ -0.548322354718134, -0.26018959167190703 ], "target_alternate_panel_score": -0.4042559731950205, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5330591292953066, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.05041982949539758, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.6682875934523518, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.5712931773573726, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.04055936409569001, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.7157513373508678, "rank": 2.0, "portrait_scores": [ 0.630331263416744, 0.8011714112849917 ], "portrait_score_gap": 0.17084014786824775, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.702240568434428, "rank": 3.0, "portrait_scores": [ 0.9489535335637574, 0.4555276033050985 ], "portrait_score_gap": 0.49342593025865894, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.3022286793885628, "rank": 6.0, "portrait_scores": [ 0.8398200972119692, -0.2353627384348436 ], "portrait_score_gap": 1.0751828356468127, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.5848113826263373, "rank": 4.0, "portrait_scores": [ -0.6740792904889853, 1.8437020557416597 ], "portrait_score_gap": 2.517781346230645, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.5, 12.0 ] }, { "identity": "emma_stone", "mean_score": 0.2791139626448253, "rank": 7.0, "portrait_scores": [ 1.0069412088883922, -0.44871328359874163 ], "portrait_score_gap": 1.4556544924871337, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "emma_watson", "mean_score": -1.0120986343378737, "rank": 12.0, "portrait_scores": [ -0.15120092017815692, -1.8729963484975904 ], "portrait_score_gap": 1.7217954283194334, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.21534275018090138, "rank": 10.0, "portrait_scores": [ 0.23028625248408863, -0.6609717528458914 ], "portrait_score_gap": 0.89125800532998, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.5, 12.0 ] }, { "identity": "keira_knightley", "mean_score": -0.1847842097376341, "rank": 9.0, "portrait_scores": [ -0.43413630809608905, 0.06456788862082086 ], "portrait_score_gap": 0.4987041967169099, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "lily_collins", "mean_score": -0.028898460614838417, "rank": 8.0, "portrait_scores": [ 0.033355082615060316, -0.09115200384473715 ], "portrait_score_gap": 0.12450708645979747, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "margot_robbie", "mean_score": -0.5510518742526286, "rank": 11.0, "portrait_scores": [ -0.01501030764015705, -1.0870934408651 ], "portrait_score_gap": 1.072083133224943, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.5466764741467869, "rank": 5.0, "portrait_scores": [ 1.5498030137452972, -0.4564500654517235 ], "portrait_score_gap": 2.0062530791970206, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.0503504289650538, "rank": 1.0, "portrait_scores": [ 1.304625904153473, 0.7960749537766348 ], "portrait_score_gap": 0.5085509503768381, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.1328125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.081246875833081, "rank": 13.0, "portrait_scores": [ -1.1908087659180528, -0.9716849857481095 ], "portrait_score_gap": 0.21912378016994338, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 12.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "11.0": 1024, "12.0": 4096, "13.0": 3072 }, "target_primary_score": -1.081246875833081, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -1.0119662240726313, -1.2337701193465822 ], "target_alternate_panel_score": -1.1228681717096067, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.0939697110947684, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.707679554335944, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.8251087401440347, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.8386195090604747, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.37290399472530955, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -6.704217334108087e-05, "rank": 6.0, "portrait_scores": [ -0.6747724357686773, 0.6746383514219951 ], "portrait_score_gap": 1.3494107871906724, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.5, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.7136596860895247, "rank": 2.0, "portrait_scores": [ 0.28748195660687953, 1.1398374155721698 ], "portrait_score_gap": 0.8523554589652902, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.25390625, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.03059651948247402, "rank": 5.0, "portrait_scores": [ -0.2532567024674372, 0.31444974143238524 ], "portrait_score_gap": 0.5677064438998225, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.36135995335827986, "rank": 10.0, "portrait_scores": [ -1.3561156599911803, 0.6333957532746206 ], "portrait_score_gap": 1.989511413265801, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 8.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.14138894321062326, "rank": 8.0, "portrait_scores": [ -0.38169674561207667, 0.09891885919083016 ], "portrait_score_gap": 0.4806156048029068, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "emma_watson", "mean_score": -0.2288548054050908, "rank": 9.0, "portrait_scores": [ 0.9735951615389479, -1.4313047723491295 ], "portrait_score_gap": 2.4048999338880774, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "gal_gadot", "mean_score": -0.019581740371938994, "rank": 7.0, "portrait_scores": [ 0.24671834980404506, -0.28588183054792304 ], "portrait_score_gap": 0.5326001803519681, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -1.4563715539775914, "rank": 13.0, "portrait_scores": [ -2.2646772635009675, -0.6480658444542153 ], "portrait_score_gap": 1.6166114190467522, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": 0.143374810069331, "rank": 4.0, "portrait_scores": [ 0.5471672674568812, -0.26041764731821915 ], "portrait_score_gap": 0.8075849147751003, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.7804320601515438, "rank": 11.0, "portrait_scores": [ -0.17110989767966633, -1.3897542226234212 ], "portrait_score_gap": 1.2186443249437549, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "natalie_portman", "mean_score": 0.8262534308478345, "rank": 1.0, "portrait_scores": [ 1.5157240818686533, 0.1367827798270158 ], "portrait_score_gap": 1.3789413020416375, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5079456226299373, "rank": 3.0, "portrait_scores": [ 0.12614776885193157, 0.8897434764079428 ], "portrait_score_gap": 0.7635957075560112, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.831055721527555, "rank": 12.0, "portrait_scores": [ -0.9972277581647653, -0.6648836848903448 ], "portrait_score_gap": 0.3323440732744205, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 128, "9.0": 896, "10.0": 2304, "11.0": 2816, "12.0": 1664, "13.0": 384 }, "target_primary_score": -0.831055721527555, "target_primary_rank": 12.0, "target_alternate_photo_scores": [ -1.2702443301564137, -0.0907550380629466 ], "target_alternate_panel_score": -0.6804996841096802, "target_alternate_panel_rank": 11.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.8238744941790113, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.31913973075140034, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.3941593701992048, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.6804326419363391, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.3936883373514962, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } } }, { "encoder": "openai/clip-vit-base-patch16", "variants": [ { "variant": "portrait", "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 12, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6926278892240548, "brier_score": 0.2289070261440679, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.45454545454545453, "mean_cross_entropy": 0.6930408616871878, "brier_score": 0.2291135053424804, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.6940968408521674, "brier_score": 0.21922482449407477, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6909289033572292, "brier_score": 0.24889086703929153, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 2, "accuracy": 0.5, "mean_cross_entropy": 0.6934319263034472, "brier_score": 0.2501423730839177, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.006638534400911377, "right_score": -0.006216134029205748, "margin": -0.0004224003717056288, "p_left": 0.49989439990864365, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6929360026768516, "brier": 0.24989441106002294 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.006450842074515536, "right_score": 0.0018247584928000962, "margin": -0.008275600567315633, "p_left": 0.4979311116655731, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6890179409474528, "brier": 0.24793539196451342 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.0006396200704294381, "right_score": -0.004335000573849723, "margin": 0.004974620644279161, "p_left": 0.5012436525963638, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6956375842352145, "brier": 0.25124519926814426 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.016330556470451986, "right_score": -0.010338103086953414, "margin": -0.0059924533834985715, "p_left": 0.4985018911371507, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.690155442548674, "brier": 0.24850413546731565 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.006638534400911377, "right_score": -0.010338103086953414, "margin": 0.0036995686860420377, "p_left": 0.5009248911166101, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6949986757530485, "brier": 0.2509257465401877 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.006450842074515536, "right_score": -0.0021084802544085266, "margin": -0.0043423618201070095, "p_left": 0.4989144112508041, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6909783566613121, "brier": 0.24891558975373648 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.006638534400911377, "right_score": 0.003178020225977012, "margin": -0.009816554626888388, "p_left": 0.4975458810507918, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6882509487912288, "brier": 0.24755190375060865 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.008724430873682777, "right_score": 0.00402075708498895, "margin": -0.012745187958671727, "p_left": 0.4968137461414228, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6995400793788646, "brier": 0.2531964060722285 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.002175560622086061, "right_score": -0.0038982068904720463, "margin": 0.006073767512558108, "p_left": 0.5015184372101309, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6901149081280528, "brier": 0.24848386844143022 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.004335000573849723, "right_score": -0.005304499572813406, "margin": 0.0009694989989636827, "p_left": 0.5002423747307564, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6926625485514976, "brier": 0.24975768401475373 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.003178020225977012, "right_score": 0.0020878128061420443, "margin": 0.0010902074198349677, "p_left": 0.5002725518279636, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6926022254190477, "brier": 0.24972752245653532 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0018247584928000962, "right_score": -0.0021084802544085266, "margin": 0.003933238747208623, "p_left": 0.500983308419124, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6911824949809748, "brier": 0.24901765847632315 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.003178020225977012, "right_score": 0.0006396200704294381, "margin": 0.0025384001555475736, "p_left": 0.5006345996981346, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6944171860719216, "brier": 0.2506350024149115 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.010338103086953414, "right_score": 0.0006396200704294381, "margin": -0.010977723157382853, "p_left": 0.49725559677136105, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.698651105863713, "brier": 0.25275193497772036 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.0038982068904720463, "right_score": -0.0035178423924650927, "margin": -0.0003803644980069536, "p_left": 0.4999049088766448, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6933373808935925, "brier": 0.250095100165677 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.002053432006307835, "right_score": 0.0006396200704294381, "margin": 0.0014138119358783968, "p_left": 0.5003534529250941, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6938543363858874, "brier": 0.2503535778540644 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.006216134029205748, "right_score": -0.006638534400911377, "margin": 0.0004224003717056288, "p_left": 0.5001056000913563, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6929360026768516, "brier": 0.249894411060023 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.006216134029205748, "right_score": -0.016330556470451986, "margin": 0.010114422441246239, "p_left": 0.5025285840538445, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931599681981023, "brier": 6.3937373173566085e-06 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.006216134029205748, "right_score": -0.005304499572813406, "margin": -0.000911634456392342, "p_left": 0.49977209140168605, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6926914672164183, "brier": 0.24977214334401523 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.005304499572813406, "right_score": 0.002053432006307835, "margin": -0.00735793157912124, "p_left": 0.49816052540417916, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6894749821497593, "brier": 0.24816390907096783 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.003178020225977012, "right_score": -0.0026883576970835417, "margin": 0.005866377923060554, "p_left": 0.5014665902747835, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6902182933909886, "brier": 0.24853556061225063 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.0035178423924650927, "right_score": -0.008724430873682777, "margin": 0.0052065884812176845, "p_left": 0.5013016441798305, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6957538633671783, "brier": 0.2513033384574014 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.002053432006307835, "right_score": -0.006638534400911377, "margin": 0.008691966407219212, "p_left": 0.5021729779210644, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6888106411116105, "brier": 0.247831743911981 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.0066689238371282315, "right_score": -0.0057755090336212445, "margin": -0.000893414803506987, "p_left": 0.4997766463139798, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.693593987735447, "brier": 0.2502234035728893 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.004335000573849723, "right_score": 0.0006396200704294381, "margin": -0.004974620644279161, "p_left": 0.49875634740363617, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6956375842352145, "brier": 0.25124519926814426 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.010338103086953414, "right_score": 0.0020878128061420443, "margin": -0.012425915893095458, "p_left": 0.4968935609970149, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6993794388055483, "brier": 0.2531160889662644 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.00402075708498895, "right_score": -0.0022665165995480454, "margin": 0.006287273684536995, "p_left": 0.5015718132433477, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6962957586203733, "brier": 0.2515742838402196 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.0014693917966952063, "right_score": -0.005304499572813406, "margin": 0.006773891369508612, "p_left": 0.5016934663669048, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6965398619342692, "brier": 0.2516963341952407 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.0014693917966952063, "right_score": -0.016330556470451986, "margin": 0.017799948267147193, "p_left": 0.5044498695767005, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.684286810673326, "brier": 0.24556993176254918 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.002053432006307835, "right_score": -0.0026883576970835417, "margin": 0.004741789703391376, "p_left": 0.5011854452046632, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6955208859802068, "brier": 0.2511868504849965 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.006216134029205748, "right_score": 0.0020878128061420443, "margin": -0.008303946835347792, "p_left": 0.4979240252203117, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931557999768111, "brier": 4.309671285901863e-06 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.0020878128061420443, "right_score": 0.0014693917966952063, "margin": 0.0006184210094468379, "p_left": 0.5001546052474344, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934564388702361, "brier": 0.2501546291502169 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.010338103086953414, "right_score": -0.016330556470451986, "margin": 0.0059924533834985715, "p_left": 0.5014981088628493, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.690155442548674, "brier": 0.24850413546731565 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.016330556470451986, "right_score": -0.004335000573849723, "margin": -0.011995555896602263, "p_left": 0.4970011469853499, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6871673891739637, "brier": 0.2470101401047534 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.00402075708498895, "right_score": 0.0018247584928000962, "margin": 0.0021959985921888537, "p_left": 0.5005489994274223, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6942457826571458, "brier": 0.2505493008277936 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.004335000573849723, "right_score": -0.0026883576970835417, "margin": -0.0016466428767661816, "p_left": 0.4995883393738243, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6939708409273856, "brier": 0.2504118300906468 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.0059459087196547696, "right_score": 0.0043347647034655944, "margin": -0.010280673423120364, "p_left": 0.49742985428121955, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931603920325183, "brier": 6.605649015765464e-06 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.0026883576970835417, "right_score": 0.0014693917966952063, "margin": -0.004157749493778748, "p_left": 0.4989605641239385, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6910704666716061, "brier": 0.24896164455087896 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0018247584928000962, "right_score": -0.0059459087196547696, "margin": 0.0077706672124548655, "p_left": 0.5019426570277917, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6892693948433435, "brier": 0.2480611168885359 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.010338103086953414, "right_score": -0.006638534400911377, "margin": -0.0036995686860420377, "p_left": 0.49907510888338985, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6949986757530485, "brier": 0.2509257465401878 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 5, "regressions": 2, "both_correct": 0, "both_wrong": 4, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "0add2ec437a1", "34bec29a8d38", "974972ffb84a", "fe73774e1a9e" ], "regressions": [ "70f39a6b70be", "396ff40ef0fb" ], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba" ] } }, "other": { "fixes": 2, "regressions": 2, "both_correct": 0, "both_wrong": 3, "net_fixes": 0, "trial_ids": { "fixes": [ "0add2ec437a1", "34bec29a8d38" ], "regressions": [ "70f39a6b70be", "396ff40ef0fb" ], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.6219970053934322, "brier_score": 0.19386630798510582, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 7, "accuracy": 0.6363636363636364, "mean_cross_entropy": 0.6840996180672846, "brier_score": 0.22464449520875174, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6847674243011896, "brier_score": 0.2145615286264152, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6827640055994748, "brier_score": 0.24481042837342476, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5437434958347607, "brier_score": 0.17609112131320637, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.1441808319436378, "right_score": 0.2256551794006486, "margin": -0.3698360113442864, "p_left": 0.40858064760213547, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5252299490726595, "brier": 0.1669381455949804 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.01580503486413258, "right_score": 0.03045324105649268, "margin": -0.0146482061923601, "p_left": 0.49633801393102145, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6858498984670575, "brier": 0.24635142407299085 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.12274478288576612, "right_score": 0.27357670608917906, "margin": -0.15083192320341293, "p_left": 0.46236334582594757, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6205723109700949, "brier": 0.21377986356336479 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.08484497451773984, "right_score": 0.28184105269778387, "margin": -0.3666860272155237, "p_left": 0.40934203627544824, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5265181706944143, "brier": 0.1675609026621304 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.1441808319436378, "right_score": 0.28184105269778387, "margin": -0.42602188464142166, "p_left": 0.39507666779037964, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5026535526018743, "brier": 0.15608557343235 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.01580503486413258, "right_score": -0.019399097203755197, "margin": 0.03520413206788778, "p_left": 0.5088001241802486, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7109041549591885, "brier": 0.25887756636583636 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.1441808319436378, "right_score": 0.06532792397591351, "margin": -0.20950875591955131, "p_left": 0.4478135604631541, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5938695369465322, "brier": 0.200536984934687 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.11798138579753292, "right_score": 0.08525057464540728, "margin": 0.032730811152125644, "p_left": 0.5081819723514016, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6769156822565576, "brier": 0.2418849723201575 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.16269314562450723, "right_score": 0.17506768793771177, "margin": -0.012374542313204534, "p_left": 0.49690640389828133, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6993535927566035, "brier": 0.25310316643855924 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.27357670608917906, "right_score": 0.044604095246084344, "margin": 0.22897261084309473, "p_left": 0.5569943595019053, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.585200165673622, "brier": 0.19625399751312708 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06532792397591351, "right_score": -0.060468401183432045, "margin": 0.12579632515934555, "p_left": 0.5314076740841813, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6322258045032999, "brier": 0.21957876790719683 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.03045324105649268, "right_score": -0.019399097203755197, "margin": 0.049852338260247875, "p_left": 0.5124605040437954, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6685316362196545, "brier": 0.23769476011723004 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06532792397591351, "right_score": 0.12274478288576612, "margin": -0.05741685890985261, "p_left": 0.4856497274294147, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6648507814731197, "brier": 0.2358556577522648 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.28184105269778387, "right_score": 0.12274478288576612, "margin": 0.15909626981201774, "p_left": 0.5396903837467287, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6167596672914709, "brier": 0.21188494281523385 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.17506768793771177, "right_score": 0.1569007226791014, "margin": 0.018166965258610362, "p_left": 0.5045416164062695, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6841049521916707, "brier": 0.2454790098733122 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.14912422090234062, "right_score": 0.12274478288576612, "margin": 0.026379438016574497, "p_left": 0.5065944770977369, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7064238813900164, "brier": 0.25663796422592944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.2256551794006486, "right_score": -0.1441808319436378, "margin": 0.3698360113442864, "p_left": 0.5914193523978645, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5252299490726595, "brier": 0.16693814559498046 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.2256551794006486, "right_score": -0.08484497451773984, "margin": 0.3105001539183885, "p_left": 0.577007338333131, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7051503714324228, "brier": 0.005930130157153307 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.2256551794006486, "right_score": 0.044604095246084344, "margin": 0.18105108415456428, "p_left": 0.5451395340890248, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7877645753910609, "brier": 0.29717711162679905 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.044604095246084344, "right_score": 0.14912422090234062, "margin": -0.10452012565625628, "p_left": 0.4738937306604894, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6422520536854891, "brier": 0.22457526795931648 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06532792397591351, "right_score": -0.03915905769270277, "margin": 0.10448698166861628, "p_left": 0.5260980059243151, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.642267760550382, "brier": 0.2245830999889105 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.1569007226791014, "right_score": 0.11798138579753292, "margin": 0.038919336881568484, "p_left": 0.5097286062460838, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7127961764000524, "brier": 0.2598232520255752 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.14912422090234062, "right_score": -0.1441808319436378, "margin": 0.2933050528459784, "p_left": 0.5728050713653648, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5572098097342435, "brier": 0.18249550705115103 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.10753129359122938, "right_score": 0.02568289790413717, "margin": 0.0818483956870922, "p_left": 0.520450683329859, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.653060144061854, "brier": 0.2299675471187992 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.27357670608917906, "right_score": 0.12274478288576612, "margin": 0.15083192320341293, "p_left": 0.5376366541740525, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6205723109700949, "brier": 0.21377986356336473 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.28184105269778387, "right_score": -0.060468401183432045, "margin": 0.3423094538812159, "p_left": 0.5847514075902206, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5365684663549133, "brier": 0.1724313934983031 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.08525057464540728, "right_score": 0.19226688743817444, "margin": -0.10701631279276716, "p_left": 0.47327142599731803, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6410699029624615, "brier": 0.22398584266553487 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17214790961568646, "right_score": 0.044604095246084344, "margin": 0.1275438143696021, "p_left": 0.5318427985976575, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7589511390340549, "brier": 0.2828567624201885 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17214790961568646, "right_score": -0.08484497451773984, "margin": 0.2569928841334263, "p_left": 0.5638969329649134, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5728837871536208, "brier": 0.19018588507740924 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.14912422090234062, "right_score": -0.03915905769270277, "margin": 0.1882832785950434, "p_left": 0.5469322534808638, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7917136138658412, "brier": 0.2991348898976559 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.2256551794006486, "right_score": -0.060468401183432045, "margin": 0.28612358058408066, "p_left": 0.571046857535315, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7033458007516116, "brier": 0.005047655965643351 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.060468401183432045, "right_score": 0.17214790961568646, "margin": -0.23261631079911851, "p_left": 0.4421067392170663, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5835876238183586, "brier": 0.19545836886114706 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.28184105269778387, "right_score": -0.08484497451773984, "margin": 0.3666860272155237, "p_left": 0.5906579637245518, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5265181706944143, "brier": 0.16756090266213033 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.08484497451773984, "right_score": 0.27357670608917906, "margin": -0.3584216806069189, "p_left": 0.4113416861811875, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5299093760227782, "brier": 0.16920198279038254 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.08525057464540728, "right_score": 0.03045324105649268, "margin": 0.0547973335889146, "p_left": 0.5136959064560166, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7209211438739509, "brier": 0.26388348430966857 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.27357670608917906, "right_score": -0.03915905769270277, "margin": 0.31273576378188184, "p_left": 0.5775528891931694, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5489552577611818, "brier": 0.17846156142903863 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.045787233056468535, "right_score": 0.18750193277098498, "margin": -0.23328916582745352, "p_left": 0.44194078709156387, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6999347888211294, "brier": 0.0033708722035471167 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.03915905769270277, "right_score": 0.17214790961568646, "margin": -0.21130696730838922, "p_left": 0.44736894675220423, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5930646732693086, "brier": 0.20013897451817655 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.03045324105649268, "right_score": -0.045787233056468535, "margin": 0.07624047411296121, "p_left": 0.5190508914720711, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6557533438372365, "brier": 0.23131204499380956 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.28184105269778387, "right_score": -0.1441808319436378, "margin": 0.42602188464142166, "p_left": 0.6049233322096204, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5026535526018743, "brier": 0.15608557343235 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 5, "regressions": 0, "both_correct": 2, "both_wrong": 4, "net_fixes": 5, "trial_ids": { "fixes": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "e5aa020b210d", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "34bec29a8d38", "b20ebcae7654", "c469d02ac7ba" ] } }, "other": { "fixes": 2, "regressions": 0, "both_correct": 2, "both_wrong": 3, "net_fixes": 2, "trial_ids": { "fixes": [ "d013ade6e6a0", "e5aa020b210d" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "34bec29a8d38", "b20ebcae7654" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 7, "regressions": 0, "both_correct": 2, "both_wrong": 2, "net_fixes": 7, "trial_ids": { "fixes": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "34bec29a8d38", "c469d02ac7ba" ] } }, "other": { "fixes": 4, "regressions": 0, "both_correct": 2, "both_wrong": 1, "net_fixes": 4, "trial_ids": { "fixes": [ "0add2ec437a1", "d013ade6e6a0", "b20ebcae7654", "e5aa020b210d" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "34bec29a8d38" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.6320205750438171, "fold_losses": [ 0.3743683513685698, 0.6304440661124163, 0.2179039278832674, 0.12548507929642855, 0.32509037736122565, 0.48538734533974714, 0.49019737309019396, 0.8474959953040533, 0.7331032500767469, 0.7633257074100377, 0.8082980208424667, 0.9587795429203139, 1.3979376599048852, 0.7436147267495867, 0.3857446776616644, 0.20588164058360636, 1.2273084167545147, 0.47010401002552793, 1.5523163170214314, 0.8355171277509585, 0.40245348809073583, 0.18494448722097453, 0.2717325855260917, 0.7310596267561661 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.4289731316752093, "brier_score": 0.10957827229955115, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 7, "accuracy": 0.6363636363636364, "mean_cross_entropy": 0.6738723320521806, "brier_score": 0.21868488674150274, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6562435260943088, "brier_score": 0.19923433297090873, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.7091299439679242, "brier_score": 0.2575859942826908, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.24174179585762834, "brier_score": 0.05170753402997744, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.1854433410692162, "right_score": 0.006303477925840045, "margin": -1.1917468189950562, "p_left": 0.23294666416536342, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2651989417866118, "brier": 0.054264148345770605 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.412981092759068, "right_score": -0.3621385436612092, "margin": -0.05084254909785885, "p_left": 0.4872921000637722, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6680489918145508, "brier": 0.23745359078456138 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.41299220808639675, "right_score": 1.0737655425659578, "margin": -0.660773334479561, "p_left": 0.3405659140864639, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4163732573639536, "brier": 0.11598514183754872 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7146734246933677, "right_score": 0.948653491368409, "margin": -1.6633269160617767, "p_left": 0.159315903412465, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.17353931788964083, "brier": 0.025381557080129874 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.1854433410692162, "right_score": 0.948653491368409, "margin": -2.134096832437625, "p_left": 0.10582669255183436, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.11185566639030733, "brier": 0.011199288856460474 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.412981092759068, "right_score": -0.46855975596944754, "margin": 0.05557866321037952, "p_left": 0.5138910902026552, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7213225859537857, "brier": 0.26408405258967355 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.1854433410692162, "right_score": 0.11542688232613056, "margin": -1.3008702233953469, "p_left": 0.21401859621500036, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2408221461394325, "brier": 0.04580395952583936 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.013402616976340653, "right_score": 0.06082423505989064, "margin": -0.07422685203623129, "p_left": 0.48145180234436236, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7309491517274802, "brier": 0.26889223329191025 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.14259163945223305, "right_score": 0.27318098455077355, "margin": -0.1305893450985405, "p_left": 0.4673989808950212, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7605720372498436, "brier": 0.283663845551662 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.0737655425659578, "right_score": 0.2054009291387146, "margin": 0.8683646134272431, "p_left": 0.7044052941653334, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.35040138653748754, "brier": 0.08737623011748309 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.11542688232613056, "right_score": -0.7751134079814872, "margin": 0.8905402903076177, "p_left": 0.7090016569341694, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34389741545109087, "brier": 0.08468003566705885 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.3621385436612092, "right_score": -0.46855975596944754, "margin": 0.10642121230823837, "p_left": 0.5265802216759135, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6413515911594234, "brier": 0.22412628650842725 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.11542688232613056, "right_score": 0.41299220808639675, "margin": -0.2975653257602662, "p_left": 0.4261527671396656, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5553920629280176, "brier": 0.18160618094079406 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.948653491368409, "right_score": 0.41299220808639675, "margin": 0.5356612832820122, "p_left": 0.6308025456903213, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.46076238824774873, "brier": 0.13630676026874725 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.27318098455077355, "right_score": -0.0662949724074468, "margin": 0.33947595695822036, "p_left": 0.5840632210016012, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5377460468733466, "brier": 0.1730034041235628 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.02108393870029955, "right_score": 0.41299220808639675, "margin": -0.4340761467866963, "p_left": 0.3931534065726685, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4994792489787984, "brier": 0.15456960109969398 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.006303477925840045, "right_score": -1.1854433410692162, "margin": 1.1917468189950562, "p_left": 0.7670533358346366, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.26519894178661185, "brier": 0.054264148345770584 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.006303477925840045, "right_score": -0.7146734246933677, "margin": 0.7209769026192078, "p_left": 0.6728221014000204, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7567627722914241, "brier": 0.029867478732318938 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.006303477925840045, "right_score": 0.2054009291387146, "margin": -0.19909745121287456, "p_left": 0.4503894085031117, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5985452669771384, "brier": 0.20285061929178283 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.2054009291387146, "right_score": -0.02108393870029955, "margin": 0.22648486783901417, "p_left": 0.5563804180958515, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8127878813238743, "brier": 0.3095591696405145 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.11542688232613056, "right_score": -0.3398387608599189, "margin": 0.45526564318604945, "p_left": 0.6118904559600113, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49120200602045794, "brier": 0.15062901817492788 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.0662949724074468, "right_score": -0.013402616976340653, "margin": -0.05289235543110615, "p_left": 0.4867799930242127, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6670506622464744, "brier": 0.2369547616086526 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.02108393870029955, "right_score": -1.1854433410692162, "margin": 1.1643594023689168, "p_left": 0.7621239383172855, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.27164608781100985, "brier": 0.05658502072167858 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.3383711378879495, "right_score": 0.017728968853186644, "margin": 0.32064216903476284, "p_left": 0.5794807456863833, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5456228424896356, "brier": 0.17683644324848025 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.0737655425659578, "right_score": 0.41299220808639675, "margin": 0.660773334479561, "p_left": 0.659434085913536, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4163732573639536, "brier": 0.11598514183754877 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.948653491368409, "right_score": -0.7751134079814872, "margin": 1.7237668993498962, "p_left": 0.8486134007467954, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16415155474444876, "brier": 0.022917902433450377 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.06082423505989064, "right_score": 0.7256014617008741, "margin": -0.6647772266409835, "p_left": 0.3396672917351126, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41501146754746837, "brier": 0.11537386907466611 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.27361034189284084, "right_score": 0.2054009291387146, "margin": 0.06820941275412623, "p_left": 0.5170457448895194, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7278333397310208, "brier": 0.267336302308358 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.27361034189284084, "right_score": -0.7146734246933677, "margin": 0.9882837665862085, "p_left": 0.72874880086702, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31642618678428436, "brier": 0.07357721303107957 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.02108393870029955, "right_score": -0.3398387608599189, "margin": 0.3187548221596193, "p_left": 0.5790207628299472, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8651717644017827, "brier": 0.33526504378817396 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.006303477925840045, "right_score": -0.7751134079814872, "margin": 0.7814168859073273, "p_left": 0.6859854049219665, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7676073700602004, "brier": 0.03459057084398783 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.7751134079814872, "right_score": 0.27361034189284084, "margin": -1.048723749874328, "p_left": 0.25947025117112166, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.30038947210570827, "brier": 0.06732481124280495 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.948653491368409, "right_score": -0.7146734246933677, "margin": 1.6633269160617767, "p_left": 0.840684096587535, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17353931788964072, "brier": 0.02538155708012989 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7146734246933677, "right_score": 1.0737655425659578, "margin": -1.7884389672593255, "p_left": 0.1432642173078917, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.15462571283011192, "brier": 0.020524635960842818 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.06082423505989064, "right_score": -0.3621385436612092, "margin": 0.42296277872109983, "p_left": 0.6041919989711013, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9268260311900485, "brier": 0.3650479716206953 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.0737655425659578, "right_score": -0.3398387608599189, "margin": 1.4136043034258767, "p_left": 0.8043338145462654, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21774090374988786, "brier": 0.03828525613001526 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5573592732336569, "right_score": 0.6735738462434677, "margin": -1.2309331194771245, "p_left": 0.22601814972764678, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8716734146664359, "brier": 0.07506605427866218 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.3398387608599189, "right_score": 0.27361034189284084, "margin": -0.6134491027527598, "p_left": 0.35127281299549207, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4327430096605831, "brier": 0.12339258914976595 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.3621385436612092, "right_score": -0.5573592732336569, "margin": 0.19522072957244768, "p_left": 0.5486507691187684, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6002931617076739, "brier": 0.2037161282170793 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.948653491368409, "right_score": -1.1854433410692162, "margin": 2.134096832437625, "p_left": 0.8941733074481656, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1118556663903072, "brier": 0.011199288856460484 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 5, "regressions": 0, "both_correct": 2, "both_wrong": 4, "net_fixes": 5, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "fe73774e1a9e" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "c469d02ac7ba" ] } }, "other": { "fixes": 2, "regressions": 0, "both_correct": 2, "both_wrong": 3, "net_fixes": 2, "trial_ids": { "fixes": [ "b20ebcae7654", "e5aa020b210d" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "semantic_bt_tuned", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6058947507107211, "fold_losses": [ 0.7189138463180482, 0.862528473911577, 0.012956381259303106, 0.02875134178728414, 0.3084284976166184, 0.19669979870669563, 0.3750644082386938, 0.38324845399676055, 0.589275804801903, 0.27100099964862917, 1.2483631457662172, 1.1428209915621068, 1.3137492467426766, 0.8884768258439122, 0.6780032306823264, 0.03820678283549661, 1.949127557480403, 0.17691592059716665, 0.9165369731790542, 0.7461045385030495, 0.4440661182866313, 0.033274110873456074, 0.2584830958987909, 0.9604774725205032 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.382008249051607, "brier_score": 0.0966426598534706, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.233378610516842, "brier_score": 0.4463367275497861, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.0515950815381339, "brier_score": 0.3716947179716129, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5969456684742582, "brier_score": 0.5956207467061325, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2512082207788717, "brier_score": 0.07378107487924734, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.176021553148216, "right_score": -0.501418926464471, "margin": -0.6746026266837452, "p_left": 0.3374670030189513, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4116849155982283, "brier": 0.11388397812659287 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.377058013611375, "right_score": -2.420135256135749, "margin": 1.0430772425243742, "p_left": 0.7394433255475688, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3449348811734736, "brier": 0.5467764316968478 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.1839689484679448, "right_score": 1.4876179315635425, "margin": -0.30364898309559774, "p_left": 0.4246657021738499, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.9476749442737873, "right_score": 2.402811337098627, "margin": -4.350486281372414, "p_left": 0.012736233502465381, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.012818034625402548, "brier": 0.00016221164382932158 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.176021553148216, "right_score": 2.402811337098627, "margin": -3.578832890246843, "p_left": 0.0271505275042335, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027525913290559173, "brier": 0.0007371511437581397 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.377058013611375, "right_score": -2.2481420937145673, "margin": 0.8710840801031923, "p_left": 0.7049712224381213, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.220682376342896, "brier": 0.4969844244658991 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.176021553148216, "right_score": 0.13784496726368295, "margin": -1.313866520411899, "p_left": 0.2118405546161929, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23805486773498735, "brier": 0.0448764205800962 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.5591332815684273, "right_score": -0.7133654814755538, "margin": -0.8457678000928736, "p_left": 0.30032141095473314, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2029020079839134, "brier": 0.4895501279683753 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.08113737662587367, "right_score": 0.3631700579311891, "margin": -0.28203268130531545, "p_left": 0.42995550776731006, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8440735459565317, "brier": 0.3249507231248253 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.29543592810435054, "margin": 1.783053859667893, "p_left": 0.8560735462448692, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15539898802081753, "brier": 0.020714824090527804 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.9249117475935976, "margin": 1.0627567148572805, "p_left": 0.743217004543786, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2967672115723152, "brier": 0.065937506755466 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -2.2481420937145673, "margin": -0.1719931624211819, "p_left": 0.45710739346596463, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.78283691905207, "brier": 0.29473238222931897 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.13784496726368295, "right_score": 1.1839689484679448, "margin": -1.0461239812042618, "p_left": 0.2599700971489528, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3010646843421818, "brier": 0.06758445141163597 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.402811337098627, "right_score": 1.1839689484679448, "margin": 1.2188423886306823, "p_left": 0.7718597671711172, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25895239419956795, "brier": 0.052047965835216876 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.3631700579311891, "right_score": -0.13957407779101405, "margin": 0.5027441357222031, "p_left": 0.623103996242335, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4730418459677973, "brier": 0.1420505976484978 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3703770135456499, "right_score": 1.1839689484679448, "margin": -1.5543459620135947, "p_left": 0.17445946230961398, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19171690999496488, "brier": 0.03043610398935962 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.501418926464471, "right_score": -1.176021553148216, "margin": 0.6746026266837452, "p_left": 0.6625329969810487, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4116849155982284, "brier": 0.11388397812659287 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.501418926464471, "right_score": -1.9476749442737873, "margin": 1.4462560178093162, "p_left": 0.8094215626308593, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9345634155344207, "brier": 0.09574170342092278 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.29543592810435054, "margin": -0.2059829983601204, "p_left": 0.44868555725984277, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5954499561237059, "brier": 0.20131872929357564 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.29543592810435054, "right_score": -0.3703770135456499, "margin": 0.07494108544129935, "p_left": 0.5187265079153722, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7313195798502383, "brier": 0.2690771900140767 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.057344187165848215, "margin": 0.19518915442953116, "p_left": 0.5486429500561503, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6003074132475881, "brier": 0.20372318653401486 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.13957407779101405, "right_score": -1.5591332815684273, "margin": 1.4195592037774132, "p_left": 0.8052693041663582, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6361377219497029, "brier": 0.6484586522325707 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3703770135456499, "right_score": -1.176021553148216, "margin": 0.8056445396025662, "p_left": 0.6911806063283058, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3693541198600183, "brier": 0.09536941790775284 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.2547528147367135, "right_score": 0.3685822126168096, "margin": -1.6233350273535232, "p_left": 0.1647454425611767, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.8033537687367593, "brier": 0.6976501757223245 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.4876179315635425, "right_score": 1.1839689484679448, "margin": 0.30364898309559774, "p_left": 0.5753342978261501, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.402811337098627, "right_score": -0.9249117475935976, "margin": 3.3277230846922246, "p_left": 0.9653677264878943, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.03524618651146527, "brier": 0.0011993943686173006 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.7133654814755538, "right_score": -0.31359736199695565, "margin": -0.39976811947859814, "p_left": 0.40136805294921685, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5131083153739083, "brier": 0.16109631392824533 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.06094111590212581, "right_score": -0.29543592810435054, "margin": 0.35637704400647635, "p_left": 0.5881631365285638, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8871279704944108, "brier": 0.34593587517111796 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.06094111590212581, "right_score": -1.9476749442737873, "margin": 2.008616060175913, "p_left": 0.8816987451064758, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1259048401560463, "brier": 0.013995186909382576 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3703770135456499, "right_score": -0.057344187165848215, "margin": -0.3130328263798017, "p_left": 0.4223746334959473, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5488297752901937, "brier": 0.1784003310208358 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.9249117475935976, "margin": 0.4234928211291267, "p_left": 0.6043187484443668, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7153999015994471, "brier": 0.010882401276999088 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.9249117475935976, "right_score": 0.06094111590212581, "margin": -0.9858528634957234, "p_left": 0.27173199255289543, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3170861564354553, "brier": 0.07383827577676681 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.9476749442737873, "margin": 4.350486281372414, "p_left": 0.9872637664975346, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.01281803462540232, "brier": 0.00016221164382932318 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.9476749442737873, "right_score": 1.4876179315635425, "margin": -3.43529287583733, "p_left": 0.031210497022679466, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.031707921879038925, "brier": 0.0009740951244026838 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.7133654814755538, "right_score": -2.420135256135749, "margin": 1.7067697746601955, "p_left": 0.8464168396113582, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8735130971651364, "brier": 0.7164214663776796 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.057344187165848215, "margin": 1.5449621187293907, "p_left": 0.8241849138254784, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19336036426644232, "brier": 0.030910944526554445 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.13009458642499722, "right_score": -0.5909243129983202, "margin": 0.46082972657332294, "p_left": 0.613210991878727, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7194610699935612, "brier": 0.012816728682165183 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.057344187165848215, "right_score": 0.06094111590212581, "margin": -0.11828530306797402, "p_left": 0.4704631047797302, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6357524370097754, "brier": 0.2213355329589834 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -0.13009458642499722, "margin": -2.290040669710752, "p_left": 0.09195115411081337, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.386497776506353, "brier": 0.8245527065206839 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.176021553148216, "margin": 3.578832890246843, "p_left": 0.9728494724957664, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.02752591329055898, "brier": 0.0007371511437581471 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 9, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 5, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "consistent_bt", "selected": { "ridge": 0.1, "consistency": 0.1, "mean_cross_entropy": 0.6109166363162718, "fold_losses": [ 0.5177778067016785, 0.9275989903923381, 0.039033664665995754, 0.05849012776521651, 0.31342053318864216, 0.27633647912585935, 0.4492951102465713, 0.5079517450316822, 0.6375559170641422, 0.3879236709270066, 1.0150188612224396, 1.3769517714544939, 0.9615840859985116, 0.7940164109065572, 0.596409368721866, 0.09318643328112675, 2.0544564837931243, 0.2305869777519458, 0.8629918913352478, 0.7500958974883426, 0.46174771446993745, 0.08966980153411769, 0.42315094783761387, 0.8367485806860648 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.42197404504672703, "brier_score": 0.11090902925836625, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.0859307796627728, "brier_score": 0.3835735359881786, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8247057965769973, "brier_score": 0.27750073612716347, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.6083807458343244, "brier_score": 0.5957191357102087, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2674614008689228, "brier_score": 0.077597014808838, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.0803133389622654, "right_score": -0.1810473837205192, "margin": -0.8992659552417462, "p_left": 0.28920136715328437, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3413661061042845, "brier": 0.08363743076332879 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.456412041920701, "right_score": -2.134916385097236, "margin": 0.678504343176535, "p_left": 0.6634048005898459, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0888742593290535, "brier": 0.44010592944565313 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.8866103024741911, "right_score": 0.9988101791903153, "margin": -0.11219987671612419, "p_left": 0.4719794201403607, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6386200190285669, "brier": 0.22276457303603112 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.470917242566171, "right_score": 1.8226244530851612, "margin": -3.2935416956513324, "p_left": 0.03579341285514762, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03644970531350696, "brier": 0.001281168403819047 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.0803133389622654, "right_score": 1.8226244530851612, "margin": -2.9029377920474264, "p_left": 0.052008528456139784, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.053409773029332676, "brier": 0.002704887032173102 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.456412041920701, "right_score": -1.6423596882616498, "margin": 0.18594764634094885, "p_left": 0.5463534267801844, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7904368571924157, "brier": 0.2985020669544503 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.0803133389622654, "right_score": 0.18766306604504648, "margin": -1.267976405007312, "p_left": 0.2196038549706713, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2479536099822745, "brier": 0.04822585311797964 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.3400904407192387, "right_score": -0.4466081925332326, "margin": -0.8934822481860061, "p_left": 0.29039173671995716, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2365244512574263, "brier": 0.5035438873153186 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.12765076343419546, "right_score": -0.10884771895629641, "margin": 0.23649848239049187, "p_left": 0.558850575719023, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5818731479658042, "brier": 0.19461281454343748 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.9988101791903153, "right_score": -0.4208937721750325, "margin": 1.4197039513653478, "p_left": 0.8052920011270379, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21655033301646043, "brier": 0.037911204825113405 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.18766306604504648, "right_score": -0.6583794203392574, "margin": 0.8460424863843039, "p_left": 0.6997363052869247, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.35705172164350274, "brier": 0.0901582863627469 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.134916385097236, "right_score": -1.6423596882616498, "margin": -0.4925566968355861, "p_left": 0.37929146043956335, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9694503445413516, "brier": 0.3852790910832501 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.18766306604504648, "right_score": 0.8866103024741911, "margin": -0.6989472364291447, "p_left": 0.3320456803820311, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4035354915892061, "brier": 0.11025433386036594 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.8226244530851612, "right_score": 0.8866103024741911, "margin": 0.93601415061097, "p_left": 0.7182938317540959, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33087655722198317, "brier": 0.07935836522778963 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.10884771895629641, "right_score": -0.847589171230831, "margin": 0.7387414522745346, "p_left": 0.676720585220303, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3904968161973221, "brier": 0.10450958002030339 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.2819409600148173, "right_score": 0.8866103024741911, "margin": -1.1685512624890084, "p_left": 0.23711695021128035, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.27065053626265306, "brier": 0.0562244480774988 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.1810473837205192, "right_score": -1.0803133389622654, "margin": 0.8992659552417462, "p_left": 0.7107986328467156, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3413661061042844, "brier": 0.08363743076332879 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.1810473837205192, "right_score": -1.470917242566171, "margin": 1.2898698588456519, "p_left": 0.7841251606223446, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.888121557145297, "brier": 0.08072710689867313 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.1810473837205192, "right_score": -0.4208937721750325, "margin": 0.23984638845451328, "p_left": 0.5596757937694042, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8202439910552437, "brier": 0.3132369941314127 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.4208937721750325, "right_score": -0.2819409600148173, "margin": -0.1389528121602152, "p_left": 0.46531758268490053, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6260823208389248, "brier": 0.21652045275571924 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.18766306604504648, "right_score": -0.013052812952860926, "margin": 0.2007158789979074, "p_left": 0.5500111828998336, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5978166684171718, "brier": 0.20248993551520697 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.847589171230831, "right_score": -1.3400904407192387, "margin": 0.49250126948840767, "p_left": 0.620695490243356, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9694159406752765, "brier": 0.3852628916084401 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.2819409600148173, "right_score": -1.0803133389622654, "margin": 0.7983723789474482, "p_left": 0.6896262095912857, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3716055534069793, "brier": 0.0963318897726725 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.0003295477787946, "right_score": 0.1934331586011072, "margin": -1.1937627063799017, "p_left": 0.2325866542688777, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.458492416859746, "brier": 0.588923243206235 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.9988101791903153, "right_score": 0.8866103024741911, "margin": 0.11219987671612419, "p_left": 0.5280205798596393, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6386200190285669, "brier": 0.2227645730360311 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.8226244530851612, "right_score": -0.6583794203392574, "margin": 2.4810038734244184, "p_left": 0.9227993448840229, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08034346260474212, "brier": 0.005959941150336036 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.4466081925332326, "right_score": 0.21246490493664044, "margin": -0.6590730974698731, "p_left": 0.34094785807084105, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41695262480366646, "brier": 0.11624544192309437 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.21083692993621256, "right_score": -0.4208937721750325, "margin": 0.631730702111245, "p_left": 0.6528817900032535, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0580898942328485, "brier": 0.42625463171785244 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.21083692993621256, "right_score": -1.470917242566171, "margin": 1.6817541725023835, "p_left": 0.8431366727711693, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1706262074651963, "brier": 0.02460610342929921 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.2819409600148173, "right_score": -0.013052812952860926, "margin": -0.26888814706195635, "p_left": 0.4331800736452534, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5677136159001027, "brier": 0.18764497620330717 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.1810473837205192, "right_score": -0.6583794203392574, "margin": 0.47733203661873824, "p_left": 0.6171176764665771, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7213615679628387, "brier": 0.013716550140929831 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.6583794203392574, "right_score": 0.21083692993621256, "margin": -0.86921635027547, "p_left": 0.2954173893678483, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3501496931522558, "brier": 0.08727143394091488 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.8226244530851612, "right_score": -1.470917242566171, "margin": 3.2935416956513324, "p_left": 0.9642065871448523, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.036449705313506886, "brier": 0.0012811684038190525 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.470917242566171, "right_score": 0.9988101791903153, "margin": -2.4697274217564864, "p_left": 0.07800783741446315, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.08121855591119846, "brier": 0.006085222698081318 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.4466081925332326, "right_score": -2.134916385097236, "margin": 1.6883081925640033, "p_left": 0.8440015411495622, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.85790915094526, "brier": 0.7123386014628361 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.9988101791903153, "right_score": -0.013052812952860926, "margin": 1.0118629921431763, "p_left": 0.7333845814899811, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31008504689562777, "brier": 0.07108378138727253 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.2982539951347632, "right_score": -0.40333820027264944, "margin": 0.7015921954074127, "p_left": 0.6685406878057302, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7534541176643208, "brier": 0.028405963446028602 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.013052812952860926, "right_score": 0.21083692993621256, "margin": -0.22388974288907348, "p_left": 0.4442602072572621, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5874550929412494, "brier": 0.1973671317522655 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.134916385097236, "right_score": 0.2982539951347632, "margin": -2.4331703802319993, "p_left": 0.08067801024391051, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.5172892285216326, "brier": 0.8451529208490954 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.8226244530851612, "right_score": -1.0803133389622654, "margin": 2.9029377920474264, "p_left": 0.9479914715438602, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.05340977302933281, "brier": 0.0027048870321731023 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 1, "regressions": 0, "both_correct": 2, "both_wrong": 8, "net_fixes": 1, "trial_ids": { "fixes": [ "34bec29a8d38" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 1, "regressions": 0, "both_correct": 2, "both_wrong": 4, "net_fixes": 1, "trial_ids": { "fixes": [ "34bec29a8d38" ], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "semantic_bt_matched", "selected": { "ridge": 0.1, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.382008249051607, "brier_score": 0.0966426598534706, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.233378610516842, "brier_score": 0.4463367275497861, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.0515950815381339, "brier_score": 0.3716947179716129, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5969456684742582, "brier_score": 0.5956207467061325, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2512082207788717, "brier_score": 0.07378107487924734, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.176021553148216, "right_score": -0.501418926464471, "margin": -0.6746026266837452, "p_left": 0.3374670030189513, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4116849155982283, "brier": 0.11388397812659287 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.377058013611375, "right_score": -2.420135256135749, "margin": 1.0430772425243742, "p_left": 0.7394433255475688, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3449348811734736, "brier": 0.5467764316968478 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.1839689484679448, "right_score": 1.4876179315635425, "margin": -0.30364898309559774, "p_left": 0.4246657021738499, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.9476749442737873, "right_score": 2.402811337098627, "margin": -4.350486281372414, "p_left": 0.012736233502465381, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.012818034625402548, "brier": 0.00016221164382932158 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.176021553148216, "right_score": 2.402811337098627, "margin": -3.578832890246843, "p_left": 0.0271505275042335, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027525913290559173, "brier": 0.0007371511437581397 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.377058013611375, "right_score": -2.2481420937145673, "margin": 0.8710840801031923, "p_left": 0.7049712224381213, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.220682376342896, "brier": 0.4969844244658991 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.176021553148216, "right_score": 0.13784496726368295, "margin": -1.313866520411899, "p_left": 0.2118405546161929, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23805486773498735, "brier": 0.0448764205800962 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.5591332815684273, "right_score": -0.7133654814755538, "margin": -0.8457678000928736, "p_left": 0.30032141095473314, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2029020079839134, "brier": 0.4895501279683753 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.08113737662587367, "right_score": 0.3631700579311891, "margin": -0.28203268130531545, "p_left": 0.42995550776731006, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8440735459565317, "brier": 0.3249507231248253 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.29543592810435054, "margin": 1.783053859667893, "p_left": 0.8560735462448692, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15539898802081753, "brier": 0.020714824090527804 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.9249117475935976, "margin": 1.0627567148572805, "p_left": 0.743217004543786, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2967672115723152, "brier": 0.065937506755466 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -2.2481420937145673, "margin": -0.1719931624211819, "p_left": 0.45710739346596463, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.78283691905207, "brier": 0.29473238222931897 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.13784496726368295, "right_score": 1.1839689484679448, "margin": -1.0461239812042618, "p_left": 0.2599700971489528, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3010646843421818, "brier": 0.06758445141163597 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.402811337098627, "right_score": 1.1839689484679448, "margin": 1.2188423886306823, "p_left": 0.7718597671711172, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25895239419956795, "brier": 0.052047965835216876 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.3631700579311891, "right_score": -0.13957407779101405, "margin": 0.5027441357222031, "p_left": 0.623103996242335, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4730418459677973, "brier": 0.1420505976484978 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3703770135456499, "right_score": 1.1839689484679448, "margin": -1.5543459620135947, "p_left": 0.17445946230961398, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19171690999496488, "brier": 0.03043610398935962 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.501418926464471, "right_score": -1.176021553148216, "margin": 0.6746026266837452, "p_left": 0.6625329969810487, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4116849155982284, "brier": 0.11388397812659287 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.501418926464471, "right_score": -1.9476749442737873, "margin": 1.4462560178093162, "p_left": 0.8094215626308593, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9345634155344207, "brier": 0.09574170342092278 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.29543592810435054, "margin": -0.2059829983601204, "p_left": 0.44868555725984277, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5954499561237059, "brier": 0.20131872929357564 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.29543592810435054, "right_score": -0.3703770135456499, "margin": 0.07494108544129935, "p_left": 0.5187265079153722, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7313195798502383, "brier": 0.2690771900140767 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.057344187165848215, "margin": 0.19518915442953116, "p_left": 0.5486429500561503, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6003074132475881, "brier": 0.20372318653401486 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.13957407779101405, "right_score": -1.5591332815684273, "margin": 1.4195592037774132, "p_left": 0.8052693041663582, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6361377219497029, "brier": 0.6484586522325707 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3703770135456499, "right_score": -1.176021553148216, "margin": 0.8056445396025662, "p_left": 0.6911806063283058, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3693541198600183, "brier": 0.09536941790775284 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.2547528147367135, "right_score": 0.3685822126168096, "margin": -1.6233350273535232, "p_left": 0.1647454425611767, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.8033537687367593, "brier": 0.6976501757223245 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.4876179315635425, "right_score": 1.1839689484679448, "margin": 0.30364898309559774, "p_left": 0.5753342978261501, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.402811337098627, "right_score": -0.9249117475935976, "margin": 3.3277230846922246, "p_left": 0.9653677264878943, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.03524618651146527, "brier": 0.0011993943686173006 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.7133654814755538, "right_score": -0.31359736199695565, "margin": -0.39976811947859814, "p_left": 0.40136805294921685, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5131083153739083, "brier": 0.16109631392824533 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.06094111590212581, "right_score": -0.29543592810435054, "margin": 0.35637704400647635, "p_left": 0.5881631365285638, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8871279704944108, "brier": 0.34593587517111796 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.06094111590212581, "right_score": -1.9476749442737873, "margin": 2.008616060175913, "p_left": 0.8816987451064758, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1259048401560463, "brier": 0.013995186909382576 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3703770135456499, "right_score": -0.057344187165848215, "margin": -0.3130328263798017, "p_left": 0.4223746334959473, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5488297752901937, "brier": 0.1784003310208358 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.9249117475935976, "margin": 0.4234928211291267, "p_left": 0.6043187484443668, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7153999015994471, "brier": 0.010882401276999088 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.9249117475935976, "right_score": 0.06094111590212581, "margin": -0.9858528634957234, "p_left": 0.27173199255289543, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3170861564354553, "brier": 0.07383827577676681 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.9476749442737873, "margin": 4.350486281372414, "p_left": 0.9872637664975346, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.01281803462540232, "brier": 0.00016221164382932318 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.9476749442737873, "right_score": 1.4876179315635425, "margin": -3.43529287583733, "p_left": 0.031210497022679466, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.031707921879038925, "brier": 0.0009740951244026838 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.7133654814755538, "right_score": -2.420135256135749, "margin": 1.7067697746601955, "p_left": 0.8464168396113582, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8735130971651364, "brier": 0.7164214663776796 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.057344187165848215, "margin": 1.5449621187293907, "p_left": 0.8241849138254784, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19336036426644232, "brier": 0.030910944526554445 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.13009458642499722, "right_score": -0.5909243129983202, "margin": 0.46082972657332294, "p_left": 0.613210991878727, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7194610699935612, "brier": 0.012816728682165183 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.057344187165848215, "right_score": 0.06094111590212581, "margin": -0.11828530306797402, "p_left": 0.4704631047797302, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6357524370097754, "brier": 0.2213355329589834 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -0.13009458642499722, "margin": -2.290040669710752, "p_left": 0.09195115411081337, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.386497776506353, "brier": 0.8245527065206839 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.176021553148216, "margin": 3.578832890246843, "p_left": 0.9728494724957664, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.02752591329055898, "brier": 0.0007371511437581471 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 9, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "b20ebcae7654", "e5aa020b210d", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 2, "both_wrong": 5, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [ "70f39a6b70be", "396ff40ef0fb" ], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "b20ebcae7654", "e5aa020b210d" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.0059459087196547696, "rank": 10.0, "portrait_scores": [ -0.009642117338916411, -0.002249700100393129 ], "portrait_score_gap": 0.007392417238523282, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.00402075708498895, "rank": 2.0, "portrait_scores": [ 0.000838912001872391, 0.007202602168105509 ], "portrait_score_gap": 0.006363690166233118, "single_photo_rank_min": 1.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.0057755090336212445, "rank": 9.0, "portrait_scores": [ -0.00930049262768177, -0.002250525439560719 ], "portrait_score_gap": 0.007049967188121051, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.0021084802544085266, "rank": 5.0, "portrait_scores": [ -0.004279786803456181, 6.282629463912797e-05 ], "portrait_score_gap": 0.004342613098095308, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 5.5, 10.0 ] }, { "identity": "emma_stone", "mean_score": -0.0066689238371282315, "rank": 12.0, "portrait_scores": [ -0.004284282350333867, -0.009053565323922597 ], "portrait_score_gap": 0.004769282973588729, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "emma_watson", "mean_score": -0.0038982068904720463, "rank": 8.0, "portrait_scores": [ -0.007817900794385185, 2.1487013441092524e-05 ], "portrait_score_gap": 0.007839387807826278, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": 0.002175560622086061, "rank": 3.0, "portrait_scores": [ -0.0017219610664405771, 0.006073082310612699 ], "portrait_score_gap": 0.007795043377053276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.008724430873682777, "rank": 13.0, "portrait_scores": [ -0.006252669656427401, -0.011196192090938153 ], "portrait_score_gap": 0.004943522434510753, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 13.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.006450842074515536, "rank": 11.0, "portrait_scores": [ -0.010714569341540886, -0.0021871148074901854 ], "portrait_score_gap": 0.0085274545340507, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.5, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.0043347647034655944, "rank": 1.0, "portrait_scores": [ 0.004407039028129015, 0.004262490378802173 ], "portrait_score_gap": 0.00014454864932684271, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": -0.0035178423924650927, "rank": 7.0, "portrait_scores": [ -0.005410572839986854, -0.0016251119449433313 ], "portrait_score_gap": 0.003785460895043523, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.0022665165995480454, "rank": 6.0, "portrait_scores": [ -0.0010918872303096855, -0.0034411459687864058 ], "portrait_score_gap": 0.00234925873847672, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 10.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0018247584928000962, "rank": 4.0, "portrait_scores": [ 0.0005703360910877847, 0.0030791808945124077 ], "portrait_score_gap": 0.002508844803424623, "single_photo_rank_min": 2.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 3.0, 4.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "2.0": 1024, "3.0": 4096, "4.0": 3072 }, "target_primary_score": 0.0018247584928000962, "target_primary_rank": 4.0, "target_alternate_photo_scores": [ -0.011105909795185503, -0.0034348302903154364 ], "target_alternate_panel_score": -0.00727037004275047, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.0008195279682349337, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.005161889788341943, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.01129112712773942, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.0013244613230957002, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 8.307486458193851e-06, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.045787233056468535, "rank": 13.0, "portrait_scores": [ -0.05788836417688211, -0.03368610193605496 ], "portrait_score_gap": 0.024202262240827152, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.08525057464540728, "rank": 8.0, "portrait_scores": [ 0.08830794348310426, 0.08219320580771029 ], "portrait_score_gap": 0.006114737675393972, "single_photo_rank_min": 6.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.02568289790413717, "rank": 10.0, "portrait_scores": [ 0.10847247797033244, -0.057106682162058095 ], "portrait_score_gap": 0.16557916013239055, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.019399097203755197, "rank": 12.0, "portrait_scores": [ -0.019874094155495483, -0.018924100252014908 ], "portrait_score_gap": 0.0009499939034805747, "single_photo_rank_min": 10.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 12.0 ] }, { "identity": "emma_stone", "mean_score": 0.10753129359122938, "rank": 7.0, "portrait_scores": [ 0.08798330646757549, 0.12707928071488328 ], "portrait_score_gap": 0.03909597424730779, "single_photo_rank_min": 5.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "emma_watson", "mean_score": 0.17506768793771177, "rank": 3.0, "portrait_scores": [ 0.19044808873629937, 0.15968728713912417 ], "portrait_score_gap": 0.030760801597175202, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.078125, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "gal_gadot", "mean_score": 0.16269314562450723, "rank": 4.0, "portrait_scores": [ 0.168438168885425, 0.15694812236358946 ], "portrait_score_gap": 0.01149004652183555, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 4.0, 5.0 ] }, { "identity": "keira_knightley", "mean_score": 0.11798138579753292, "rank": 6.0, "portrait_scores": [ 0.007811057692995836, 0.22815171390207 ], "portrait_score_gap": 0.22034065620907417, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 11.0 ] }, { "identity": "lily_collins", "mean_score": 0.01580503486413258, "rank": 11.0, "portrait_scores": [ 0.06635994022216074, -0.03474987049389558 ], "portrait_score_gap": 0.10110981071605632, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.5, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.18750193277098498, "rank": 2.0, "portrait_scores": [ 0.21848835608655537, 0.1565155094554146 ], "portrait_score_gap": 0.06197284663114078, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "natalie_portman", "mean_score": 0.1569007226791014, "rank": 5.0, "portrait_scores": [ 0.18937121923615843, 0.12443022612204437 ], "portrait_score_gap": 0.06494099311411407, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 5.0, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.19226688743817444, "rank": 1.0, "portrait_scores": [ 0.15239304786600857, 0.23214072701034028 ], "portrait_score_gap": 0.0797476791443317, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 2.5, 6.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.03045324105649268, "rank": 9.0, "portrait_scores": [ 0.0036891898677969825, 0.05721729224518838 ], "portrait_score_gap": 0.0535281023773914, "single_photo_rank_min": 8.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 11.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 512, "9.0": 2560, "10.0": 3584, "11.0": 1536 }, "target_primary_score": 0.03045324105649268, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ 0.09456458771775719, 0.11343733918623501 ], "target_alternate_panel_score": 0.10400096345199611, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.08819592858786353, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.1234000606557513, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": 0.01875038880658883, "prefers_target": true, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.14978819650846464, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0020519876642351204, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5573592732336569, "rank": 13.0, "portrait_scores": [ -0.5598457993974082, -0.5548727470699055 ], "portrait_score_gap": 0.0049730523275026695, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.5, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.06082423505989064, "rank": 6.0, "portrait_scores": [ 0.11993745604626549, 0.0017110140735157875 ], "portrait_score_gap": 0.1182264419727497, "single_photo_rank_min": 4.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 6.5, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.017728968853186644, "rank": 7.0, "portrait_scores": [ 0.4760898433553939, -0.4406319056490206 ], "portrait_score_gap": 0.9167217490044145, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.46855975596944754, "rank": 12.0, "portrait_scores": [ -0.3723087939323175, -0.5648107180065776 ], "portrait_score_gap": 0.19250192407426003, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.3383711378879495, "rank": 3.0, "portrait_scores": [ 0.20401545804822127, 0.47272681772767766 ], "portrait_score_gap": 0.26871135967945636, "single_photo_rank_min": 3.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.27318098455077355, "rank": 4.0, "portrait_scores": [ 0.4532403437952076, 0.09312162530633944 ], "portrait_score_gap": 0.3601187184888682, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 8.0 ] }, { "identity": "gal_gadot", "mean_score": 0.14259163945223305, "rank": 5.0, "portrait_scores": [ 0.3233549006567723, -0.03817162175230623 ], "portrait_score_gap": 0.3615265224090785, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.013402616976340653, "rank": 8.0, "portrait_scores": [ -0.7218233621917541, 0.6950181282390728 ], "portrait_score_gap": 1.4168414904308269, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 8.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.412981092759068, "rank": 11.0, "portrait_scores": [ -0.13751631264378888, -0.6884458728743472 ], "portrait_score_gap": 0.5509295602305584, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.6735738462434677, "rank": 2.0, "portrait_scores": [ 0.780652146411644, 0.5664955460752913 ], "portrait_score_gap": 0.21415660033635275, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 1.5, 3.0 ] }, { "identity": "natalie_portman", "mean_score": -0.0662949724074468, "rank": 9.0, "portrait_scores": [ 0.11368995300408836, -0.246279897818982 ], "portrait_score_gap": 0.35996985082307037, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.7256014617008741, "rank": 1.0, "portrait_scores": [ 0.6798036070066402, 0.7713993163951081 ], "portrait_score_gap": 0.09159570938846784, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.375, "single_photo_rank_quantiles": [ 1.0, 2.0, 3.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.3621385436612092, "rank": 10.0, "portrait_scores": [ -0.4880160555937668, -0.2362610317286515 ], "portrait_score_gap": 0.2517550238651153, "single_photo_rank_min": 7.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 256, "8.0": 1024, "9.0": 2048, "10.0": 2560, "11.0": 1792, "12.0": 512 }, "target_primary_score": -0.3621385436612092, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ -0.1069543304445751, 0.1501818653485918 ], "target_alternate_panel_score": 0.021613767452008346, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.43459486021107635, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.49017352342145587, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.03921046760788229, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5789730406856652, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.07273472287815451, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.13009458642499722, "rank": 4.0, "portrait_scores": [ 0.18799318441134627, -0.4481823572613407 ], "portrait_score_gap": 0.636175541672687, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.7133654814755538, "rank": 8.0, "portrait_scores": [ -0.30938440250615096, -1.1173465604449566 ], "portrait_score_gap": 0.8079621579388057, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 7.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.3685822126168096, "rank": 1.0, "portrait_scores": [ 1.6188186464055905, -0.8816542211719712 ], "portrait_score_gap": 2.5004728675775616, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -2.2481420937145673, "rank": 12.0, "portrait_scores": [ -1.9692674536691548, -2.52701673375998 ], "portrait_score_gap": 0.5577492800908253, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.2547528147367135, "rank": 9.0, "portrait_scores": [ 0.6197110612078303, -3.1292166906812575 ], "portrait_score_gap": 3.748927751889088, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.3631700579311891, "rank": 2.0, "portrait_scores": [ 1.091934396303953, -0.3655942804415747 ], "portrait_score_gap": 1.4575286767455276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.08113737662587367, "rank": 3.0, "portrait_scores": [ -0.36320541688355734, 0.5254801701353047 ], "portrait_score_gap": 0.888685587018862, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.033203125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.5591332815684273, "rank": 11.0, "portrait_scores": [ -3.207521366292256, 0.0892548031554013 ], "portrait_score_gap": 3.296776169447657, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.377058013611375, "rank": 10.0, "portrait_scores": [ -1.2412704739968592, -1.5128455532258906 ], "portrait_score_gap": 0.27157507922903146, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.5909243129983202, "rank": 7.0, "portrait_scores": [ -0.7647706640864682, -0.4170779619101721 ], "portrait_score_gap": 0.3476927021762961, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": -0.13957407779101405, "rank": 5.0, "portrait_scores": [ -0.682942375998363, 0.40379422041633495 ], "portrait_score_gap": 1.086736596414698, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.31359736199695565, "rank": 6.0, "portrait_scores": [ -1.5338127578833025, 0.9066180338893912 ], "portrait_score_gap": 2.4404307917726937, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.420135256135749, "rank": 13.0, "portrait_scores": [ -0.40310570063197304, -4.4371648116395255 ], "portrait_score_gap": 4.0340591110075525, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.420135256135749, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -3.368770113158943, -3.3457628357664864 ], "target_alternate_panel_score": -3.3572664744627145, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.9802084608513395, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.1091243807481472, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.643900992987161, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.2271718880377174, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.1358325064457129, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.2982539951347632, "rank": 1.0, "portrait_scores": [ 0.2410358800595792, 0.3554721102099472 ], "portrait_score_gap": 0.11443623015036797, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0234375, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.4466081925332326, "rank": 7.0, "portrait_scores": [ -0.23728789771125025, -0.6559284873552149 ], "portrait_score_gap": 0.41864058964396467, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.1934331586011072, "rank": 3.0, "portrait_scores": [ 1.2622237751753123, -0.8753574579730978 ], "portrait_score_gap": 2.13758123314841, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -1.6423596882616498, "rank": 12.0, "portrait_scores": [ -1.4685505513353818, -1.8161688251879178 ], "portrait_score_gap": 0.34761827385253596, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.0003295477787946, "rank": 9.0, "portrait_scores": [ 0.5205223826030391, -2.5211814781606283 ], "portrait_score_gap": 3.0417038607636675, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 7.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": -0.10884771895629641, "rank": 5.0, "portrait_scores": [ 0.5179309669676208, -0.7356264048802136 ], "portrait_score_gap": 1.2535573718478343, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.12765076343419546, "rank": 4.0, "portrait_scores": [ -0.25549410859769695, 0.5107956354660879 ], "portrait_score_gap": 0.7662897440637848, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.3400904407192387, "rank": 10.0, "portrait_scores": [ -2.9615736798331227, 0.2813927983946455 ], "portrait_score_gap": 3.242966478227768, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.456412041920701, "rank": 11.0, "portrait_scores": [ -1.6050302714769495, -1.3077938123644521 ], "portrait_score_gap": 0.2972364591124974, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.40333820027264944, "rank": 6.0, "portrait_scores": [ -0.24567143258816004, -0.5610049679571388 ], "portrait_score_gap": 0.3153335353689788, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "natalie_portman", "mean_score": -0.847589171230831, "rank": 8.0, "portrait_scores": [ -0.8724657195876679, -0.822712622873994 ], "portrait_score_gap": 0.049753096713673894, "single_photo_rank_min": 7.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.21246490493664044, "rank": 2.0, "portrait_scores": [ -0.5827645297901508, 1.0076943396634317 ], "portrait_score_gap": 1.5904588694535824, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.0, 9.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.134916385097236, "rank": 13.0, "portrait_scores": [ -0.4308175656799993, -3.8390152045144728 ], "portrait_score_gap": 3.4081976388344737, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.134916385097236, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -2.650423135031142, -3.44346762464777 ], "target_alternate_panel_score": -3.046945379839456, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.590533337918755, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.404585691577806, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.6003371873062235, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.3451993749742193, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.7911822746370201, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.13009458642499722, "rank": 4.0, "portrait_scores": [ 0.18799318441134627, -0.4481823572613407 ], "portrait_score_gap": 0.636175541672687, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.7133654814755538, "rank": 8.0, "portrait_scores": [ -0.30938440250615096, -1.1173465604449566 ], "portrait_score_gap": 0.8079621579388057, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 7.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.3685822126168096, "rank": 1.0, "portrait_scores": [ 1.6188186464055905, -0.8816542211719712 ], "portrait_score_gap": 2.5004728675775616, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -2.2481420937145673, "rank": 12.0, "portrait_scores": [ -1.9692674536691548, -2.52701673375998 ], "portrait_score_gap": 0.5577492800908253, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.2547528147367135, "rank": 9.0, "portrait_scores": [ 0.6197110612078303, -3.1292166906812575 ], "portrait_score_gap": 3.748927751889088, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.3631700579311891, "rank": 2.0, "portrait_scores": [ 1.091934396303953, -0.3655942804415747 ], "portrait_score_gap": 1.4575286767455276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.08113737662587367, "rank": 3.0, "portrait_scores": [ -0.36320541688355734, 0.5254801701353047 ], "portrait_score_gap": 0.888685587018862, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.033203125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.5591332815684273, "rank": 11.0, "portrait_scores": [ -3.207521366292256, 0.0892548031554013 ], "portrait_score_gap": 3.296776169447657, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.377058013611375, "rank": 10.0, "portrait_scores": [ -1.2412704739968592, -1.5128455532258906 ], "portrait_score_gap": 0.27157507922903146, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.5909243129983202, "rank": 7.0, "portrait_scores": [ -0.7647706640864682, -0.4170779619101721 ], "portrait_score_gap": 0.3476927021762961, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": -0.13957407779101405, "rank": 5.0, "portrait_scores": [ -0.682942375998363, 0.40379422041633495 ], "portrait_score_gap": 1.086736596414698, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.31359736199695565, "rank": 6.0, "portrait_scores": [ -1.5338127578833025, 0.9066180338893912 ], "portrait_score_gap": 2.4404307917726937, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.420135256135749, "rank": 13.0, "portrait_scores": [ -0.40310570063197304, -4.4371648116395255 ], "portrait_score_gap": 4.0340591110075525, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.420135256135749, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -3.368770113158943, -3.3457628357664864 ], "target_alternate_panel_score": -3.3572664744627145, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.9802084608513395, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.1091243807481472, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.643900992987161, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.2271718880377174, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.1358325064457129, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, "consistency_ablation": [ { "name": "gamma_0", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.382008249051607, "brier_score": 0.0966426598534706, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.233378610516842, "brier_score": 0.4463367275497861, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.0515950815381339, "brier_score": 0.3716947179716129, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5969456684742582, "brier_score": 0.5956207467061325, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2512082207788717, "brier_score": 0.07378107487924734, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.176021553148216, "right_score": -0.501418926464471, "margin": -0.6746026266837452, "p_left": 0.3374670030189513, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4116849155982283, "brier": 0.11388397812659287 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.377058013611375, "right_score": -2.420135256135749, "margin": 1.0430772425243742, "p_left": 0.7394433255475688, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3449348811734736, "brier": 0.5467764316968478 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.1839689484679448, "right_score": 1.4876179315635425, "margin": -0.30364898309559774, "p_left": 0.4246657021738499, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.9476749442737873, "right_score": 2.402811337098627, "margin": -4.350486281372414, "p_left": 0.012736233502465381, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.012818034625402548, "brier": 0.00016221164382932158 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.176021553148216, "right_score": 2.402811337098627, "margin": -3.578832890246843, "p_left": 0.0271505275042335, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.027525913290559173, "brier": 0.0007371511437581397 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.377058013611375, "right_score": -2.2481420937145673, "margin": 0.8710840801031923, "p_left": 0.7049712224381213, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.220682376342896, "brier": 0.4969844244658991 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.176021553148216, "right_score": 0.13784496726368295, "margin": -1.313866520411899, "p_left": 0.2118405546161929, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23805486773498735, "brier": 0.0448764205800962 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.5591332815684273, "right_score": -0.7133654814755538, "margin": -0.8457678000928736, "p_left": 0.30032141095473314, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2029020079839134, "brier": 0.4895501279683753 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.08113737662587367, "right_score": 0.3631700579311891, "margin": -0.28203268130531545, "p_left": 0.42995550776731006, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8440735459565317, "brier": 0.3249507231248253 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.29543592810435054, "margin": 1.783053859667893, "p_left": 0.8560735462448692, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15539898802081753, "brier": 0.020714824090527804 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.9249117475935976, "margin": 1.0627567148572805, "p_left": 0.743217004543786, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2967672115723152, "brier": 0.065937506755466 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -2.2481420937145673, "margin": -0.1719931624211819, "p_left": 0.45710739346596463, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.78283691905207, "brier": 0.29473238222931897 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.13784496726368295, "right_score": 1.1839689484679448, "margin": -1.0461239812042618, "p_left": 0.2599700971489528, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3010646843421818, "brier": 0.06758445141163597 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.402811337098627, "right_score": 1.1839689484679448, "margin": 1.2188423886306823, "p_left": 0.7718597671711172, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25895239419956795, "brier": 0.052047965835216876 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.3631700579311891, "right_score": -0.13957407779101405, "margin": 0.5027441357222031, "p_left": 0.623103996242335, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4730418459677973, "brier": 0.1420505976484978 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3703770135456499, "right_score": 1.1839689484679448, "margin": -1.5543459620135947, "p_left": 0.17445946230961398, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19171690999496488, "brier": 0.03043610398935962 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.501418926464471, "right_score": -1.176021553148216, "margin": 0.6746026266837452, "p_left": 0.6625329969810487, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4116849155982284, "brier": 0.11388397812659287 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.501418926464471, "right_score": -1.9476749442737873, "margin": 1.4462560178093162, "p_left": 0.8094215626308593, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.9345634155344207, "brier": 0.09574170342092278 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.29543592810435054, "margin": -0.2059829983601204, "p_left": 0.44868555725984277, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5954499561237059, "brier": 0.20131872929357564 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.29543592810435054, "right_score": -0.3703770135456499, "margin": 0.07494108544129935, "p_left": 0.5187265079153722, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7313195798502383, "brier": 0.2690771900140767 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.13784496726368295, "right_score": -0.057344187165848215, "margin": 0.19518915442953116, "p_left": 0.5486429500561503, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6003074132475881, "brier": 0.20372318653401486 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.13957407779101405, "right_score": -1.5591332815684273, "margin": 1.4195592037774132, "p_left": 0.8052693041663582, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6361377219497029, "brier": 0.6484586522325707 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3703770135456499, "right_score": -1.176021553148216, "margin": 0.8056445396025662, "p_left": 0.6911806063283058, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3693541198600183, "brier": 0.09536941790775284 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.2547528147367135, "right_score": 0.3685822126168096, "margin": -1.6233350273535232, "p_left": 0.1647454425611767, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.8033537687367593, "brier": 0.6976501757223245 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.4876179315635425, "right_score": 1.1839689484679448, "margin": 0.30364898309559774, "p_left": 0.5753342978261501, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5528040196012971, "brier": 0.180340958602809 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.402811337098627, "right_score": -0.9249117475935976, "margin": 3.3277230846922246, "p_left": 0.9653677264878943, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.03524618651146527, "brier": 0.0011993943686173006 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.7133654814755538, "right_score": -0.31359736199695565, "margin": -0.39976811947859814, "p_left": 0.40136805294921685, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5131083153739083, "brier": 0.16109631392824533 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.06094111590212581, "right_score": -0.29543592810435054, "margin": 0.35637704400647635, "p_left": 0.5881631365285638, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8871279704944108, "brier": 0.34593587517111796 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.06094111590212581, "right_score": -1.9476749442737873, "margin": 2.008616060175913, "p_left": 0.8816987451064758, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1259048401560463, "brier": 0.013995186909382576 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3703770135456499, "right_score": -0.057344187165848215, "margin": -0.3130328263798017, "p_left": 0.4223746334959473, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5488297752901937, "brier": 0.1784003310208358 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.501418926464471, "right_score": -0.9249117475935976, "margin": 0.4234928211291267, "p_left": 0.6043187484443668, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7153999015994471, "brier": 0.010882401276999088 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.9249117475935976, "right_score": 0.06094111590212581, "margin": -0.9858528634957234, "p_left": 0.27173199255289543, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3170861564354553, "brier": 0.07383827577676681 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.9476749442737873, "margin": 4.350486281372414, "p_left": 0.9872637664975346, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.01281803462540232, "brier": 0.00016221164382932318 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.9476749442737873, "right_score": 1.4876179315635425, "margin": -3.43529287583733, "p_left": 0.031210497022679466, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.031707921879038925, "brier": 0.0009740951244026838 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.7133654814755538, "right_score": -2.420135256135749, "margin": 1.7067697746601955, "p_left": 0.8464168396113582, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8735130971651364, "brier": 0.7164214663776796 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.4876179315635425, "right_score": -0.057344187165848215, "margin": 1.5449621187293907, "p_left": 0.8241849138254784, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19336036426644232, "brier": 0.030910944526554445 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.13009458642499722, "right_score": -0.5909243129983202, "margin": 0.46082972657332294, "p_left": 0.613210991878727, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7194610699935612, "brier": 0.012816728682165183 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.057344187165848215, "right_score": 0.06094111590212581, "margin": -0.11828530306797402, "p_left": 0.4704631047797302, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6357524370097754, "brier": 0.2213355329589834 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.420135256135749, "right_score": -0.13009458642499722, "margin": -2.290040669710752, "p_left": 0.09195115411081337, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.386497776506353, "brier": 0.8245527065206839 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.402811337098627, "right_score": -1.176021553148216, "margin": 3.578832890246843, "p_left": 0.9728494724957664, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.02752591329055898, "brier": 0.0007371511437581471 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.13009458642499722, "rank": 4.0, "portrait_scores": [ 0.18799318441134627, -0.4481823572613407 ], "portrait_score_gap": 0.636175541672687, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.7133654814755538, "rank": 8.0, "portrait_scores": [ -0.30938440250615096, -1.1173465604449566 ], "portrait_score_gap": 0.8079621579388057, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 7.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.3685822126168096, "rank": 1.0, "portrait_scores": [ 1.6188186464055905, -0.8816542211719712 ], "portrait_score_gap": 2.5004728675775616, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -2.2481420937145673, "rank": 12.0, "portrait_scores": [ -1.9692674536691548, -2.52701673375998 ], "portrait_score_gap": 0.5577492800908253, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.2547528147367135, "rank": 9.0, "portrait_scores": [ 0.6197110612078303, -3.1292166906812575 ], "portrait_score_gap": 3.748927751889088, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.3631700579311891, "rank": 2.0, "portrait_scores": [ 1.091934396303953, -0.3655942804415747 ], "portrait_score_gap": 1.4575286767455276, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.08113737662587367, "rank": 3.0, "portrait_scores": [ -0.36320541688355734, 0.5254801701353047 ], "portrait_score_gap": 0.888685587018862, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.033203125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.5591332815684273, "rank": 11.0, "portrait_scores": [ -3.207521366292256, 0.0892548031554013 ], "portrait_score_gap": 3.296776169447657, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.377058013611375, "rank": 10.0, "portrait_scores": [ -1.2412704739968592, -1.5128455532258906 ], "portrait_score_gap": 0.27157507922903146, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.5909243129983202, "rank": 7.0, "portrait_scores": [ -0.7647706640864682, -0.4170779619101721 ], "portrait_score_gap": 0.3476927021762961, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "natalie_portman", "mean_score": -0.13957407779101405, "rank": 5.0, "portrait_scores": [ -0.682942375998363, 0.40379422041633495 ], "portrait_score_gap": 1.086736596414698, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.31359736199695565, "rank": 6.0, "portrait_scores": [ -1.5338127578833025, 0.9066180338893912 ], "portrait_score_gap": 2.4404307917726937, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.420135256135749, "rank": 13.0, "portrait_scores": [ -0.40310570063197304, -4.4371648116395255 ], "portrait_score_gap": 4.0340591110075525, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.420135256135749, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -3.368770113158943, -3.3457628357664864 ], "target_alternate_panel_score": -3.3572664744627145, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.9802084608513395, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.1091243807481472, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.643900992987161, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.2271718880377174, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.1358325064457129, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.01", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.389002020354497, "brier_score": 0.09903847290773145, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.195036929466897, "brier_score": 0.4313313510437169, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.9965525360797043, "brier_score": 0.350653432721321, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5920057162412813, "brier_score": 0.5926871876885088, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.25246821078191906, "brier_score": 0.07375555690202039, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.162979960757378, "right_score": -0.43139574202865977, "margin": -0.7315842187287183, "p_left": 0.32484717784187617, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3928162105377229, "brier": 0.10552568895183152 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.4221197172726114, "right_score": -2.3507977255764647, "margin": 0.9286780083038533, "p_left": 0.7168070050882198, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2616266529495015, "brier": 0.5138122825435432 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.121134877015584, "right_score": 1.3871498653421468, "margin": -0.2660149883265628, "p_left": 0.4338856699332984, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5689592245932388, "brier": 0.18825677457346718 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.8352294398235567, "right_score": 2.2707285897575398, "margin": -4.105958029581096, "p_left": 0.016207226918306547, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.016340000570125985, "brier": 0.00026267420438148034 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.162979960757378, "right_score": 2.2707285897575398, "margin": -3.433708550514918, "p_left": 0.03125843691551912, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.031757407426588766, "brier": 0.0009770898784014885 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.4221197172726114, "right_score": -2.1163324836968207, "margin": 0.6942127664242093, "p_left": 0.6669034214558969, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0993228053930368, "brier": 0.4447601735495816 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.162979960757378, "right_score": 0.14600975768271313, "margin": -1.3089897184400912, "p_left": 0.21265594989039, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23908995950004103, "brier": 0.045222553023784064 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.5426069099298372, "right_score": -0.6511873941392068, "margin": -0.8914195157906304, "p_left": 0.2908169760919566, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.235061157819102, "brier": 0.5029405613993565 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.08128227856564119, "right_score": 0.2572728015119413, "margin": -0.17599052294630013, "p_left": 0.4561155789558497, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7850090389614973, "brier": 0.2958102634545305 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.3871498653421468, "right_score": -0.32391519590143586, "margin": 1.7110650612435827, "p_left": 0.8469743765442888, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16608483680151087, "brier": 0.023416841434009125 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.14600975768271313, "right_score": -0.8690908929802009, "margin": 1.015100650662914, "p_left": 0.7340171676137429, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30922286152160194, "brier": 0.07074686712421573 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.3507977255764647, "right_score": -2.1163324836968207, "margin": -0.23446524187964402, "p_left": 0.44165075231818895, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8172358623662291, "brier": 0.31175388238684437 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.14600975768271313, "right_score": 1.121134877015584, "margin": -0.9751251193328709, "p_left": 0.27386013670932496, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.32001263355109816, "brier": 0.07499937447845016 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.2707285897575398, "right_score": 1.121134877015584, "margin": 1.1495937127419558, "p_left": 0.7594366990979478, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2751783059128947, "brier": 0.05787070174089133 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.2572728015119413, "right_score": -0.3303150276836988, "margin": 0.5875878291956401, "p_left": 0.6428114905054747, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4419037695699689, "brier": 0.1275836313149206 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3466233677067513, "right_score": 1.121134877015584, "margin": -1.4677582447223352, "p_left": 0.18728358887796662, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.20737304809348614, "brier": 0.03507514266301122 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.43139574202865977, "right_score": -1.162979960757378, "margin": 0.7315842187287183, "p_left": 0.6751528221581239, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.39281621053772287, "brier": 0.10552568895183148 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.43139574202865977, "right_score": -1.8352294398235567, "margin": 1.4038336977948969, "p_left": 0.8027915338100603, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.92157705684004, "brier": 0.09168271294704888 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.43139574202865977, "right_score": -0.32391519590143586, "margin": -0.10748054612722391, "p_left": 0.4731557007577672, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6408502214521359, "brier": 0.22387631715957373 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.32391519590143586, "right_score": -0.3466233677067513, "margin": 0.022708171805315425, "p_left": 0.5056767990114058, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7045657227110642, "brier": 0.25570902505842175 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.14600975768271313, "right_score": -0.047395290164848114, "margin": 0.19340504784756124, "p_left": 0.5482011066226904, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6011130764677504, "brier": 0.20412224005696158 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.3303150276836988, "right_score": -1.5426069099298372, "margin": 1.2122918822461384, "p_left": 0.7707042199632786, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4727424929652497, "brier": 0.5939849946692058 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3466233677067513, "right_score": -1.162979960757378, "margin": 0.8163565930506268, "p_left": 0.6934623990904036, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3660582597448804, "brier": 0.09396530077141098 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.202315775425999, "right_score": 0.3262346943835124, "margin": -1.5285504698095114, "p_left": 0.17820586835196905, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.7248158332195862, "brier": 0.6753455948111412 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.3871498653421468, "right_score": 1.121134877015584, "margin": 0.2660149883265628, "p_left": 0.5661143300667016, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5689592245932388, "brier": 0.18825677457346712 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.2707285897575398, "right_score": -0.8690908929802009, "margin": 3.1398194827377406, "p_left": 0.9585057016658176, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.042379767984325856, "brier": 0.0017217767942461315 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.6511873941392068, "right_score": -0.18668745680812632, "margin": -0.46449993733108047, "p_left": 0.38591885805858445, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48762820656738476, "brier": 0.14893336500524185 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.09160679956484996, "right_score": -0.32391519590143586, "margin": 0.4155219954662858, "p_left": 0.6024112077188809, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.922336993024788, "brier": 0.3628992631853207 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.09160679956484996, "right_score": -1.8352294398235567, "margin": 1.9268362393884066, "p_left": 0.872898825729768, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.13593562249346425, "brier": 0.016154708500871896 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3466233677067513, "right_score": -0.047395290164848114, "margin": -0.29922807754190317, "p_left": 0.42574619688404847, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.554683814681788, "brier": 0.18125982416123096 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.43139574202865977, "right_score": -0.8690908929802009, "margin": 0.43769515095154116, "p_left": 0.6077096945285485, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7169055626241152, "brier": 0.011601378295433232 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.8690908929802009, "right_score": 0.09160679956484996, "margin": -0.9606976925450509, "p_left": 0.2767385269523086, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.32398447206790043, "brier": 0.07658421229973364 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.2707285897575398, "right_score": -1.8352294398235567, "margin": 4.105958029581096, "p_left": 0.9837927730816936, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.016340000570125746, "brier": 0.0002626742043814772 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.8352294398235567, "right_score": 1.3871498653421468, "margin": -3.2223793051657035, "p_left": 0.038332181184344664, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03908619063355964, "brier": 0.001469356114349427 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.6511873941392068, "right_score": -2.3507977255764647, "margin": 1.699610331437258, "p_left": 0.8454838351100484, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8674565609979448, "brier": 0.7148429154323955 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.3871498653421468, "right_score": -0.047395290164848114, "margin": 1.4345451555069948, "p_left": 0.8076085159374596, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21367784785231936, "brier": 0.037014483139786725 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.022054147161358506, "right_score": -0.5370279883916551, "margin": 0.5149738412302967, "p_left": 0.625971733204674, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.725936984141809, "brier": 0.015868877566589574 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.047395290164848114, "right_score": 0.09160679956484996, "margin": -0.13900208972969808, "p_left": 0.46530532258799906, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6260593914214865, "brier": 0.21650904322872186 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.3507977255764647, "right_score": -0.022054147161358506, "margin": -2.3287435784151063, "p_left": 0.08877024280851536, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.42170378865145, "brier": 0.8303396703912521 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.2707285897575398, "right_score": -1.162979960757378, "margin": 3.433708550514918, "p_left": 0.9687415630844809, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.031757407426588724, "brier": 0.0009770898784014876 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.01, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.022054147161358506, "rank": 4.0, "portrait_scores": [ 0.1998238798657015, -0.2439321741884185 ], "portrait_score_gap": 0.44375605405412, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01953125, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.6511873941392068, "rank": 8.0, "portrait_scores": [ -0.30128650769950915, -1.0010882805789045 ], "portrait_score_gap": 0.6998017728793954, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.5, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.3262346943835124, "rank": 1.0, "portrait_scores": [ 1.5255677291933014, -0.8730983404262767 ], "portrait_score_gap": 2.398666069619578, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -2.1163324836968207, "rank": 12.0, "portrait_scores": [ -1.8925823537252475, -2.340082613668394 ], "portrait_score_gap": 0.44750025994314657, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.202315775425999, "rank": 9.0, "portrait_scores": [ 0.6039444731891328, -3.008576024041131 ], "portrait_score_gap": 3.612520497230264, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.2572728015119413, "rank": 2.0, "portrait_scores": [ 0.9803523863820206, -0.4658067833581379 ], "portrait_score_gap": 1.4461591697401586, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.08128227856564119, "rank": 3.0, "portrait_scores": [ -0.368155723843513, 0.5307202809747954 ], "portrait_score_gap": 0.8988760048183084, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.5426069099298372, "rank": 11.0, "portrait_scores": [ -3.1952653047076804, 0.11005148484800603 ], "portrait_score_gap": 3.3053167895556865, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.4221197172726114, "rank": 10.0, "portrait_scores": [ -1.3472006244550006, -1.4970388100902223 ], "portrait_score_gap": 0.14983818563522178, "single_photo_rank_min": 9.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 10.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.5370279883916551, "rank": 7.0, "portrait_scores": [ -0.639977070874973, -0.43407890590833714 ], "portrait_score_gap": 0.20589816496663588, "single_photo_rank_min": 3.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "natalie_portman", "mean_score": -0.3303150276836988, "rank": 6.0, "portrait_scores": [ -0.7553215044602455, 0.09469144909284788 ], "portrait_score_gap": 0.8500129535530934, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 10.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.18668745680812632, "rank": 5.0, "portrait_scores": [ -1.308711144486383, 0.9353362308701304 ], "portrait_score_gap": 2.2440473753565136, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.3507977255764647, "rank": 13.0, "portrait_scores": [ -0.3834412270172489, -4.3181542241356805 ], "portrait_score_gap": 3.9347129971184316, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.3507977255764647, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -3.2444434605810324, -3.393205306364057 ], "target_alternate_panel_score": -3.3188243834725446, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.8967046661999332, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.202491899775724, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.667636989333338, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.296770236311186, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.0542376488744944, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 20, "accuracy": 0.9090909090909091, "mean_cross_entropy": 0.42197404504672703, "brier_score": 0.11090902925836625, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 1.0859307796627728, "brier_score": 0.3835735359881786, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.8247057965769973, "brier_score": 0.27750073612716347, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.6083807458343244, "brier_score": 0.5957191357102087, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2674614008689228, "brier_score": 0.077597014808838, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.0803133389622654, "right_score": -0.1810473837205192, "margin": -0.8992659552417462, "p_left": 0.28920136715328437, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3413661061042845, "brier": 0.08363743076332879 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.456412041920701, "right_score": -2.134916385097236, "margin": 0.678504343176535, "p_left": 0.6634048005898459, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0888742593290535, "brier": 0.44010592944565313 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.8866103024741911, "right_score": 0.9988101791903153, "margin": -0.11219987671612419, "p_left": 0.4719794201403607, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6386200190285669, "brier": 0.22276457303603112 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.470917242566171, "right_score": 1.8226244530851612, "margin": -3.2935416956513324, "p_left": 0.03579341285514762, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03644970531350696, "brier": 0.001281168403819047 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.0803133389622654, "right_score": 1.8226244530851612, "margin": -2.9029377920474264, "p_left": 0.052008528456139784, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.053409773029332676, "brier": 0.002704887032173102 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.456412041920701, "right_score": -1.6423596882616498, "margin": 0.18594764634094885, "p_left": 0.5463534267801844, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7904368571924157, "brier": 0.2985020669544503 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.0803133389622654, "right_score": 0.18766306604504648, "margin": -1.267976405007312, "p_left": 0.2196038549706713, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2479536099822745, "brier": 0.04822585311797964 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -1.3400904407192387, "right_score": -0.4466081925332326, "margin": -0.8934822481860061, "p_left": 0.29039173671995716, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.2365244512574263, "brier": 0.5035438873153186 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.12765076343419546, "right_score": -0.10884771895629641, "margin": 0.23649848239049187, "p_left": 0.558850575719023, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5818731479658042, "brier": 0.19461281454343748 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.9988101791903153, "right_score": -0.4208937721750325, "margin": 1.4197039513653478, "p_left": 0.8052920011270379, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21655033301646043, "brier": 0.037911204825113405 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.18766306604504648, "right_score": -0.6583794203392574, "margin": 0.8460424863843039, "p_left": 0.6997363052869247, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.35705172164350274, "brier": 0.0901582863627469 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -2.134916385097236, "right_score": -1.6423596882616498, "margin": -0.4925566968355861, "p_left": 0.37929146043956335, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9694503445413516, "brier": 0.3852790910832501 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.18766306604504648, "right_score": 0.8866103024741911, "margin": -0.6989472364291447, "p_left": 0.3320456803820311, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4035354915892061, "brier": 0.11025433386036594 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.8226244530851612, "right_score": 0.8866103024741911, "margin": 0.93601415061097, "p_left": 0.7182938317540959, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.33087655722198317, "brier": 0.07935836522778963 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.10884771895629641, "right_score": -0.847589171230831, "margin": 0.7387414522745346, "p_left": 0.676720585220303, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3904968161973221, "brier": 0.10450958002030339 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.2819409600148173, "right_score": 0.8866103024741911, "margin": -1.1685512624890084, "p_left": 0.23711695021128035, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.27065053626265306, "brier": 0.0562244480774988 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.1810473837205192, "right_score": -1.0803133389622654, "margin": 0.8992659552417462, "p_left": 0.7107986328467156, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3413661061042844, "brier": 0.08363743076332879 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.1810473837205192, "right_score": -1.470917242566171, "margin": 1.2898698588456519, "p_left": 0.7841251606223446, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.888121557145297, "brier": 0.08072710689867313 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.1810473837205192, "right_score": -0.4208937721750325, "margin": 0.23984638845451328, "p_left": 0.5596757937694042, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8202439910552437, "brier": 0.3132369941314127 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.4208937721750325, "right_score": -0.2819409600148173, "margin": -0.1389528121602152, "p_left": 0.46531758268490053, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6260823208389248, "brier": 0.21652045275571924 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.18766306604504648, "right_score": -0.013052812952860926, "margin": 0.2007158789979074, "p_left": 0.5500111828998336, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5978166684171718, "brier": 0.20248993551520697 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.847589171230831, "right_score": -1.3400904407192387, "margin": 0.49250126948840767, "p_left": 0.620695490243356, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9694159406752765, "brier": 0.3852628916084401 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.2819409600148173, "right_score": -1.0803133389622654, "margin": 0.7983723789474482, "p_left": 0.6896262095912857, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3716055534069793, "brier": 0.0963318897726725 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.0003295477787946, "right_score": 0.1934331586011072, "margin": -1.1937627063799017, "p_left": 0.2325866542688777, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.458492416859746, "brier": 0.588923243206235 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.9988101791903153, "right_score": 0.8866103024741911, "margin": 0.11219987671612419, "p_left": 0.5280205798596393, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6386200190285669, "brier": 0.2227645730360311 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.8226244530851612, "right_score": -0.6583794203392574, "margin": 2.4810038734244184, "p_left": 0.9227993448840229, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08034346260474212, "brier": 0.005959941150336036 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.4466081925332326, "right_score": 0.21246490493664044, "margin": -0.6590730974698731, "p_left": 0.34094785807084105, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41695262480366646, "brier": 0.11624544192309437 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.21083692993621256, "right_score": -0.4208937721750325, "margin": 0.631730702111245, "p_left": 0.6528817900032535, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0580898942328485, "brier": 0.42625463171785244 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.21083692993621256, "right_score": -1.470917242566171, "margin": 1.6817541725023835, "p_left": 0.8431366727711693, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1706262074651963, "brier": 0.02460610342929921 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.2819409600148173, "right_score": -0.013052812952860926, "margin": -0.26888814706195635, "p_left": 0.4331800736452534, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5677136159001027, "brier": 0.18764497620330717 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.1810473837205192, "right_score": -0.6583794203392574, "margin": 0.47733203661873824, "p_left": 0.6171176764665771, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7213615679628387, "brier": 0.013716550140929831 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.6583794203392574, "right_score": 0.21083692993621256, "margin": -0.86921635027547, "p_left": 0.2954173893678483, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3501496931522558, "brier": 0.08727143394091488 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.8226244530851612, "right_score": -1.470917242566171, "margin": 3.2935416956513324, "p_left": 0.9642065871448523, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.036449705313506886, "brier": 0.0012811684038190525 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.470917242566171, "right_score": 0.9988101791903153, "margin": -2.4697274217564864, "p_left": 0.07800783741446315, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.08121855591119846, "brier": 0.006085222698081318 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.4466081925332326, "right_score": -2.134916385097236, "margin": 1.6883081925640033, "p_left": 0.8440015411495622, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.85790915094526, "brier": 0.7123386014628361 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.9988101791903153, "right_score": -0.013052812952860926, "margin": 1.0118629921431763, "p_left": 0.7333845814899811, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31008504689562777, "brier": 0.07108378138727253 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.2982539951347632, "right_score": -0.40333820027264944, "margin": 0.7015921954074127, "p_left": 0.6685406878057302, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7534541176643208, "brier": 0.028405963446028602 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.013052812952860926, "right_score": 0.21083692993621256, "margin": -0.22388974288907348, "p_left": 0.4442602072572621, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5874550929412494, "brier": 0.1973671317522655 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -2.134916385097236, "right_score": 0.2982539951347632, "margin": -2.4331703802319993, "p_left": 0.08067801024391051, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.5172892285216326, "brier": 0.8451529208490954 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.8226244530851612, "right_score": -1.0803133389622654, "margin": 2.9029377920474264, "p_left": 0.9479914715438602, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.05340977302933281, "brier": 0.0027048870321731023 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 0.1, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.2982539951347632, "rank": 1.0, "portrait_scores": [ 0.2410358800595792, 0.3554721102099472 ], "portrait_score_gap": 0.11443623015036797, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0234375, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.4466081925332326, "rank": 7.0, "portrait_scores": [ -0.23728789771125025, -0.6559284873552149 ], "portrait_score_gap": 0.41864058964396467, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.1934331586011072, "rank": 3.0, "portrait_scores": [ 1.2622237751753123, -0.8753574579730978 ], "portrait_score_gap": 2.13758123314841, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.5, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -1.6423596882616498, "rank": 12.0, "portrait_scores": [ -1.4685505513353818, -1.8161688251879178 ], "portrait_score_gap": 0.34761827385253596, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.0003295477787946, "rank": 9.0, "portrait_scores": [ 0.5205223826030391, -2.5211814781606283 ], "portrait_score_gap": 3.0417038607636675, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 7.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": -0.10884771895629641, "rank": 5.0, "portrait_scores": [ 0.5179309669676208, -0.7356264048802136 ], "portrait_score_gap": 1.2535573718478343, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.12765076343419546, "rank": 4.0, "portrait_scores": [ -0.25549410859769695, 0.5107956354660879 ], "portrait_score_gap": 0.7662897440637848, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -1.3400904407192387, "rank": 10.0, "portrait_scores": [ -2.9615736798331227, 0.2813927983946455 ], "portrait_score_gap": 3.242966478227768, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.456412041920701, "rank": 11.0, "portrait_scores": [ -1.6050302714769495, -1.3077938123644521 ], "portrait_score_gap": 0.2972364591124974, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.40333820027264944, "rank": 6.0, "portrait_scores": [ -0.24567143258816004, -0.5610049679571388 ], "portrait_score_gap": 0.3153335353689788, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "natalie_portman", "mean_score": -0.847589171230831, "rank": 8.0, "portrait_scores": [ -0.8724657195876679, -0.822712622873994 ], "portrait_score_gap": 0.049753096713673894, "single_photo_rank_min": 7.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.21246490493664044, "rank": 2.0, "portrait_scores": [ -0.5827645297901508, 1.0076943396634317 ], "portrait_score_gap": 1.5904588694535824, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.0, 9.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -2.134916385097236, "rank": 13.0, "portrait_scores": [ -0.4308175656799993, -3.8390152045144728 ], "portrait_score_gap": 3.4081976388344737, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "3.0": 32, "4.0": 224, "5.0": 672, "6.0": 1120, "7.0": 1120, "8.0": 672, "9.0": 224, "10.0": 32, "13.0": 4096 }, "target_primary_score": -2.134916385097236, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -2.650423135031142, -3.44346762464777 ], "target_alternate_panel_score": -3.046945379839456, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.590533337918755, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.404585691577806, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -2.6003371873062235, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -3.3451993749742193, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.7911822746370201, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.48730140568793495, "brier_score": 0.13555214336872926, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.9550764675086781, "brier_score": 0.33530873596032884, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6810822091036941, "brier_score": 0.21205318178277366, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.5030649843186459, "brier_score": 0.5818198443154392, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.31880547483145205, "brier_score": 0.09091472375528044, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.814766449930493, "right_score": 0.16449765755681608, "margin": -0.9792641074873091, "p_left": 0.2730378252195104, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3188808319899551, "brier": 0.07454965400059992 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -1.0264766102813012, "right_score": -1.7570153074659243, "margin": 0.730538697184623, "p_left": 0.6749234748378296, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1236946623317867, "brier": 0.4555216968871703 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.5680954597292999, "right_score": 0.5107857094182159, "margin": 0.057309750311083985, "p_left": 0.5143235174367033, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7222125504788527, "brier": 0.2645286805884629 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -1.0172811111100093, "right_score": 1.1744602722900663, "margin": -2.1917413834000756, "p_left": 0.10049457094649605, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.10591018997496214, "brier": 0.01009915878972033 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.814766449930493, "right_score": 1.1744602722900663, "margin": -1.9892267222205593, "p_left": 0.12033869553197993, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.12821832688203813, "brier": 0.014481401642338566 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -1.0264766102813012, "right_score": -1.0774369768118293, "margin": 0.050960366530528045, "p_left": 0.512737335223868, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7189519485748853, "brier": 0.2628995749324732 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.814766449930493, "right_score": 0.21443806298495555, "margin": -1.0292045129154486, "p_left": 0.2632383549929514, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3054908515753684, "brier": 0.0692944315393951 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.6225637639562793, "right_score": -0.3414237043629525, "margin": -0.28114005959332683, "p_left": 0.4301742974520005, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8435648095121449, "brier": 0.32470133128432127 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.20895250283630074, "right_score": -0.397370274908483, "margin": 0.6063227777447837, "p_left": 0.6471015223397129, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4352520843645631, "brier": 0.12453733553494818 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5107857094182159, "right_score": -0.4020631442030237, "margin": 0.9128488536212396, "p_left": 0.713582772394902, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3374568398310436, "brier": 0.08203482826899053 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21443806298495555, "right_score": -0.46486629958882214, "margin": 0.6793043625737777, "p_left": 0.6635834206581492, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41010070491684647, "brier": 0.11317611485607176 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.7570153074659243, "right_score": -1.0774369768118293, "margin": -0.679578330654095, "p_left": 0.3363554213216257, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0895868765443102, "brier": 0.44042412680919685 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21443806298495555, "right_score": 0.5680954597292999, "margin": -0.3536573967443443, "p_left": 0.41249579376690565, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5318718734437599, "brier": 0.17015277987538957 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.1744602722900663, "right_score": 0.5680954597292999, "margin": 0.6063648125607665, "p_left": 0.6471111213989437, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43523725054374174, "brier": 0.12453056064031108 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.397370274908483, "right_score": -0.7908503348425331, "margin": 0.3934800599340501, "p_left": 0.5971201714215847, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5156368936826258, "brier": 0.1623121562753733 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.2511561786236193, "right_score": 0.5680954597292999, "margin": -0.8192516383529191, "p_left": 0.3059225393854083, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.36517170998535997, "brier": 0.09358860010401669 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.16449765755681608, "right_score": -0.814766449930493, "margin": 0.9792641074873091, "p_left": 0.7269621747804895, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3188808319899551, "brier": 0.07454965400059996 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.16449765755681608, "right_score": -1.0172811111100093, "margin": 1.1817787686668253, "p_left": 0.7652674803621147, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8584192430692914, "brier": 0.0703668361376649 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.16449765755681608, "right_score": -0.4020631442030237, "margin": 0.5665608017598398, "p_left": 0.6379692199834623, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0160260431010197, "brier": 0.40700472564630735 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.4020631442030237, "right_score": -0.2511561786236193, "margin": -0.15090696557940442, "p_left": 0.46234469158350106, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6205376148259827, "brier": 0.2137626138354427 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21443806298495555, "right_score": 0.055287016232973635, "margin": 0.1591510467519819, "p_left": 0.5397039916605625, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6167344533119565, "brier": 0.21187241529321954 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.7908503348425331, "right_score": -0.6225637639562793, "margin": -0.16828657088625376, "p_left": 0.45802736726177545, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6125397719202171, "brier": 0.20978906916075332 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.2511561786236193, "right_score": -0.814766449930493, "margin": 0.5636102713068737, "p_left": 0.6372874751776498, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4505344298007975, "brier": 0.131560375663004 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.6479321496699127, "right_score": 0.15438811256531682, "margin": -0.8023202622352296, "p_left": 0.3095294111839859, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1727021633142534, "brier": 0.4767496340199332 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5107857094182159, "right_score": 0.5680954597292999, "margin": -0.057309750311083985, "p_left": 0.48567648256329665, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7222125504788527, "brier": 0.2645286805884629 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.1744602722900663, "right_score": -0.46486629958882214, "margin": 1.6393265718788885, "p_left": 0.8374432831129951, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17740173923225577, "brier": 0.026424686205081864 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.3414237043629525, "right_score": 0.3204516982868358, "margin": -0.6618754026497883, "p_left": 0.34031845427124846, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41599806687696, "brier": 0.11581665031757182 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.2625690052436416, "right_score": -0.4020631442030237, "margin": 0.6646321494466653, "p_left": 0.6603001676475659, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0796928973322526, "brier": 0.4359963113954037 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.2625690052436416, "right_score": -1.0172811111100093, "margin": 1.2798501163536509, "p_left": 0.7824242618304059, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.24535815124537397, "brier": 0.04733920184004376 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.2511561786236193, "right_score": 0.055287016232973635, "margin": -0.3064431948565929, "p_left": 0.4239831512229152, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5516183673649527, "brier": 0.17976171252091336 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.16449765755681608, "right_score": -0.46486629958882214, "margin": 0.6293639571456382, "p_left": 0.6523452274221511, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7418633458473488, "brier": 0.02320906831830693 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.46486629958882214, "right_score": 0.2625690052436416, "margin": -0.7274353048324638, "p_left": 0.3257577842408401, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3941658620654125, "brier": 0.10611813399350174 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.1744602722900663, "right_score": -1.0172811111100093, "margin": 2.1917413834000756, "p_left": 0.899505429053504, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.10591018997496215, "brier": 0.01009915878972033 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -1.0172811111100093, "right_score": 0.5107857094182159, "margin": -1.5280668205282253, "p_left": 0.17827670910551086, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1963515696803753, "brier": 0.03178258500949094 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.3414237043629525, "right_score": -1.7570153074659243, "margin": 1.4155916031029718, "p_left": 0.8046463884679326, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.632943970157363, "brier": 0.6474558104744871 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5107857094182159, "right_score": 0.055287016232973635, "margin": 0.4554986931852423, "p_left": 0.6119457993529838, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49111156354047747, "brier": 0.15058606263979474 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.2872680154118502, "right_score": -0.28840147779913267, "margin": 0.5756694932109829, "p_left": 0.6400703492421401, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7340119345839035, "brier": 0.019619702736815105 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.055287016232973635, "right_score": 0.2625690052436416, "margin": -0.20728198901066794, "p_left": 0.4483642515255097, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5948673264710135, "brier": 0.2010305020460305 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.7570153074659243, "right_score": 0.2872680154118502, "margin": -2.0442833228777744, "p_left": 0.11463129539671309, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.1660344282411237, "brier": 0.7838777430909023 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.1744602722900663, "right_score": -0.814766449930493, "margin": 1.9892267222205593, "p_left": 0.8796613044680202, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12821832688203827, "brier": 0.014481401642338547 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 1.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.2872680154118502, "rank": 2.0, "portrait_scores": [ 0.10013050541069624, 0.47440552541300424 ], "portrait_score_gap": 0.374275020002308, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.078125, "single_photo_rank_quantiles": [ 1.0, 3.0, 5.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.3414237043629525, "rank": 6.0, "portrait_scores": [ -0.1385913099523234, -0.5442560987735816 ], "portrait_score_gap": 0.4056647888212582, "single_photo_rank_min": 3.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.15438811256531682, "rank": 4.0, "portrait_scores": [ 1.173957063848778, -0.8651808387181443 ], "portrait_score_gap": 2.0391379025669223, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 4.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -1.0774369768118293, "rank": 12.0, "portrait_scores": [ -0.6981312807277706, -1.456742672895888 ], "portrait_score_gap": 0.7586113921681175, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.6479321496699127, "rank": 9.0, "portrait_scores": [ 0.2585517628048406, -1.554416062144666 ], "portrait_score_gap": 1.8129678249495067, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 8.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": -0.397370274908483, "rank": 7.0, "portrait_scores": [ -0.10818059206117764, -0.6865599577557883 ], "portrait_score_gap": 0.5783793656946106, "single_photo_rank_min": 3.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.20895250283630074, "rank": 3.0, "portrait_scores": [ 0.09408042342863421, 0.32382458224396726 ], "portrait_score_gap": 0.22974415881533305, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 3.5, 5.0 ] }, { "identity": "keira_knightley", "mean_score": -0.6225637639562793, "rank": 8.0, "portrait_scores": [ -1.8254386472781856, 0.580311119365627 ], "portrait_score_gap": 2.4057497666438126, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 7.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -1.0264766102813012, "rank": 11.0, "portrait_scores": [ -1.2577136444525372, -0.7952395761100651 ], "portrait_score_gap": 0.46247406834247207, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.28840147779913267, "rank": 5.0, "portrait_scores": [ 0.028912060353972757, -0.6057150159522381 ], "portrait_score_gap": 0.6346270763062108, "single_photo_rank_min": 3.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 9.0 ] }, { "identity": "natalie_portman", "mean_score": -0.7908503348425331, "rank": 10.0, "portrait_scores": [ -0.5657595545966657, -1.0159411150884006 ], "portrait_score_gap": 0.45018156049173486, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 12.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.3204516982868358, "rank": 1.0, "portrait_scores": [ -0.21140377539848787, 0.8523071719721594 ], "portrait_score_gap": 1.0637109473706472, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.7570153074659243, "rank": 13.0, "portrait_scores": [ -0.7741417718129209, -2.7398888431189277 ], "portrait_score_gap": 1.9657470713060068, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 128, "8.0": 640, "9.0": 1280, "10.0": 1280, "11.0": 640, "12.0": 128, "13.0": 4096 }, "target_primary_score": -1.7570153074659243, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -1.466775076627436, -2.6181978241440227 ], "target_alternate_panel_score": -2.0424864503857294, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -1.0160098401044282, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.9650494735739001, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -1.701062746022777, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -2.3297544657975795, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.39067210461270485, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_10", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 17, "accuracy": 0.7727272727272727, "mean_cross_entropy": 0.5796584772338963, "brier_score": 0.174255800397405, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.36363636363636365, "mean_cross_entropy": 0.8053660866815062, "brier_score": 0.2814506921277932, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.6530827044035634, "brier_score": 0.199171376697791, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.1099328512373918, "brier_score": 0.44600932298779755, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.45233583376390957, "brier_score": 0.1363273422623368, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5281324550278776, "right_score": 0.35622073034935686, "margin": -0.8843531853772345, "p_left": 0.292276504815768, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3457018050954974, "brier": 0.08542555526732165 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.5635410731351249, "right_score": -1.141966127855504, "margin": 0.5784250547203791, "p_left": 0.6407049309891841, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0236113088343326, "brier": 0.4105028085938552 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.41804101612159383, "right_score": 0.2849983238641038, "margin": 0.13304269225749005, "p_left": 0.5332116992629833, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7618794415722923, "brier": 0.28431471623091814 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3458441183957045, "right_score": 0.43267801426958724, "margin": -0.7785221326652918, "p_left": 0.3146384876782387, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3778088247194664, "brier": 0.09899737792844918 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5281324550278776, "right_score": 0.43267801426958724, "margin": -0.9608104692974648, "p_left": 0.2767159547670827, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.32395326366838245, "brier": 0.07657171962265817 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.5635410731351249, "right_score": -0.7066593719670242, "margin": 0.1431182988318993, "p_left": 0.5357186272451111, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.767264503746858, "brier": 0.2869944475773863 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5281324550278776, "right_score": 0.1216984368420102, "margin": -0.6498308918698879, "p_left": 0.3430276464689176, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4201133412442749, "brier": 0.1176679662420047 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.16036770871262973, "right_score": -0.46710867574867315, "margin": 0.3067409670360434, "p_left": 0.576089569480534, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5514921278051457, "brier": 0.179700053103199 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.13242638244208504, "right_score": -0.39238444698827823, "margin": 0.5248108294303633, "p_left": 0.6282720092824894, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.46478207049002906, "brier": 0.13818169908287767 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.2849983238641038, "right_score": -0.2586716019281891, "margin": 0.5436699257922929, "p_left": 0.6326657222722826, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.45781308120179987, "brier": 0.13493447159374383 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.1216984368420102, "right_score": -0.4002592197309365, "margin": 0.5219576565729467, "p_left": 0.6276054178338364, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.46584362553809155, "brier": 0.1386777248267116 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.141966127855504, "right_score": -0.7066593719670242, "margin": -0.4353067558884798, "p_left": 0.3928598418069346, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9343023673464168, "brier": 0.3686191716907004 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.1216984368420102, "right_score": 0.41804101612159383, "margin": -0.29634257927958363, "p_left": 0.4264518125827846, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5559133225471605, "brier": 0.18186114845514242 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.43267801426958724, "right_score": 0.41804101612159383, "margin": 0.014636998147993407, "p_left": 0.5036591842080733, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.68585546146124, "brier": 0.2463542054209953 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.39238444698827823, "right_score": -0.2648450424105462, "margin": -0.12753940457773205, "p_left": 0.4681582993790947, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7589487937204265, "brier": 0.2828555945193367 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.16485284231213027, "right_score": 0.41804101612159383, "margin": -0.5828938584337241, "p_left": 0.35826698906125437, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.44358293260234694, "brier": 0.12835523545101696 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.35622073034935686, "right_score": -0.5281324550278776, "margin": 0.8843531853772345, "p_left": 0.707723495184232, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3457018050954973, "brier": 0.08542555526732162 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.35622073034935686, "right_score": -0.3458441183957045, "margin": 0.7020648487450614, "p_left": 0.668645416621925, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7535338037338115, "brier": 0.028441276547582658 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.35622073034935686, "right_score": -0.2586716019281891, "margin": 0.6148923322775459, "p_left": 0.6490559998357616, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.047128611936348, "brier": 0.4212736909228001 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2586716019281891, "right_score": -0.16485284231213027, "margin": -0.09381875961605884, "p_left": 0.4765624989028151, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6473376424325367, "brier": 0.22711181536049566 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.1216984368420102, "right_score": 0.12901926312229578, "margin": -0.007320826280285575, "p_left": 0.4981698016039683, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.696814292997306, "brier": 0.25183354802220054 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.2648450424105462, "right_score": -0.16036770871262973, "margin": -0.10447733369791645, "p_left": 0.4739043994976756, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6422723327545393, "brier": 0.2245853798632525 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.16485284231213027, "right_score": -0.5281324550278776, "margin": 0.3632796127157474, "p_left": 0.5898341033384757, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5279139624033444, "brier": 0.16823606278415218 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.22047229863845477, "right_score": 0.060858777382992, "margin": -0.2813310760214468, "p_left": 0.4301274752948062, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.843673660054417, "brier": 0.3247546944138717 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.2849983238641038, "right_score": 0.41804101612159383, "margin": -0.13304269225749005, "p_left": 0.46678830073701666, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7618794415722923, "brier": 0.28431471623091825 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.43267801426958724, "right_score": -0.4002592197309365, "margin": 0.8329372340005237, "p_left": 0.6969756340764429, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.36100482711610726, "brier": 0.09182376634337382 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.46710867574867315, "right_score": -0.04109193075408307, "margin": -0.4260167449945901, "p_left": 0.3950778961208629, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5026555831595746, "brier": 0.15608654400328734 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": -0.04489554717410965, "right_score": -0.2586716019281891, "margin": 0.21377605475407946, "p_left": 0.5532414060426016, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8057368885134109, "brier": 0.3060760533599948 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": -0.04489554717410965, "right_score": -0.3458441183957045, "margin": 0.30094857122159485, "p_left": 0.5746743865403666, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5539516828619089, "brier": 0.18090187746481345 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.16485284231213027, "right_score": 0.12901926312229578, "margin": -0.29387210543442605, "p_left": 0.4270561769262202, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.556967607084489, "brier": 0.1823769782508391 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.35622073034935686, "right_score": -0.4002592197309365, "margin": 0.7564799500802933, "p_left": 0.6805890054599067, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7630366462735774, "brier": 0.03261238889299823 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.4002592197309365, "right_score": -0.04489554717410965, "margin": -0.3553636725568268, "p_left": 0.41208235157381407, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5311683945889076, "brier": 0.1698118644786045 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.43267801426958724, "right_score": -0.3458441183957045, "margin": 0.7785221326652918, "p_left": 0.6853615123217612, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3778088247194664, "brier": 0.0989973779284492 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3458441183957045, "right_score": 0.2849983238641038, "margin": -0.6308424422598082, "p_left": 0.34731954112997643, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4266676127043825, "brier": 0.12063086365073739 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.46710867574867315, "right_score": -1.141966127855504, "margin": 0.6748574521068309, "p_left": 0.6625899692600018, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0864563797923301, "brier": 0.43902546736397013 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.2849983238641038, "right_score": 0.12901926312229578, "margin": 0.155979060741808, "p_left": 0.5389158969065548, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6181957556717645, "brier": 0.21259855012548676 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.03128269942578364, "right_score": -0.0896226176893357, "margin": 0.05833991826355206, "p_left": 0.5145808442525402, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6935725634975178, "brier": 0.000212601019116836 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.12901926312229578, "right_score": -0.04489554717410965, "margin": 0.17391481029640543, "p_left": 0.5433694436519952, "prediction": "left", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7838806256450657, "brier": 0.29525035229467883 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.141966127855504, "right_score": -0.03128269942578364, "margin": -1.1106834284297205, "p_left": 0.24774349833141054, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3953613489764882, "brier": 0.5658898443026645 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.43267801426958724, "right_score": -0.5281324550278776, "margin": 0.9608104692974648, "p_left": 0.7232840452329172, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3239532636683824, "brier": 0.0765717196226582 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 10.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.03128269942578364, "rank": 3.0, "portrait_scores": [ -0.24380369517039666, 0.18123829631882937 ], "portrait_score_gap": 0.42504199148922606, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.46710867574867315, "rank": 10.0, "portrait_scores": [ -0.28311829746516215, -0.6510990540321842 ], "portrait_score_gap": 0.367980756567022, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 9.0, 12.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.060858777382992, "rank": 2.0, "portrait_scores": [ 0.8132784725238006, -0.6915609177578166 ], "portrait_score_gap": 1.5048393902816173, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.5, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.7066593719670242, "rank": 12.0, "portrait_scores": [ -0.43572621213010365, -0.9775925318039449 ], "portrait_score_gap": 0.5418663196738412, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.22047229863845477, "rank": 7.0, "portrait_scores": [ 0.08897157642077887, -0.5299161736976884 ], "portrait_score_gap": 0.6188877501184673, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.5, 10.0 ] }, { "identity": "emma_watson", "mean_score": -0.39238444698827823, "rank": 9.0, "portrait_scores": [ -0.3051452222052147, -0.47962367177134185 ], "portrait_score_gap": 0.17447844956612718, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 10.0 ] }, { "identity": "gal_gadot", "mean_score": 0.13242638244208504, "rank": 1.0, "portrait_scores": [ -0.0507775401920247, 0.3156303050761948 ], "portrait_score_gap": 0.36640784526821946, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "keira_knightley", "mean_score": -0.16036770871262973, "rank": 6.0, "portrait_scores": [ -0.8183226533439851, 0.49758723591872567 ], "portrait_score_gap": 1.3159098892627108, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.5635410731351249, "rank": 11.0, "portrait_scores": [ -0.5851145119462411, -0.5419676343240089 ], "portrait_score_gap": 0.04314687762223224, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": -0.0896226176893357, "rank": 5.0, "portrait_scores": [ 0.026207399307033252, -0.20545263468570465 ], "portrait_score_gap": 0.2316600339927379, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 4.5, 7.0 ] }, { "identity": "natalie_portman", "mean_score": -0.2648450424105462, "rank": 8.0, "portrait_scores": [ -0.18717672728954116, -0.3425133575315512 ], "portrait_score_gap": 0.15533663024201005, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.04109193075408307, "rank": 4.0, "portrait_scores": [ -0.40484988655844645, 0.3226660250502803 ], "portrait_score_gap": 0.7275159116087268, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 4.0, 9.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.141966127855504, "rank": 13.0, "portrait_scores": [ -0.7732193606393987, -1.510712895071609 ], "portrait_score_gap": 0.7374935344322104, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "11.0": 1024, "12.0": 2048, "13.0": 5120 }, "target_primary_score": -1.141966127855504, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.7033754804040215, -1.334337185246392 ], "target_alternate_panel_score": -1.018856332825207, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.455315259690082, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.3121969608581827, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.5517476570765338, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9875736333994233, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.12127698386696507, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_100", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 17, "accuracy": 0.7727272727272727, "mean_cross_entropy": 0.6445786889822528, "brier_score": 0.20509441748377152, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.2727272727272727, "mean_cross_entropy": 0.7553661941123493, "brier_score": 0.2599271419917426, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.6937833637148421, "brier_score": 0.219193008818012, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8785318549073637, "brier_score": 0.3413954083392038, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.5980724947626912, "brier_score": 0.20347691377002688, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.27689355769429463, "right_score": 0.3348247415288379, "margin": -0.6117182992231325, "p_left": 0.3516673303484557, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4333513352719347, "brier": 0.12366991123440987 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.2870579346419938, "right_score": -0.7032839236609537, "margin": 0.4162259890189599, "p_left": 0.6025798104209371, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9227611459801157, "brier": 0.36310242792693254 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.3876639751020863, "right_score": 0.27651634610611786, "margin": 0.11114762899596842, "p_left": 0.5277583363970003, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7502644252645316, "brier": 0.27852886163652935 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.05027242486246586, "right_score": 0.02620828421925725, "margin": -0.07648070908172311, "p_left": 0.48088913725052734, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6556378102467834, "brier": 0.23125436232555652 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.27689355769429463, "right_score": 0.02620828421925725, "margin": -0.3031018419135519, "p_left": 0.42479938780983634, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5530364082675152, "brier": 0.18045451988361172 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.2870579346419938, "right_score": -0.5424153885675643, "margin": 0.2553574539255705, "p_left": 0.5634947106347379, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.828954786100613, "brier": 0.31752628891332696 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.27689355769429463, "right_score": 0.0903191013582376, "margin": -0.36721265905253225, "p_left": 0.40921471271469795, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.52630263167274, "brier": 0.16745668110217277 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.021945498938953645, "right_score": -0.419110068839775, "margin": 0.39716456990082133, "p_left": 0.5980062273390105, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.514154111475341, "brier": 0.16159899325821528 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.11708725989285543, "right_score": -0.32267319970392344, "margin": 0.4397604595967789, "p_left": 0.6082019515835564, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4972482949581668, "brier": 0.1535057107429339 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.27651634610611786, "right_score": -0.19967804264536598, "margin": 0.47619438875148384, "p_left": 0.6168488333124674, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4831312878298833, "brier": 0.14680481653401736 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0903191013582376, "right_score": -0.3077223331779957, "margin": 0.3980414345362333, "p_left": 0.5982170028924307, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5138017097659031, "brier": 0.161429576764741 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.7032839236609537, "right_score": -0.5424153885675643, "margin": -0.1608685350933894, "p_left": 0.4598693729122802, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7768128017564851, "brier": 0.29174109431817336 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0903191013582376, "right_score": 0.3876639751020863, "margin": -0.29734487374384866, "p_left": 0.4262066788051525, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5554860151072709, "brier": 0.18165213305811845 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.02620828421925725, "right_score": 0.3876639751020863, "margin": -0.36145569088282903, "p_left": 0.41060722995784865, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8901181662770022, "brier": 0.34738383737796025 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.32267319970392344, "right_score": 0.044482104970745236, "margin": -0.36715530467466867, "p_left": 0.40922857866838996, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.893481407000305, "brier": 0.34901087226217065 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.09305888448548418, "right_score": 0.3876639751020863, "margin": -0.48072285958757044, "p_left": 0.38208144700017704, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4813986214721002, "brier": 0.1459862321417491 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.3348247415288379, "right_score": -0.27689355769429463, "margin": 0.6117182992231325, "p_left": 0.6483326696515442, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4333513352719347, "brier": 0.12366991123440992 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.3348247415288379, "right_score": -0.05027242486246586, "margin": 0.38509716639130376, "p_left": 0.5951018868716079, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.71157123285211, "brier": 0.009044368886540111 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.3348247415288379, "right_score": -0.19967804264536598, "margin": 0.5345027841742038, "p_left": 0.6305327011684478, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9956930436428073, "brier": 0.39757148724277913 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.19967804264536598, "right_score": -0.09305888448548418, "margin": -0.10661915815988181, "p_left": 0.4733704319713341, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6412578845621797, "brier": 0.22407956586472744 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0903191013582376, "right_score": 0.04575741073288439, "margin": 0.044561690625353215, "p_left": 0.511138579519798, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6711145327465133, "brier": 0.23898548843392084 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.044482104970745236, "right_score": -0.021945498938953645, "margin": 0.06642760390969887, "p_left": 0.5166007969998386, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7269124594518848, "brier": 0.2668763834608685 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.09305888448548418, "right_score": -0.27689355769429463, "margin": 0.18383467320881047, "p_left": 0.5458296724126143, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6054483071936569, "brier": 0.20627068646083327 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.11457635992670391, "right_score": -0.02000963573973147, "margin": -0.09456672418697244, "p_left": 0.4763759219088068, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7415479845314038, "brier": 0.27418217515685195 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.27651634610611786, "right_score": 0.3876639751020863, "margin": -0.11114762899596842, "p_left": 0.47224166360299963, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7502644252645316, "brier": 0.27852886163652946 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.02620828421925725, "right_score": -0.3077223331779957, "margin": 0.33393061739725294, "p_left": 0.5827154481243854, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5400562939019019, "brier": 0.17412639723403253 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.419110068839775, "right_score": -0.3339818236489499, "margin": -0.08512824519082507, "p_left": 0.47873078166428307, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6514886368399874, "brier": 0.22918316131289546 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": -0.23366461618181475, "right_score": -0.19967804264536598, "margin": -0.033986573536448766, "p_left": 0.49150417438508276, "prediction": "right", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6762982727407537, "brier": 0.24157635343796185 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": -0.23366461618181475, "right_score": -0.05027242486246586, "margin": -0.1833921913193489, "p_left": 0.4542800209116475, "prediction": "right", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.789041484926185, "brier": 0.2978102955761918 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.09305888448548418, "right_score": 0.04575741073288439, "margin": -0.13881629521836858, "p_left": 0.46535154786898214, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6261458468907083, "brier": 0.21655206310405758 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.3348247415288379, "right_score": -0.3077223331779957, "margin": 0.6425470747068336, "p_left": 0.6553290031522281, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7438914119192546, "brier": 0.024127099220264882 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3077223331779957, "right_score": -0.23366461618181475, "margin": -0.07405771699618094, "p_left": 0.4814940280482615, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6568037336320401, "brier": 0.23183649904614007 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.02620828421925725, "right_score": -0.05027242486246586, "margin": 0.07648070908172311, "p_left": 0.5191108627494727, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6556378102467834, "brier": 0.23125436232555646 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.05027242486246586, "right_score": 0.27651634610611786, "margin": -0.3267887709685837, "p_left": 0.41902216917003815, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5430426797773903, "brier": 0.17557957825596407 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.419110068839775, "right_score": -0.7032839236609537, "margin": 0.28417385482117874, "p_left": 0.5705692016249839, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8452946719012796, "brier": 0.32554921384297153 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.27651634610611786, "right_score": 0.04575741073288439, "margin": 0.23075893537323347, "p_left": 0.5574350930261583, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5844092074909739, "brier": 0.19586369688476513 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.21103583505110243, "right_score": -0.04767730188865413, "margin": -0.1633585331624483, "p_left": 0.45925094557130086, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6964792293610346, "brier": 0.0016604854368330849 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.04575741073288439, "right_score": -0.23366461618181475, "margin": 0.27942202691469914, "p_left": 0.5694045204452377, "prediction": "left", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8425861921219295, "brier": 0.32422150790347115 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.7032839236609537, "right_score": -0.21103583505110243, "margin": -0.49224808860985125, "p_left": 0.3793641186100036, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9692587999915747, "brier": 0.3851888972687376 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.02620828421925725, "right_score": -0.27689355769429463, "margin": 0.3031018419135519, "p_left": 0.5752006121901637, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5530364082675152, "brier": 0.1804545198836117 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 0.1, "consistency": 100.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.21103583505110243, "rank": 7.0, "portrait_scores": [ -0.3467058024943152, -0.07536586760788966 ], "portrait_score_gap": 0.27133993488642555, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 7.0, 11.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.419110068839775, "rank": 11.0, "portrait_scores": [ -0.325619719848689, -0.512600417830861 ], "portrait_score_gap": 0.18698069798217198, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 12.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.02000963573973147, "rank": 3.0, "portrait_scores": [ 0.5295471764157658, -0.5695664478952287 ], "portrait_score_gap": 1.0991136243109945, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.5, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.5424153885675643, "rank": 12.0, "portrait_scores": [ -0.2531803863575209, -0.8316503907776077 ], "portrait_score_gap": 0.5784700044200868, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.11457635992670391, "rank": 6.0, "portrait_scores": [ -0.025338184272620717, -0.2038145355807871 ], "portrait_score_gap": 0.1784763513081664, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.5, 7.0 ] }, { "identity": "emma_watson", "mean_score": -0.32267319970392344, "rank": 9.0, "portrait_scores": [ -0.22374432486040832, -0.42160207454743853 ], "portrait_score_gap": 0.1978577496870302, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.11708725989285543, "rank": 1.0, "portrait_scores": [ -0.12161600760348563, 0.3557905273891965 ], "portrait_score_gap": 0.4774065349926821, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.021945498938953645, "rank": 4.0, "portrait_scores": [ -0.4597457786406234, 0.4158547807627161 ], "portrait_score_gap": 0.8756005594033395, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.2870579346419938, "rank": 8.0, "portrait_scores": [ -0.2606148496697166, -0.31350101961427107 ], "portrait_score_gap": 0.05288616994455447, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "margot_robbie", "mean_score": -0.04767730188865413, "rank": 5.0, "portrait_scores": [ -0.0268269954903132, -0.06852760828699506 ], "portrait_score_gap": 0.04170061279668186, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "natalie_portman", "mean_score": 0.044482104970745236, "rank": 2.0, "portrait_scores": [ -0.04376296320405636, 0.13272717314554683 ], "portrait_score_gap": 0.1764901363496032, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0703125, "single_photo_rank_quantiles": [ 1.0, 3.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.3339818236489499, "rank": 10.0, "portrait_scores": [ -0.6313776658114278, -0.03658598148647202 ], "portrait_score_gap": 0.5947916843249559, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.7032839236609537, "rank": 13.0, "portrait_scores": [ -0.4267166288910396, -0.9798512184308679 ], "portrait_score_gap": 0.5531345895398283, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 128, "9.0": 640, "10.0": 1280, "11.0": 1280, "12.0": 640, "13.0": 4224 }, "target_primary_score": -0.7032839236609537, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.3156021239198784, -0.6749049758810746 ], "target_alternate_panel_score": -0.4952535499004765, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.20819561525848268, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.04716183866708784, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.07614348106070151, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.2842177148493741, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.06562076848857276, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } ] }, { "variant": "full", "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 13, "accuracy": 0.5909090909090909, "mean_cross_entropy": 0.6925276480341477, "brier_score": 0.2288569021636729, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6922395076672613, "brier_score": 0.22871283169881818, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.6925776061161542, "brier_score": 0.2184652138363438, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6915633107694755, "brier_score": 0.24920806742376692, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.6940539017202587, "brier_score": 0.25045335929011886, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.006883674632906349, "right_score": -0.0076911357897173835, "margin": 0.0008074611568110344, "p_left": 0.5002018652782348, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6935509926375386, "brier": 0.2502019060278254 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.006042520248854092, "right_score": -0.00011281449579252523, "margin": -0.005929705753061567, "p_left": 0.4985175779054031, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.690186722853265, "brier": 0.2485197754806696 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.004227162498453013, "right_score": -0.0020419652813698777, "margin": 0.0062691277798228905, "p_left": 0.5015672768118711, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6962866571872017, "brier": 0.2515697331684761 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.005645409277289262, "right_score": -0.006348040317190647, "margin": 0.0007026310399013847, "p_left": 0.5001756577527486, "prediction": "left", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934985577911921, "brier": 0.25017568860839473 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.006883674632906349, "right_score": -0.006348040317190647, "margin": -0.000535634315715702, "p_left": 0.4998660914242727, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6928793992651021, "brier": 0.24986610935577933 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.006042520248854092, "right_score": -0.00216220665221373, "margin": -0.003880313596640362, "p_left": 0.4990299228180306, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912089058646455, "brier": 0.24903086386776957 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.006883674632906349, "right_score": 0.00483201650663787, "margin": -0.01171569113954422, "p_left": 0.4970711107160192, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6873064920694109, "brier": 0.24707968910845704 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.0009339228876092543, "right_score": 0.0013210483588317447, "margin": -0.0003871254712224904, "p_left": 0.49990321863340303, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6933407620288227, "brier": 0.2500967907332299 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0019282540335639462, "right_score": -0.0045846148950199495, "margin": 0.006512868928583895, "p_left": 0.501628211476767, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6898960482689925, "brier": 0.24837443959584612 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.0020419652813698777, "right_score": -0.0060435887393002746, "margin": 0.004001623457930397, "p_left": 0.5010004045295273, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.691148370453432, "brier": 0.24900059627969537 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.00483201650663787, "right_score": 0.0013004709690664774, "margin": 0.003531545537571393, "p_left": 0.5008828854667945, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.691382966767085, "brier": 0.249117894019953 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.00011281449579252523, "right_score": -0.00216220665221373, "margin": 0.0020493921564212047, "p_left": 0.5005123478597832, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6921230094826691, "brier": 0.24948791464054626 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.00483201650663787, "right_score": 0.004227162498453013, "margin": 0.0006048540081848575, "p_left": 0.5001512134974361, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934496532950835, "brier": 0.2501512363629579 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.006348040317190647, "right_score": 0.004227162498453013, "margin": -0.010575202815643659, "p_left": 0.49735622393491574, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6984487612669507, "brier": 0.2526507656169666 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.0045846148950199495, "right_score": -0.004019419381127919, "margin": -0.0005651955138920306, "p_left": 0.49985870112528846, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6934298182476369, "brier": 0.2501413188400836 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.0018617103404967878, "right_score": 0.004227162498453013, "margin": -0.002365452157956225, "p_left": 0.4994086372362518, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6919651539012931, "brier": 0.24940898694617014 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.0076911357897173835, "right_score": -0.006883674632906349, "margin": -0.0008074611568110344, "p_left": 0.4997981347217652, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6935509926375386, "brier": 0.25020190602782527 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.0076911357897173835, "right_score": -0.005645409277289262, "margin": -0.002045726512428121, "p_left": 0.49948856855025453, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931477036844745, "brier": 2.615621277887529e-07 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.0076911357897173835, "right_score": -0.0060435887393002746, "margin": -0.001647547050417109, "p_left": 0.4995881133305649, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6923237463361088, "brier": 0.24958828298119337 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.0060435887393002746, "right_score": 0.0018617103404967878, "margin": -0.007905299079797062, "p_left": 0.4980236855223158, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6892023427188985, "brier": 0.2480275913412305 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.00483201650663787, "right_score": -0.0055398772937816565, "margin": 0.010371893800419526, "p_left": 0.5025929502051731, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6879746806220877, "brier": 0.24741377318559338 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.004019419381127919, "right_score": 0.0009339228876092543, "margin": -0.0049533422687371735, "p_left": 0.4987616669647516, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6906735763723953, "brier": 0.2487632004334578 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.0018617103404967878, "right_score": -0.006883674632906349, "margin": 0.008745384973403138, "p_left": 0.5021863323088236, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6887840482625696, "brier": 0.247818447740141 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.006721541770849666, "right_score": -0.003535409689602233, "margin": -0.003186132081247433, "p_left": 0.49920346765351703, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6947415155297372, "brier": 0.25079716681026193 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.0020419652813698777, "right_score": 0.004227162498453013, "margin": -0.0062691277798228905, "p_left": 0.4984327231881289, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6962866571872017, "brier": 0.2515697331684761 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.006348040317190647, "right_score": 0.0013004709690664774, "margin": -0.007648511286257124, "p_left": 0.4980878814999604, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.696978748650862, "brier": 0.2519157746971978 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.0013210483588317447, "right_score": -0.0007135445536791125, "margin": 0.0020345929125108572, "p_left": 0.5005086480526624, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6941649944621514, "brier": 0.25050890677550386 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.001109405741877672, "right_score": -0.0060435887393002746, "margin": 0.007152994481177947, "p_left": 0.5017882409956393, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6967300734531554, "brier": 0.25179143880149785 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.001109405741877672, "right_score": -0.005645409277289262, "margin": 0.0067548150191669345, "p_left": 0.5016886973338731, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6897754764802616, "brier": 0.24831415436481233 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.0018617103404967878, "right_score": -0.0055398772937816565, "margin": 0.007401587634278444, "p_left": 0.5018503884610144, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6968548222988917, "brier": 0.25185381239847104 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.0076911357897173835, "right_score": 0.0013004709690664774, "margin": -0.008991606758783861, "p_left": 0.4977521134552304, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931572866499139, "brier": 5.0529939181561885e-06 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.0013004709690664774, "right_score": 0.001109405741877672, "margin": 0.00019106522718880534, "p_left": 0.5000477663066519, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6932427177367798, "brier": 0.250047768588272 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.006348040317190647, "right_score": -0.005645409277289262, "margin": -0.0007026310399013847, "p_left": 0.49982434224725136, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6934985577911921, "brier": 0.25017568860839473 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.005645409277289262, "right_score": -0.0020419652813698777, "margin": -0.0036034439959193847, "p_left": 0.49909913997581123, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6913470816621865, "brier": 0.2490999515245944 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.0013210483588317447, "right_score": -0.00011281449579252523, "margin": 0.00143386285462427, "p_left": 0.5003584656522401, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6938643689825712, "brier": 0.2503585941498639 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.0020419652813698777, "right_score": -0.0055398772937816565, "margin": 0.003497912012411779, "p_left": 0.5008744771114725, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6913997539765155, "brier": 0.249126287598746 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.006258333934324074, "right_score": 0.005757556704005823, "margin": -0.012015890638329897, "p_left": 0.49699606348310094, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931652281548522, "brier": 9.02363459755968e-06 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.0055398772937816565, "right_score": 0.001109405741877672, "margin": -0.0066492830356593285, "p_left": 0.49833768536573547, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6898280656525455, "brier": 0.24834044865567875 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.00011281449579252523, "right_score": -0.006258333934324074, "margin": 0.006145519438531549, "p_left": 0.5015363750242272, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6900791417593967, "brier": 0.24846598542398787 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.006348040317190647, "right_score": -0.006883674632906349, "margin": 0.000535634315715702, "p_left": 0.5001339085757274, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6928793992651021, "brier": 0.24986610935577927 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 6, "regressions": 0, "both_correct": 0, "both_wrong": 5, "net_fixes": 6, "trial_ids": { "fixes": [ "1b6d9789fab8", "0add2ec437a1", "34bec29a8d38", "974972ffb84a", "b20ebcae7654", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba" ] } }, "other": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 3, "trial_ids": { "fixes": [ "0add2ec437a1", "34bec29a8d38", "b20ebcae7654" ], "regressions": [], "both_correct": [], "both_wrong": [ "d013ade6e6a0", "70f39a6b70be", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.6233107438657765, "brier_score": 0.19447990744519492, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 7, "accuracy": 0.6363636363636364, "mean_cross_entropy": 0.6838358940792807, "brier_score": 0.22451277006174455, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 4, "accuracy": 0.5714285714285714, "mean_cross_entropy": 0.679271628974334, "brier_score": 0.21181490954641763, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6929644242891743, "brier_score": 0.24990849109239846, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5561843120989012, "brier_score": 0.18209331978336396, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.10863039016919479, "right_score": 0.2002280778561575, "margin": -0.30885846802535233, "p_left": 0.42339339830097295, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5505950444867587, "brier": 0.17926196972484632 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.025852115578457346, "right_score": 0.02934396217927062, "margin": -0.003491846600813274, "p_left": 0.49912703923679685, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6914027813828498, "brier": 0.24912780129729095 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.14155846122992993, "right_score": 0.3087798212336246, "margin": -0.16722136000369464, "p_left": 0.45829180519631846, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6130278084953247, "brier": 0.2100313787101003 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.034462496220780015, "right_score": 0.28570557082447967, "margin": -0.3201680670452597, "p_left": 0.42063478917056846, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5458222388920391, "brier": 0.17693362586056857 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.10863039016919479, "right_score": 0.28570557082447967, "margin": -0.39433596099367446, "p_left": 0.402673943579592, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5152921565214825, "brier": 0.16214630483794046 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.025852115578457346, "right_score": 0.010686420598060133, "margin": 0.015165694980397213, "p_left": 0.5037913510783513, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7007587775626613, "brier": 0.2538057254213506 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.10863039016919479, "right_score": 0.12434544355560544, "margin": -0.23297583372480024, "p_left": 0.44201806531995086, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5834286922502717, "brier": 0.19537997006919233 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.14147511927912115, "right_score": 0.06135644389453036, "margin": 0.08011867538459079, "p_left": 0.5200189615118608, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6538900036255897, "brier": 0.23038179730815259 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.17220131265606628, "right_score": 0.14772863457473537, "margin": 0.024472678081330912, "p_left": 0.5061178641848717, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6809857036477075, "brier": 0.24391956407731283 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.3087798212336246, "right_score": 0.04768428181759183, "margin": 0.26109553941603275, "p_left": 0.5649055805617463, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5710966758508111, "brier": 0.189307153826311 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.12434544355560544, "right_score": -0.04347148512773118, "margin": 0.1678169286833366, "p_left": 0.5418560472610521, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6127549082785249, "brier": 0.20989588143126736 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.02934396217927062, "right_score": 0.010686420598060133, "margin": 0.01865754158121049, "p_left": 0.5046642500924667, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.68386192212046, "brier": 0.24535750513645835 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.12434544355560544, "right_score": 0.14155846122992993, "margin": -0.017213017674324496, "p_left": 0.49569685182848316, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6845777072627525, "brier": 0.24571536891266918 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.28570557082447967, "right_score": 0.14155846122992993, "margin": 0.14414710959454974, "p_left": 0.535974507932486, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6236686788697909, "brier": 0.21531965728849847 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.14772863457473537, "right_score": 0.1477386461398867, "margin": -1.0011565151318846e-05, "p_left": 0.4999974971087121, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6931521863550498, "brier": 0.2500025028975524 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.12469526342413896, "right_score": 0.14155846122992993, "margin": -0.016863197805790972, "p_left": 0.49578430044871963, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6847511271659165, "brier": 0.2458020725714263 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.2002280778561575, "right_score": -0.10863039016919479, "margin": 0.30885846802535233, "p_left": 0.576606601699027, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5505950444867587, "brier": 0.17926196972484637 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.2002280778561575, "right_score": -0.034462496220780015, "margin": 0.23469057407693752, "p_left": 0.5584048128265118, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7000163956509333, "brier": 0.0034111221612998744 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.2002280778561575, "right_score": 0.04768428181759183, "margin": 0.1525437960385657, "p_left": 0.5380621699650632, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7723249639792988, "brier": 0.2895108987475125 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04768428181759183, "right_score": 0.12469526342413896, "margin": -0.07701098160654712, "p_left": 0.4807567641327104, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6553828430468458, "brier": 0.23112706625935453 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.12434544355560544, "right_score": -0.013608916458112063, "margin": 0.1379543600137175, "p_left": 0.5344339967048256, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6265470421947791, "brier": 0.21675170342424233 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.1477386461398867, "right_score": 0.14147511927912115, "margin": 0.006263526860765534, "p_left": 0.5015658765958524, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6962838479534037, "brier": 0.2515683285653658 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.12469526342413896, "right_score": -0.10863039016919479, "margin": 0.23332565359333374, "p_left": 0.5580682118352457, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.583274080639606, "brier": 0.19530370539049727 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.11690529484827952, "right_score": 0.058482671677521886, "margin": 0.05842262317075764, "p_left": 0.5146015028712699, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.664362458673795, "brier": 0.23561170101482978 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.3087798212336246, "right_score": 0.14155846122992993, "margin": 0.16722136000369464, "p_left": 0.5417081948036816, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6130278084953247, "brier": 0.21003137871010025 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.28570557082447967, "right_score": -0.04347148512773118, "margin": 0.32917705595221086, "p_left": 0.5815591284160413, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5420426296200123, "brier": 0.17509276301194301 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.06135644389453036, "right_score": 0.15565733232930845, "margin": -0.09430088843477809, "p_left": 0.476442232900753, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.647107906909857, "brier": 0.22699720129145537 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17172252392285378, "right_score": 0.04768428181759183, "margin": 0.12403824210526196, "p_left": 0.5309698635056589, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7570882556826732, "brier": 0.281928995951218 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17172252392285378, "right_score": -0.034462496220780015, "margin": 0.2061850201436338, "p_left": 0.5513644157097035, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5953593169150232, "brier": 0.20127388749149572 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.12469526342413896, "right_score": -0.013608916458112063, "margin": 0.13830417988225102, "p_left": 0.5345210358411612, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7646883730630345, "brier": 0.2857127377567079 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.2002280778561575, "right_score": -0.04347148512773118, "margin": 0.24369956298388867, "p_left": 0.5606251460053987, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7005525672277906, "brier": 0.00367540832817591 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.04347148512773118, "right_score": 0.17172252392285378, "margin": -0.21519400905058494, "p_left": 0.4464081512986074, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5913275989536385, "brier": 0.19928023754584034 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.28570557082447967, "right_score": -0.034462496220780015, "margin": 0.3201680670452597, "p_left": 0.5793652108294315, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5458222388920391, "brier": 0.17693362586056863 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.034462496220780015, "right_score": 0.3087798212336246, "margin": -0.3432423174544046, "p_left": 0.41502209501362064, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5361812017176092, "brier": 0.17224333934949476 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.06135644389453036, "right_score": 0.02934396217927062, "margin": 0.03201248171525974, "p_left": 0.508002437033041, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7092815158212858, "brier": 0.25806647603150884 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.3087798212336246, "right_score": -0.013608916458112063, "margin": 0.32238873769173665, "p_left": 0.5799062952640668, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5448887483834168, "brier": 0.17647872075876145 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.017638301294970682, "right_score": 0.20719938358299897, "margin": -0.1895610822880283, "p_left": 0.45275112916352656, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6976321470666078, "brier": 0.0022324557953217507 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.013608916458112063, "right_score": 0.17172252392285378, "margin": -0.18533144038096586, "p_left": 0.45379930506261634, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6047687976302994, "brier": 0.20593380927531352 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.02934396217927062, "right_score": 0.017638301294970682, "margin": 0.011705660884299939, "p_left": 0.5029263818061396, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6873114778321012, "brier": 0.24708218190433573 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.28570557082447967, "right_score": -0.10863039016919479, "margin": 0.39433596099367446, "p_left": 0.5973260564204079, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5152921565214825, "brier": 0.1621463048379405 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated", "paired_against_semantic_matched": { "heldout": { "fixes": 7, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 7, "trial_ids": { "fixes": [ "1b6d9789fab8", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "e5aa020b210d", "396ff40ef0fb", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "70f39a6b70be", "b20ebcae7654", "c469d02ac7ba" ] } }, "other": { "fixes": 4, "regressions": 0, "both_correct": 0, "both_wrong": 3, "net_fixes": 4, "trial_ids": { "fixes": [ "d013ade6e6a0", "34bec29a8d38", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "70f39a6b70be", "b20ebcae7654" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 9, "regressions": 0, "both_correct": 0, "both_wrong": 2, "net_fixes": 9, "trial_ids": { "fixes": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "34bec29a8d38", "c469d02ac7ba" ] } }, "other": { "fixes": 6, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 6, "trial_ids": { "fixes": [ "0add2ec437a1", "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [], "both_correct": [], "both_wrong": [ "34bec29a8d38" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.6143569802030641, "fold_losses": [ 0.43009545259434645, 0.5885297379231711, 0.2119363005688523, 0.11836832559674669, 0.29961203546426635, 0.42061235050905665, 0.4338895838281087, 0.9016514096942299, 0.684488383836272, 0.686619031307367, 0.7599270169348659, 0.9149713973145267, 1.4765389219804481, 0.6838696615229648, 0.44160048648187433, 0.19220614113169532, 1.304267807805476, 0.4883078371710705, 1.3489101181439425, 0.7954933131921965, 0.4133232523421361, 0.1775785349057803, 0.25580607454256676, 0.7159643500815742 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.42139488957372545, "brier_score": 0.10630457609868955, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 8, "accuracy": 0.7272727272727273, "mean_cross_entropy": 0.6612051619498636, "brier_score": 0.21280796721440096, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.7142857142857143, "mean_cross_entropy": 0.6456175780644942, "brier_score": 0.19443558608562417, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6923803297206025, "brier_score": 0.24955272947195453, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.23976518558226717, "brier_score": 0.05074977274982879, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.172005335439315, "right_score": -0.10447434345676701, "margin": -1.067530991982548, "p_left": 0.2558729047178062, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2955434317461184, "brier": 0.06547094336872754 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.40968247210190695, "right_score": -0.31855413515256603, "margin": -0.09112833694934092, "p_left": 0.4772336685929999, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6486206998280608, "brier": 0.2277519744387333 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.39836450263900786, "right_score": 1.150357988539636, "margin": -0.7519934859006281, "p_left": 0.32038708546799716, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3862318862277737, "brier": 0.10264788453467771 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6617890514212527, "right_score": 1.0169688769186234, "margin": -1.6787579283398761, "p_left": 0.15726000970450602, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1710968023689474, "brier": 0.024730710652261328 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.172005335439315, "right_score": 1.0169688769186234, "margin": -2.1889742123579383, "p_left": 0.10074498718868632, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.10618862198622922, "brier": 0.010149552443648572 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.40968247210190695, "right_score": -0.44765208508567, "margin": 0.03796961298376306, "p_left": 0.5094912629839001, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7123121876662466, "brier": 0.2595813470569297 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.172005335439315, "right_score": 0.20115149344534472, "margin": -1.3731568288846596, "p_left": 0.20211029201442304, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22578490162867548, "brier": 0.040848570138155356 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.03690108158438388, "right_score": -0.07769897376093807, "margin": 0.11460005534532194, "p_left": 0.5286186994964223, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6374879019241732, "brier": 0.22220033046444426 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.15000570235260247, "right_score": 0.21384209731613885, "margin": -0.06383639496353638, "p_left": 0.48404631860892955, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7255746772391942, "brier": 0.26620820134099826 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 1.150357988539636, "right_score": 0.1457694940793372, "margin": 1.0045884944602987, "p_left": 0.7319597743577126, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31202971959403447, "brier": 0.07184556256236832 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.20115149344534472, "right_score": -0.7623160135330355, "margin": 0.9634675069783802, "p_left": 0.7238155203841825, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3232187251642058, "brier": 0.07627786678065991 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.31855413515256603, "right_score": -0.44765208508567, "margin": 0.12909794993310397, "p_left": 0.5322297374245752, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6306800455911299, "brier": 0.21880901854988183 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.20115149344534472, "right_score": 0.39836450263900786, "margin": -0.19721300919366314, "p_left": 0.4508559244746089, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5993944392510455, "brier": 0.20327106463385428 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.0169688769186234, "right_score": 0.39836450263900786, "margin": 0.6186043742796155, "p_left": 0.6499010687849552, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43093512954563384, "brier": 0.12256926163791665 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.21384209731613885, "right_score": -0.020016674956573933, "margin": 0.2338587722727128, "p_left": 0.558199689799722, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5830385135953652, "brier": 0.19518751409306187 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.15887969403793384, "right_score": 0.39836450263900786, "margin": -0.5572441966769417, "p_left": 0.36418533889417004, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.452848171466751, "brier": 0.1326309610654615 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.10447434345676701, "right_score": -1.172005335439315, "margin": 1.067530991982548, "p_left": 0.7441270952821938, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2955434317461185, "brier": 0.06547094336872754 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.10447434345676701, "right_score": -0.6617890514212527, "margin": 0.5573147079644857, "p_left": 0.635830988146785, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7314798468474659, "brier": 0.018450057340932035 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.10447434345676701, "right_score": 0.1457694940793372, "margin": -0.25024383753610424, "p_left": 0.437763483296111, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5758326693926626, "brier": 0.19163686730754442 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.1457694940793372, "right_score": -0.15887969403793384, "margin": 0.30464918811727104, "p_left": 0.5755786542290253, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8570285771570463, "brier": 0.3312907872040959 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.20115149344534472, "right_score": -0.32470980788053605, "margin": 0.5258613013258808, "p_left": 0.6285173100222687, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4643917095295621, "brier": 0.1379993889530912 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.020016674956573933, "right_score": 0.03690108158438388, "margin": -0.056917756540957806, "p_left": 0.48577440113168335, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6650932015149289, "brier": 0.2359767687948456 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.15887969403793384, "right_score": -1.172005335439315, "margin": 1.0131256414013812, "p_left": 0.7336313966015735, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3097485609709929, "brier": 0.07095223287642823 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.34766819296433304, "right_score": 0.05276504346572605, "margin": 0.29490314949860696, "p_left": 0.5731960791151088, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5565274233950104, "brier": 0.18216158688271644 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 1.150357988539636, "right_score": 0.39836450263900786, "margin": 0.7519934859006281, "p_left": 0.6796129145320029, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3862318862277736, "brier": 0.10264788453467769 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.0169688769186234, "right_score": -0.7623160135330355, "margin": 1.7792848904516587, "p_left": 0.8556085419061533, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1559423182961932, "brier": 0.02084889317046708 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.07769897376093807, "right_score": 0.449858856453495, "margin": -0.527557830214433, "p_left": 0.37108666525412426, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4637618143725624, "brier": 0.13770531312942647 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.2715618901468906, "right_score": 0.1457694940793372, "margin": 0.12579239606755338, "p_left": 0.5314066956869337, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7580200417150369, "brier": 0.2823930762209053 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.2715618901468906, "right_score": -0.6617890514212527, "margin": 0.9333509415681432, "p_left": 0.717754624099106, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3316275175090574, "brier": 0.07966245221743698 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.15887969403793384, "right_score": -0.32470980788053605, "margin": 0.1658301138426022, "p_left": 0.5413627834511298, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7794957593240472, "brier": 0.2930736633059548 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.10447434345676701, "right_score": -0.7623160135330355, "margin": 0.6578416700762685, "p_left": 0.658775383047162, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7462934827671388, "brier": 0.025209622261773014 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.7623160135330355, "right_score": 0.2715618901468906, "margin": -1.033877903679926, "p_left": 0.2623329826761539, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3042627522393741, "brier": 0.06881859379976726 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.0169688769186234, "right_score": -0.6617890514212527, "margin": 1.6787579283398761, "p_left": 0.842739990295494, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.17109680236894742, "brier": 0.024730710652261335 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6617890514212527, "right_score": 1.150357988539636, "margin": -1.8121470399608888, "p_left": 0.14037883669625412, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.15126349457298305, "brier": 0.019706217792193583 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.07769897376093807, "right_score": -0.31855413515256603, "margin": 0.24085516139162796, "p_left": 0.5599243795753557, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8208087022353946, "brier": 0.313515310842847 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 1.150357988539636, "right_score": -0.32470980788053605, "margin": 1.4750677964201722, "p_left": 0.8138264443535322, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.20600814903768794, "brier": 0.034660592822048426 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.3666018438878577, "right_score": 0.6670370497727397, "margin": -1.0336388936605974, "p_left": 0.2623792371812373, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8211449048084731, "brier": 0.05646362692257068 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.32470980788053605, "right_score": 0.2715618901468906, "margin": -0.5962716980274266, "p_left": 0.3551971329813628, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4388106414307493, "brier": 0.12616500327817992 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.31855413515256603, "right_score": -0.3666018438878577, "margin": 0.04804770873529168, "p_left": 0.5120096168402948, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6694118712278248, "brier": 0.23813461405635591 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.0169688769186234, "right_score": -1.172005335439315, "margin": 2.1889742123579383, "p_left": 0.8992550128113136, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.10618862198622914, "brier": 0.01014955244364859 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 8, "regressions": 0, "both_correct": 0, "both_wrong": 3, "net_fixes": 8, "trial_ids": { "fixes": [ "1b6d9789fab8", "d013ade6e6a0", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "34bec29a8d38", "c469d02ac7ba" ] } }, "other": { "fixes": 5, "regressions": 0, "both_correct": 0, "both_wrong": 2, "net_fixes": 5, "trial_ids": { "fixes": [ "d013ade6e6a0", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "34bec29a8d38" ] } }, "target": { "fixes": 3, "regressions": 0, "both_correct": 0, "both_wrong": 1, "net_fixes": 3, "trial_ids": { "fixes": [ "1b6d9789fab8", "974972ffb84a", "fe73774e1a9e" ], "regressions": [], "both_correct": [], "both_wrong": [ "c469d02ac7ba" ] } } } }, { "name": "semantic_bt_tuned", "selected": { "ridge": 1.0, "consistency": 0.0, "mean_cross_entropy": 0.6160207767377772, "fold_losses": [ 0.5859098688107738, 0.6432171371184643, 0.216951062615758, 0.2754518908609063, 0.5626737054557178, 0.47023613661358366, 0.6235046445788839, 0.6571623931765784, 0.4762218151854869, 0.5367358585591491, 0.792324816241669, 1.0784280975624152, 0.9481019833921712, 0.6549999109599929, 0.8857109435590665, 0.34985329677502974, 1.2538528948029428, 0.364598092521656, 0.7435466781902008, 0.7067897319798959, 0.5542838326825236, 0.2965791293816204, 0.5250884129505637, 0.5822763077316027 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5317102193123868, "brier_score": 0.15301175117822444, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.010636637793838, "brier_score": 0.3772109732126972, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8797446109401741, "brier_score": 0.309520954622169, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2724206915011664, "brier_score": 0.5125910103937537, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.36803611913974005, "brier_score": 0.1011224482297331, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.1202243928859767, "margin": -0.5173669920198206, "p_left": 0.37346812484778835, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.0025331266354489956, "right_score": -0.8393385093802779, "margin": 0.8368053827448289, "p_left": 0.6977919664036878, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1966396457896504, "brier": 0.48691362837752533 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.174867975786096, "right_score": 0.45103328257425607, "margin": -0.2761653067881601, "p_left": 0.43139415182525515, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.1861009142290313 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.859079675898924, "margin": -1.5672691690555438, "p_left": 0.1726060418174397, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18947432715388196, "brier": 0.029792845671883738 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.859079675898924, "margin": -1.2562222750327678, "p_left": 0.2216248919699548, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2505467270003285, "brier": 0.04911759274069414 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.0025331266354489956, "right_score": -0.37462365215645116, "margin": 0.3720905255210022, "p_left": 0.5919640263529361, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8963999377611955, "brier": 0.35042140849597964 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.06883873998264661, "margin": -0.46598133911649053, "p_left": 0.3855678467111719, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48705676569112055, "brier": 0.14866256441748973 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.244337813821957, "right_score": 0.3599551634000384, "margin": -0.6042929772219954, "p_left": 0.3533621435112404, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.040261845628836, "brier": 0.41814051744437764 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0662966295701812, "right_score": 0.13193798447273392, "margin": -0.06564135490255271, "p_left": 0.4835955511250581, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7265063597770781, "brier": 0.26667355481783256 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.2376283470924767, "margin": 0.6886616296667327, "p_left": 0.665669133504202, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40696252842627256, "brier": 0.11177712829183112 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.16062358048092495, "margin": 0.22946232046357157, "p_left": 0.557115192780389, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5849832511364297, "brier": 0.19614695246595198 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": -0.37462365215645116, "margin": -0.4647148572238267, "p_left": 0.38586792641078876, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9522601276247856, "brier": 0.3771582038109844 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06883873998264661, "right_score": 0.174867975786096, "margin": -0.10602923580344939, "p_left": 0.4735174965325325, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6415371797409833, "brier": 0.22421881952243694 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.859079675898924, "right_score": 0.174867975786096, "margin": 0.684211700112828, "p_left": 0.6646780560445341, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40845248180205285, "brier": 0.11244080609807261 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.13193798447273392, "right_score": 0.20707827403818344, "margin": -0.07514028956544952, "p_left": 0.4812237610965914, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7314229172632362, "brier": 0.2691287860507665 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3414369310004953, "right_score": 0.174867975786096, "margin": -0.5163049067865914, "p_left": 0.3737166752152307, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46795241476966365, "brier": 0.13966415333392623 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1202243928859767, "right_score": -0.3971425991338439, "margin": 0.5173669920198206, "p_left": 0.6265318751522116, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.7081894931566199, "margin": 0.8284138860425967, "p_left": 0.6960194499561154, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7765846167202332, "brier": 0.038423624761098035 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.2376283470924767, "margin": 0.3578527399784534, "p_left": 0.5885205437582018, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8879961841895156, "brier": 0.34635643042544956 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2376283470924767, "right_score": -0.3414369310004953, "margin": 0.10380858390801861, "p_left": 0.5259288655567775, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7463978958818103, "brier": 0.27660117162583897 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.13256827664104387, "margin": 0.20140701662369048, "p_left": 0.5501822327701553, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5975057233246752, "brier": 0.20233602371564272 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.20707827403818344, "right_score": -0.244337813821957, "margin": 0.4514160878601404, "p_left": 0.610975868727525, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9441139031682542, "brier": 0.3732915121673538 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.3971425991338439, "margin": 0.055705668133348596, "p_left": 0.5139228168702084, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6656821865334857, "brier": 0.236271027959393 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.49595921613685756, "right_score": 0.23941716687082182, "margin": -0.7353763830076794, "p_left": 0.3240160277788788, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1269622959600016, "brier": 0.45695433069984553 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.45103328257425607, "right_score": 0.174867975786096, "margin": 0.2761653067881601, "p_left": 0.5686058481747448, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.18610091422903133 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.859079675898924, "right_score": -0.16062358048092495, "margin": 1.019703256379849, "p_left": 0.7349147933763889, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30800071386846106, "brier": 0.07027016677068261 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3599551634000384, "right_score": 0.09725595794298547, "margin": 0.26269920545705294, "p_left": 0.5652997001457829, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8330984511918679, "brier": 0.3195637509849121 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.2376283470924767, "margin": 0.5411735074699813, "p_left": 0.6320853632309412, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9999043330227885, "brier": 0.3995319064107909 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.7081894931566199, "margin": 1.0117346535341245, "p_left": 0.7333594864800296, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31011926555790437, "brier": 0.07109716345019353 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.13256827664104387, "margin": -0.20886865435945146, "p_left": 0.4479718478670522, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5941562337646624, "brier": 0.20067877648142135 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.16062358048092495, "margin": 0.28084797336690165, "p_left": 0.5697541036356324, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7029743950459738, "brier": 0.004865634974010544 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.16062358048092495, "right_score": 0.3035451603775046, "margin": -0.4641687408584296, "p_left": 0.3859973497829098, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4877560345298307, "brier": 0.14899395403943 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.7081894931566199, "margin": 1.5672691690555438, "p_left": 0.8273939581825605, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1894743271538819, "brier": 0.02979284567188369 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.45103328257425607, "margin": -1.159222775730876, "p_left": 0.2388085395414515, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2728703621483618, "brier": 0.05702951855792101 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3599551634000384, "right_score": -0.8393385093802779, "margin": 1.1992936727803163, "p_left": 0.768399107982165, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4627396817456582, "brier": 0.590437189147787 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.13256827664104387, "margin": 0.5836015592152999, "p_left": 0.6418957033408885, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4433294443448714, "brier": 0.1282386872857169 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.3798258533937844, "right_score": -0.23176226517134157, "margin": 0.611588118565126, "p_left": 0.6483029882243924, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7391911767709232, "brier": 0.02199377631628426 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.13256827664104387, "right_score": 0.3035451603775046, "margin": -0.4361134370185485, "p_left": 0.39266744806662623, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49867877643922714, "brier": 0.1541877247711566 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": 0.3798258533937844, "margin": -1.2191643627740623, "p_left": 0.2280835406349222, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4780433108445714, "brier": 0.5958550202387178 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.3971425991338439, "margin": 1.2562222750327678, "p_left": 0.7783751080300452, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25054672700032854, "brier": 0.049117592740694146 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 11, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 7, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "consistent_bt", "selected": { "ridge": 1.0, "consistency": 0.1, "mean_cross_entropy": 0.6175934060809684, "fold_losses": [ 0.5635730767500121, 0.6631726177539041, 0.23130403211852077, 0.28076797295009526, 0.5528860637553809, 0.48307190367168173, 0.6251552374286717, 0.6571274845332367, 0.4861071977845454, 0.5465349186375383, 0.786406614223135, 1.0863569299403846, 0.9113046940909887, 0.6624489267622008, 0.8370239935725786, 0.3616043450843871, 1.2241377812215084, 0.38126837484374243, 0.7424415070002868, 0.7076375017874439, 0.5639583761286637, 0.31972519791075, 0.5495238000271772, 0.5987031979664059 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.537889153740508, "brier_score": 0.1556001116699543, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.9872854052204229, "brier_score": 0.3667224529067475, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8507485390991081, "brier_score": 0.2958819095610288, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2603591374630527, "brier_score": 0.5084035395981851, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3755271441445376, "brier_score": 0.1040213782607071, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.1383639539656737, "margin": -0.5463927304335155, "p_left": 0.36670172733940004, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4568137629850261, "brier": 0.1344701568336997 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.012064661072830885, "right_score": -0.8093960293656783, "margin": 0.8214606904385092, "p_left": 0.6945463161442109, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1859571195374503, "brier": 0.4823945852694942 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.175316082258721, "right_score": 0.40099994434125463, "margin": -0.22568386208253363, "p_left": 0.4438172960633887, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5866584345075525, "brier": 0.19697379228501763 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6665261101000797, "right_score": 0.8251507367526193, "margin": -1.491676846852699, "p_left": 0.18367017527195886, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2029368087129817, "brier": 0.03373473328443209 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.8251507367526193, "margin": -1.233179513220461, "p_left": 0.2256254210847683, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.25569957037259006, "brier": 0.050906830639679004 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.012064661072830885, "right_score": -0.32728625106482434, "margin": 0.3393509121376552, "p_left": 0.5840328431220481, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8771489716501898, "brier": 0.3410943618452228 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.0696335543067845, "margin": -0.4776623307746264, "p_left": 0.3828042835103552, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4825690987478617, "brier": 0.14653911947387643 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.21330345767726425, "right_score": 0.3452807228167657, "margin": -0.5585841804940299, "p_left": 0.3638751162787647, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.010944557359507, "brier": 0.40465486768935516 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.09357621247170457, "right_score": 0.10816787852889424, "margin": -0.014591666057189673, "p_left": 0.49635214820952556, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.700469627942222, "brier": 0.2536611586131597 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.40099994434125463, "right_score": -0.2436158504616248, "margin": 0.6446157948028794, "p_left": 0.6557961206947117, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4219053299676887, "brier": 0.11847631052880947 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0696335543067845, "right_score": -0.15289731173500143, "margin": 0.22253086604178593, "p_left": 0.5554042700966442, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5880590158123018, "brier": 0.19766536304829774 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8093960293656783, "right_score": -0.32728625106482434, "margin": -0.482109778300854, "p_left": 0.3817540557237506, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.962978710909049, "brier": 0.38222804761403123 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0696335543067845, "right_score": 0.175316082258721, "margin": -0.1056825279519365, "p_left": 0.47360393113397375, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6417013669585916, "brier": 0.22430068358555375 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8251507367526193, "right_score": 0.175316082258721, "margin": 0.6498346544938983, "p_left": 0.6569732014743254, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.42011205056181133, "brier": 0.11766738450677378 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.10816787852889424, "right_score": 0.18621017697538225, "margin": -0.07804229844648801, "p_left": 0.48049932195352746, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7329294616996797, "brier": 0.2698809544907448 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.31212044802441685, "right_score": 0.175316082258721, "margin": -0.4874365302831378, "p_left": 0.3804976412694458, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47883877044552453, "brier": 0.14477845501161185 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1383639539656737, "right_score": -0.40802877646784186, "margin": 0.5463927304335155, "p_left": 0.6332982726605999, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4568137629850261, "brier": 0.13447015683369973 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.6665261101000797, "margin": 0.8048900640657534, "p_left": 0.691019540318562, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7720322093279754, "brier": 0.03648846478351475 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.2436158504616248, "margin": 0.3819798044272985, "p_left": 0.594350518804801, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9022658390887589, "brier": 0.35325253920353616 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2436158504616248, "right_score": -0.31212044802441685, "margin": 0.06850459756279206, "p_left": 0.5171194549521316, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7279859746593611, "brier": 0.2674125306899897 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0696335543067845, "right_score": -0.10969199922312739, "margin": 0.1793255535299119, "p_left": 0.5447116341124956, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.607498736068087, "brier": 0.2072874961125141 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.18621017697538225, "right_score": -0.21330345767726425, "margin": 0.3995136346526465, "p_left": 0.5985708000046159, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9127240998860848, "brier": 0.35828700261816593 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.31212044802441685, "right_score": -0.40802877646784186, "margin": 0.095908328443425, "p_left": 0.5239587197535264, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6463423768606547, "brier": 0.22661530049870166 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.4329683211657527, "right_score": 0.1743640285925066, "margin": -0.6073323497582592, "p_left": 0.35266796489516494, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0422282740615614, "brier": 0.4190387636729674 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.40099994434125463, "right_score": 0.175316082258721, "margin": 0.22568386208253363, "p_left": 0.5561827039366113, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5866584345075525, "brier": 0.19697379228501763 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8251507367526193, "right_score": -0.15289731173500143, "margin": 0.9780480484876207, "p_left": 0.7267207348460247, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31921300888367254, "brier": 0.07468155676309672 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3452807228167657, "right_score": 0.16020782739207254, "margin": 0.18507289542469316, "p_left": 0.5461366097984767, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7899590288608884, "brier": 0.29826519656217354 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.2834162243870383, "right_score": -0.2436158504616248, "margin": 0.5270320748486631, "p_left": 0.6287906249336642, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9909890222835912, "brier": 0.395377650004468 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.2834162243870383, "right_score": -0.6665261101000797, "margin": 0.949942334487118, "p_left": 0.721103580874529, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3269724892216227, "brier": 0.0777832126010104 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.31212044802441685, "right_score": -0.10969199922312739, "margin": -0.20242844880128946, "p_left": 0.4495649943921943, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5970463940805371, "brier": 0.20210868418285366 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.15289731173500143, "margin": 0.2912612657006751, "p_left": 0.5723048835802026, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7037140492016809, "brier": 0.0052279961895466505 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.15289731173500143, "right_score": 0.2834162243870383, "margin": -0.4363135361220397, "p_left": 0.3926197295128116, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49860020880915296, "brier": 0.15415025200271334 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8251507367526193, "right_score": -0.6665261101000797, "margin": 1.491676846852699, "p_left": 0.8163298247280412, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2029368087129817, "brier": 0.03373473328443208 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6665261101000797, "right_score": 0.40099994434125463, "margin": -1.0675260544413343, "p_left": 0.25587384483647074, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2955446951314519, "brier": 0.06547142447139831 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3452807228167657, "right_score": -0.8093960293656783, "margin": 1.1546767521824441, "p_left": 0.7603641077893023, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4286346234143417, "brier": 0.5781535764142217 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.40099994434125463, "right_score": -0.10969199922312739, "margin": 0.510691943564382, "p_left": 0.624968668179239, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4700537614155492, "brier": 0.14064849984725372 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.39136331813372094, "right_score": -0.22287017515185603, "margin": 0.614233493285577, "p_left": 0.6489059132353077, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7395842913327311, "brier": 0.022172970996440975 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.10969199922312739, "right_score": 0.2834162243870383, "margin": -0.3931082236101657, "p_left": 0.4029692836067714, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5157867156681659, "brier": 0.16238424353055458 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8093960293656783, "right_score": 0.39136331813372094, "margin": -1.2007593474993992, "p_left": 0.23134016034724877, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4638660959913699, "brier": 0.5908379490949932 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8251507367526193, "right_score": -0.40802877646784186, "margin": 1.233179513220461, "p_left": 0.7743745789152318, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25569957037258995, "brier": 0.050906830639678984 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 11, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 7, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } }, { "name": "semantic_bt_matched", "selected": { "ridge": 1.0, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5317102193123868, "brier_score": 0.15301175117822444, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.010636637793838, "brier_score": 0.3772109732126972, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8797446109401741, "brier_score": 0.309520954622169, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2724206915011664, "brier_score": 0.5125910103937537, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.36803611913974005, "brier_score": 0.1011224482297331, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.1202243928859767, "margin": -0.5173669920198206, "p_left": 0.37346812484778835, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.0025331266354489956, "right_score": -0.8393385093802779, "margin": 0.8368053827448289, "p_left": 0.6977919664036878, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1966396457896504, "brier": 0.48691362837752533 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.174867975786096, "right_score": 0.45103328257425607, "margin": -0.2761653067881601, "p_left": 0.43139415182525515, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.1861009142290313 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.859079675898924, "margin": -1.5672691690555438, "p_left": 0.1726060418174397, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18947432715388196, "brier": 0.029792845671883738 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.859079675898924, "margin": -1.2562222750327678, "p_left": 0.2216248919699548, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2505467270003285, "brier": 0.04911759274069414 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.0025331266354489956, "right_score": -0.37462365215645116, "margin": 0.3720905255210022, "p_left": 0.5919640263529361, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8963999377611955, "brier": 0.35042140849597964 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.06883873998264661, "margin": -0.46598133911649053, "p_left": 0.3855678467111719, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48705676569112055, "brier": 0.14866256441748973 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.244337813821957, "right_score": 0.3599551634000384, "margin": -0.6042929772219954, "p_left": 0.3533621435112404, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.040261845628836, "brier": 0.41814051744437764 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0662966295701812, "right_score": 0.13193798447273392, "margin": -0.06564135490255271, "p_left": 0.4835955511250581, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7265063597770781, "brier": 0.26667355481783256 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.2376283470924767, "margin": 0.6886616296667327, "p_left": 0.665669133504202, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40696252842627256, "brier": 0.11177712829183112 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.16062358048092495, "margin": 0.22946232046357157, "p_left": 0.557115192780389, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5849832511364297, "brier": 0.19614695246595198 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": -0.37462365215645116, "margin": -0.4647148572238267, "p_left": 0.38586792641078876, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9522601276247856, "brier": 0.3771582038109844 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06883873998264661, "right_score": 0.174867975786096, "margin": -0.10602923580344939, "p_left": 0.4735174965325325, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6415371797409833, "brier": 0.22421881952243694 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.859079675898924, "right_score": 0.174867975786096, "margin": 0.684211700112828, "p_left": 0.6646780560445341, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40845248180205285, "brier": 0.11244080609807261 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.13193798447273392, "right_score": 0.20707827403818344, "margin": -0.07514028956544952, "p_left": 0.4812237610965914, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7314229172632362, "brier": 0.2691287860507665 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3414369310004953, "right_score": 0.174867975786096, "margin": -0.5163049067865914, "p_left": 0.3737166752152307, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46795241476966365, "brier": 0.13966415333392623 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1202243928859767, "right_score": -0.3971425991338439, "margin": 0.5173669920198206, "p_left": 0.6265318751522116, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.7081894931566199, "margin": 0.8284138860425967, "p_left": 0.6960194499561154, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7765846167202332, "brier": 0.038423624761098035 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.2376283470924767, "margin": 0.3578527399784534, "p_left": 0.5885205437582018, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8879961841895156, "brier": 0.34635643042544956 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2376283470924767, "right_score": -0.3414369310004953, "margin": 0.10380858390801861, "p_left": 0.5259288655567775, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7463978958818103, "brier": 0.27660117162583897 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.13256827664104387, "margin": 0.20140701662369048, "p_left": 0.5501822327701553, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5975057233246752, "brier": 0.20233602371564272 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.20707827403818344, "right_score": -0.244337813821957, "margin": 0.4514160878601404, "p_left": 0.610975868727525, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9441139031682542, "brier": 0.3732915121673538 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.3971425991338439, "margin": 0.055705668133348596, "p_left": 0.5139228168702084, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6656821865334857, "brier": 0.236271027959393 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.49595921613685756, "right_score": 0.23941716687082182, "margin": -0.7353763830076794, "p_left": 0.3240160277788788, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1269622959600016, "brier": 0.45695433069984553 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.45103328257425607, "right_score": 0.174867975786096, "margin": 0.2761653067881601, "p_left": 0.5686058481747448, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.18610091422903133 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.859079675898924, "right_score": -0.16062358048092495, "margin": 1.019703256379849, "p_left": 0.7349147933763889, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30800071386846106, "brier": 0.07027016677068261 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3599551634000384, "right_score": 0.09725595794298547, "margin": 0.26269920545705294, "p_left": 0.5652997001457829, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8330984511918679, "brier": 0.3195637509849121 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.2376283470924767, "margin": 0.5411735074699813, "p_left": 0.6320853632309412, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9999043330227885, "brier": 0.3995319064107909 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.7081894931566199, "margin": 1.0117346535341245, "p_left": 0.7333594864800296, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31011926555790437, "brier": 0.07109716345019353 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.13256827664104387, "margin": -0.20886865435945146, "p_left": 0.4479718478670522, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5941562337646624, "brier": 0.20067877648142135 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.16062358048092495, "margin": 0.28084797336690165, "p_left": 0.5697541036356324, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7029743950459738, "brier": 0.004865634974010544 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.16062358048092495, "right_score": 0.3035451603775046, "margin": -0.4641687408584296, "p_left": 0.3859973497829098, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4877560345298307, "brier": 0.14899395403943 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.7081894931566199, "margin": 1.5672691690555438, "p_left": 0.8273939581825605, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1894743271538819, "brier": 0.02979284567188369 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.45103328257425607, "margin": -1.159222775730876, "p_left": 0.2388085395414515, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2728703621483618, "brier": 0.05702951855792101 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3599551634000384, "right_score": -0.8393385093802779, "margin": 1.1992936727803163, "p_left": 0.768399107982165, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4627396817456582, "brier": 0.590437189147787 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.13256827664104387, "margin": 0.5836015592152999, "p_left": 0.6418957033408885, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4433294443448714, "brier": 0.1282386872857169 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.3798258533937844, "right_score": -0.23176226517134157, "margin": 0.611588118565126, "p_left": 0.6483029882243924, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7391911767709232, "brier": 0.02199377631628426 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.13256827664104387, "right_score": 0.3035451603775046, "margin": -0.4361134370185485, "p_left": 0.39266744806662623, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49867877643922714, "brier": 0.1541877247711566 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": 0.3798258533937844, "margin": -1.2191643627740623, "p_left": 0.2280835406349222, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4780433108445714, "brier": 0.5958550202387178 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.3971425991338439, "margin": 1.2562222750327678, "p_left": 0.7783751080300452, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25054672700032854, "brier": 0.049117592740694146 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "paired_against_semantic_matched": { "heldout": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 11, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "974972ffb84a", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb", "c469d02ac7ba", "fe73774e1a9e" ] } }, "other": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 7, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "0add2ec437a1", "d013ade6e6a0", "34bec29a8d38", "70f39a6b70be", "b20ebcae7654", "e5aa020b210d", "396ff40ef0fb" ] } }, "target": { "fixes": 0, "regressions": 0, "both_correct": 0, "both_wrong": 4, "net_fixes": 0, "trial_ids": { "fixes": [], "regressions": [], "both_correct": [], "both_wrong": [ "1b6d9789fab8", "974972ffb84a", "c469d02ac7ba", "fe73774e1a9e" ] } } } } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.006258333934324074, "rank": 12.0, "portrait_scores": [ -0.01651002041669066, 0.003993352548042513 ], "portrait_score_gap": 0.020503372964733174, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.0013210483588317447, "rank": 3.0, "portrait_scores": [ -0.002233464457180626, 0.004875561174844115 ], "portrait_score_gap": 0.007109025632024741, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 4.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.003535409689602233, "rank": 8.0, "portrait_scores": [ -0.0066098017896289475, -0.0004610175895755185 ], "portrait_score_gap": 0.006148784200053429, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.00216220665221373, "rank": 7.0, "portrait_scores": [ -0.001717606619682557, -0.002606806684744903 ], "portrait_score_gap": 0.0008892000650623458, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "emma_stone", "mean_score": -0.006721541770849666, "rank": 13.0, "portrait_scores": [ -0.003762681503884961, -0.00968040203781437 ], "portrait_score_gap": 0.00591772053392941, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": -0.0045846148950199495, "rank": 10.0, "portrait_scores": [ -0.010240766068066991, 0.0010715362780270915 ], "portrait_score_gap": 0.011312302346094084, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0019282540335639462, "rank": 2.0, "portrait_scores": [ -0.001366168906587922, 0.0052226769737158145 ], "portrait_score_gap": 0.0065888458803037364, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.0009339228876092543, "rank": 4.0, "portrait_scores": [ -0.0007998195278131229, 0.0026676653030316315 ], "portrait_score_gap": 0.0034674848308447544, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "lily_collins", "mean_score": -0.006042520248854092, "rank": 11.0, "portrait_scores": [ -0.006478354876574201, -0.005606685621133983 ], "portrait_score_gap": 0.0008716692554402179, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.005757556704005823, "rank": 1.0, "portrait_scores": [ 0.004514939579220938, 0.007000173828790707 ], "portrait_score_gap": 0.0024852342495697696, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": -0.004019419381127919, "rank": 9.0, "portrait_scores": [ -0.00662398836879293, -0.0014148503934629081 ], "portrait_score_gap": 0.0052091379753300224, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.0, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.0007135445536791125, "rank": 6.0, "portrait_scores": [ 0.004443840509806633, -0.005870929617164858 ], "portrait_score_gap": 0.01031477012697149, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 5.5, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.00011281449579252523, "rank": 5.0, "portrait_scores": [ -0.0016329574381319248, 0.0014073284465468744 ], "portrait_score_gap": 0.003040285884678799, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "2.0": 128, "3.0": 640, "4.0": 1344, "5.0": 1664, "6.0": 1600, "7.0": 1408, "8.0": 960, "9.0": 384, "10.0": 64 }, "target_primary_score": -0.00011281449579252523, "target_primary_rank": 5.0, "target_alternate_photo_scores": [ -0.008859994913873086, -0.003857072798025145 ], "target_alternate_panel_score": -0.006358533855949116, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.00031601360709502396, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.004196327203735386, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.0076795822147808605, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.00010019992162504151, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.687803557847922e-05, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.017638301294970682, "rank": 12.0, "portrait_scores": [ 0.004019971722874254, 0.031256630867067106 ], "portrait_score_gap": 0.027236659144192852, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.5, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.06135644389453036, "rank": 8.0, "portrait_scores": [ 0.07284183933156325, 0.049871048457497465 ], "portrait_score_gap": 0.02297079087406579, "single_photo_rank_min": 8.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.5, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.058482671677521886, "rank": 9.0, "portrait_scores": [ 0.15223429864784316, -0.03526895529279938 ], "portrait_score_gap": 0.18750325394064254, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": 0.010686420598060133, "rank": 13.0, "portrait_scores": [ 0.01446985838069684, 0.006902982815423425 ], "portrait_score_gap": 0.0075668755652734156, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 12.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.11690529484827952, "rank": 7.0, "portrait_scores": [ 0.11926332801565788, 0.11454726168090115 ], "portrait_score_gap": 0.004716066334756727, "single_photo_rank_min": 5.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.14772863457473537, "rank": 5.0, "portrait_scores": [ 0.1511003940538574, 0.14435687509561335 ], "portrait_score_gap": 0.006743518958244044, "single_photo_rank_min": 3.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.17220131265606628, "rank": 2.0, "portrait_scores": [ 0.17239643096959206, 0.1720061943425405 ], "portrait_score_gap": 0.0003902366270515656, "single_photo_rank_min": 2.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 3.0, 4.0 ] }, { "identity": "keira_knightley", "mean_score": 0.14147511927912115, "rank": 6.0, "portrait_scores": [ 0.08268975041672358, 0.20026048814151876 ], "portrait_score_gap": 0.11757073772479518, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 8.0 ] }, { "identity": "lily_collins", "mean_score": 0.025852115578457346, "rank": 11.0, "portrait_scores": [ 0.026614349743295546, 0.025089881413619147 ], "portrait_score_gap": 0.0015244683296763983, "single_photo_rank_min": 9.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 10.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.20719938358299897, "rank": 1.0, "portrait_scores": [ 0.2291865367471198, 0.18521223041887813 ], "portrait_score_gap": 0.043974306328241675, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": 0.1477386461398867, "rank": 4.0, "portrait_scores": [ 0.1487063305343982, 0.14677096174537518 ], "portrait_score_gap": 0.0019353687890230065, "single_photo_rank_min": 3.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.15565733232930845, "rank": 3.0, "portrait_scores": [ 0.1149614253830554, 0.19635323927556148 ], "portrait_score_gap": 0.08139181389250608, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 4.0, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.02934396217927062, "rank": 10.0, "portrait_scores": [ 0.01846813465288577, 0.04021978970565547 ], "portrait_score_gap": 0.0217516550527697, "single_photo_rank_min": 9.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 10.0, 12.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "9.0": 2048, "10.0": 3072, "11.0": 2048, "12.0": 1024 }, "target_primary_score": 0.02934396217927062, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ 0.10148625435755892, 0.10475237723363268 ], "target_alternate_panel_score": 0.1031193157955958, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.07726720021713845, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.09243289519753567, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": 0.04176287190106544, "prefers_target": true, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.08548101450062512, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0011425488231075554, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.3666018438878577, "rank": 11.0, "portrait_scores": [ -0.40914515286237624, -0.3240585349133392 ], "portrait_score_gap": 0.08508661794903705, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.07769897376093807, "rank": 9.0, "portrait_scores": [ -0.05400054919018421, -0.10139739833169194 ], "portrait_score_gap": 0.04739684914150773, "single_photo_rank_min": 6.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.05276504346572605, "rank": 6.0, "portrait_scores": [ 0.536315109604242, -0.4307850226727899 ], "portrait_score_gap": 0.9671001322770318, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 7.5, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.44765208508567, "rank": 13.0, "portrait_scores": [ -0.4305094983854742, -0.46479467178586575 ], "portrait_score_gap": 0.034285173400391555, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 13.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.34766819296433304, "rank": 3.0, "portrait_scores": [ 0.3747213270518393, 0.3206150588768268 ], "portrait_score_gap": 0.05410626817501252, "single_photo_rank_min": 2.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 5.0 ] }, { "identity": "emma_watson", "mean_score": 0.21384209731613885, "rank": 4.0, "portrait_scores": [ 0.25209823772545975, 0.1755859569068179 ], "portrait_score_gap": 0.07651228081864184, "single_photo_rank_min": 4.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 5.5, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.15000570235260247, "rank": 5.0, "portrait_scores": [ 0.28792896731315776, 0.012082437392047188 ], "portrait_score_gap": 0.2758465299211106, "single_photo_rank_min": 4.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.03690108158438388, "rank": 7.0, "portrait_scores": [ -0.377840115932221, 0.45164227910098875 ], "portrait_score_gap": 0.8294823950332098, "single_photo_rank_min": 2.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 6.5, 11.0 ] }, { "identity": "lily_collins", "mean_score": -0.40968247210190695, "rank": 12.0, "portrait_scores": [ -0.4120143981885208, -0.4073505460152931 ], "portrait_score_gap": 0.004663852173227734, "single_photo_rank_min": 10.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 11.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.6670370497727397, "rank": 1.0, "portrait_scores": [ 0.7652415492262185, 0.5688325503192609 ], "portrait_score_gap": 0.1964089989069575, "single_photo_rank_min": 1.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.75, "single_photo_rank_quantiles": [ 1.0, 1.0, 2.0 ] }, { "identity": "natalie_portman", "mean_score": -0.020016674956573933, "rank": 8.0, "portrait_scores": [ 0.022174844308231828, -0.062208194221379694 ], "portrait_score_gap": 0.08438303852961151, "single_photo_rank_min": 5.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.449858856453495, "rank": 2.0, "portrait_scores": [ 0.3273169399880661, 0.5724007729189239 ], "portrait_score_gap": 0.24508383293085778, "single_photo_rank_min": 1.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.31855413515256603, "rank": 10.0, "portrait_scores": [ -0.3032614627524739, -0.33384680755265816 ], "portrait_score_gap": 0.030585344800184266, "single_photo_rank_min": 8.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 9.0, 11.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 1536, "9.0": 3584, "10.0": 2560, "11.0": 512 }, "target_primary_score": -0.31855413515256603, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ -0.05691497042431225, 0.12452621913970024 ], "target_alternate_panel_score": 0.033805624357694, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.44348809645960097, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.481457709443364, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": 0.11150459811863206, "prefers_target": true, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.40040746824555173, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.035107280653465085, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.3798258533937844, "rank": 1.0, "portrait_scores": [ 0.7873578021790224, -0.02770609539145366 ], "portrait_score_gap": 0.815063897570476, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3599551634000384, "rank": 2.0, "portrait_scores": [ 0.09461817750995859, 0.6252921492901182 ], "portrait_score_gap": 0.5306739717801596, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.23941716687082182, "rank": 3.0, "portrait_scores": [ 1.0424851487988127, -0.5636508150571691 ], "portrait_score_gap": 1.6061359638559818, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.37462365215645116, "rank": 11.0, "portrait_scores": [ -0.5811769774975934, -0.1680703268153089 ], "portrait_score_gap": 0.4131066506822846, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.49595921613685756, "rank": 12.0, "portrait_scores": [ -0.01681240740244498, -0.9751060248712702 ], "portrait_score_gap": 0.9582936174688252, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.13193798447273392, "rank": 5.0, "portrait_scores": [ 0.31816832522057836, -0.054292356275110504 ], "portrait_score_gap": 0.3724606814956889, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0662966295701812, "rank": 7.0, "portrait_scores": [ 0.2659173843065428, -0.13332412516618042 ], "portrait_score_gap": 0.39924150947272324, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.244337813821957, "rank": 10.0, "portrait_scores": [ -0.5751237329039705, 0.08644810526005649 ], "portrait_score_gap": 0.661571838164027, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.0025331266354489956, "rank": 8.0, "portrait_scores": [ 0.3223232838074066, -0.3273895370783046 ], "portrait_score_gap": 0.6497128208857113, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.23176226517134157, "rank": 9.0, "portrait_scores": [ -0.26615431927661387, -0.19737021106606925 ], "portrait_score_gap": 0.06878410821054462, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.20707827403818344, "rank": 4.0, "portrait_scores": [ 0.2803411990077478, 0.13381534906861908 ], "portrait_score_gap": 0.1465258499391287, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.09725595794298547, "rank": 6.0, "portrait_scores": [ -0.46897348854749654, 0.6634854044334675 ], "portrait_score_gap": 1.132458892980964, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8393385093802779, "rank": 13.0, "portrait_scores": [ -0.5745789139272125, -1.1040981048333434 ], "portrait_score_gap": 0.5295191909061309, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "10.0": 512, "11.0": 1536, "12.0": 1536, "13.0": 4608 }, "target_primary_score": -0.8393385093802779, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.4383148833399725, -0.6698268072415561 ], "target_alternate_panel_score": -0.5540708452907643, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5515377186553153, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.17944719313431318, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9140260086908027, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9338966986845487, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.14156884813177417, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.39136331813372094, "rank": 1.0, "portrait_scores": [ 0.7593631533747329, 0.02336348289270901 ], "portrait_score_gap": 0.735999670482024, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3452807228167657, "rank": 2.0, "portrait_scores": [ 0.11168431678718208, 0.5788771288463493 ], "portrait_score_gap": 0.4671928120591673, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.064453125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.1743640285925066, "rank": 4.0, "portrait_scores": [ 0.9288758539147842, -0.580147796729771 ], "portrait_score_gap": 1.5090236506445551, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 6.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.32728625106482434, "rank": 11.0, "portrait_scores": [ -0.5532602267590956, -0.1013122753705531 ], "portrait_score_gap": 0.4519479513885425, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.4329683211657527, "rank": 12.0, "portrait_scores": [ 0.01754641849382503, -0.8834830608253303 ], "portrait_score_gap": 0.9010294793191553, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.10816787852889424, "rank": 6.0, "portrait_scores": [ 0.2655533770269189, -0.049217619969130405 ], "portrait_score_gap": 0.31477099699604927, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.09357621247170457, "rank": 7.0, "portrait_scores": [ 0.29090770762570406, -0.10375528268229492 ], "portrait_score_gap": 0.394662990307999, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -0.21330345767726425, "rank": 9.0, "portrait_scores": [ -0.5440786643784081, 0.11747174902387963 ], "portrait_score_gap": 0.6615504134022878, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 9.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": 0.012064661072830885, "rank": 8.0, "portrait_scores": [ 0.32870223162099277, -0.304572909475331 ], "portrait_score_gap": 0.6332751410963238, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.22287017515185603, "rank": 10.0, "portrait_scores": [ -0.2085235237962932, -0.23721682650741888 ], "portrait_score_gap": 0.02869330271112569, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.18621017697538225, "rank": 3.0, "portrait_scores": [ 0.26927849352959415, 0.10314186042117035 ], "portrait_score_gap": 0.1661366331084238, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.16020782739207254, "rank": 5.0, "portrait_scores": [ -0.358031092650626, 0.6784467474347711 ], "portrait_score_gap": 1.0364778400853971, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8093960293656783, "rank": 13.0, "portrait_scores": [ -0.5714490027811948, -1.047343055950162 ], "portrait_score_gap": 0.47589405316896716, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "11.0": 1024, "12.0": 2048, "13.0": 5120 }, "target_primary_score": -0.8093960293656783, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.45990054053268736, -0.6918943345225459 ], "target_alternate_panel_score": -0.5758974375276167, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5879620986004476, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.24861118646279234, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9211781603443824, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9672607556613376, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.12453731249506285, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.3798258533937844, "rank": 1.0, "portrait_scores": [ 0.7873578021790224, -0.02770609539145366 ], "portrait_score_gap": 0.815063897570476, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3599551634000384, "rank": 2.0, "portrait_scores": [ 0.09461817750995859, 0.6252921492901182 ], "portrait_score_gap": 0.5306739717801596, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.23941716687082182, "rank": 3.0, "portrait_scores": [ 1.0424851487988127, -0.5636508150571691 ], "portrait_score_gap": 1.6061359638559818, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.37462365215645116, "rank": 11.0, "portrait_scores": [ -0.5811769774975934, -0.1680703268153089 ], "portrait_score_gap": 0.4131066506822846, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.49595921613685756, "rank": 12.0, "portrait_scores": [ -0.01681240740244498, -0.9751060248712702 ], "portrait_score_gap": 0.9582936174688252, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.13193798447273392, "rank": 5.0, "portrait_scores": [ 0.31816832522057836, -0.054292356275110504 ], "portrait_score_gap": 0.3724606814956889, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0662966295701812, "rank": 7.0, "portrait_scores": [ 0.2659173843065428, -0.13332412516618042 ], "portrait_score_gap": 0.39924150947272324, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.244337813821957, "rank": 10.0, "portrait_scores": [ -0.5751237329039705, 0.08644810526005649 ], "portrait_score_gap": 0.661571838164027, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.0025331266354489956, "rank": 8.0, "portrait_scores": [ 0.3223232838074066, -0.3273895370783046 ], "portrait_score_gap": 0.6497128208857113, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.23176226517134157, "rank": 9.0, "portrait_scores": [ -0.26615431927661387, -0.19737021106606925 ], "portrait_score_gap": 0.06878410821054462, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.20707827403818344, "rank": 4.0, "portrait_scores": [ 0.2803411990077478, 0.13381534906861908 ], "portrait_score_gap": 0.1465258499391287, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.09725595794298547, "rank": 6.0, "portrait_scores": [ -0.46897348854749654, 0.6634854044334675 ], "portrait_score_gap": 1.132458892980964, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8393385093802779, "rank": 13.0, "portrait_scores": [ -0.5745789139272125, -1.1040981048333434 ], "portrait_score_gap": 0.5295191909061309, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "10.0": 512, "11.0": 1536, "12.0": 1536, "13.0": 4608 }, "target_primary_score": -0.8393385093802779, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.4383148833399725, -0.6698268072415561 ], "target_alternate_panel_score": -0.5540708452907643, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5515377186553153, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.17944719313431318, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9140260086908027, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9338966986845487, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.14156884813177417, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, "consistency_ablation": [ { "name": "gamma_0", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5317102193123868, "brier_score": 0.15301175117822444, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.010636637793838, "brier_score": 0.3772109732126972, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8797446109401741, "brier_score": 0.309520954622169, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2724206915011664, "brier_score": 0.5125910103937537, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.36803611913974005, "brier_score": 0.1011224482297331, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.1202243928859767, "margin": -0.5173669920198206, "p_left": 0.37346812484778835, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.0025331266354489956, "right_score": -0.8393385093802779, "margin": 0.8368053827448289, "p_left": 0.6977919664036878, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1966396457896504, "brier": 0.48691362837752533 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.174867975786096, "right_score": 0.45103328257425607, "margin": -0.2761653067881601, "p_left": 0.43139415182525515, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.1861009142290313 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.859079675898924, "margin": -1.5672691690555438, "p_left": 0.1726060418174397, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.18947432715388196, "brier": 0.029792845671883738 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.859079675898924, "margin": -1.2562222750327678, "p_left": 0.2216248919699548, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2505467270003285, "brier": 0.04911759274069414 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.0025331266354489956, "right_score": -0.37462365215645116, "margin": 0.3720905255210022, "p_left": 0.5919640263529361, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8963999377611955, "brier": 0.35042140849597964 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.3971425991338439, "right_score": 0.06883873998264661, "margin": -0.46598133911649053, "p_left": 0.3855678467111719, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48705676569112055, "brier": 0.14866256441748973 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.244337813821957, "right_score": 0.3599551634000384, "margin": -0.6042929772219954, "p_left": 0.3533621435112404, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.040261845628836, "brier": 0.41814051744437764 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0662966295701812, "right_score": 0.13193798447273392, "margin": -0.06564135490255271, "p_left": 0.4835955511250581, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7265063597770781, "brier": 0.26667355481783256 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.2376283470924767, "margin": 0.6886616296667327, "p_left": 0.665669133504202, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40696252842627256, "brier": 0.11177712829183112 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.16062358048092495, "margin": 0.22946232046357157, "p_left": 0.557115192780389, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5849832511364297, "brier": 0.19614695246595198 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": -0.37462365215645116, "margin": -0.4647148572238267, "p_left": 0.38586792641078876, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9522601276247856, "brier": 0.3771582038109844 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06883873998264661, "right_score": 0.174867975786096, "margin": -0.10602923580344939, "p_left": 0.4735174965325325, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6415371797409833, "brier": 0.22421881952243694 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.859079675898924, "right_score": 0.174867975786096, "margin": 0.684211700112828, "p_left": 0.6646780560445341, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40845248180205285, "brier": 0.11244080609807261 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.13193798447273392, "right_score": 0.20707827403818344, "margin": -0.07514028956544952, "p_left": 0.4812237610965914, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7314229172632362, "brier": 0.2691287860507665 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3414369310004953, "right_score": 0.174867975786096, "margin": -0.5163049067865914, "p_left": 0.3737166752152307, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46795241476966365, "brier": 0.13966415333392623 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1202243928859767, "right_score": -0.3971425991338439, "margin": 0.5173669920198206, "p_left": 0.6265318751522116, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4675556278042503, "brier": 0.13947844027732323 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.7081894931566199, "margin": 0.8284138860425967, "p_left": 0.6960194499561154, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7765846167202332, "brier": 0.038423624761098035 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.2376283470924767, "margin": 0.3578527399784534, "p_left": 0.5885205437582018, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8879961841895156, "brier": 0.34635643042544956 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2376283470924767, "right_score": -0.3414369310004953, "margin": 0.10380858390801861, "p_left": 0.5259288655567775, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7463978958818103, "brier": 0.27660117162583897 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06883873998264661, "right_score": -0.13256827664104387, "margin": 0.20140701662369048, "p_left": 0.5501822327701553, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5975057233246752, "brier": 0.20233602371564272 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.20707827403818344, "right_score": -0.244337813821957, "margin": 0.4514160878601404, "p_left": 0.610975868727525, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9441139031682542, "brier": 0.3732915121673538 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.3971425991338439, "margin": 0.055705668133348596, "p_left": 0.5139228168702084, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6656821865334857, "brier": 0.236271027959393 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.49595921613685756, "right_score": 0.23941716687082182, "margin": -0.7353763830076794, "p_left": 0.3240160277788788, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1269622959600016, "brier": 0.45695433069984553 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.45103328257425607, "right_score": 0.174867975786096, "margin": 0.2761653067881601, "p_left": 0.5686058481747448, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5645677946004993, "brier": 0.18610091422903133 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.859079675898924, "right_score": -0.16062358048092495, "margin": 1.019703256379849, "p_left": 0.7349147933763889, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30800071386846106, "brier": 0.07027016677068261 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3599551634000384, "right_score": 0.09725595794298547, "margin": 0.26269920545705294, "p_left": 0.5652997001457829, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8330984511918679, "brier": 0.3195637509849121 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.2376283470924767, "margin": 0.5411735074699813, "p_left": 0.6320853632309412, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9999043330227885, "brier": 0.3995319064107909 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.3035451603775046, "right_score": -0.7081894931566199, "margin": 1.0117346535341245, "p_left": 0.7333594864800296, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31011926555790437, "brier": 0.07109716345019353 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3414369310004953, "right_score": -0.13256827664104387, "margin": -0.20886865435945146, "p_left": 0.4479718478670522, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5941562337646624, "brier": 0.20067877648142135 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1202243928859767, "right_score": -0.16062358048092495, "margin": 0.28084797336690165, "p_left": 0.5697541036356324, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7029743950459738, "brier": 0.004865634974010544 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.16062358048092495, "right_score": 0.3035451603775046, "margin": -0.4641687408584296, "p_left": 0.3859973497829098, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4877560345298307, "brier": 0.14899395403943 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.7081894931566199, "margin": 1.5672691690555438, "p_left": 0.8273939581825605, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1894743271538819, "brier": 0.02979284567188369 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7081894931566199, "right_score": 0.45103328257425607, "margin": -1.159222775730876, "p_left": 0.2388085395414515, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2728703621483618, "brier": 0.05702951855792101 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3599551634000384, "right_score": -0.8393385093802779, "margin": 1.1992936727803163, "p_left": 0.768399107982165, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4627396817456582, "brier": 0.590437189147787 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.45103328257425607, "right_score": -0.13256827664104387, "margin": 0.5836015592152999, "p_left": 0.6418957033408885, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4433294443448714, "brier": 0.1282386872857169 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.3798258533937844, "right_score": -0.23176226517134157, "margin": 0.611588118565126, "p_left": 0.6483029882243924, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7391911767709232, "brier": 0.02199377631628426 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.13256827664104387, "right_score": 0.3035451603775046, "margin": -0.4361134370185485, "p_left": 0.39266744806662623, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49867877643922714, "brier": 0.1541877247711566 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8393385093802779, "right_score": 0.3798258533937844, "margin": -1.2191643627740623, "p_left": 0.2280835406349222, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4780433108445714, "brier": 0.5958550202387178 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.859079675898924, "right_score": -0.3971425991338439, "margin": 1.2562222750327678, "p_left": 0.7783751080300452, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25054672700032854, "brier": 0.049117592740694146 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 0.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.3798258533937844, "rank": 1.0, "portrait_scores": [ 0.7873578021790224, -0.02770609539145366 ], "portrait_score_gap": 0.815063897570476, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3599551634000384, "rank": 2.0, "portrait_scores": [ 0.09461817750995859, 0.6252921492901182 ], "portrait_score_gap": 0.5306739717801596, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.23941716687082182, "rank": 3.0, "portrait_scores": [ 1.0424851487988127, -0.5636508150571691 ], "portrait_score_gap": 1.6061359638559818, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.37462365215645116, "rank": 11.0, "portrait_scores": [ -0.5811769774975934, -0.1680703268153089 ], "portrait_score_gap": 0.4131066506822846, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.49595921613685756, "rank": 12.0, "portrait_scores": [ -0.01681240740244498, -0.9751060248712702 ], "portrait_score_gap": 0.9582936174688252, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.13193798447273392, "rank": 5.0, "portrait_scores": [ 0.31816832522057836, -0.054292356275110504 ], "portrait_score_gap": 0.3724606814956889, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0662966295701812, "rank": 7.0, "portrait_scores": [ 0.2659173843065428, -0.13332412516618042 ], "portrait_score_gap": 0.39924150947272324, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.244337813821957, "rank": 10.0, "portrait_scores": [ -0.5751237329039705, 0.08644810526005649 ], "portrait_score_gap": 0.661571838164027, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.0025331266354489956, "rank": 8.0, "portrait_scores": [ 0.3223232838074066, -0.3273895370783046 ], "portrait_score_gap": 0.6497128208857113, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.23176226517134157, "rank": 9.0, "portrait_scores": [ -0.26615431927661387, -0.19737021106606925 ], "portrait_score_gap": 0.06878410821054462, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.20707827403818344, "rank": 4.0, "portrait_scores": [ 0.2803411990077478, 0.13381534906861908 ], "portrait_score_gap": 0.1465258499391287, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.09725595794298547, "rank": 6.0, "portrait_scores": [ -0.46897348854749654, 0.6634854044334675 ], "portrait_score_gap": 1.132458892980964, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8393385093802779, "rank": 13.0, "portrait_scores": [ -0.5745789139272125, -1.1040981048333434 ], "portrait_score_gap": 0.5295191909061309, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "10.0": 512, "11.0": 1536, "12.0": 1536, "13.0": 4608 }, "target_primary_score": -0.8393385093802779, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.4383148833399725, -0.6698268072415561 ], "target_alternate_panel_score": -0.5540708452907643, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5515377186553153, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.17944719313431318, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9140260086908027, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9338966986845487, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.14156884813177417, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.01", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5324179653392558, "brier_score": 0.15330514961968203, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.0076945826857961, "brier_score": 0.37590552165006846, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.876217355386598, "brier_score": 0.3078767318605964, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2706490372841923, "brier_score": 0.5119631012290128, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3689124285899755, "brier_score": 0.10146204234348814, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.39849979709959077, "right_score": 0.12224346983511122, "margin": -0.520743266934702, "p_left": 0.3726784494499539, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46629603001386327, "brier": 0.13888922668442183 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.0005660291449432753, "right_score": -0.8350060811942714, "margin": 0.8344400520493281, "p_left": 0.697292936162393, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1949897271262937, "brier": 0.48621743882197105 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.17495572118976774, "right_score": 0.4449103683560026, "margin": -0.26995464716623485, "p_left": 0.4329182291229927, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5672517689383171, "brier": 0.187418193106988 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.7035238435806821, "right_score": 0.8550962648159455, "margin": -1.5586201083966276, "p_left": 0.17384474165658356, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.19097255902805058, "brier": 0.03022199420164428 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.39849979709959077, "right_score": 0.8550962648159455, "margin": -1.2535960619155362, "p_left": 0.22207826408925876, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2511293563796711, "brier": 0.04931875538089856 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.0005660291449432753, "right_score": -0.3687103462750547, "margin": 0.3681443171301114, "p_left": 0.5910105040429592, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.894065805525754, "brier": 0.34929341588911267 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.39849979709959077, "right_score": 0.06899272329543589, "margin": -0.46749252039502665, "p_left": 0.38520990185049375, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4864743732549657, "brier": 0.14838666848366702 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.24072295357529608, "right_score": 0.35865930017990566, "margin": -0.5993822537552017, "p_left": 0.3544850370929999, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0370891423831279, "brier": 0.41668956733682566 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.06986212375851444, "right_score": 0.1294294298872608, "margin": -0.05956730612874636, "p_left": 0.4851125752502327, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7233743010609162, "brier": 0.2651090601654472 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.4449103683560026, "right_score": -0.23845876741422029, "margin": 0.683369135770223, "p_left": 0.6644902382961858, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.408735091235934, "brier": 0.11256680019855018 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06899272329543589, "right_score": -0.15962807647655242, "margin": 0.2286207997719883, "p_left": 0.5569075478023021, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5853560352330685, "brier": 0.19633092119456919 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8350060811942714, "right_score": -0.3687103462750547, "margin": -0.4662957349192167, "p_left": 0.38549336736493117, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.9532312914058025, "brier": 0.3776184015524914 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06899272329543589, "right_score": 0.17495572118976774, "margin": -0.10596299789433185, "p_left": 0.4735340095846038, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6415685450967791, "brier": 0.22423445823327162 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8550962648159455, "right_score": 0.17495572118976774, "margin": 0.6801405436261777, "p_left": 0.663770064563653, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4098194777850962, "brier": 0.11305056948353011 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.1294294298872608, "right_score": 0.20462030123041197, "margin": -0.07519087134315117, "p_left": 0.4812111334966231, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7314491582069926, "brier": 0.2691418880078586 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.33766107350263375, "right_score": 0.17495572118976774, "margin": -0.5126167946924015, "p_left": 0.3745802883982379, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4693323160640412, "brier": 0.1403103924565071 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.12224346983511122, "right_score": -0.39849979709959077, "margin": 0.520743266934702, "p_left": 0.6273215505500462, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4662960300138633, "brier": 0.13888922668442175 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.12224346983511122, "right_score": -0.7035238435806821, "margin": 0.8257673134157933, "p_left": 0.6954592073974686, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7760665782429503, "brier": 0.03820430175644664 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.12224346983511122, "right_score": -0.23845876741422029, "margin": 0.3607022372493315, "p_left": 0.5892104152589257, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.889674154849253, "brier": 0.34716891344959566 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.23845876741422029, "right_score": -0.33766107350263375, "margin": 0.09920230608841346, "p_left": 0.5247802577767083, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7439779667139905, "brier": 0.27539431895218847 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06899272329543589, "right_score": -0.12984911283263564, "margin": 0.19884183612807155, "p_left": 0.549547316046227, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.598660401391033, "brier": 0.20290762048115774 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.20462030123041197, "right_score": -0.24072295357529608, "margin": 0.445343254805708, "p_left": 0.609531482322704, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9404079334971038, "brier": 0.37152862794251285 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.33766107350263375, "right_score": -0.39849979709959077, "margin": 0.06083872359695702, "p_left": 0.5152049912719611, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6631904162110471, "brier": 0.23502620048761935 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.48818041505355453, "right_score": 0.23130146528108453, "margin": -0.719481880334639, "p_left": 0.3275070866167795, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.1162455856164222, "brier": 0.4522467185506517 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.4449103683560026, "right_score": 0.17495572118976774, "margin": 0.26995464716623485, "p_left": 0.5670817708770073, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5672517689383171, "brier": 0.187418193106988 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8550962648159455, "right_score": -0.15962807647655242, "margin": 1.014724341292498, "p_left": 0.7339436920209692, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.30932296717816987, "brier": 0.07078595901543289 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.35865930017990566, "right_score": 0.10535727247938584, "margin": 0.2533020277005198, "p_left": 0.5629890748783519, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8277970839206779, "brier": 0.31695669843238256 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.3014221234140517, "right_score": -0.23845876741422029, "margin": 0.539880890828272, "p_left": 0.6317847094720893, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.999087483267275, "brier": 0.3991519191227323 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.3014221234140517, "right_score": -0.7035238435806821, "margin": 1.0049459669947338, "p_left": 0.7320299027454265, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3119339151100764, "brier": 0.0718079730226256 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.33766107350263375, "right_score": -0.12984911283263564, "margin": -0.2078119606699981, "p_left": 0.4482331752379236, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5946297408583241, "brier": 0.20091297938387112 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.12224346983511122, "right_score": -0.15962807647655242, "margin": 0.28187154631166367, "p_left": 0.5700049986040965, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7030459218669536, "brier": 0.00490069982955955 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.15962807647655242, "right_score": 0.3014221234140517, "margin": -0.46105019989060414, "p_left": 0.38673671684404426, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.488960935815655, "brier": 0.14956528815531048 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8550962648159455, "right_score": -0.7035238435806821, "margin": 1.5586201083966276, "p_left": 0.8261552583434165, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.19097255902805066, "brier": 0.030221994201644252 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.7035238435806821, "right_score": 0.4449103683560026, "margin": -1.1484342119366846, "p_left": 0.24077519683606882, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2754573620884491, "brier": 0.057972695411447686 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.35865930017990566, "right_score": -0.8350060811942714, "margin": 1.193665381374177, "p_left": 0.7673959737287622, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4584177291968372, "brier": 0.5888965804951152 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.4449103683560026, "right_score": -0.12984911283263564, "margin": 0.5747594811886383, "p_left": 0.6398606736899476, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4465048236873861, "brier": 0.1297003343550584 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.3814552002604174, "right_score": -0.23093251937011616, "margin": 0.6123877196305336, "p_left": 0.6484852806172366, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.739309832881789, "brier": 0.022047878559979514 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.12984911283263564, "right_score": 0.3014221234140517, "margin": -0.4312712362466874, "p_left": 0.39382281298227934, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5005829478317894, "brier": 0.15509640802527536 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8350060811942714, "right_score": 0.3814552002604174, "margin": -1.216461281454689, "p_left": 0.2285597987877002, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.475957401407836, "brier": 0.5951199840464737 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8550962648159455, "right_score": -0.39849979709959077, "margin": 1.2535960619155362, "p_left": 0.7779217359107412, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25112935637967104, "brier": 0.04931875538089857 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 0.01, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.3814552002604174, "rank": 1.0, "portrait_scores": [ 0.7841046575277215, -0.021194257006886708 ], "portrait_score_gap": 0.8052989145346082, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.35865930017990566, "rank": 2.0, "portrait_scores": [ 0.09710390738425828, 0.620214692975553 ], "portrait_score_gap": 0.5231107855912946, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.23130146528108453, "rank": 3.0, "portrait_scores": [ 1.0281403846548691, -0.5655374540927001 ], "portrait_score_gap": 1.5936778387475692, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3687103462750547, "rank": 11.0, "portrait_scores": [ -0.5774527729763594, -0.15996791957374995 ], "portrait_score_gap": 0.4174848534026095, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.48818041505355453, "rank": 12.0, "portrait_scores": [ -0.012657143436211063, -0.963703686670898 ], "portrait_score_gap": 0.951046543234687, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.1294294298872608, "rank": 5.0, "portrait_scores": [ 0.31233554079366194, -0.053476681019140354 ], "portrait_score_gap": 0.36581222181280226, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.06986212375851444, "rank": 7.0, "portrait_scores": [ 0.269425444253127, -0.1297011967360981 ], "portrait_score_gap": 0.39912664098922507, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.24072295357529608, "rank": 10.0, "portrait_scores": [ -0.5717695927934994, 0.09032368564290721 ], "portrait_score_gap": 0.6620932784364066, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.5, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.0005660291449432753, "rank": 8.0, "portrait_scores": [ 0.32347814388555335, -0.3246102021754399 ], "portrait_score_gap": 0.6480883460609932, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.23093251937011616, "rank": 9.0, "portrait_scores": [ -0.25895746101000827, -0.20290757773022408 ], "portrait_score_gap": 0.05604988327978419, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.20462030123041197, "rank": 4.0, "portrait_scores": [ 0.27936011871858024, 0.1298804837422437 ], "portrait_score_gap": 0.14947963497633654, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.10535727247938584, "rank": 6.0, "portrait_scores": [ -0.454958149392705, 0.6656726943514767 ], "portrait_score_gap": 1.1206308437441816, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8350060811942714, "rank": 13.0, "portrait_scores": [ -0.5733894630627333, -1.0966226993258095 ], "portrait_score_gap": 0.5232332362630762, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "11.0": 1024, "12.0": 2048, "13.0": 5120 }, "target_primary_score": -0.8350060811942714, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.44063532753151746, -0.6719373783024293 ], "target_alternate_panel_score": -0.5562863529169734, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5557203237720301, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.18757600664191865, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.914945653096879, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9377415531773907, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.13937158362850186, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_0.1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.537889153740508, "brier_score": 0.1556001116699543, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.9872854052204229, "brier_score": 0.3667224529067475, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 0.8507485390991081, "brier_score": 0.2958819095610288, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2603591374630527, "brier_score": 0.5084035395981851, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.3755271441445376, "brier_score": 0.1040213782607071, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.1383639539656737, "margin": -0.5463927304335155, "p_left": 0.36670172733940004, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4568137629850261, "brier": 0.1344701568336997 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.012064661072830885, "right_score": -0.8093960293656783, "margin": 0.8214606904385092, "p_left": 0.6945463161442109, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1859571195374503, "brier": 0.4823945852694942 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.175316082258721, "right_score": 0.40099994434125463, "margin": -0.22568386208253363, "p_left": 0.4438172960633887, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5866584345075525, "brier": 0.19697379228501763 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.6665261101000797, "right_score": 0.8251507367526193, "margin": -1.491676846852699, "p_left": 0.18367017527195886, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2029368087129817, "brier": 0.03373473328443209 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.8251507367526193, "margin": -1.233179513220461, "p_left": 0.2256254210847683, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.25569957037259006, "brier": 0.050906830639679004 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.012064661072830885, "right_score": -0.32728625106482434, "margin": 0.3393509121376552, "p_left": 0.5840328431220481, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8771489716501898, "brier": 0.3410943618452228 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.40802877646784186, "right_score": 0.0696335543067845, "margin": -0.4776623307746264, "p_left": 0.3828042835103552, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4825690987478617, "brier": 0.14653911947387643 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.21330345767726425, "right_score": 0.3452807228167657, "margin": -0.5585841804940299, "p_left": 0.3638751162787647, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.010944557359507, "brier": 0.40465486768935516 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.09357621247170457, "right_score": 0.10816787852889424, "margin": -0.014591666057189673, "p_left": 0.49635214820952556, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.700469627942222, "brier": 0.2536611586131597 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.40099994434125463, "right_score": -0.2436158504616248, "margin": 0.6446157948028794, "p_left": 0.6557961206947117, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4219053299676887, "brier": 0.11847631052880947 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.0696335543067845, "right_score": -0.15289731173500143, "margin": 0.22253086604178593, "p_left": 0.5554042700966442, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5880590158123018, "brier": 0.19766536304829774 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8093960293656783, "right_score": -0.32728625106482434, "margin": -0.482109778300854, "p_left": 0.3817540557237506, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.962978710909049, "brier": 0.38222804761403123 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.0696335543067845, "right_score": 0.175316082258721, "margin": -0.1056825279519365, "p_left": 0.47360393113397375, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6417013669585916, "brier": 0.22430068358555375 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8251507367526193, "right_score": 0.175316082258721, "margin": 0.6498346544938983, "p_left": 0.6569732014743254, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.42011205056181133, "brier": 0.11766738450677378 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.10816787852889424, "right_score": 0.18621017697538225, "margin": -0.07804229844648801, "p_left": 0.48049932195352746, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7329294616996797, "brier": 0.2698809544907448 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.31212044802441685, "right_score": 0.175316082258721, "margin": -0.4874365302831378, "p_left": 0.3804976412694458, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47883877044552453, "brier": 0.14477845501161185 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.1383639539656737, "right_score": -0.40802877646784186, "margin": 0.5463927304335155, "p_left": 0.6332982726605999, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4568137629850261, "brier": 0.13447015683369973 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.6665261101000797, "margin": 0.8048900640657534, "p_left": 0.691019540318562, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7720322093279754, "brier": 0.03648846478351475 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.2436158504616248, "margin": 0.3819798044272985, "p_left": 0.594350518804801, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9022658390887589, "brier": 0.35325253920353616 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.2436158504616248, "right_score": -0.31212044802441685, "margin": 0.06850459756279206, "p_left": 0.5171194549521316, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7279859746593611, "brier": 0.2674125306899897 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.0696335543067845, "right_score": -0.10969199922312739, "margin": 0.1793255535299119, "p_left": 0.5447116341124956, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.607498736068087, "brier": 0.2072874961125141 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.18621017697538225, "right_score": -0.21330345767726425, "margin": 0.3995136346526465, "p_left": 0.5985708000046159, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9127240998860848, "brier": 0.35828700261816593 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.31212044802441685, "right_score": -0.40802877646784186, "margin": 0.095908328443425, "p_left": 0.5239587197535264, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6463423768606547, "brier": 0.22661530049870166 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.4329683211657527, "right_score": 0.1743640285925066, "margin": -0.6073323497582592, "p_left": 0.35266796489516494, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0422282740615614, "brier": 0.4190387636729674 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.40099994434125463, "right_score": 0.175316082258721, "margin": 0.22568386208253363, "p_left": 0.5561827039366113, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5866584345075525, "brier": 0.19697379228501763 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8251507367526193, "right_score": -0.15289731173500143, "margin": 0.9780480484876207, "p_left": 0.7267207348460247, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31921300888367254, "brier": 0.07468155676309672 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.3452807228167657, "right_score": 0.16020782739207254, "margin": 0.18507289542469316, "p_left": 0.5461366097984767, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7899590288608884, "brier": 0.29826519656217354 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.2834162243870383, "right_score": -0.2436158504616248, "margin": 0.5270320748486631, "p_left": 0.6287906249336642, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9909890222835912, "brier": 0.395377650004468 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.2834162243870383, "right_score": -0.6665261101000797, "margin": 0.949942334487118, "p_left": 0.721103580874529, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3269724892216227, "brier": 0.0777832126010104 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.31212044802441685, "right_score": -0.10969199922312739, "margin": -0.20242844880128946, "p_left": 0.4495649943921943, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5970463940805371, "brier": 0.20210868418285366 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.1383639539656737, "right_score": -0.15289731173500143, "margin": 0.2912612657006751, "p_left": 0.5723048835802026, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7037140492016809, "brier": 0.0052279961895466505 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.15289731173500143, "right_score": 0.2834162243870383, "margin": -0.4363135361220397, "p_left": 0.3926197295128116, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49860020880915296, "brier": 0.15415025200271334 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8251507367526193, "right_score": -0.6665261101000797, "margin": 1.491676846852699, "p_left": 0.8163298247280412, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2029368087129817, "brier": 0.03373473328443208 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.6665261101000797, "right_score": 0.40099994434125463, "margin": -1.0675260544413343, "p_left": 0.25587384483647074, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2955446951314519, "brier": 0.06547142447139831 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.3452807228167657, "right_score": -0.8093960293656783, "margin": 1.1546767521824441, "p_left": 0.7603641077893023, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.4286346234143417, "brier": 0.5781535764142217 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.40099994434125463, "right_score": -0.10969199922312739, "margin": 0.510691943564382, "p_left": 0.624968668179239, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4700537614155492, "brier": 0.14064849984725372 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.39136331813372094, "right_score": -0.22287017515185603, "margin": 0.614233493285577, "p_left": 0.6489059132353077, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7395842913327311, "brier": 0.022172970996440975 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.10969199922312739, "right_score": 0.2834162243870383, "margin": -0.3931082236101657, "p_left": 0.4029692836067714, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5157867156681659, "brier": 0.16238424353055458 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8093960293656783, "right_score": 0.39136331813372094, "margin": -1.2007593474993992, "p_left": 0.23134016034724877, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4638660959913699, "brier": 0.5908379490949932 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8251507367526193, "right_score": -0.40802877646784186, "margin": 1.233179513220461, "p_left": 0.7743745789152318, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25569957037258995, "brier": 0.050906830639678984 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 0.1, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.39136331813372094, "rank": 1.0, "portrait_scores": [ 0.7593631533747329, 0.02336348289270901 ], "portrait_score_gap": 0.735999670482024, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.3452807228167657, "rank": 2.0, "portrait_scores": [ 0.11168431678718208, 0.5788771288463493 ], "portrait_score_gap": 0.4671928120591673, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.064453125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.1743640285925066, "rank": 4.0, "portrait_scores": [ 0.9288758539147842, -0.580147796729771 ], "portrait_score_gap": 1.5090236506445551, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 6.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.32728625106482434, "rank": 11.0, "portrait_scores": [ -0.5532602267590956, -0.1013122753705531 ], "portrait_score_gap": 0.4519479513885425, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.4329683211657527, "rank": 12.0, "portrait_scores": [ 0.01754641849382503, -0.8834830608253303 ], "portrait_score_gap": 0.9010294793191553, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.10816787852889424, "rank": 6.0, "portrait_scores": [ 0.2655533770269189, -0.049217619969130405 ], "portrait_score_gap": 0.31477099699604927, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] }, { "identity": "gal_gadot", "mean_score": 0.09357621247170457, "rank": 7.0, "portrait_scores": [ 0.29090770762570406, -0.10375528268229492 ], "portrait_score_gap": 0.394662990307999, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -0.21330345767726425, "rank": 9.0, "portrait_scores": [ -0.5440786643784081, 0.11747174902387963 ], "portrait_score_gap": 0.6615504134022878, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 9.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": 0.012064661072830885, "rank": 8.0, "portrait_scores": [ 0.32870223162099277, -0.304572909475331 ], "portrait_score_gap": 0.6332751410963238, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.22287017515185603, "rank": 10.0, "portrait_scores": [ -0.2085235237962932, -0.23721682650741888 ], "portrait_score_gap": 0.02869330271112569, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.18621017697538225, "rank": 3.0, "portrait_scores": [ 0.26927849352959415, 0.10314186042117035 ], "portrait_score_gap": 0.1661366331084238, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.16020782739207254, "rank": 5.0, "portrait_scores": [ -0.358031092650626, 0.6784467474347711 ], "portrait_score_gap": 1.0364778400853971, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 5.5, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8093960293656783, "rank": 13.0, "portrait_scores": [ -0.5714490027811948, -1.047343055950162 ], "portrait_score_gap": 0.47589405316896716, "single_photo_rank_min": 11.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "11.0": 1024, "12.0": 2048, "13.0": 5120 }, "target_primary_score": -0.8093960293656783, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.45990054053268736, -0.6918943345225459 ], "target_alternate_panel_score": -0.5758974375276167, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.5879620986004476, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.24861118646279234, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9211781603443824, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.9672607556613376, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.12453731249506285, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_1", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.5630007884937331, "brier_score": 0.16662319697616015, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.9338916373101772, "brier_score": 0.34139572665979023, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.7697941217261942, "brier_score": 0.25671861369510607, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2620866684781435, "brier_score": 0.5107499525891587, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.4053898594868622, "brier_score": 0.11566781052547921, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.42400256814521875, "right_score": 0.2194251986640749, "margin": -0.6434277668092936, "p_left": 0.34447209958930464, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4223144131285856, "brier": 0.11866102739546382 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": 0.015941712525307516, "right_score": -0.8161781883070665, "margin": 0.832119900832374, "p_left": 0.6968029850353004, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1933724703656172, "brier": 0.485534399954105 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.1748211790820644, "right_score": 0.2601502817077501, "margin": -0.08532910262568572, "p_left": 0.478680658383683, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6513924852370174, "brier": 0.22913517271063624 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.4885565345098273, "right_score": 0.6989251425082067, "margin": -1.187481677018034, "p_left": 0.23370963835066902, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.26619411886208016, "brier": 0.054620195058000504 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.42400256814521875, "right_score": 0.6989251425082067, "margin": -1.1229277106534254, "p_left": 0.245468627196667, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.28165842071976543, "brier": 0.06025484693781629 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": 0.015941712525307516, "right_score": -0.22106703166208214, "margin": 0.23700874418738965, "p_left": 0.5589763701522731, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8186568225496282, "brier": 0.312454582388611 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.42400256814521875, "right_score": 0.06478030849409848, "margin": -0.48878287663931724, "p_left": 0.38018033270214874, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47832670244775416, "brier": 0.1445370853735165 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.1092099208334803, "right_score": 0.23699381818056528, "margin": -0.34620373901404555, "p_left": 0.41430330618478023, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8811569497597795, "brier": 0.34304061714607925 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.12058168422457266, "right_score": -0.011774759312362056, "margin": 0.1323564435369347, "p_left": 0.5330408901405371, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.62915714079728, "brier": 0.2180508102807419 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.2601502817077501, "right_score": -0.23365528264128968, "margin": 0.49380556434903977, "p_left": 0.6210025153955859, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4764201465003627, "brier": 0.14363909333647307 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.06478030849409848, "right_score": -0.14778430687553185, "margin": 0.21256461536963034, "p_left": 0.5529419609226807, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5925022360913098, "brier": 0.19986089030365797 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8161781883070665, "right_score": -0.22106703166208214, "margin": -0.5951111566449843, "p_left": 0.3554629788912848, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.03433417330144, "brier": 0.4154279715796963 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.06478030849409848, "right_score": 0.1748211790820644, "margin": -0.11004087058796592, "p_left": 0.4725175088660109, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6396396063458414, "brier": 0.2232727961849407 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.6989251425082067, "right_score": 0.1748211790820644, "margin": 0.5241039634261423, "p_left": 0.6281069083855119, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4650448907198278, "brier": 0.13830447159058207 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.011774759312362056, "right_score": 0.11763628534176142, "margin": -0.12941104465412348, "p_left": 0.4676923149113854, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7599446460479332, "brier": 0.28335147160439966 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.27078157162628924, "right_score": 0.1748211790820644, "margin": -0.44560275070835365, "p_left": 0.3904067586674875, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49496336172412736, "brier": 0.15241743721325382 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.2194251986640749, "right_score": -0.42400256814521875, "margin": 0.6434277668092936, "p_left": 0.6555279004106954, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.42231441312858564, "brier": 0.11866102739546378 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.2194251986640749, "right_score": -0.4885565345098273, "margin": 0.7079817331739022, "p_left": 0.6699550433438648, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7545355349221168, "brier": 0.02888471675801496 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.2194251986640749, "right_score": -0.23365528264128968, "margin": 0.4530804813053646, "p_left": 0.6113713958644764, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9451311365760434, "brier": 0.3737749836812783 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.23365528264128968, "right_score": -0.27078157162628924, "margin": 0.03712628898499956, "p_left": 0.5092805062798199, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7118826103248709, "brier": 0.2593666340766297 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.06478030849409848, "right_score": -0.030100367135878894, "margin": 0.09488067562997737, "p_left": 0.5237023901550961, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6468317137294585, "brier": 0.22685941314396826 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.11763628534176142, "right_score": -0.1092099208334803, "margin": 0.2268462061752417, "p_left": 0.5564696022572475, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8129889390113962, "brier": 0.3096584182363392 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.27078157162628924, "right_score": -0.42400256814521875, "margin": 0.1532209965189295, "p_left": 0.5382304846604619, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6194684004122223, "brier": 0.21323108529691195 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.2619556884274239, "right_score": 0.02224487042692469, "margin": -0.2842005588543486, "p_left": 0.4294242553654656, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.845309908487528, "brier": 0.3255566803652534 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.2601502817077501, "right_score": 0.1748211790820644, "margin": 0.08532910262568572, "p_left": 0.521319341616317, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6513924852370174, "brier": 0.22913517271063624 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.6989251425082067, "right_score": -0.14778430687553185, "margin": 0.8467094493837386, "p_left": 0.6998764191495714, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3568515035965121, "brier": 0.09007416378248373 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.23699381818056528, "right_score": 0.26792328497705176, "margin": -0.030929466796486477, "p_left": 0.49226824966098, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6778020213851672, "brier": 0.24232802962428493 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.17677852047784087, "right_score": -0.23365528264128968, "margin": 0.4104338031191306, "p_left": 0.6011918921356696, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9192749104483343, "brier": 0.36143169116966656 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.17677852047784087, "right_score": -0.4885565345098273, "margin": 0.6653350549876682, "p_left": 0.6604578143015283, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.41482202639858445, "brier": 0.11528889586889546 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.27078157162628924, "right_score": -0.030100367135878894, "margin": -0.24068120449041036, "p_left": 0.44011848542940074, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5800300987633159, "brier": 0.19370428121666963 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.2194251986640749, "right_score": -0.14778430687553185, "margin": 0.36720950553960674, "p_left": 0.5907845248980063, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7099086749076313, "brier": 0.008241829960956718 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.14778430687553185, "right_score": 0.17677852047784087, "margin": -0.3245628273533727, "p_left": 0.41956415612144454, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5439760026795608, "brier": 0.1760340811018999 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.6989251425082067, "right_score": -0.4885565345098273, "margin": 1.187481677018034, "p_left": 0.766290361649331, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2661941188620802, "brier": 0.05462019505800052 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.4885565345098273, "right_score": 0.2601502817077501, "margin": -0.7487068162175774, "p_left": 0.32110314436377796, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3872860692418646, "brier": 0.10310722932030524 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.23699381818056528, "right_score": -0.8161781883070665, "margin": 1.0531720064876318, "p_left": 0.7413835461346955, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.3524091879658255, "brier": 0.5496495624792561 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.2601502817077501, "right_score": -0.030100367135878894, "margin": 0.290250648843629, "p_left": 0.5720574948107503, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5585157772502642, "brier": 0.18313478774765102 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.3902558229129481, "right_score": -0.18057260564232883, "margin": 0.5708284285552769, "p_left": 0.6389543087318237, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7333365457708415, "brier": 0.019308299915138973 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.030100367135878894, "right_score": 0.17677852047784087, "margin": -0.20687888761371975, "p_left": 0.4484639541793424, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5950480828221453, "brier": 0.2011199181981713 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8161781883070665, "right_score": 0.3902558229129481, "margin": -1.2064340112200145, "p_left": 0.23033261967030366, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.468230842279691, "brier": 0.5923878763435774 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.6989251425082067, "right_score": -0.42400256814521875, "margin": 1.1229277106534254, "p_left": 0.7545313728033329, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.28165842071976543, "brier": 0.06025484693781632 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 1.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.3902558229129481, "rank": 1.0, "portrait_scores": [ 0.6611157374079529, 0.11939590841794323 ], "portrait_score_gap": 0.5417198289900097, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.126953125, "single_photo_rank_quantiles": [ 1.0, 3.0, 7.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.23699381818056528, "rank": 3.0, "portrait_scores": [ 0.10481110664518528, 0.3691765297159453 ], "portrait_score_gap": 0.26436542307076, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.02224487042692469, "rank": 6.0, "portrait_scores": [ 0.6772960863377309, -0.6328063454838815 ], "portrait_score_gap": 1.3101024318216123, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.22106703166208214, "rank": 11.0, "portrait_scores": [ -0.5200521536296849, 0.07791809030552063 ], "portrait_score_gap": 0.5979702439352056, "single_photo_rank_min": 3.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.0, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.2619556884274239, "rank": 12.0, "portrait_scores": [ 0.13099241037228765, -0.6549037872271355 ], "portrait_score_gap": 0.7858961975994232, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.001953125, "single_photo_rank_quantiles": [ 3.0, 10.5, 12.0 ] }, { "identity": "emma_watson", "mean_score": -0.011774759312362056, "rank": 8.0, "portrait_scores": [ 0.03962944507014224, -0.06317896369486635 ], "portrait_score_gap": 0.10280840876500859, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "gal_gadot", "mean_score": 0.12058168422457266, "rank": 4.0, "portrait_scores": [ 0.26857180527805075, -0.027408436828905425 ], "portrait_score_gap": 0.29598024210695617, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 5.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.1092099208334803, "rank": 9.0, "portrait_scores": [ -0.4063821062591253, 0.1879622645921647 ], "portrait_score_gap": 0.59434437085129, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 8.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": 0.015941712525307516, "rank": 7.0, "portrait_scores": [ 0.26985923199438167, -0.23797580694376663 ], "portrait_score_gap": 0.5078350389381483, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.18057260564232883, "rank": 10.0, "portrait_scores": [ -0.08553430645005738, -0.27561090483460027 ], "portrait_score_gap": 0.19007659838454288, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.11763628534176142, "rank": 5.0, "portrait_scores": [ 0.18712724698350183, 0.048145323700021 ], "portrait_score_gap": 0.13898192328348083, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 5.5, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.26792328497705176, "rank": 2.0, "portrait_scores": [ -0.1266685697365228, 0.6625151396906264 ], "portrait_score_gap": 0.7891837094271492, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.0, 11.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8161781883070665, "rank": 13.0, "portrait_scores": [ -0.6642389141802727, -0.9681174624338602 ], "portrait_score_gap": 0.3038785482535875, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "13.0": 8192 }, "target_primary_score": -0.8161781883070665, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.5581282533989742, -0.8308722146045686 ], "target_alternate_panel_score": -0.6945002340017714, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.7104419465270789, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.4734332023396893, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.9314940521823367, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.0847560569147194, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.08720832300130268, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_10", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 16, "accuracy": 0.7272727272727273, "mean_cross_entropy": 0.6002482598050803, "brier_score": 0.1840418512529746, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.8969681737601384, "brier_score": 0.323580237768768, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.7188286893240761, "brier_score": 0.2315141882671751, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.2532471426322622, "brier_score": 0.5077123367719539, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.455994876431611, "brier_score": 0.13691558992914094, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.3981379974242783, "right_score": 0.33528308014035035, "margin": -0.7334210775646286, "p_left": 0.3244444449120122, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.39221988205026703, "brier": 0.1052641978342637 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.08709872879279984, "right_score": -0.8748608091234149, "margin": 0.7877620803306151, "p_left": 0.6873506039602721, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1626728567163287, "brier": 0.4724508527645509 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.17874669696988932, "right_score": 0.14197988641090553, "margin": 0.03676681055898379, "p_left": 0.5091906673356528, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7116995511176362, "brier": 0.2592751357017274 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.24411950760783882, "right_score": 0.5170971920534471, "margin": -0.7612166996612859, "p_left": 0.31838216585299944, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.38328613868755873, "brier": 0.10136720353324684 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.3981379974242783, "right_score": 0.5170971920534471, "margin": -0.9152351894777253, "p_left": 0.2859297512455217, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.33677393387098215, "brier": 0.08175582264732593 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.08709872879279984, "right_score": -0.20149712670498748, "margin": 0.11439839791218764, "p_left": 0.52856845001179, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7519813624460291, "brier": 0.2793846063478661 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.3981379974242783, "right_score": 0.04706970701108649, "margin": -0.4452077044353648, "p_left": 0.39050077953129336, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49511760903011665, "brier": 0.1524908588145478 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.04312396719311541, "right_score": 0.08888601757676598, "margin": -0.1320099847698814, "p_left": 0.46704534730836805, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.761328922583431, "brier": 0.28404066182565807 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.016718047790444134, "right_score": -0.15483494209358137, "margin": 0.1715529898840255, "p_left": 0.5427833711578657, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6110449867930781, "brier": 0.209047045689766 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.14197988641090553, "right_score": -0.14399314456483348, "margin": 0.285973030975739, "p_left": 0.5710099796614794, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5603485919631309, "brier": 0.18403243755004434 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.04706970701108649, "right_score": -0.2122319162642665, "margin": 0.259301623275353, "p_left": 0.5644646075922887, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5718775942715257, "brier": 0.18969107803973909 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8748608091234149, "right_score": -0.20149712670498748, "margin": -0.6733636824184275, "p_left": 0.33774406568194476, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.085466872445868, "brier": 0.4385829225394802 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.04706970701108649, "right_score": 0.17874669696988932, "margin": -0.13167698995880284, "p_left": 0.46712823528455555, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6294744752913242, "brier": 0.2182087882000631 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5170971920534471, "right_score": 0.17874669696988932, "margin": 0.33835049508355775, "p_left": 0.583789782908995, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5382143217276087, "brier": 0.17323094481094153 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.15483494209358137, "right_score": 0.04584213706592037, "margin": -0.20067707915950173, "p_left": 0.44999842003541013, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7985112072563575, "brier": 0.3025017379635451 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.3525939389400149, "right_score": 0.17874669696988932, "margin": -0.5313406359099042, "p_left": 0.37020426030671755, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.46235973487718024, "brier": 0.1370511943492439 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.33528308014035035, "right_score": -0.3981379974242783, "margin": 0.7334210775646286, "p_left": 0.6755555550879878, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.39221988205026714, "brier": 0.1052641978342637 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.33528308014035035, "right_score": -0.24411950760783882, "margin": 0.5794025877481892, "p_left": 0.6409299302017525, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7345364351684875, "brier": 0.019861245226670828 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.33528308014035035, "right_score": -0.14399314456483348, "margin": 0.47927622470518383, "p_left": 0.617576951213846, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9612278255989277, "brier": 0.38140129067058914 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.14399314456483348, "right_score": -0.3525939389400149, "margin": 0.20860079437518142, "p_left": 0.5519619112919828, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8028770307435611, "brier": 0.3046619515170987 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.04706970701108649, "right_score": 0.08553979549582993, "margin": -0.038470088484743446, "p_left": 0.4903836638224526, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7125672068594182, "brier": 0.259708810099027 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.04584213706592037, "right_score": -0.04312396719311541, "margin": 0.08896610425903578, "p_left": 0.5222268675867412, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7386192775399398, "brier": 0.27272090122945974 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.3525939389400149, "right_score": -0.3981379974242783, "margin": 0.045544058484263406, "p_left": 0.5113840468984661, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6706344115696485, "brier": 0.2387455496253204 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.1153701426058363, "right_score": -0.020729308327351426, "margin": -0.09464083427848488, "p_left": 0.4763574357789269, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.741586791044754, "brier": 0.27420153506402073 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.14197988641090553, "right_score": 0.17874669696988932, "margin": -0.03676681055898379, "p_left": 0.49080933266434723, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7116995511176362, "brier": 0.2592751357017274 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5170971920534471, "right_score": -0.2122319162642665, "margin": 0.7293291083177136, "p_left": 0.6746580327585125, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3935493346194716, "brier": 0.10584739564856109 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.08888601757676598, "right_score": 0.2338376146015055, "margin": -0.14495159702473953, "p_left": 0.4638254171507861, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6232954566737914, "brier": 0.21513401759510076 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.04536014671972321, "right_score": -0.14399314456483348, "margin": 0.1893532912845567, "p_left": 0.5471973864645528, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7922989801675794, "brier": 0.2994249797536372 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.04536014671972321, "right_score": -0.24411950760783882, "margin": 0.289479654327562, "p_left": 0.5718687389259528, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5588457913384874, "brier": 0.18329637670885396 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.3525939389400149, "right_score": 0.08553979549582993, "margin": -0.4381337344358448, "p_left": 0.39218575271400724, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.49788595802723007, "brier": 0.15380966463185244 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.33528308014035035, "right_score": -0.2122319162642665, "margin": 0.5475149964046169, "p_left": 0.6335588592395036, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.730159870548221, "brier": 0.017837968881357538 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.2122319162642665, "right_score": 0.04536014671972321, "margin": -0.2575920629839897, "p_left": 0.4359557243061937, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.572622527562075, "brier": 0.190057393555338 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5170971920534471, "right_score": -0.24411950760783882, "margin": 0.7612166996612859, "p_left": 0.6816178341470006, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3832861386875588, "brier": 0.10136720353324681 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.24411950760783882, "right_score": 0.14197988641090553, "margin": -0.3860993940187444, "p_left": 0.4046566437734619, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.518616970588661, "brier": 0.16374699935000242 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.08888601757676598, "right_score": -0.8748608091234149, "margin": 0.9637468267001809, "p_left": 0.7238713547527563, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2868884158903573, "brier": 0.5239897382315908 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.14197988641090553, "right_score": 0.08553979549582993, "margin": 0.0564400909150756, "p_left": 0.5141062783173987, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6653252677458815, "brier": 0.23609270877056918 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.344196176249795, "right_score": -0.15730848721995624, "margin": 0.5015046634697512, "p_left": 0.6228128674935333, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7242615102552277, "brier": 0.015083000421984159 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.08553979549582993, "right_score": 0.04536014671972321, "margin": 0.040179648776106726, "p_left": 0.5100435610331423, "prediction": "left", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7134387918969473, "brier": 0.26014443415136873 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8748608091234149, "right_score": 0.344196176249795, "margin": -1.2190569853732098, "p_left": 0.22810244620662665, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4779604254764953, "brier": 0.5958258335521937 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5170971920534471, "right_score": -0.3981379974242783, "margin": 0.9152351894777253, "p_left": 0.7140702487544783, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3367739338709821, "brier": 0.0817558226473259 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 10.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.344196176249795, "rank": 1.0, "portrait_scores": [ 0.5650293159578083, 0.12336303654178168 ], "portrait_score_gap": 0.44166627941602654, "single_photo_rank_min": 1.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.2578125, "single_photo_rank_quantiles": [ 1.0, 2.0, 6.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.08888601757676598, "rank": 3.0, "portrait_scores": [ 0.038985643134431244, 0.1387863920191007 ], "portrait_score_gap": 0.09980074888466946, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0078125, "single_photo_rank_quantiles": [ 2.0, 5.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.020729308327351426, "rank": 6.0, "portrait_scores": [ 0.5698227734878929, -0.6112813901425957 ], "portrait_score_gap": 1.1811041636304886, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 6.5, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.20149712670498748, "rank": 12.0, "portrait_scores": [ -0.5695031798189831, 0.1665089264090082 ], "portrait_score_gap": 0.7360121062279913, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 8.5, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.1153701426058363, "rank": 9.0, "portrait_scores": [ 0.2647769358477147, -0.49551722105938734 ], "portrait_score_gap": 0.760294156907102, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.0, 12.0 ] }, { "identity": "emma_watson", "mean_score": -0.15483494209358137, "rank": 10.0, "portrait_scores": [ -0.2012661703792185, -0.10840371380794422 ], "portrait_score_gap": 0.09286245657127429, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 9.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": 0.016718047790444134, "rank": 5.0, "portrait_scores": [ 0.023839867960091257, 0.00959622762079701 ], "portrait_score_gap": 0.014243640339294247, "single_photo_rank_min": 4.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "keira_knightley", "mean_score": -0.04312396719311541, "rank": 7.0, "portrait_scores": [ -0.2829570847056311, 0.19670915031940026 ], "portrait_score_gap": 0.47966623502503136, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.08709872879279984, "rank": 8.0, "portrait_scores": [ 0.042057408778827673, -0.21625486636442737 ], "portrait_score_gap": 0.25831227514325505, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.15730848721995624, "rank": 11.0, "portrait_scores": [ -0.07056861556472555, -0.24404835887518694 ], "portrait_score_gap": 0.1734797433104614, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.04584213706592037, "rank": 4.0, "portrait_scores": [ 0.05910598515295276, 0.03257828897888798 ], "portrait_score_gap": 0.02652769617406478, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.5, 8.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.2338376146015055, "rank": 2.0, "portrait_scores": [ -0.08648090681495141, 0.5541561360179624 ], "portrait_score_gap": 0.6406370428329138, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 4.0, 10.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8748608091234149, "rank": 13.0, "portrait_scores": [ -0.781201504785683, -0.9685201134611467 ], "portrait_score_gap": 0.1873186086754638, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "13.0": 8192 }, "target_primary_score": -0.8748608091234149, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.6260636458136953, -0.9762434236945963 ], "target_alternate_panel_score": -0.8011535347541459, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.7140548059613461, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.5996564080491584, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.8900395523309118, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.1453497110039408, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.06734099288827264, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } }, { "name": "gamma_100", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 14, "accuracy": 0.6363636363636364, "mean_cross_entropy": 0.6141987818619592, "brier_score": 0.19078058371974502, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 0.8705854659237146, "brier_score": 0.3121345530335646, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 0.7089219082846103, "brier_score": 0.2265805132022256, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.1939125812019231, "brier_score": 0.4832426326962425, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.47614952278207157, "brier_score": 0.14588757668431762, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.3851954550574669, "right_score": 0.3744585837569667, "margin": -0.7596540388144336, "p_left": 0.3187213828409519, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3837839270479761, "brier": 0.10158331988004864 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.1354487481877154, "right_score": -0.8170198238831458, "margin": 0.6815710756954303, "p_left": 0.6640892549700156, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.0909097940991448, "brier": 0.44101453856663037 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.172863978066021, "right_score": 0.113892583217342, "margin": 0.058971394848679, "p_left": 0.5147385776886904, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7230675181862337, "brier": 0.2649558033609759 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.15982608636172202, "right_score": 0.44480258637241477, "margin": -0.6046286727341368, "p_left": 0.35328544175640253, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4358502591955319, "brier": 0.12481060335701648 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.3851954550574669, "right_score": 0.44480258637241477, "margin": -0.8299980414298817, "p_left": 0.3036454843056776, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3618963866985444, "brier": 0.09220058013922951 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.1354487481877154, "right_score": -0.20737583144029217, "margin": 0.07192708325257677, "p_left": 0.5179740224228617, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7297572709955874, "brier": 0.26829708790491924 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.3851954550574669, "right_score": 0.034750132699050536, "margin": -0.4199455877565174, "p_left": 0.39652977058054795, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5050585695379421, "brier": 0.15723585895666198 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.03154374702734633, "right_score": 0.05783549261807762, "margin": -0.08937923964542395, "p_left": 0.4776700536174501, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.738835049231477, "brier": 0.2728285728879975 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.029212322765313402, "right_score": -0.17464996701129143, "margin": 0.14543764424597802, "p_left": 0.5362954564852668, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6230700449939655, "brier": 0.21502190367620713 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.113892583217342, "right_score": -0.10246184376787013, "margin": 0.21635442698521212, "p_left": 0.553878602499565, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5908097453092911, "brier": 0.19902430130774112 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.034750132699050536, "right_score": -0.24469472671097317, "margin": 0.2794448594100237, "p_left": 0.5694101185763537, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5631543337018664, "brier": 0.18540764598442977 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.8170198238831458, "right_score": -0.20737583144029217, "margin": -0.6096439924428536, "p_left": 0.3521404123325759, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.0437252842508733, "brier": 0.41972204533260465 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.034750132699050536, "right_score": 0.172863978066021, "margin": -0.13811384536697047, "p_left": 0.46552632127667165, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6264727944006372, "brier": 0.2167147558013909 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.44480258637241477, "right_score": 0.172863978066021, "margin": 0.27193860830639377, "p_left": 0.5675687684352821, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5663933591084951, "brier": 0.18699677003257864 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.17464996701129143, "right_score": 0.028858623692691543, "margin": -0.20350859070398297, "p_left": 0.449297721048677, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.80006953514529, "brier": 0.3032730000421807 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.40373901950662017, "right_score": 0.172863978066021, "margin": -0.5766029975726412, "p_left": 0.3597146178780399, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4458412924509829, "brier": 0.12939460631514424 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.3744585837569667, "right_score": -0.3851954550574669, "margin": 0.7596540388144336, "p_left": 0.6812786171590481, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.38378392704797604, "brier": 0.10158331988004861 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.3744585837569667, "right_score": -0.15982608636172202, "margin": 0.5342846701186887, "p_left": 0.6304818876067885, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7284131860803996, "brier": 0.01702552299343058 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.3744585837569667, "right_score": -0.10246184376787013, "margin": 0.4769204275248369, "p_left": 0.617020415362613, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9597735950410229, "brier": 0.38071419297425146 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.10246184376787013, "right_score": -0.40373901950662017, "margin": 0.30127717573875, "p_left": 0.5747547033125514, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8550891078791493, "brier": 0.330342968979899 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.034750132699050536, "right_score": 0.12869334467508245, "margin": -0.09394321197603192, "p_left": 0.47653145426714405, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7412215470126964, "brier": 0.2740193183716711 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.028858623692691543, "right_score": -0.03154374702734633, "margin": 0.06040237072003787, "p_left": 0.5150960032125423, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7238043524064337, "brier": 0.26532389252553545 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.40373901950662017, "right_score": -0.3851954550574669, "margin": -0.018543564449153283, "p_left": 0.49536424172606486, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7024619451415007, "brier": 0.2546572485287095 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.07433009940422544, "right_score": -0.021659777213552944, "margin": -0.0526703221906725, "p_left": 0.4868354626924773, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7198290719344255, "brier": 0.2633378423500438 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.113892583217342, "right_score": 0.172863978066021, "margin": -0.058971394848679, "p_left": 0.48526142231130964, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7230675181862337, "brier": 0.2649558033609759 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.44480258637241477, "right_score": -0.24469472671097317, "margin": 0.6894973130833879, "p_left": 0.6658550922162954, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4066832113703598, "brier": 0.11165281939778049 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.05783549261807762, "right_score": 0.22069494314106314, "margin": -0.1628594505229855, "p_left": 0.45937489002981574, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6150291978829564, "brier": 0.2110252895899053 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.026455922617774873, "right_score": -0.10246184376787013, "margin": 0.128917766385645, "p_left": 0.5321848784440603, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7596821005234989, "brier": 0.2832207448445192 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.026455922617774873, "right_score": -0.15982608636172202, "margin": 0.1862820089794969, "p_left": 0.5464362977296466, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6043375422408231, "brier": 0.20572003201718975 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.40373901950662017, "right_score": 0.12869334467508245, "margin": -0.5324323641817026, "p_left": 0.36994975660183543, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4619557113508765, "brier": 0.13686282240975728 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.3744585837569667, "right_score": -0.24469472671097317, "margin": 0.6191533104679399, "p_left": 0.650025957774737, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7403196370857495, "brier": 0.022507788006227183 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.24469472671097317, "right_score": 0.026455922617774873, "margin": -0.27115064932874805, "p_left": 0.43262463413912644, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.566734173374918, "brier": 0.187164074064013 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.44480258637241477, "right_score": -0.15982608636172202, "margin": 0.6046286727341368, "p_left": 0.6467145582435975, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.435850259195532, "brier": 0.12481060335701648 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.15982608636172202, "right_score": 0.113892583217342, "margin": -0.273718669579064, "p_left": 0.43199439584120586, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5656239938329088, "brier": 0.18661915803820844 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.05783549261807762, "right_score": -0.8170198238831458, "margin": 0.8748553165012234, "p_left": 0.7057549830490605, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2233424677313152, "brier": 0.4980900960985796 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.113892583217342, "right_score": 0.12869334467508245, "margin": -0.014800761457740452, "p_left": 0.4962998771818435, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7005749438563458, "brier": 0.253713813727026 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 0.32321530337679016, "right_score": -0.1508543564435714, "margin": 0.47406965982036153, "p_left": 0.6163465368673064, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7209807436867466, "brier": 0.0135365166410155 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.12869334467508245, "right_score": 0.026455922617774873, "margin": 0.10223742205730757, "p_left": 0.5255371155170482, "prediction": "left", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7455718842592708, "brier": 0.27618925978597925 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.8170198238831458, "right_score": 0.32321530337679016, "margin": -1.140235127259936, "p_left": 0.24227719396394337, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.4176727787263586, "brier": 0.5741438507871555 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.44480258637241477, "right_score": -0.3851954550574669, "margin": 0.8299980414298817, "p_left": 0.6963545156943225, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3618963866985444, "brier": 0.09220058013922944 } ], "probability_status": "Bradley-Terry fitted scale; no held-out calibration", "ridge": 1.0, "consistency": 100.0, "sensitivity": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 0.32321530337679016, "rank": 1.0, "portrait_scores": [ 0.5280180328139967, 0.11841257393958354 ], "portrait_score_gap": 0.4096054588744132, "single_photo_rank_min": 1.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.515625, "single_photo_rank_quantiles": [ 1.0, 1.0, 5.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.05783549261807762, "rank": 3.0, "portrait_scores": [ 0.03373650870912767, 0.08193447652702757 ], "portrait_score_gap": 0.048197967817899895, "single_photo_rank_min": 2.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.5, 6.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.021659777213552944, "rank": 5.0, "portrait_scores": [ 0.5228194580722525, -0.5661390124993584 ], "portrait_score_gap": 1.0889584705716109, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 6.5, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.20737583144029217, "rank": 12.0, "portrait_scores": [ -0.5791047168201438, 0.1643530539395594 ], "portrait_score_gap": 0.7434577707597032, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 9.0, 12.0 ] }, { "identity": "emma_stone", "mean_score": -0.07433009940422544, "rank": 8.0, "portrait_scores": [ 0.29458774140092336, -0.44324794020937425 ], "portrait_score_gap": 0.7378356816102976, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 7.0, 12.0 ] }, { "identity": "emma_watson", "mean_score": -0.17464996701129143, "rank": 11.0, "portrait_scores": [ -0.24350676807384825, -0.10579316594873464 ], "portrait_score_gap": 0.1377136021251136, "single_photo_rank_min": 6.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 10.0, 11.0 ] }, { "identity": "gal_gadot", "mean_score": -0.029212322765313402, "rank": 6.0, "portrait_scores": [ -0.07852259095863198, 0.02009794542800518 ], "portrait_score_gap": 0.09862053638663716, "single_photo_rank_min": 4.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 10.0 ] }, { "identity": "keira_knightley", "mean_score": -0.03154374702734633, "rank": 7.0, "portrait_scores": [ -0.25321084469102917, 0.1901233506363365 ], "portrait_score_gap": 0.4433341953273657, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 7.0, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.1354487481877154, "rank": 9.0, "portrait_scores": [ -0.03878259322905089, -0.23211490314637995 ], "portrait_score_gap": 0.19333230991732905, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "margot_robbie", "mean_score": -0.1508543564435714, "rank": 10.0, "portrait_scores": [ -0.07302749050585432, -0.22868122238128846 ], "portrait_score_gap": 0.15565373187543413, "single_photo_rank_min": 4.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.028858623692691543, "rank": 4.0, "portrait_scores": [ 0.029017264550636218, 0.028699982834746868 ], "portrait_score_gap": 0.0003172817158893504, "single_photo_rank_min": 3.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 5.5, 7.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.22069494314106314, "rank": 2.0, "portrait_scores": [ -0.07503204494989568, 0.516421931232022 ], "portrait_score_gap": 0.5914539761819176, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.5, 9.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.8170198238831458, "rank": 13.0, "portrait_scores": [ -0.7415839553979406, -0.8924556923683509 ], "portrait_score_gap": 0.15087173697041023, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "13.0": 8192 }, "target_primary_score": -0.8170198238831458, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -0.5902430843262887, -0.9499496614591753 ], "target_alternate_panel_score": -0.7700963728927319, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.6346476247050166, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.5627205414524398, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.8279318655108096, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -1.093311676269522, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 0.059855443956072625, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } ] } ], "bootstrap": { "iterations": 1000, "interpretation": "Resampling the observed training comparisons with fixed features and selected penalties measures conditional fit sensitivity; these are not population confidence intervals or new human judgments.", "identities": [ { "identity": "alexandra_daddario", "rank_quantiles": [ 1.0, 3.0, 6.0 ], "score_quantiles": [ -0.3655288696232892, 0.2886994058211903, 0.890873203706713 ], "top_share": 0.135 }, { "identity": "ana_de_armas", "rank_quantiles": [ 2.0, 6.0, 11.0 ], "score_quantiles": [ -1.473582054072134, -0.2918501795920375, 0.8526758555784086 ], "top_share": 0.038 }, { "identity": "anne_hathaway", "rank_quantiles": [ 1.0, 3.0, 7.0 ], "score_quantiles": [ -0.48141666621447543, 0.2463551330682593, 0.9873759726571771 ], "top_share": 0.249 }, { "identity": "elizabeth_olsen", "rank_quantiles": [ 8.0, 11.0, 13.0 ], "score_quantiles": [ -2.425963636528404, -1.4918787089328678, -0.45417277849087256 ], "top_share": 0.0 }, { "identity": "emma_stone", "rank_quantiles": [ 5.0, 9.0, 13.0 ], "score_quantiles": [ -1.7250695381423444, -1.0288550071890072, -0.2686429493351853 ], "top_share": 0.0 }, { "identity": "emma_watson", "rank_quantiles": [ 2.0, 5.0, 9.0 ], "score_quantiles": [ -0.9625059778733069, -0.1113519748153449, 0.7764335342723903 ], "top_share": 0.049 }, { "identity": "gal_gadot", "rank_quantiles": [ 1.0, 3.0, 8.0 ], "score_quantiles": [ -0.8506394212512322, 0.1744748978626371, 1.026554963543837 ], "top_share": 0.286 }, { "identity": "keira_knightley", "rank_quantiles": [ 5.0, 10.0, 12.0 ], "score_quantiles": [ -2.0874579513150797, -1.1434728015417543, -0.014753747739032366 ], "top_share": 0.0 }, { "identity": "lily_collins", "rank_quantiles": [ 6.0, 11.0, 13.0 ], "score_quantiles": [ -2.269500538139463, -1.3681413650230607, -0.41599265443512473 ], "top_share": 0.003 }, { "identity": "margot_robbie", "rank_quantiles": [ 2.0, 6.0, 10.0 ], "score_quantiles": [ -1.1707707600781405, -0.31098245598091273, 0.4179177239417052 ], "top_share": 0.013 }, { "identity": "natalie_portman", "rank_quantiles": [ 5.0, 8.0, 11.0 ], "score_quantiles": [ -1.1904154184236304, -0.827407558418809, -0.3261596825551611 ], "top_share": 0.001 }, { "identity": "scarlett_johansson", "rank_quantiles": [ 1.0, 3.0, 7.0 ], "score_quantiles": [ -0.619549139800398, 0.2488920381255847, 0.9931892001746583 ], "top_share": 0.226 }, { "identity": "tongdak_cheonsa", "rank_quantiles": [ 8.0, 13.0, 13.0 ], "score_quantiles": [ -3.5629857647735803, -2.107265957331977, -0.6331003489892761 ], "top_share": 0.0 } ] }, "fixed_head_full": { "models": [ { "name": "zero_shot", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 13, "accuracy": 0.5909090909090909, "mean_cross_entropy": 0.6925276480341477, "brier_score": 0.2288569021636729, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.5454545454545454, "mean_cross_entropy": 0.6922395076672613, "brier_score": 0.22871283169881818, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 3, "accuracy": 0.42857142857142855, "mean_cross_entropy": 0.6925776061161542, "brier_score": 0.2184652138363438, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6915633107694755, "brier_score": 0.24920806742376692, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 1, "accuracy": 0.25, "mean_cross_entropy": 0.6940539017202587, "brier_score": 0.25045335929011886, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.006883674632906349, "right_score": -0.0076911357897173835, "margin": 0.0008074611568110344, "p_left": 0.5002018652782348, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6935509926375386, "brier": 0.2502019060278254 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.006042520248854092, "right_score": -0.00011281449579252523, "margin": -0.005929705753061567, "p_left": 0.4985175779054031, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.690186722853265, "brier": 0.2485197754806696 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.004227162498453013, "right_score": -0.0020419652813698777, "margin": 0.0062691277798228905, "p_left": 0.5015672768118711, "prediction": "left", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6962866571872017, "brier": 0.2515697331684761 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.005645409277289262, "right_score": -0.006348040317190647, "margin": 0.0007026310399013847, "p_left": 0.5001756577527486, "prediction": "left", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934985577911921, "brier": 0.25017568860839473 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.006883674632906349, "right_score": -0.006348040317190647, "margin": -0.000535634315715702, "p_left": 0.4998660914242727, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6928793992651021, "brier": 0.24986610935577933 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.006042520248854092, "right_score": -0.00216220665221373, "margin": -0.003880313596640362, "p_left": 0.4990299228180306, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6912089058646455, "brier": 0.24903086386776957 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.006883674632906349, "right_score": 0.00483201650663787, "margin": -0.01171569113954422, "p_left": 0.4970711107160192, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6873064920694109, "brier": 0.24707968910845704 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.0009339228876092543, "right_score": 0.0013210483588317447, "margin": -0.0003871254712224904, "p_left": 0.49990321863340303, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6933407620288227, "brier": 0.2500967907332299 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.0019282540335639462, "right_score": -0.0045846148950199495, "margin": 0.006512868928583895, "p_left": 0.501628211476767, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6898960482689925, "brier": 0.24837443959584612 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": -0.0020419652813698777, "right_score": -0.0060435887393002746, "margin": 0.004001623457930397, "p_left": 0.5010004045295273, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.691148370453432, "brier": 0.24900059627969537 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.00483201650663787, "right_score": 0.0013004709690664774, "margin": 0.003531545537571393, "p_left": 0.5008828854667945, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.691382966767085, "brier": 0.249117894019953 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.00011281449579252523, "right_score": -0.00216220665221373, "margin": 0.0020493921564212047, "p_left": 0.5005123478597832, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6921230094826691, "brier": 0.24948791464054626 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.00483201650663787, "right_score": 0.004227162498453013, "margin": 0.0006048540081848575, "p_left": 0.5001512134974361, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6934496532950835, "brier": 0.2501512363629579 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": -0.006348040317190647, "right_score": 0.004227162498453013, "margin": -0.010575202815643659, "p_left": 0.49735622393491574, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6984487612669507, "brier": 0.2526507656169666 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": -0.0045846148950199495, "right_score": -0.004019419381127919, "margin": -0.0005651955138920306, "p_left": 0.49985870112528846, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6934298182476369, "brier": 0.2501413188400836 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.0018617103404967878, "right_score": 0.004227162498453013, "margin": -0.002365452157956225, "p_left": 0.4994086372362518, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6919651539012931, "brier": 0.24940898694617014 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.0076911357897173835, "right_score": -0.006883674632906349, "margin": -0.0008074611568110344, "p_left": 0.4997981347217652, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6935509926375386, "brier": 0.25020190602782527 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.0076911357897173835, "right_score": -0.005645409277289262, "margin": -0.002045726512428121, "p_left": 0.49948856855025453, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931477036844745, "brier": 2.615621277887529e-07 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.0076911357897173835, "right_score": -0.0060435887393002746, "margin": -0.001647547050417109, "p_left": 0.4995881133305649, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6923237463361088, "brier": 0.24958828298119337 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": -0.0060435887393002746, "right_score": 0.0018617103404967878, "margin": -0.007905299079797062, "p_left": 0.4980236855223158, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6892023427188985, "brier": 0.2480275913412305 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.00483201650663787, "right_score": -0.0055398772937816565, "margin": 0.010371893800419526, "p_left": 0.5025929502051731, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6879746806220877, "brier": 0.24741377318559338 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.004019419381127919, "right_score": 0.0009339228876092543, "margin": -0.0049533422687371735, "p_left": 0.4987616669647516, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6906735763723953, "brier": 0.2487632004334578 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.0018617103404967878, "right_score": -0.006883674632906349, "margin": 0.008745384973403138, "p_left": 0.5021863323088236, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6887840482625696, "brier": 0.247818447740141 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.006721541770849666, "right_score": -0.003535409689602233, "margin": -0.003186132081247433, "p_left": 0.49920346765351703, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6947415155297372, "brier": 0.25079716681026193 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": -0.0020419652813698777, "right_score": 0.004227162498453013, "margin": -0.0062691277798228905, "p_left": 0.4984327231881289, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6962866571872017, "brier": 0.2515697331684761 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": -0.006348040317190647, "right_score": 0.0013004709690664774, "margin": -0.007648511286257124, "p_left": 0.4980878814999604, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.696978748650862, "brier": 0.2519157746971978 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.0013210483588317447, "right_score": -0.0007135445536791125, "margin": 0.0020345929125108572, "p_left": 0.5005086480526624, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6941649944621514, "brier": 0.25050890677550386 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.001109405741877672, "right_score": -0.0060435887393002746, "margin": 0.007152994481177947, "p_left": 0.5017882409956393, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6967300734531554, "brier": 0.25179143880149785 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.001109405741877672, "right_score": -0.005645409277289262, "margin": 0.0067548150191669345, "p_left": 0.5016886973338731, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6897754764802616, "brier": 0.24831415436481233 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.0018617103404967878, "right_score": -0.0055398772937816565, "margin": 0.007401587634278444, "p_left": 0.5018503884610144, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6968548222988917, "brier": 0.25185381239847104 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.0076911357897173835, "right_score": 0.0013004709690664774, "margin": -0.008991606758783861, "p_left": 0.4977521134552304, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931572866499139, "brier": 5.0529939181561885e-06 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.0013004709690664774, "right_score": 0.001109405741877672, "margin": 0.00019106522718880534, "p_left": 0.5000477663066519, "prediction": "left", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6932427177367798, "brier": 0.250047768588272 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": -0.006348040317190647, "right_score": -0.005645409277289262, "margin": -0.0007026310399013847, "p_left": 0.49982434224725136, "prediction": "right", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.6934985577911921, "brier": 0.25017568860839473 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.005645409277289262, "right_score": -0.0020419652813698777, "margin": -0.0036034439959193847, "p_left": 0.49909913997581123, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6913470816621865, "brier": 0.2490999515245944 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.0013210483588317447, "right_score": -0.00011281449579252523, "margin": 0.00143386285462427, "p_left": 0.5003584656522401, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6938643689825712, "brier": 0.2503585941498639 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": -0.0020419652813698777, "right_score": -0.0055398772937816565, "margin": 0.003497912012411779, "p_left": 0.5008744771114725, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6913997539765155, "brier": 0.249126287598746 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.006258333934324074, "right_score": 0.005757556704005823, "margin": -0.012015890638329897, "p_left": 0.49699606348310094, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6931652281548522, "brier": 9.02363459755968e-06 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.0055398772937816565, "right_score": 0.001109405741877672, "margin": -0.0066492830356593285, "p_left": 0.49833768536573547, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6898280656525455, "brier": 0.24834044865567875 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.00011281449579252523, "right_score": -0.006258333934324074, "margin": 0.006145519438531549, "p_left": 0.5015363750242272, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6900791417593967, "brier": 0.24846598542398787 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": -0.006348040317190647, "right_score": -0.006883674632906349, "margin": 0.000535634315715702, "p_left": 0.5001339085757274, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6928793992651021, "brier": 0.24986610935577927 } ], "probability_status": "Raw sigmoid of cosine contrast; uncalibrated" }, { "name": "pair_centroid", "selected": null, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.6334174847718116, "brier_score": 0.1994635463500992, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 8, "accuracy": 0.7272727272727273, "mean_cross_entropy": 0.6804607044399216, "brier_score": 0.22282537719124398, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.7142857142857143, "mean_cross_entropy": 0.679714137310847, "brier_score": 0.21203589214846047, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6819538386980705, "brier_score": 0.2444043472768111, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.5671850930046762, "brier_score": 0.18751451881406, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.1246007584636746, "right_score": 0.21193979526563828, "margin": -0.3365405537293129, "p_left": 0.41665006190854237, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5389680358381743, "brier": 0.17359727408839218 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.021458069618402416, "right_score": 0.02076683342587906, "margin": -0.042224903044281475, "p_left": 0.48944534238791204, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6722575802877235, "brier": 0.23955674318522044 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.11069241240348489, "right_score": 0.23701500712253468, "margin": -0.12632259471904977, "p_left": 0.4684612798293576, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6319792331092329, "brier": 0.21945597069935968 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": 0.007233588884448874, "right_score": 0.25544597959964976, "margin": -0.2482123907152009, "p_left": 0.4382635394779224, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.576722470526563, "brier": 0.19207493003571646 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.1246007584636746, "right_score": 0.25544597959964976, "margin": -0.38004673806332434, "p_left": 0.4061156244627528, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5210706325447348, "brier": 0.16492990043277164 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.021458069618402416, "right_score": -0.030191111284014804, "margin": 0.008733041665612389, "p_left": 0.5021832465407943, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.697523234614549, "brier": 0.25218801310625216 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.1246007584636746, "right_score": 0.09052574175964918, "margin": -0.2151265002233238, "p_left": 0.44642483467460126, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5913577360075494, "brier": 0.19929513301424506 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.12126658405690686, "right_score": 0.0570395156613495, "margin": 0.06422706839555736, "p_left": 0.5160512497055404, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6615491972978235, "brier": 0.23420639291156922 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.16589917852340358, "right_score": 0.15503423903162647, "margin": 0.01086493949177711, "p_left": 0.5027162081529986, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6877294666052489, "brier": 0.24729116963373185 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.23701500712253468, "right_score": 0.025522720840629426, "margin": 0.21149228628190525, "p_left": 0.552676869205474, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5929817715606103, "brier": 0.20009798334381662 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.09052574175964918, "right_score": -0.04550801485003754, "margin": 0.13603375660968672, "p_left": 0.5339560916546296, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.627441668768495, "brier": 0.21719692450582795 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.02076683342587906, "right_score": -0.030191111284014804, "margin": 0.050957944709893864, "p_left": 0.5127367301616064, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6679927621078657, "brier": 0.2374254941336032 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.09052574175964918, "right_score": 0.11069241240348489, "margin": -0.020166670643835705, "p_left": 0.4949585032003454, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.683114681202197, "brier": 0.24498391989032633 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.25544597959964976, "right_score": 0.11069241240348489, "margin": 0.1447535671961649, "p_left": 0.5361253341813528, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6233873128179794, "brier": 0.21517970558836164 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.15503423903162647, "right_score": 0.1447821400090164, "margin": 0.010252099022610067, "p_left": 0.5025630023068813, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6880342691828996, "brier": 0.24744356667394374 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.11665191209183828, "right_score": 0.11069241240348489, "margin": 0.005959499688353395, "p_left": 0.501489870512616, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.6961313698521193, "brier": 0.25149209022676033 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 0.21193979526563828, "right_score": -0.1246007584636746, "margin": 0.3365405537293129, "p_left": 0.5833499380914576, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5389680358381743, "brier": 0.17359727408839218 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 0.21193979526563828, "right_score": 0.007233588884448874, "margin": 0.20470620638118942, "p_left": 0.5509985861070118, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6983761390720897, "brier": 0.002600855784914297 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 0.21193979526563828, "right_score": 0.025522720840629426, "margin": 0.18641707442500885, "p_left": 0.5464697726353832, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7906933581429731, "brier": 0.2986292124041674 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.025522720840629426, "right_score": 0.11665191209183828, "margin": -0.09112919125120886, "p_left": 0.4772334554603267, "prediction": "right", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6486202921265374, "brier": 0.22775177101060362 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.09052574175964918, "right_score": -0.037034081797160626, "margin": 0.1275598235568098, "p_left": 0.5318467846596907, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6313998298800797, "brier": 0.21916743303347 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.1447821400090164, "right_score": 0.12126658405690686, "margin": 0.023515555952109538, "p_left": 0.5058786180933326, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7049740796148779, "brier": 0.2559131762440198 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.11665191209183828, "right_score": -0.1246007584636746, "margin": 0.2412526705555129, "p_left": 0.5600223271037934, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5797786262195203, "brier": 0.19358035264716136 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.11216167927396731, "right_score": 0.011082199246506046, "margin": 0.10107948002746125, "p_left": 0.5252483766489551, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6438840298873053, "brier": 0.22538910387445243 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.23701500712253468, "right_score": 0.11069241240348489, "margin": 0.12632259471904977, "p_left": 0.5315387201706423, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6319792331092329, "brier": 0.2194559706993598 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.25544597959964976, "right_score": -0.04550801485003754, "margin": 0.3009539944496873, "p_left": 0.5746757121055013, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5539493762276879, "brier": 0.18090074987296242 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.0570395156613495, "right_score": 0.13437257207941272, "margin": -0.07733305641806322, "p_left": 0.48067636519380746, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6552280163499159, "brier": 0.23104976805593055 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.14336234488399252, "right_score": 0.025522720840629426, "margin": 0.1178396240433631, "p_left": 0.5294258628272672, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7538017613330881, "brier": 0.2802917442303964 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.14336234488399252, "right_score": 0.007233588884448874, "margin": 0.13612875599954366, "p_left": 0.5339797318899855, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.627397396004461, "brier": 0.21717489028932982 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.11665191209183828, "right_score": -0.037034081797160626, "margin": 0.1536859938889989, "p_left": 0.5383460523202556, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7729396995604985, "brier": 0.28981647204880334 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 0.21193979526563828, "right_score": -0.04550801485003754, "margin": 0.2574478101156758, "p_left": 0.5640088038266341, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7014093230420535, "brier": 0.004097126967316528 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.04550801485003754, "right_score": 0.14336234488399252, "margin": -0.18887035973403005, "p_left": 0.45292227337309177, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6031643904447961, "brier": 0.20513858571744967 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.25544597959964976, "right_score": 0.007233588884448874, "margin": 0.2482123907152009, "p_left": 0.5617364605220776, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.576722470526563, "brier": 0.19207493003571646 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": 0.007233588884448874, "right_score": 0.23701500712253468, "margin": -0.2297814182380858, "p_left": 0.44280607515485665, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5848419401417693, "brier": 0.19607722019404855 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.0570395156613495, "right_score": 0.02076683342587906, "margin": 0.03627268223547044, "p_left": 0.5090671764346815, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7114479760969669, "brier": 0.2591493901231791 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.23701500712253468, "right_score": -0.037034081797160626, "margin": 0.2740490889196953, "p_left": 0.5680866790605559, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5654812679239367, "brier": 0.18654911680493927 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.013588513042144132, "right_score": 0.19909433425865783, "margin": -0.21268284730080195, "p_left": 0.4470288126640144, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.6987908049341561, "brier": 0.0028059466877840816 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.037034081797160626, "right_score": 0.14336234488399252, "margin": -0.18039642668115313, "p_left": 0.4550228011553255, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6070113221763326, "brier": 0.2070457495712389 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.02076683342587906, "right_score": -0.013588513042144132, "margin": 0.03435534646802319, "p_left": 0.5085879919403254, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6761170362997255, "brier": 0.24148576166524166 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.25544597959964976, "right_score": -0.1246007584636746, "margin": 0.38004673806332434, "p_left": 0.5938843755372473, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5210706325447348, "brier": 0.16492990043277161 } ], "probability_status": "Raw sigmoid of normalized prototype projection; uncalibrated" }, { "name": "identity_bt", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6237422105476452, "fold_losses": [ 0.5912621843305084, 0.5861368997395501, 0.4026945148438192, 0.3323741325990308, 0.442539724452386, 0.5624759226065226, 0.5203061041714794, 0.9678367479395096, 0.6145126435604417, 0.7801887677874529, 0.70593162468813, 0.7793918906659547, 0.9437037716914924, 0.6006008485165383, 0.5668647964666637, 0.3934462332447004, 1.008398736644656, 0.5269021741740239, 1.0074142217309268, 0.7082368331289262, 0.5153421091732847, 0.3828070031866322, 0.43538533281613256, 0.5950598349847225 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 18, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5215362065355041, "brier_score": 0.14664515692026486, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 9, "accuracy": 0.8181818181818182, "mean_cross_entropy": 0.5999291980724863, "brier_score": 0.18331301665466668, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 6, "accuracy": 0.8571428571428571, "mean_cross_entropy": 0.6110543575305215, "brier_score": 0.17829255884093187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.577678879156416, "brier_score": 0.19335393228213624, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.40385947416758644, "brier_score": 0.11204020819206394, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.5552034817852926, "right_score": -0.09129337128269767, "margin": -0.4639101105025949, "p_left": 0.38605864786248933, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.3839854052468287, "right_score": 0.0, "margin": -0.3839854052468287, "p_left": 0.4051660265356517, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5194729485687224, "brier": 0.16415950905868842 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.09092352020600739, "right_score": 0.5639172371946242, "margin": -0.4729937169886168, "p_left": 0.3839079161510299, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4843588398528423, "brier": 0.1473852880834262 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5144123302720128, "margin": -0.8757880570215799, "p_left": 0.2940513566767732, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34821278730304145, "brier": 0.08646620036345087 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.5144123302720128, "margin": -1.0696158120573054, "p_left": 0.2554761529614165, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196509 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.3839854052468287, "right_score": -0.3613757267495671, "margin": -0.022609678497261643, "p_left": 0.4943478211553035, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6819062396455225, "brier": 0.24437976828099595 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.5552034817852926, "right_score": 0.21958607998633384, "margin": -0.7747895617716264, "p_left": 0.3154439397449703, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3789847380381482, "brier": 0.09950487912182847 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": 0.5639172371946242, "right_score": 0.21958607998633384, "margin": 0.34433115720829033, "p_left": 0.5852422276637617, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5357294530687823, "brier": 0.1720240097133189 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": -0.04655164691805275, "right_score": 0.22928675300045034, "margin": -0.2758383999185031, "p_left": 0.4314743416674672, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.8405472333379712, "brier": 0.32322142418243993 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.5639172371946242, "right_score": 0.04735007533955107, "margin": 0.5165671618550731, "p_left": 0.6263447042117808, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4678544137260341, "brier": 0.13961828007058158 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.3839854052468287, "margin": 0.6035714852331626, "p_left": 0.6464729800527182, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.43622387583895805, "brier": 0.12498135383280574 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": 0.0, "right_score": -0.3613757267495671, "margin": 0.3613757267495671, "p_left": 0.589373417869286, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5286953100191659, "brier": 0.16861418995235203 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.21958607998633384, "right_score": 0.09092352020600739, "margin": 0.12866255978032645, "p_left": 0.5321213406311366, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.759546291534882, "brier": 0.28315312115507807 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.5144123302720128, "right_score": 0.09092352020600739, "margin": 0.42348881006600547, "p_left": 0.6043177893281838, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5036550781392832, "brier": 0.15656441184213554 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.22928675300045034, "right_score": -0.09129337128269767, "margin": 0.320580124283148, "p_left": 0.5794656263723439, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.545648933971355, "brier": 0.17684915940240506 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.04655164691805275, "right_score": 0.09092352020600739, "margin": -0.13747516712406013, "p_left": 0.46568523530362693, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6267701666806329, "brier": 0.2168627383797944 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.09129337128269767, "right_score": -0.5552034817852926, "margin": 0.4639101105025949, "p_left": 0.6139413521375107, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48785587308845046, "brier": 0.14904127958941354 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3613757267495671, "margin": 0.27008235546686943, "p_left": 0.5671131230007795, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7022376614223658, "brier": 0.004504171278917763 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.09129337128269767, "right_score": 0.04735007533955107, "margin": -0.13864344662224876, "p_left": 0.46539455276808567, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6262262859691204, "brier": 0.21659208974620647 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.04735007533955107, "right_score": -0.04655164691805275, "margin": 0.09390172225760382, "p_left": 0.5234581961445747, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7411998286648287, "brier": 0.274008483110932 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.21958607998633384, "right_score": -0.22706636401654093, "margin": 0.44665244400287474, "p_left": 0.6098430283691514, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49455368547234, "brier": 0.15222246251215482 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.09129337128269767, "right_score": 0.5639172371946242, "margin": -0.6552106084773218, "p_left": 0.34181630111695643, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.41827120898583914, "brier": 0.11683838370927783 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.5552034817852926, "margin": 0.5086518348672397, "p_left": 0.624490379951733, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4708193539359016, "brier": 0.14100747474879388 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.04735007533955107, "right_score": -0.22706636401654093, "margin": 0.274416439356092, "p_left": 0.5681768114526454, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5653226209322291, "brier": 0.18647126616720414 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.5639172371946242, "right_score": 0.09092352020600739, "margin": 0.4729937169886168, "p_left": 0.6160920838489701, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48435883985284234, "brier": 0.1473852880834262 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.5144123302720128, "right_score": -0.3839854052468287, "margin": 0.8983977355188415, "p_left": 0.7106201255237109, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34161727392220165, "brier": 0.08374071175191283 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 0.21958607998633384, "right_score": 0.5144123302720128, "margin": -0.294826250285679, "p_left": 0.4268227337937082, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5565602450039485, "brier": 0.18217764608313472 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.22928675300045034, "right_score": 0.04735007533955107, "margin": 0.18193667766089927, "p_left": 0.545359119207845, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7882474446553732, "brier": 0.29741656890315643 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.22928675300045034, "right_score": -0.3613757267495671, "margin": 0.5906624797500174, "p_left": 0.6435171348703433, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44080662468532317, "brier": 0.127080033131049 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.04655164691805275, "right_score": -0.22706636401654093, "margin": 0.18051471709848818, "p_left": 0.5450065319974984, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7874722161726936, "brier": 0.2970321199199402 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.09129337128269767, "right_score": -0.3839854052468287, "margin": 0.29269203396413107, "p_left": 0.5726550593090842, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7038177512532197, "brier": 0.005278757643206546 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.3839854052468287, "right_score": 0.22928675300045034, "margin": -0.6132721582472791, "p_left": 0.35131313621006827, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4328051690222217, "brier": 0.12342091967375397 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.3613757267495671, "margin": 0.8757880570215799, "p_left": 0.7059486433232268, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3482127873030414, "brier": 0.08646620036345087 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.3613757267495671, "right_score": 0.5639172371946242, "margin": -0.9252929639441912, "p_left": 0.2838806450631104, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3339084290746344, "brier": 0.08058822064144766 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 0.21958607998633384, "right_score": 0.0, "margin": 0.21958607998633384, "p_left": 0.5546769948635638, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8089554058814965, "brier": 0.307666568630874 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.5639172371946242, "right_score": -0.22706636401654093, "margin": 0.7909836012111651, "p_left": 0.6880424898072077, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3739046845150584, "brier": 0.0973174881656861 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.5552034817852926, "right_score": 0.09092352020600739, "margin": -0.6461270019912999, "p_left": 0.3438628385403446, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7444489252985238, "brier": 0.0243788131886785 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.22706636401654093, "right_score": 0.22928675300045034, "margin": -0.45635311701699127, "p_left": 0.38785132162817176, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4907800874585346, "brier": 0.15042864768871952 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": 0.0, "right_score": -0.5552034817852926, "margin": 0.5552034817852926, "p_left": 0.6353419937988055, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4535918521562793, "brier": 0.1329754614866304 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.5144123302720128, "right_score": -0.5552034817852926, "margin": 1.0696158120573054, "p_left": 0.7445238470385834, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.29501039642601157, "brier": 0.06526806473196511 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "visual_bt", "selected": { "ridge": 10.0, "consistency": 0.0, "mean_cross_entropy": 0.6320205750438171, "fold_losses": [ 0.3743683513685698, 0.6304440661124163, 0.2179039278832674, 0.12548507929642855, 0.32509037736122565, 0.48538734533974714, 0.49019737309019396, 0.8474959953040533, 0.7331032500767469, 0.7633257074100377, 0.8082980208424667, 0.9587795429203139, 1.3979376599048852, 0.7436147267495867, 0.3857446776616644, 0.20588164058360636, 1.2273084167545147, 0.47010401002552793, 1.5523163170214314, 0.8355171277509585, 0.40245348809073583, 0.18494448722097453, 0.2717325855260917, 0.7310596267561661 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.4446341971279795, "brier_score": 0.11521331755773499, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 8, "accuracy": 0.7272727272727273, "mean_cross_entropy": 0.651173058449653, "brier_score": 0.20763341633616736, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 5, "accuracy": 0.7142857142857143, "mean_cross_entropy": 0.6427332930871021, "brier_score": 0.19274937602247436, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 3, "accuracy": 0.75, "mean_cross_entropy": 0.6680525891747549, "brier_score": 0.23740149696355334, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.2669540188509706, "brier_score": 0.060214945637106346, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -1.173227342088397, "right_score": -0.022602202916935765, "margin": -1.1506251391714613, "p_left": 0.24037491735303235, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.2749302797269295, "brier": 0.05778010089247713 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.6793420338305063, "right_score": -0.368851350626314, "margin": -0.31049068320419226, "p_left": 0.42299497318453594, "prediction": "right", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5499043005267802, "brier": 0.17892474733938626 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 0.309671025570845, "right_score": 0.892127649875238, "margin": -0.5824566243043929, "p_left": 0.35836752055505344, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.44373960113482347, "brier": 0.12842727978877666 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.47819853747650676, "right_score": 0.8812886831562179, "margin": -1.3594872206327246, "p_left": 0.20432365500611563, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22856277758914317, "brier": 0.04174815599505816 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -1.173227342088397, "right_score": 0.8812886831562179, "margin": -2.054516025244615, "p_left": 0.11359685678800006, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.12058341695298617, "brier": 0.012904245872113397 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.6793420338305063, "right_score": -0.6151609892063097, "margin": -0.0641810446241966, "p_left": 0.48396024438810403, "prediction": "right", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6615714707089582, "brier": 0.2342175181481934 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -1.173227342088397, "right_score": 0.1324540866978254, "margin": -1.3056814287862224, "p_left": 0.21321039462404867, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.23979440382045156, "brier": 0.04545867237574256 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.0761646690868942, "right_score": -0.014177080267333414, "margin": -0.06198758881956079, "p_left": 0.48450806307462774, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7246212057368983, "brier": 0.26573193703507203 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 0.14093534830924198, "right_score": 0.2097694817586376, "margin": -0.06883413344939562, "p_left": 0.48279825812015487, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7281563976357021, "brier": 0.26749764180354596 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 0.892127649875238, "right_score": 0.10194849474574966, "margin": 0.7901791551294883, "p_left": 0.6878697973686371, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3741557069691597, "brier": 0.09742526339469568 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 0.1324540866978254, "right_score": -0.8357333816639461, "margin": 0.9681874683617715, "p_left": 0.7247580744125418, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3219173702799738, "brier": 0.07575811760109187 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -0.368851350626314, "right_score": -0.6151609892063097, "margin": 0.24630963857999566, "p_left": 0.5612679697279142, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5775568231056306, "brier": 0.19248579438666646 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 0.1324540866978254, "right_score": 0.309671025570845, "margin": -0.17721693887301962, "p_left": 0.45581135326970096, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6084593151617127, "brier": 0.20776398976955612 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 0.8812886831562179, "right_score": 0.309671025570845, "margin": 0.5716176575853729, "p_left": 0.6391363573443991, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44763745559411605, "brier": 0.13022256859066922 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.2097694817586376, "right_score": -0.14523760109561928, "margin": 0.35500708285425686, "p_left": 0.5878312545576733, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5313153543155107, "brier": 0.16988307471950154 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.18381440243220146, "right_score": 0.309671025570845, "margin": -0.4934854280030465, "p_left": 0.3790728342979939, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4765414894311567, "brier": 0.14369621370271432 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": -0.022602202916935765, "right_score": -1.173227342088397, "margin": 1.1506251391714613, "p_left": 0.7596250826469676, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2749302797269295, "brier": 0.057780100892477156 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": -0.022602202916935765, "right_score": -0.47819853747650676, "margin": 0.455596334559571, "p_left": 0.6119689858148791, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7188718418067809, "brier": 0.012537053784412604 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": -0.022602202916935765, "right_score": 0.10194849474574966, "margin": -0.12455069766268542, "p_left": 0.4689025161459517, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6328096891774175, "brier": 0.21986956964800447 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.10194849474574966, "right_score": -0.18381440243220146, "margin": 0.2857628971779511, "p_left": 0.5709585050264169, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8462016398515224, "brier": 0.3259936144620009 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 0.1324540866978254, "right_score": -0.3430987495153674, "margin": 0.4755528362131928, "p_left": 0.6166971933491726, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4833771480747078, "brier": 0.14692104158640157 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": -0.14523760109561928, "right_score": -0.0761646690868942, "margin": -0.06907293200872508, "p_left": 0.48273862938602136, "prediction": "right", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6592069797774582, "brier": 0.2330365843014945 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.18381440243220146, "right_score": -1.173227342088397, "margin": 0.9894129396561955, "p_left": 0.7289719513459568, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3161200232336301, "brier": 0.07345620315721839 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": 0.35557602843431857, "right_score": -0.09692656501363592, "margin": 0.4525025934479545, "p_left": 0.6112340829492493, "prediction": "left", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.49227527869693355, "brier": 0.15113893826031116 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 0.892127649875238, "right_score": 0.309671025570845, "margin": 0.5824566243043929, "p_left": 0.6416324794449466, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44373960113482336, "brier": 0.1284272797887766 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 0.8812886831562179, "right_score": -0.8357333816639461, "margin": 1.717022064820164, "p_left": 0.8477448617068543, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.16517555908959203, "brier": 0.023181627136664936 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": -0.014177080267333414, "right_score": 0.4116274122738632, "margin": -0.42580449254119657, "p_left": 0.39512862375120233, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5027394447957999, "brier": 0.15612662930751922 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 0.1452530091726118, "right_score": 0.10194849474574966, "margin": 0.043304514426862145, "p_left": 0.5108244370878428, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7150338295808492, "brier": 0.2609416055261114 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 0.1452530091726118, "right_score": -0.47819853747650676, "margin": 0.6234515466491186, "p_left": 0.6510031413928989, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.4292408112968864, "brier": 0.1217988073176249 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.18381440243220146, "right_score": -0.3430987495153674, "margin": 0.15928434708316594, "p_left": 0.5397371064324787, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.7759574449918772, "brier": 0.29131614406010486 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": -0.022602202916935765, "right_score": -0.8357333816639461, "margin": 0.8131311787470104, "p_left": 0.692776337848022, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.7736136660671025, "brier": 0.0371627164340947 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": -0.8357333816639461, "right_score": 0.1452530091726118, "margin": -0.9809863908365579, "p_left": 0.2726961060073739, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3184108777968904, "brier": 0.07436316623158488 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 0.8812886831562179, "right_score": -0.47819853747650676, "margin": 1.3594872206327246, "p_left": 0.7956763449938843, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22856277758914323, "brier": 0.0417481559950582 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.47819853747650676, "right_score": 0.892127649875238, "margin": -1.3703261873517447, "p_left": 0.20256715106921164, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.22635764983598564, "brier": 0.04103345069229681 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": -0.014177080267333414, "right_score": -0.368851350626314, "margin": 0.3546742703589806, "p_left": 0.5877506165026163, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.8861268130016823, "brier": 0.3454507871992055 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 0.892127649875238, "right_score": -0.3430987495153674, "margin": 1.2352263993906054, "p_left": 0.77473200712037, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2552381066944891, "brier": 0.05074566861601701 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": -0.4391355859423503, "right_score": 0.6792978458539041, "margin": -1.1184334317962543, "p_left": 0.24630198147372742, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 0.8419802130295557, "brier": 0.06436268460415696 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": -0.3430987495153674, "right_score": 0.1452530091726118, "margin": -0.4883517586879792, "p_left": 0.3802819279826576, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.47849062691332533, "brier": 0.14461434475020718 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -0.368851350626314, "right_score": -0.4391355859423503, "margin": 0.07028423531603628, "p_left": 0.517563829166018, "prediction": "left", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.6586224200649262, "brier": 0.2327446589289551 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 0.8812886831562179, "right_score": -1.173227342088397, "margin": 2.054516025244615, "p_left": 0.8864031432119999, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12058341695298624, "brier": 0.012904245872113413 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "semantic_bt_tuned", "selected": { "ridge": 0.1, "consistency": 0.0, "mean_cross_entropy": 0.6058947507107211, "fold_losses": [ 0.7189138463180482, 0.862528473911577, 0.012956381259303106, 0.02875134178728414, 0.3084284976166184, 0.19669979870669563, 0.3750644082386938, 0.38324845399676055, 0.589275804801903, 0.27100099964862917, 1.2483631457662172, 1.1428209915621068, 1.3137492467426766, 0.8884768258439122, 0.6780032306823264, 0.03820678283549661, 1.949127557480403, 0.17691592059716665, 0.9165369731790542, 0.7461045385030495, 0.4440661182866313, 0.033274110873456074, 0.2584830958987909, 0.9604774725205032 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.445369684936586, "brier_score": 0.11910381860456508, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 2.009180177979975, "brier_score": 0.6170827186745439, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.6339575693454638, "brier_score": 0.5310645687792478, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 2.7596253952489973, "brier_score": 0.789119018465136, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.11502369200502349, "brier_score": 0.02272571219897284, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.7772078568033206, "right_score": 2.349863138207501, "margin": -3.1270709950108215, "p_left": 0.04200431208956988, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.04291200215874474, "brier": 0.0017643622341179865 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.24244242045976644, "right_score": -1.9206913081991126, "margin": 1.6782488877393462, "p_left": 0.8426725157307828, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8494257590107208, "brier": 0.7100969687680464 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.8557963182461654, "right_score": 2.728411430237043, "margin": -0.8726151119908774, "p_left": 0.2947104431030656, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34914684148908653, "brier": 0.08685424527400526 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.46971746554167193, "right_score": 2.7517298898608176, "margin": -3.2214473554024896, "p_left": 0.03836655027029346, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03912193031357186, "brier": 0.0014719921796429551 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.7772078568033206, "right_score": 2.7517298898608176, "margin": -3.528937746664138, "p_left": 0.02849998435306833, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.028913994058691123, "brier": 0.0008122491081251396 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.24244242045976644, "right_score": -0.9557421954865252, "margin": 0.7132997750267587, "p_left": 0.6711298790561516, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1120923753262606, "brier": 0.4504153145619247 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.7772078568033206, "right_score": 1.7258291519014666, "margin": -2.503037008704787, "p_left": 0.0756455484827382, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.07865967536008436, "brier": 0.005722249005254296 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.7675995197970344, "right_score": 1.7898858215767315, "margin": -2.557485341373766, "p_left": 0.07192522012601181, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.632128309055475, "brier": 0.8613227970381517 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 1.1938887519875307, "right_score": 1.381862026956093, "margin": -0.1879732749685623, "p_left": 0.45314456571751976, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.79154407479088, "brier": 0.2990508660042801 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 2.728411430237043, "right_score": 0.9119389286688726, "margin": 1.8164725015681702, "p_left": 0.8601423166030105, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15065741900351504, "brier": 0.01956017160517254 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 1.7258291519014666, "right_score": 0.36483188626726687, "margin": 1.3609972656341998, "p_left": 0.7959217317055155, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22825442497529203, "brier": 0.041647939590075594 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.9206913081991126, "right_score": -0.9557421954865252, "margin": -0.9649491127125874, "p_left": 0.2758883950696157, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.287758860732637, "brier": 0.524337616394857 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 1.7258291519014666, "right_score": 1.8557963182461654, "margin": -0.12996716634469885, "p_left": 0.46755386745211763, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6302735460500207, "brier": 0.2186066189694324 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.7517298898608176, "right_score": 1.8557963182461654, "margin": 0.8959335716146521, "p_left": 0.710113134048331, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34233097791059586, "brier": 0.08403439505128092 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 1.381862026956093, "right_score": 1.3975201524984366, "margin": -0.01565812554234358, "p_left": 0.496085548591841, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7010068901299779, "brier": 0.25392977433798586 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.05083587808232801, "right_score": 1.8557963182461654, "margin": -1.8049604401638375, "p_left": 0.14124830587520196, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.15227546266755526, "brier": 0.01995108391261461 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 2.349863138207501, "right_score": -0.7772078568033206, "margin": 3.1270709950108215, "p_left": 0.9579956879104301, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.04291200215874458, "brier": 0.0017643622341179876 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 2.349863138207501, "right_score": -0.46971746554167193, "margin": 2.819580603749173, "p_left": 0.9437247971476428, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.4677109856591943, "brier": 0.19689169560371675 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 2.349863138207501, "right_score": 0.9119389286688726, "margin": 1.4379242095386284, "p_left": 0.8081329975600784, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6509528426042275, "brier": 0.6530789417454377 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.9119389286688726, "right_score": 0.05083587808232801, "margin": 0.8611030505865446, "p_left": 0.7028910616950244, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2136564117928645, "brier": 0.4940558446107585 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 1.7258291519014666, "right_score": 0.7476936828420898, "margin": 0.9781354690593768, "p_left": 0.726738096027131, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31918911941293093, "brier": 0.07467206816287747 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 1.3975201524984366, "right_score": -0.7675995197970344, "margin": 2.165119672295471, "p_left": 0.8970732243925478, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.2737374600059206, "brier": 0.8047403699220423 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.05083587808232801, "right_score": -0.7772078568033206, "margin": 0.8280437348856486, "p_left": 0.6959411290342147, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.36249020694616263, "brier": 0.09245179701298809 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.124909314800902, "right_score": 2.05468807433605, "margin": -3.179597389136952, "p_left": 0.03994076941017512, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 3.2203576870294155, "brier": 0.9217137262407266 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 2.728411430237043, "right_score": 1.8557963182461654, "margin": 0.8726151119908774, "p_left": 0.7052895568969344, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3491468414890866, "brier": 0.08685424527400526 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.7517298898608176, "right_score": 0.36483188626726687, "margin": 2.3868980035935508, "p_left": 0.9158227389548963, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08793244945951173, "brier": 0.007085811277055533 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.7898858215767315, "right_score": 0.7798978678510726, "margin": 1.0099879537256589, "p_left": 0.7330177917569565, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.320573258595995, "brier": 0.5373150830322448 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 1.296604522715134, "right_score": 0.9119389286688726, "margin": 0.3846655940462613, "p_left": 0.5949978928181876, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9038630089710775, "brier": 0.3540224924580835 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 1.296604522715134, "right_score": -0.46971746554167193, "margin": 1.7663219882568058, "p_left": 0.853999677838265, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15782446243220383, "brier": 0.0213160940713304 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.05083587808232801, "right_score": 0.7476936828420898, "margin": -0.6968578047597618, "p_left": 0.3325092607606263, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4042297626026656, "brier": 0.11056240849157818 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 2.349863138207501, "right_score": 0.36483188626726687, "margin": 1.9850312519402342, "p_left": 0.8792164754406891, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1212397629111552, "brier": 0.1438051352456588 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.36483188626726687, "right_score": 1.296604522715134, "margin": -0.9317726364478671, "p_left": 0.2825652234264611, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3320732392106148, "brier": 0.07984310549004589 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.7517298898608176, "right_score": -0.46971746554167193, "margin": 3.2214473554024896, "p_left": 0.9616334497297065, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.039121930313571696, "brier": 0.0014719921796429547 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.46971746554167193, "right_score": 2.728411430237043, "margin": -3.198128895778715, "p_left": 0.03923619650524021, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.040026682224562904, "brier": 0.0015394791161978242 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.7898858215767315, "right_score": -1.9206913081991126, "margin": 3.710577129775844, "p_left": 0.9761207666134847, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 3.734746093715283, "brier": 0.952811751014097 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 2.728411430237043, "right_score": 0.7476936828420898, "margin": 1.9807177473949529, "p_left": 0.8787576536015645, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12924612624902965, "brier": 0.014699706560198214 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 2.230252755438523, "right_score": 0.5232165809626554, "margin": 1.7070361744758675, "p_left": 0.8464514671590047, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.0202204998297848, "brier": 0.12002861909662688 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.7476936828420898, "right_score": 1.296604522715134, "margin": -0.5489108398730441, "p_left": 0.3661171393610426, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4558911040147067, "brier": 0.1340417597339131 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.9206913081991126, "right_score": 2.230252755438523, "margin": -4.150944063637636, "p_left": 0.015505338925830541, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 4.166570867537349, "brier": 0.9692297376835439 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.7517298898608176, "right_score": -0.7772078568033206, "margin": 3.528937746664138, "p_left": 0.9715000156469316, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.028913994058691106, "brier": 0.000812249108125146 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "consistent_bt", "selected": { "ridge": 0.1, "consistency": 0.1, "mean_cross_entropy": 0.6109166363162718, "fold_losses": [ 0.5177778067016785, 0.9275989903923381, 0.039033664665995754, 0.05849012776521651, 0.31342053318864216, 0.27633647912585935, 0.4492951102465713, 0.5079517450316822, 0.6375559170641422, 0.3879236709270066, 1.0150188612224396, 1.3769517714544939, 0.9615840859985116, 0.7940164109065572, 0.596409368721866, 0.09318643328112675, 2.0544564837931243, 0.2305869777519458, 0.8629918913352478, 0.7500958974883426, 0.46174771446993745, 0.08966980153411769, 0.42315094783761387, 0.8367485806860648 ] }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.4814146240102655, "brier_score": 0.13272841042387765, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.18181818181818182, "mean_cross_entropy": 1.6439576305462538, "brier_score": 0.5272111855023448, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 2, "accuracy": 0.2857142857142857, "mean_cross_entropy": 1.2123332116126757, "brier_score": 0.40965177308572187, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 2.50720646841341, "brier_score": 0.762330010335591, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.16437041918659373, "brier_score": 0.03318074118068618, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.8431347047966182, "right_score": 2.150701312644082, "margin": -2.9938360174407004, "p_left": 0.04770511943793034, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.04888054366336228, "brier": 0.002275778420587199 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.14570600598019257, "right_score": -1.5008536531246468, "margin": 1.3551476471444541, "p_left": 0.7949699313712878, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.5845986343620164, "brier": 0.63197719178447 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.395090420662358, "right_score": 2.1071278761666967, "margin": -0.7120374555043387, "p_left": 0.3291487934094773, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.39920791534770206, "brier": 0.10833892820291476 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": 0.009604713713101232, "right_score": 1.867248000145589, "margin": -1.8576432864324877, "p_left": 0.1349779826620476, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1450003188043072, "brier": 0.018219055803516026 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.8431347047966182, "right_score": 1.867248000145589, "margin": -2.710382704942207, "p_left": 0.06236346924062838, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.06439289893100299, "brier": 0.0038892022957268024 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.14570600598019257, "right_score": -0.48885424388344434, "margin": 0.3431482379032518, "p_left": 0.5849550642874296, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.879368485787697, "brier": 0.3421724272355109 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.8431347047966182, "right_score": 1.3909379059738294, "margin": -2.234072610770448, "p_left": 0.0967322113607426, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.10173621511093921, "brier": 0.00935712071473938 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.4807117273869074, "right_score": 1.6201854717254711, "margin": -2.1008971991123784, "p_left": 0.10900964877152243, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.2163188798310762, "brier": 0.7938638059822458 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 1.0578234662695367, "right_score": 0.6836924967443221, "margin": 0.37413096952521463, "p_left": 0.5924567879028937, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5234773404419666, "brier": 0.166091469726427 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 2.1071278761666967, "right_score": 0.7004975448997791, "margin": 1.4066303312669177, "p_left": 0.8032339143394269, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.21910930690791042, "brier": 0.03871689246618398 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 1.3909379059738294, "right_score": 0.1674338833323991, "margin": 1.2235040226414302, "p_left": 0.7726796043987987, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.25789079962999306, "brier": 0.051674562256286666 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.5008536531246468, "right_score": -0.48885424388344434, "margin": -1.0119994092412026, "p_left": 0.2665887455007575, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.3220480870544948, "brier": 0.5378920682261527 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 1.3909379059738294, "right_score": 1.395090420662358, "margin": -0.004152514688528619, "p_left": 0.49896187281960236, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6910730786364121, "brier": 0.24896295052764503 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 1.867248000145589, "right_score": 1.395090420662358, "margin": 0.4721575794832309, "p_left": 0.6158942992215481, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.48467992234505, "brier": 0.14753718937050558 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 0.6836924967443221, "right_score": 0.3373728347765982, "margin": 0.3463196619677239, "p_left": 0.585724822944883, "prediction": "left", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.5349051851131738, "brier": 0.1716239223240485 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": -0.254242681401725, "right_score": 1.395090420662358, "margin": -1.649333102064083, "p_left": 0.16119910310912156, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.1757819107002008, "brier": 0.025985150843185204 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 2.150701312644082, "right_score": -0.8431347047966182, "margin": 2.9938360174407004, "p_left": 0.9522948805620697, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.048880543663362364, "brier": 0.0022757784205871975 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 2.150701312644082, "right_score": 0.009604713713101232, "margin": 2.1410965989309805, "p_left": 0.8948338519585617, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1816655176758233, "brier": 0.1558937706524354 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 2.150701312644082, "right_score": 0.7004975448997791, "margin": 1.450203767744303, "p_left": 0.8100297920574536, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.660888019418491, "brier": 0.6561482640206415 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.7004975448997791, "right_score": -0.254242681401725, "margin": 0.9547402263015041, "p_left": 0.7220674759030397, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.280376913828224, "brier": 0.5213814397569868 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 1.3909379059738294, "right_score": 0.709325394297915, "margin": 0.6816125116759144, "p_left": 0.6640984982267629, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.40932479980414116, "brier": 0.112829818893516 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 0.3373728347765982, "right_score": -0.4807117273869074, "margin": 0.8180845621635056, "p_left": 0.6938295946155484, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1836134516887267, "brier": 0.48139950636437623 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": -0.254242681401725, "right_score": -0.8431347047966182, "margin": 0.5888920233948932, "p_left": 0.6431108840500591, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.44143812163364216, "brier": 0.12736984108353036 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -0.8879815789977697, "right_score": 1.3168950255868426, "margin": -2.204876604584612, "p_left": 0.09931342238202581, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.3094745470553226, "brier": 0.8112363111011791 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 2.1071278761666967, "right_score": 1.395090420662358, "margin": 0.7120374555043387, "p_left": 0.6708512065905228, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.39920791534770217, "brier": 0.10833892820291471 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 1.867248000145589, "right_score": 0.1674338833323991, "margin": 1.6998141168131897, "p_left": 0.845510455944657, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1678147441384692, "brier": 0.02386701922242776 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.6201854717254711, "right_score": 1.1894866233728574, "margin": 0.43069884835261374, "p_left": 0.6060405346122718, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9315072547032637, "brier": 0.3672851295931282 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 1.1205825331122115, "right_score": 0.7004975448997791, "margin": 0.42008498821243234, "p_left": 0.6035035866186387, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9250882836445873, "brier": 0.36421657906156074 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 1.1205825331122115, "right_score": 0.009604713713101232, "margin": 1.1109778193991102, "p_left": 0.7523113622550582, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2846049951735645, "brier": 0.06134966126794499 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": -0.254242681401725, "right_score": 0.709325394297915, "margin": -0.96356807569964, "p_left": 0.27616437571592567, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3231909506551783, "brier": 0.07626676241456695 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 2.150701312644082, "right_score": 0.1674338833323991, "margin": 1.9832674293116828, "p_left": 0.8790290412145495, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1205710575741725, "brier": 0.14366301408402066 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.1674338833323991, "right_score": 1.1205825331122115, "margin": -0.9531486497798123, "p_left": 0.2782520441432017, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3260792926470544, "brier": 0.07742420006987027 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 1.867248000145589, "right_score": 0.009604713713101232, "margin": 1.8576432864324877, "p_left": 0.8650220173379525, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.1450003188043072, "brier": 0.018219055803516 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": 0.009604713713101232, "right_score": 2.1071278761666967, "margin": -2.0975231624535953, "p_left": 0.10933778987334337, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.11579003660653911, "brier": 0.011954752294387389 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.6201854717254711, "right_score": -1.5008536531246468, "margin": 3.121039124850118, "p_left": 0.9577522941030294, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 3.1642052249497143, "brier": 0.9172894568596157 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 2.1071278761666967, "right_score": 0.709325394297915, "margin": 1.3978024818687818, "p_left": 0.8018349440414054, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.2208524977311237, "brier": 0.03926938940307292 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 2.4378337014530063, "right_score": 0.45607177643382135, "margin": 1.981761925019185, "p_left": 0.8788688590513332, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1200005482801798, "brier": 0.14354161235885896 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.709325394297915, "right_score": 1.1205825331122115, "margin": -0.4112571388142965, "p_left": 0.3986107211889958, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.5085128356384842, "brier": 0.15889050704681137 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.5008536531246468, "right_score": 2.4378337014530063, "margin": -3.9386873545776533, "p_left": 0.019101776700494927, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 3.957973927287414, "brier": 0.9621613244721257 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 1.867248000145589, "right_score": -0.8431347047966182, "margin": 2.710382704942207, "p_left": 0.9376365307593716, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.0643928989310032, "brier": 0.003889202295726806 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" }, { "name": "semantic_bt_matched", "selected": { "ridge": 0.1, "consistency": 0.0 }, "metrics": { "train": { "scheduled": 24, "decisive": 22, "ties": 2, "skips": 0, "correct": 19, "accuracy": 0.8636363636363636, "mean_cross_entropy": 0.445369684936586, "brier_score": 0.11910381860456508, "predicted_ties_on_decisive": 0 }, "heldout": { "scheduled": 12, "decisive": 11, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 2.009180177979975, "brier_score": 0.6170827186745439, "predicted_ties_on_decisive": 0 }, "other": { "scheduled": 8, "decisive": 7, "ties": 1, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 1.6339575693454638, "brier_score": 0.5310645687792478, "predicted_ties_on_decisive": 0 }, "target": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 0, "accuracy": 0.0, "mean_cross_entropy": 2.7596253952489973, "brier_score": 0.789119018465136, "predicted_ties_on_decisive": 0 }, "repeat": { "scheduled": 4, "decisive": 4, "ties": 0, "skips": 0, "correct": 4, "accuracy": 1.0, "mean_cross_entropy": 0.11502369200502349, "brier_score": 0.02272571219897284, "predicted_ties_on_decisive": 0 } }, "trials": [ { "trial_id": "e60641edcb3b", "phase": "train", "left_score": -0.7772078568033206, "right_score": 2.349863138207501, "margin": -3.1270709950108215, "p_left": 0.04200431208956988, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "natalie_portman", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.04291200215874474, "brier": 0.0017643622341179865 }, { "trial_id": "1b6d9789fab8", "phase": "test_target", "left_score": -0.24244242045976644, "right_score": -1.9206913081991126, "margin": 1.6782488877393462, "p_left": 0.8426725157307828, "prediction": "left", "left_identity": "lily_collins", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.8494257590107208, "brier": 0.7100969687680464 }, { "trial_id": "c4531359359a", "phase": "train", "left_score": 1.8557963182461654, "right_score": 2.728411430237043, "margin": -0.8726151119908774, "p_left": 0.2947104431030656, "prediction": "right", "left_identity": "margot_robbie", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.34914684148908653, "brier": 0.08685424527400526 }, { "trial_id": "42ecaecf4d00", "phase": "train", "left_score": -0.46971746554167193, "right_score": 2.7517298898608176, "margin": -3.2214473554024896, "p_left": 0.03836655027029346, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.03912193031357186, "brier": 0.0014719921796429551 }, { "trial_id": "000340fb28e8", "phase": "train", "left_score": -0.7772078568033206, "right_score": 2.7517298898608176, "margin": -3.528937746664138, "p_left": 0.02849998435306833, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.028913994058691123, "brier": 0.0008122491081251396 }, { "trial_id": "0add2ec437a1", "phase": "test_other", "left_score": -0.24244242045976644, "right_score": -0.9557421954865252, "margin": 0.7132997750267587, "p_left": 0.6711298790561516, "prediction": "left", "left_identity": "lily_collins", "right_identity": "elizabeth_olsen", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.1120923753262606, "brier": 0.4504153145619247 }, { "trial_id": "a3e23f74e09c", "phase": "train", "left_score": -0.7772078568033206, "right_score": 1.7258291519014666, "margin": -2.503037008704787, "p_left": 0.0756455484827382, "prediction": "right", "left_identity": "alexandra_daddario", "right_identity": "ana_de_armas", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.07865967536008436, "brier": 0.005722249005254296 }, { "trial_id": "d013ade6e6a0", "phase": "test_other", "left_score": -0.7675995197970344, "right_score": 1.7898858215767315, "margin": -2.557485341373766, "p_left": 0.07192522012601181, "prediction": "right", "left_identity": "keira_knightley", "right_identity": "ana_de_armas", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 2.632128309055475, "brier": 0.8613227970381517 }, { "trial_id": "34bec29a8d38", "phase": "test_other", "left_score": 1.1938887519875307, "right_score": 1.381862026956093, "margin": -0.1879732749685623, "p_left": 0.45314456571751976, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "emma_watson", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.79154407479088, "brier": 0.2990508660042801 }, { "trial_id": "81e37ad8b84c", "phase": "train", "left_score": 2.728411430237043, "right_score": 0.9119389286688726, "margin": 1.8164725015681702, "p_left": 0.8601423166030105, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "emma_stone", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15065741900351504, "brier": 0.01956017160517254 }, { "trial_id": "4aac8a542740", "phase": "train", "left_score": 1.7258291519014666, "right_score": 0.36483188626726687, "margin": 1.3609972656341998, "p_left": 0.7959217317055155, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.22825442497529203, "brier": 0.041647939590075594 }, { "trial_id": "974972ffb84a", "phase": "test_target", "left_score": -1.9206913081991126, "right_score": -0.9557421954865252, "margin": -0.9649491127125874, "p_left": 0.2758883950696157, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 1.287758860732637, "brier": 0.524337616394857 }, { "trial_id": "d09696ba6584", "phase": "train", "left_score": 1.7258291519014666, "right_score": 1.8557963182461654, "margin": -0.12996716634469885, "p_left": 0.46755386745211763, "prediction": "right", "left_identity": "ana_de_armas", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.6302735460500207, "brier": 0.2186066189694324 }, { "trial_id": "e956c2e5a05f", "phase": "train", "left_score": 2.7517298898608176, "right_score": 1.8557963182461654, "margin": 0.8959335716146521, "p_left": 0.710113134048331, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.34233097791059586, "brier": 0.08403439505128092 }, { "trial_id": "70f39a6b70be", "phase": "test_other", "left_score": 1.381862026956093, "right_score": 1.3975201524984366, "margin": -0.01565812554234358, "p_left": 0.496085548591841, "prediction": "right", "left_identity": "emma_watson", "right_identity": "natalie_portman", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 0.7010068901299779, "brier": 0.25392977433798586 }, { "trial_id": "32bd1f162f4d", "phase": "train", "left_score": 0.05083587808232801, "right_score": 1.8557963182461654, "margin": -1.8049604401638375, "p_left": 0.14124830587520196, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "margot_robbie", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.15227546266755526, "brier": 0.01995108391261461 }, { "trial_id": "0f2756272361", "phase": "repeat", "left_score": 2.349863138207501, "right_score": -0.7772078568033206, "margin": 3.1270709950108215, "p_left": 0.9579956879104301, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.04291200215874458, "brier": 0.0017643622341179876 }, { "trial_id": "071e404d1ee8", "phase": "train", "left_score": 2.349863138207501, "right_score": -0.46971746554167193, "margin": 2.819580603749173, "p_left": 0.9437247971476428, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "elizabeth_olsen", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.4677109856591943, "brier": 0.19689169560371675 }, { "trial_id": "47e30cc905fe", "phase": "train", "left_score": 2.349863138207501, "right_score": 0.9119389286688726, "margin": 1.4379242095386284, "p_left": 0.8081329975600784, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.6509528426042275, "brier": 0.6530789417454377 }, { "trial_id": "31989aab742f", "phase": "train", "left_score": 0.9119389286688726, "right_score": 0.05083587808232801, "margin": 0.8611030505865446, "p_left": 0.7028910616950244, "prediction": "left", "left_identity": "emma_stone", "right_identity": "gal_gadot", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.2136564117928645, "brier": 0.4940558446107585 }, { "trial_id": "b5d8d265ab50", "phase": "train", "left_score": 1.7258291519014666, "right_score": 0.7476936828420898, "margin": 0.9781354690593768, "p_left": 0.726738096027131, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.31918911941293093, "brier": 0.07467206816287747 }, { "trial_id": "b20ebcae7654", "phase": "test_other", "left_score": 1.3975201524984366, "right_score": -0.7675995197970344, "margin": 2.165119672295471, "p_left": 0.8970732243925478, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 2.2737374600059206, "brier": 0.8047403699220423 }, { "trial_id": "d66d725d6fec", "phase": "train", "left_score": 0.05083587808232801, "right_score": -0.7772078568033206, "margin": 0.8280437348856486, "p_left": 0.6959411290342147, "prediction": "left", "left_identity": "gal_gadot", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.36249020694616263, "brier": 0.09245179701298809 }, { "trial_id": "e5aa020b210d", "phase": "test_other", "left_score": -1.124909314800902, "right_score": 2.05468807433605, "margin": -3.179597389136952, "p_left": 0.03994076941017512, "prediction": "right", "left_identity": "emma_stone", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 3.2203576870294155, "brier": 0.9217137262407266 }, { "trial_id": "dd8c77c145bc", "phase": "repeat", "left_score": 2.728411430237043, "right_score": 1.8557963182461654, "margin": 0.8726151119908774, "p_left": 0.7052895568969344, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "margot_robbie", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.3491468414890866, "brier": 0.08685424527400526 }, { "trial_id": "969b4d8654a6", "phase": "train", "left_score": 2.7517298898608176, "right_score": 0.36483188626726687, "margin": 2.3868980035935508, "p_left": 0.9158227389548963, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "lily_collins", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.08793244945951173, "brier": 0.007085811277055533 }, { "trial_id": "396ff40ef0fb", "phase": "test_other", "left_score": 1.7898858215767315, "right_score": 0.7798978678510726, "margin": 1.0099879537256589, "p_left": 0.7330177917569565, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "scarlett_johansson", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 1.320573258595995, "brier": 0.5373150830322448 }, { "trial_id": "933eb112032c", "phase": "train", "left_score": 1.296604522715134, "right_score": 0.9119389286688726, "margin": 0.3846655940462613, "p_left": 0.5949978928181876, "prediction": "left", "left_identity": "emma_watson", "right_identity": "emma_stone", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 0.9038630089710775, "brier": 0.3540224924580835 }, { "trial_id": "921e85ee9f19", "phase": "train", "left_score": 1.296604522715134, "right_score": -0.46971746554167193, "margin": 1.7663219882568058, "p_left": 0.853999677838265, "prediction": "left", "left_identity": "emma_watson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.15782446243220383, "brier": 0.0213160940713304 }, { "trial_id": "6d1119afff61", "phase": "train", "left_score": 0.05083587808232801, "right_score": 0.7476936828420898, "margin": -0.6968578047597618, "p_left": 0.3325092607606263, "prediction": "right", "left_identity": "gal_gadot", "right_identity": "anne_hathaway", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4042297626026656, "brier": 0.11056240849157818 }, { "trial_id": "b1d82931de8a", "phase": "train", "left_score": 2.349863138207501, "right_score": 0.36483188626726687, "margin": 1.9850312519402342, "p_left": 0.8792164754406891, "prediction": "left", "left_identity": "natalie_portman", "right_identity": "lily_collins", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.1212397629111552, "brier": 0.1438051352456588 }, { "trial_id": "7dbd22b1e8e1", "phase": "train", "left_score": 0.36483188626726687, "right_score": 1.296604522715134, "margin": -0.9317726364478671, "p_left": 0.2825652234264611, "prediction": "right", "left_identity": "lily_collins", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.3320732392106148, "brier": 0.07984310549004589 }, { "trial_id": "2333a59ee3bd", "phase": "repeat", "left_score": 2.7517298898608176, "right_score": -0.46971746554167193, "margin": 3.2214473554024896, "p_left": 0.9616334497297065, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "elizabeth_olsen", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.039121930313571696, "brier": 0.0014719921796429547 }, { "trial_id": "9084ada82c9d", "phase": "train", "left_score": -0.46971746554167193, "right_score": 2.728411430237043, "margin": -3.198128895778715, "p_left": 0.03923619650524021, "prediction": "right", "left_identity": "elizabeth_olsen", "right_identity": "keira_knightley", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.040026682224562904, "brier": 0.0015394791161978242 }, { "trial_id": "c469d02ac7ba", "phase": "test_target", "left_score": 1.7898858215767315, "right_score": -1.9206913081991126, "margin": 3.710577129775844, "p_left": 0.9761207666134847, "prediction": "left", "left_identity": "ana_de_armas", "right_identity": "tongdak_cheonsa", "choice": "right", "target": 0.0, "correct": 0, "cross_entropy": 3.734746093715283, "brier": 0.952811751014097 }, { "trial_id": "ba41f8d9a95c", "phase": "train", "left_score": 2.728411430237043, "right_score": 0.7476936828420898, "margin": 1.9807177473949529, "p_left": 0.8787576536015645, "prediction": "left", "left_identity": "keira_knightley", "right_identity": "anne_hathaway", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.12924612624902965, "brier": 0.014699706560198214 }, { "trial_id": "10f32c086059", "phase": "test_other", "left_score": 2.230252755438523, "right_score": 0.5232165809626554, "margin": 1.7070361744758675, "p_left": 0.8464514671590047, "prediction": "left", "left_identity": "alexandra_daddario", "right_identity": "margot_robbie", "choice": "tie", "target": 0.5, "correct": null, "cross_entropy": 1.0202204998297848, "brier": 0.12002861909662688 }, { "trial_id": "5e839d900d72", "phase": "train", "left_score": 0.7476936828420898, "right_score": 1.296604522715134, "margin": -0.5489108398730441, "p_left": 0.3661171393610426, "prediction": "right", "left_identity": "anne_hathaway", "right_identity": "emma_watson", "choice": "right", "target": 0.0, "correct": 1, "cross_entropy": 0.4558911040147067, "brier": 0.1340417597339131 }, { "trial_id": "fe73774e1a9e", "phase": "test_target", "left_score": -1.9206913081991126, "right_score": 2.230252755438523, "margin": -4.150944063637636, "p_left": 0.015505338925830541, "prediction": "right", "left_identity": "tongdak_cheonsa", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 0, "cross_entropy": 4.166570867537349, "brier": 0.9692297376835439 }, { "trial_id": "a33fba8c36a4", "phase": "repeat", "left_score": 2.7517298898608176, "right_score": -0.7772078568033206, "margin": 3.528937746664138, "p_left": 0.9715000156469316, "prediction": "left", "left_identity": "scarlett_johansson", "right_identity": "alexandra_daddario", "choice": "left", "target": 1.0, "correct": 1, "cross_entropy": 0.028913994058691106, "brier": 0.000812249108125146 } ], "probability_status": "Portrait-fitted Bradley-Terry scale" } ], "rank_sensitivity": { "zero_shot": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.006258333934324074, "rank": 12.0, "portrait_scores": [ -0.01651002041669066, 0.003993352548042513 ], "portrait_score_gap": 0.020503372964733174, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 9.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.0013210483588317447, "rank": 3.0, "portrait_scores": [ -0.002233464457180626, 0.004875561174844115 ], "portrait_score_gap": 0.007109025632024741, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 4.0, 10.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.003535409689602233, "rank": 8.0, "portrait_scores": [ -0.0066098017896289475, -0.0004610175895755185 ], "portrait_score_gap": 0.006148784200053429, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 9.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.00216220665221373, "rank": 7.0, "portrait_scores": [ -0.001717606619682557, -0.002606806684744903 ], "portrait_score_gap": 0.0008892000650623458, "single_photo_rank_min": 5.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 8.0, 10.0 ] }, { "identity": "emma_stone", "mean_score": -0.006721541770849666, "rank": 13.0, "portrait_scores": [ -0.003762681503884961, -0.00968040203781437 ], "portrait_score_gap": 0.00591772053392941, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.0, 13.0 ] }, { "identity": "emma_watson", "mean_score": -0.0045846148950199495, "rank": 10.0, "portrait_scores": [ -0.010240766068066991, 0.0010715362780270915 ], "portrait_score_gap": 0.011312302346094084, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 10.0, 13.0 ] }, { "identity": "gal_gadot", "mean_score": 0.0019282540335639462, "rank": 2.0, "portrait_scores": [ -0.001366168906587922, 0.0052226769737158145 ], "portrait_score_gap": 0.0065888458803037364, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 2.5, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.0009339228876092543, "rank": 4.0, "portrait_scores": [ -0.0007998195278131229, 0.0026676653030316315 ], "portrait_score_gap": 0.0034674848308447544, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "lily_collins", "mean_score": -0.006042520248854092, "rank": 11.0, "portrait_scores": [ -0.006478354876574201, -0.005606685621133983 ], "portrait_score_gap": 0.0008716692554402179, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.005757556704005823, "rank": 1.0, "portrait_scores": [ 0.004514939579220938, 0.007000173828790707 ], "portrait_score_gap": 0.0024852342495697696, "single_photo_rank_min": 1.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.625, "single_photo_rank_quantiles": [ 1.0, 1.0, 3.0 ] }, { "identity": "natalie_portman", "mean_score": -0.004019419381127919, "rank": 9.0, "portrait_scores": [ -0.00662398836879293, -0.0014148503934629081 ], "portrait_score_gap": 0.0052091379753300224, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.0, 13.0 ] }, { "identity": "scarlett_johansson", "mean_score": -0.0007135445536791125, "rank": 6.0, "portrait_scores": [ 0.004443840509806633, -0.005870929617164858 ], "portrait_score_gap": 0.01031477012697149, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 5.5, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.00011281449579252523, "rank": 5.0, "portrait_scores": [ -0.0016329574381319248, 0.0014073284465468744 ], "portrait_score_gap": 0.003040285884678799, "single_photo_rank_min": 2.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 6.0, 9.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "2.0": 128, "3.0": 640, "4.0": 1344, "5.0": 1664, "6.0": 1600, "7.0": 1408, "8.0": 960, "9.0": 384, "10.0": 64 }, "target_primary_score": -0.00011281449579252523, "target_primary_rank": 5.0, "target_alternate_photo_scores": [ -0.008859994913873086, -0.003857072798025145 ], "target_alternate_panel_score": -0.006358533855949116, "target_alternate_panel_rank": 12.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -0.00031601360709502396, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -0.004196327203735386, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.0076795822147808605, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -0.00010019992162504151, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.687803557847922e-05, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "pair_centroid": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.013588513042144132, "rank": 11.0, "portrait_scores": [ -0.03242146244955037, 0.0052444363652621034 ], "portrait_score_gap": 0.03766589881481247, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.0570395156613495, "rank": 8.0, "portrait_scores": [ 0.11558078307838597, -0.00150175175568696 ], "portrait_score_gap": 0.11708253483407294, "single_photo_rank_min": 5.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.5, 12.0 ] }, { "identity": "anne_hathaway", "mean_score": 0.011082199246506046, "rank": 10.0, "portrait_scores": [ 0.10724368322500263, -0.08507928473199054 ], "portrait_score_gap": 0.19232296795699316, "single_photo_rank_min": 7.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 11.0, 13.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.030191111284014804, "rank": 13.0, "portrait_scores": [ 0.012097210632772835, -0.07247943320080244 ], "portrait_score_gap": 0.08457664383357527, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 11.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.11216167927396731, "rank": 7.0, "portrait_scores": [ 0.11613009954543342, 0.10819325900250118 ], "portrait_score_gap": 0.007936840542932241, "single_photo_rank_min": 5.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 6.5, 8.0 ] }, { "identity": "emma_watson", "mean_score": 0.15503423903162647, "rank": 3.0, "portrait_scores": [ 0.14611438803367197, 0.16395409002958097 ], "portrait_score_gap": 0.017839701995909008, "single_photo_rank_min": 3.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 4.0, 6.0 ] }, { "identity": "gal_gadot", "mean_score": 0.16589917852340358, "rank": 2.0, "portrait_scores": [ 0.16556818379152655, 0.1662301732552806 ], "portrait_score_gap": 0.0006619894637540646, "single_photo_rank_min": 2.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.5, 3.0 ] }, { "identity": "keira_knightley", "mean_score": 0.12126658405690686, "rank": 6.0, "portrait_scores": [ 0.057234591534066064, 0.18529857657974766 ], "portrait_score_gap": 0.1280639850456816, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 4.5, 9.0 ] }, { "identity": "lily_collins", "mean_score": -0.021458069618402416, "rank": 12.0, "portrait_scores": [ -0.03310440042690169, -0.009811738809903145 ], "portrait_score_gap": 0.023292661616998542, "single_photo_rank_min": 10.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 12.0, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.19909433425865783, "rank": 1.0, "portrait_scores": [ 0.2214978869083774, 0.17669078160893825 ], "portrait_score_gap": 0.044807105299439154, "single_photo_rank_min": 1.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.75, "single_photo_rank_quantiles": [ 1.0, 1.0, 2.0 ] }, { "identity": "natalie_portman", "mean_score": 0.1447821400090164, "rank": 4.0, "portrait_scores": [ 0.1516798202577523, 0.13788445976028046 ], "portrait_score_gap": 0.013795360497471848, "single_photo_rank_min": 3.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.13437257207941272, "rank": 5.0, "portrait_scores": [ 0.11121037427097377, 0.1575347698878517 ], "portrait_score_gap": 0.04632439561687793, "single_photo_rank_min": 3.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 8.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.02076683342587906, "rank": 9.0, "portrait_scores": [ 0.005015885743101252, 0.03651778110865687 ], "portrait_score_gap": 0.031501895365555616, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 9.0, 11.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 1280, "9.0": 3072, "10.0": 2560, "11.0": 1024, "12.0": 256 }, "target_primary_score": 0.02076683342587906, "target_primary_rank": 9.0, "target_alternate_photo_scores": [ 0.06297239885041195, 0.09362822347034747 ], "target_alternate_panel_score": 0.07830031116037971, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.09975838077878213, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.10849142244439451, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": 0.021260795499030205, "prefers_target": true, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.09188882420252384, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0015755634474277072, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "identity_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.5552034817852926, "rank": 13.0, "portrait_scores": [ -0.5552034817852926, -0.5552034817852926 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 13.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 13.0, 13.0, 13.0 ] }, { "identity": "ana_de_armas", "mean_score": 0.21958607998633384, "rank": 4.0, "portrait_scores": [ 0.21958607998633384, 0.21958607998633384 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 4.0, "single_photo_rank_max": 4.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 4.0, 4.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.22706636401654093, "rank": 10.0, "portrait_scores": [ -0.22706636401654093, -0.22706636401654093 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 10.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 10.0, 10.0, 10.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.3613757267495671, "rank": 11.0, "portrait_scores": [ -0.3613757267495671, -0.3613757267495671 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 11.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 11.0, 11.0, 11.0 ] }, { "identity": "emma_stone", "mean_score": 0.04735007533955107, "rank": 6.0, "portrait_scores": [ 0.04735007533955107, 0.04735007533955107 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 6.0, "single_photo_rank_max": 6.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 6.0, 6.0 ] }, { "identity": "emma_watson", "mean_score": 0.22928675300045034, "rank": 3.0, "portrait_scores": [ 0.22928675300045034, 0.22928675300045034 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 3.0, "single_photo_rank_max": 3.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 3.0, 3.0 ] }, { "identity": "gal_gadot", "mean_score": -0.04655164691805275, "rank": 8.0, "portrait_scores": [ -0.04655164691805275, -0.04655164691805275 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 8.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 8.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": 0.5639172371946242, "rank": 1.0, "portrait_scores": [ 0.5639172371946242, 0.5639172371946242 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "lily_collins", "mean_score": -0.3839854052468287, "rank": 12.0, "portrait_scores": [ -0.3839854052468287, -0.3839854052468287 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 12.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.09092352020600739, "rank": 5.0, "portrait_scores": [ 0.09092352020600739, 0.09092352020600739 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 5.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 5.0, 5.0 ] }, { "identity": "natalie_portman", "mean_score": -0.09129337128269767, "rank": 9.0, "portrait_scores": [ -0.09129337128269767, -0.09129337128269767 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 9.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 9.0, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.5144123302720128, "rank": 2.0, "portrait_scores": [ 0.5144123302720128, 0.5144123302720128 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 2.0, "single_photo_rank_max": 2.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 2.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": 0.0, "rank": 7.0, "portrait_scores": [ 0.0, 0.0 ], "portrait_score_gap": 0.0, "single_photo_rank_min": 7.0, "single_photo_rank_max": 7.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 7.0, 7.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "7.0": 8192 }, "target_primary_score": 0.0, "target_primary_rank": 7.0, "target_alternate_photo_scores": [ 0.0, 0.0 ], "target_alternate_panel_score": 0.0, "target_alternate_panel_rank": 7.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.3839854052468287, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.3613757267495671, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.21958607998633384, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.5552034817852926, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.0, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "visual_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": -0.4391355859423503, "rank": 11.0, "portrait_scores": [ -0.42671749949918003, -0.4515536723855206 ], "portrait_score_gap": 0.024836172886340546, "single_photo_rank_min": 8.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 10.0, 12.0 ] }, { "identity": "ana_de_armas", "mean_score": -0.014177080267333414, "rank": 6.0, "portrait_scores": [ 0.22247375379221362, -0.25082791432688045 ], "portrait_score_gap": 0.4733016681190941, "single_photo_rank_min": 4.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 7.0, 9.0 ] }, { "identity": "anne_hathaway", "mean_score": -0.09692656501363592, "rank": 8.0, "portrait_scores": [ 0.3913223474297452, -0.585175477457017 ], "portrait_score_gap": 0.9764978248867622, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 7.5, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.6151609892063097, "rank": 12.0, "portrait_scores": [ -0.4306959040347176, -0.7996260743779018 ], "portrait_score_gap": 0.3689301703431842, "single_photo_rank_min": 9.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 12.5, 13.0 ] }, { "identity": "emma_stone", "mean_score": 0.35557602843431857, "rank": 3.0, "portrait_scores": [ 0.3256965860026114, 0.3854554708660258 ], "portrait_score_gap": 0.05975888486341435, "single_photo_rank_min": 2.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 4.0, 5.0 ] }, { "identity": "emma_watson", "mean_score": 0.2097694817586376, "rank": 4.0, "portrait_scores": [ 0.260509084743655, 0.15902987877362024 ], "portrait_score_gap": 0.10147920597003474, "single_photo_rank_min": 4.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 5.0, 7.0 ] }, { "identity": "gal_gadot", "mean_score": 0.14093534830924198, "rank": 5.0, "portrait_scores": [ 0.24791088091256633, 0.03395981570591766 ], "portrait_score_gap": 0.21395106520664867, "single_photo_rank_min": 4.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 6.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": -0.0761646690868942, "rank": 7.0, "portrait_scores": [ -0.5492789179385797, 0.39694957976479134 ], "portrait_score_gap": 0.946228497703371, "single_photo_rank_min": 2.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 6.5, 12.0 ] }, { "identity": "lily_collins", "mean_score": -0.6793420338305063, "rank": 13.0, "portrait_scores": [ -0.687425160647916, -0.6712589070130965 ], "portrait_score_gap": 0.016166253634819494, "single_photo_rank_min": 12.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 12.0, 12.5, 13.0 ] }, { "identity": "margot_robbie", "mean_score": 0.6792978458539041, "rank": 1.0, "portrait_scores": [ 0.790594547562402, 0.568001144145406 ], "portrait_score_gap": 0.22259340341699596, "single_photo_rank_min": 1.0, "single_photo_rank_max": 1.0, "single_photo_top_share": 1.0, "single_photo_rank_quantiles": [ 1.0, 1.0, 1.0 ] }, { "identity": "natalie_portman", "mean_score": -0.14523760109561928, "rank": 9.0, "portrait_scores": [ -0.08181763996186804, -0.2086575622293705 ], "portrait_score_gap": 0.12683992226750246, "single_photo_rank_min": 6.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 7.5, 9.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.4116274122738632, "rank": 2.0, "portrait_scores": [ 0.33957765733002004, 0.4836771672177063 ], "portrait_score_gap": 0.14409950988768627, "single_photo_rank_min": 2.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 2.0, 2.0, 5.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -0.368851350626314, "rank": 10.0, "portrait_scores": [ -0.42911893457244193, -0.30858376668018617 ], "portrait_score_gap": 0.12053516789225577, "single_photo_rank_min": 8.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 8.0, 9.0, 11.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "8.0": 1536, "9.0": 3584, "10.0": 2560, "11.0": 512 }, "target_primary_score": -0.368851350626314, "target_primary_rank": 10.0, "target_alternate_photo_scores": [ -0.19798497584181135, 0.032645831213609394 ], "target_alternate_panel_score": -0.08266957231410098, "target_alternate_panel_rank": 8.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": 0.5966724615164053, "prefers_target": true, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": 0.5324914168922087, "prefers_target": true, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -0.06849249204676756, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": 0.3564660136282493, "prefers_target": true, "ties_target": false } ], "mean_within_test_score_variance": 0.045586005599813656, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_tuned": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 2.230252755438523, "rank": 1.0, "portrait_scores": [ 3.642670295009054, 0.8178352158679923 ], "portrait_score_gap": 2.8248350791410615, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.7898858215767315, "rank": 3.0, "portrait_scores": [ 1.016778491014615, 2.562993152138848 ], "portrait_score_gap": 1.546214661124233, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 2.05468807433605, "rank": 2.0, "portrait_scores": [ 4.595167820323795, -0.48579167165169457 ], "portrait_score_gap": 5.08095949197549, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.9557421954865252, "rank": 11.0, "portrait_scores": [ -1.1092114403716438, -0.8022729506014066 ], "portrait_score_gap": 0.3069384897702372, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.124909314800902, "rank": 12.0, "portrait_scores": [ 1.0208202357609357, -3.2706388653627396 ], "portrait_score_gap": 4.291459101123675, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 10.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 1.381862026956093, "rank": 5.0, "portrait_scores": [ 2.611065260888543, 0.1526587930236431 ], "portrait_score_gap": 2.4584064678649002, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "gal_gadot", "mean_score": 1.1938887519875307, "rank": 6.0, "portrait_scores": [ 0.9520642108241307, 1.4357132931509309 ], "portrait_score_gap": 0.48364908232680015, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 5.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": -0.7675995197970344, "rank": 10.0, "portrait_scores": [ -2.3734664887024675, 0.8382674491083987 ], "portrait_score_gap": 3.211733937810866, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.24244242045976644, "rank": 9.0, "portrait_scores": [ 0.5273816865072701, -1.012266527426803 ], "portrait_score_gap": 1.5396482139340733, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.5232165809626554, "rank": 8.0, "portrait_scores": [ -0.6617441661742869, 1.7081773280995978 ], "portrait_score_gap": 2.3699214942738847, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.5, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 1.3975201524984366, "rank": 4.0, "portrait_scores": [ 1.6677538899267403, 1.1272864150701332 ], "portrait_score_gap": 0.5404674748566072, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.7798978678510726, "rank": 7.0, "portrait_scores": [ -1.0748579885971403, 2.6346537242992856 ], "portrait_score_gap": 3.709511712896426, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.9206913081991126, "rank": 13.0, "portrait_scores": [ -0.21484620681227962, -3.6265364095859454 ], "portrait_score_gap": 3.4116902027736655, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "6.0": 64, "7.0": 384, "8.0": 960, "9.0": 1280, "10.0": 960, "11.0": 384, "12.0": 64, "13.0": 4096 }, "target_primary_score": -1.9206913081991126, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -1.9496789116617284, -2.8331670964779483 ], "target_alternate_panel_score": -2.3914230040698383, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -2.1489805836100717, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.4356808085833133, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -4.181308825646569, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -4.621675759508362, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 2.018646640174725, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "consistent_bt": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 2.4378337014530063, "rank": 1.0, "portrait_scores": [ 3.2993824260550975, 1.576284976850915 ], "portrait_score_gap": 1.7230974492041824, "single_photo_rank_min": 1.0, "single_photo_rank_max": 5.0, "single_photo_top_share": 0.53125, "single_photo_rank_quantiles": [ 1.0, 1.0, 4.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.6201854717254711, "rank": 2.0, "portrait_scores": [ 0.8443247809817246, 2.3960461624692178 ], "portrait_score_gap": 1.5517213814874933, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.0, 8.0 ] }, { "identity": "anne_hathaway", "mean_score": 1.3168950255868426, "rank": 3.0, "portrait_scores": [ 3.2612473596860845, -0.6274573085123996 ], "portrait_score_gap": 3.888704668198484, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 5.0, 12.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.48885424388344434, "rank": 11.0, "portrait_scores": [ -1.0880220137514423, 0.11031352598455357 ], "portrait_score_gap": 1.1983355397359958, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 10.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -0.8879815789977697, "rank": 12.0, "portrait_scores": [ 0.8966526033739692, -2.6726157613695087 ], "portrait_score_gap": 3.569268364743478, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 10.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 0.6836924967443221, "rank": 6.0, "portrait_scores": [ 1.8096335777867987, -0.44224858429815456 ], "portrait_score_gap": 2.251882162084953, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 6.0, 12.0 ] }, { "identity": "gal_gadot", "mean_score": 1.0578234662695367, "rank": 5.0, "portrait_scores": [ 0.8938478344197575, 1.221799098119316 ], "portrait_score_gap": 0.3279512636995586, "single_photo_rank_min": 2.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 5.0, 7.0 ] }, { "identity": "keira_knightley", "mean_score": -0.4807117273869074, "rank": 10.0, "portrait_scores": [ -2.133811215574857, 1.1723877608010425 ], "portrait_score_gap": 3.3061989763758994, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 9.0, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.14570600598019257, "rank": 9.0, "portrait_scores": [ 0.6163101928511965, -0.9077222048115816 ], "portrait_score_gap": 1.524032397662778, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.5, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.45607177643382135, "rank": 7.0, "portrait_scores": [ -0.18385408143114995, 1.0959976342987927 ], "portrait_score_gap": 1.2798517157299427, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 3.0, 7.0, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 0.3373728347765982, "rank": 8.0, "portrait_scores": [ 0.8871138221942821, -0.21236815264108577 ], "portrait_score_gap": 1.099481974835368, "single_photo_rank_min": 3.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 8.0, 11.0 ] }, { "identity": "scarlett_johansson", "mean_score": 1.1894866233728574, "rank": 4.0, "portrait_scores": [ -0.13283830410094466, 2.5118115508466596 ], "portrait_score_gap": 2.6446498549476045, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 3.5, 10.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.5008536531246468, "rank": 13.0, "portrait_scores": [ 0.01477973475528005, -3.0164870410045737 ], "portrait_score_gap": 3.0312667757598537, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 13.0, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "4.0": 8, "5.0": 72, "6.0": 288, "7.0": 672, "8.0": 1008, "9.0": 1008, "10.0": 672, "11.0": 288, "12.0": 72, "13.0": 4104 }, "target_primary_score": -1.5008536531246468, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -1.6254196069318085, -2.737757518935188 ], "target_alternate_panel_score": -2.181588562933498, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -2.0358825569533057, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.6927343190500537, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -3.801774034658969, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -4.619422264386504, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 1.3872383972919051, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." }, "semantic_bt_matched": { "identities": [ { "identity": "alexandra_daddario", "mean_score": 2.230252755438523, "rank": 1.0, "portrait_scores": [ 3.642670295009054, 0.8178352158679923 ], "portrait_score_gap": 2.8248350791410615, "single_photo_rank_min": 1.0, "single_photo_rank_max": 10.0, "single_photo_top_share": 0.25, "single_photo_rank_quantiles": [ 1.0, 3.0, 9.0 ] }, { "identity": "ana_de_armas", "mean_score": 1.7898858215767315, "rank": 3.0, "portrait_scores": [ 1.016778491014615, 2.562993152138848 ], "portrait_score_gap": 1.546214661124233, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.03125, "single_photo_rank_quantiles": [ 2.0, 4.0, 7.0 ] }, { "identity": "anne_hathaway", "mean_score": 2.05468807433605, "rank": 2.0, "portrait_scores": [ 4.595167820323795, -0.48579167165169457 ], "portrait_score_gap": 5.08095949197549, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.5, "single_photo_rank_quantiles": [ 1.0, 3.5, 11.0 ] }, { "identity": "elizabeth_olsen", "mean_score": -0.9557421954865252, "rank": 11.0, "portrait_scores": [ -1.1092114403716438, -0.8022729506014066 ], "portrait_score_gap": 0.3069384897702372, "single_photo_rank_min": 8.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 9.0, 11.0, 13.0 ] }, { "identity": "emma_stone", "mean_score": -1.124909314800902, "rank": 12.0, "portrait_scores": [ 1.0208202357609357, -3.2706388653627396 ], "portrait_score_gap": 4.291459101123675, "single_photo_rank_min": 2.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 4.0, 10.5, 13.0 ] }, { "identity": "emma_watson", "mean_score": 1.381862026956093, "rank": 5.0, "portrait_scores": [ 2.611065260888543, 0.1526587930236431 ], "portrait_score_gap": 2.4584064678649002, "single_photo_rank_min": 1.0, "single_photo_rank_max": 11.0, "single_photo_top_share": 0.0625, "single_photo_rank_quantiles": [ 1.0, 4.5, 10.0 ] }, { "identity": "gal_gadot", "mean_score": 1.1938887519875307, "rank": 6.0, "portrait_scores": [ 0.9520642108241307, 1.4357132931509309 ], "portrait_score_gap": 0.48364908232680015, "single_photo_rank_min": 1.0, "single_photo_rank_max": 9.0, "single_photo_top_share": 0.00390625, "single_photo_rank_quantiles": [ 3.0, 5.0, 8.0 ] }, { "identity": "keira_knightley", "mean_score": -0.7675995197970344, "rank": 10.0, "portrait_scores": [ -2.3734664887024675, 0.8382674491083987 ], "portrait_score_gap": 3.211733937810866, "single_photo_rank_min": 4.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 5.0, 10.5, 13.0 ] }, { "identity": "lily_collins", "mean_score": -0.24244242045976644, "rank": 9.0, "portrait_scores": [ 0.5273816865072701, -1.012266527426803 ], "portrait_score_gap": 1.5396482139340733, "single_photo_rank_min": 5.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 6.0, 9.0, 12.0 ] }, { "identity": "margot_robbie", "mean_score": 0.5232165809626554, "rank": 8.0, "portrait_scores": [ -0.6617441661742869, 1.7081773280995978 ], "portrait_score_gap": 2.3699214942738847, "single_photo_rank_min": 1.0, "single_photo_rank_max": 12.0, "single_photo_top_share": 0.015625, "single_photo_rank_quantiles": [ 2.0, 6.5, 11.0 ] }, { "identity": "natalie_portman", "mean_score": 1.3975201524984366, "rank": 4.0, "portrait_scores": [ 1.6677538899267403, 1.1272864150701332 ], "portrait_score_gap": 0.5404674748566072, "single_photo_rank_min": 1.0, "single_photo_rank_max": 8.0, "single_photo_top_share": 0.01171875, "single_photo_rank_quantiles": [ 2.0, 4.0, 6.0 ] }, { "identity": "scarlett_johansson", "mean_score": 0.7798978678510726, "rank": 7.0, "portrait_scores": [ -1.0748579885971403, 2.6346537242992856 ], "portrait_score_gap": 3.709511712896426, "single_photo_rank_min": 1.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.125, "single_photo_rank_quantiles": [ 1.0, 6.0, 12.0 ] }, { "identity": "tongdak_cheonsa", "mean_score": -1.9206913081991126, "rank": 13.0, "portrait_scores": [ -0.21484620681227962, -3.6265364095859454 ], "portrait_score_gap": 3.4116902027736655, "single_photo_rank_min": 6.0, "single_photo_rank_max": 13.0, "single_photo_top_share": 0.0, "single_photo_rank_quantiles": [ 7.0, 12.5, 13.0 ] } ], "exhaustive_single_photo_configurations": 8192, "target_single_photo_rank_counts": { "6.0": 64, "7.0": 384, "8.0": 960, "9.0": 1280, "10.0": 960, "11.0": 384, "12.0": 64, "13.0": 4096 }, "target_primary_score": -1.9206913081991126, "target_primary_rank": 13.0, "target_alternate_photo_scores": [ -1.9496789116617284, -2.8331670964779483 ], "target_alternate_panel_score": -2.3914230040698383, "target_alternate_panel_rank": 13.0, "target_alternate_predictions": [ { "trial_id": "1b6d9789fab8", "opponent": "lily_collins", "target_minus_opponent": -2.1489805836100717, "prefers_target": false, "ties_target": false }, { "trial_id": "974972ffb84a", "opponent": "elizabeth_olsen", "target_minus_opponent": -1.4356808085833133, "prefers_target": false, "ties_target": false }, { "trial_id": "c469d02ac7ba", "opponent": "ana_de_armas", "target_minus_opponent": -4.181308825646569, "prefers_target": false, "ties_target": false }, { "trial_id": "fe73774e1a9e", "opponent": "alexandra_daddario", "target_minus_opponent": -4.621675759508362, "prefers_target": false, "ties_target": false } ], "mean_within_test_score_variance": 2.018646640174725, "interpretation": "All 8192 choices of one of two fixed test photographs per candidate; target alternate panel replaces only the target. These are model perturbations without additional human answers." } } } } ], "accuracy_rule": "Only decisive human labels; exactly zero model margin predicts tie and counts as incorrect on a decisive label.", "probability_caution": "Zero-shot and pair-centroid logits have arbitrary uncalibrated scales; their raw cross entropy and Brier scores are descriptive, not fair probability-quality comparisons." }