pivot: code + trained checkpoints (norman, replogle k562)

.gitattributes

@@ -19,7 +19,6 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/norman/model.ptw filter=lfs diff=lfs merge=lfs -text
+models/replogle_k562/model.ptw filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,31 @@
+# large data & downloads (regenerated from public sources via src/data/preprocess.py)
+/data/
+/gears_data/
+/logs/
+
+# generated artifacts kept out of the repo
+/paper/
+/figures/
+*.h5ad
+*.npz
+*.npy
+
+# model checkpoints / large binaries
+*.pt
+*.ckpt
+
+# python
+__pycache__/
+*.pyc
+*.pyo
+.ipynb_checkpoints/
+
+# latex build artifacts
+paper/*.aux
+paper/*.bbl
+paper/*.blg
+paper/*.out
+paper/*.log
+
+# os
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bryan Cheng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,90 @@
----
-license: apache-2.0
----
+# PIVOT
+
+Perturbation-Informed Vector-field Optimization for Transcriptomic state control.
+
+PIVOT learns a perturbation-conditioned flow map over single-cell state embeddings and
+uses its Jacobian for differentiable inverse design: given a control cell state and a
+desired target state, it nominates gene-level interventions that move cells toward the
+target. The same model also does ordinary forward response prediction.
+
+## Layout
+
+```
+src/
+  data/         loading + preprocessing of perturb-seq data, splits
+  models/       perturbation encoder, flow map, the PIVOT module
+  training/     training loop and losses
+  evaluation/   inference, rewards, metrics, baselines
+  experiments/  drivers for the result tables, ablations, figures
+  utils/
+scripts/        figure generation, extra ablations, GEARS comparison
+experiments/    saved result json
+```
+
+## Setup
+
+```bash
+pip install -r requirements.txt
+```
+
+Data is not committed. Download and preprocess from the public sources first (Norman 2019
+and Replogle 2022 are pulled from scPerturb):
+
+```bash
+python -m src.data.preprocess norman
+python -m src.data.preprocess replogle_k562
+```
+
+This writes a PCA(50) embedding over 2000 highly variable genes plus the held-out splits to
+`data/processed/<dataset>/`.
+
+## Running things
+
+```bash
+# train one model
+python -m src.training.train --dataset norman --split perturbation
+
+# forward + nomination tables
+python -m src.experiments.run_tables --dataset norman --tables forward_cell forward_perturbation
+
+# ablations
+python -m src.experiments.run_ablations --dataset norman
+
+# figures
+python scripts/figures.py
+```
+
+The GEARS head-to-head runs in its own conda env (older torch + pyg), since the package is
+finicky about versions:
+
+```bash
+bash scripts/setup_gears_env.sh
+conda run -n pivot_gears python scripts/gears_ranking.py
+```
+
+## Models
+
+Trained PIVOT checkpoints live under `models/`:
+
+- `models/norman/` - trained on Norman 2019 (CRISPRa K562)
+- `models/replogle_k562/` - trained on Replogle 2022 (CRISPRi K562)
+
+each folder has `model.ptw` (a plain torch state dict), `config.json` (the training config),
+and `train_info.json` (history + run info). loading needs the matching preprocessed dataset,
+since the perturbation encoder vocabulary comes from the data:
+
+```python
+import json, torch
+from src.data.perturb_data import load_dataset
+from src.training.train import TrainConfig, make_model
+
+cfg = TrainConfig(**json.load(open("models/norman/config.json")))
+data = load_dataset(cfg.dataset)
+model = make_model(data, cfg, device="cpu")
+model.load_state_dict(torch.load("models/norman/model.ptw", map_location="cpu"))
+model.eval()
+```
+
+## License
+
+MIT, Bryan Cheng 2026. See `LICENSE`.

experiments/results/gears_norman.json

@@ -0,0 +1,56 @@
+{
+  "n_test_perts": 45,
+  "test_perts": [
+    "ARID1A+ctrl",
+    "ARRDC3+ctrl",
+    "ATL1+ctrl",
+    "BAK1+ctrl",
+    "BCL2L11+BAK1",
+    "BCL2L11+TGFBR2",
+    "BCL2L11+ctrl",
+    "BPGM+ZBTB1",
+    "C3orf72+FOXL2",
+    "CBFA2T3+ctrl",
+    "CBL+TGFBR2",
+    "CBL+ctrl",
+    "CEBPB+OSR2",
+    "CEBPE+PTPN12",
+    "CKS1B+ctrl",
+    "CNNM4+ctrl",
+    "DUSP9+IGDCC3",
+    "DUSP9+ctrl",
+    "EGR1+ctrl",
+    "ETS2+CEBPE",
+    "FOXA1+FOXF1",
+    "FOXF1+HOXB9",
+    "FOXF1+ctrl",
+    "GLB1L2+ctrl",
+    "HOXC13+ctrl",
+    "IGDCC3+PRTG",
+    "IRF1+SET",
+    "KLF1+CEBPA",
+    "KLF1+FOXA1",
+    "KMT2A+ctrl",
+    "MAP2K3+IKZF3",
+    "MAP2K3+MAP2K6",
+    "MAPK1+ctrl",
+    "NCL+ctrl",
+    "PLK4+STIL",
+    "PTPN12+UBASH3A",
+    "RREB1+ctrl",
+    "SNAI1+DLX2",
+    "TBX3+TBX2",
+    "TGFBR2+ETS2",
+    "TGFBR2+ctrl",
+    "TSC22D1+ctrl",
+    "UBASH3B+PTPN12",
+    "ZBTB10+DLX2",
+    "ZNF318+ctrl"
+  ],
+  "metrics": {
+    "mse": 0.003074304576471655,
+    "mse_de": 0.10127744511701167,
+    "pearson": 0.9916384220123291,
+    "pearson_de": 0.8981071008576287
+  }
+}

experiments/results/gears_ranking.json

@@ -0,0 +1,19 @@
+{
+  "single": {
+    "top1": 0.047619047619047616,
+    "top5": 0.19047619047619047,
+    "ndcg": 0.14115129481729458,
+    "rank": 21.666666666666668,
+    "n": 21,
+    "med_rank": 19.0
+  },
+  "combo": {
+    "top1": 0.038461538461538464,
+    "top5": 0.34615384615384615,
+    "overlap": 0.15384615384615385,
+    "ndcg": 0.26756681334907767,
+    "n": 26
+  },
+  "n_candidates_single": 101,
+  "n_candidates_combo": 128
+}

experiments/results/norman_ablation_components.json

@@ -0,0 +1,75 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "flow-map-only": {
+      "forward": {
+        "mse": 0.0017506419577291532,
+        "de_corr": 0.8443752184034662,
+        "mmd": 0.40636296707676617,
+        "pearson": 0.7724152516098098
+      },
+      "inverse": {
+        "top1": 0.09090909090909091,
+        "top5": 0.18181818181818182,
+        "ndcg": 0.1762888354358521,
+        "endpoint_dist": 0.9921409541910345
+      }
+    },
+    "no-tangent": {
+      "forward": {
+        "mse": 0.001731510258941079,
+        "de_corr": 0.8408393241722691,
+        "mmd": 0.37349285466271814,
+        "pearson": 0.7714462100349843
+      },
+      "inverse": {
+        "top1": 0.045454545454545456,
+        "top5": 0.18181818181818182,
+        "ndcg": 0.1446297312285945,
+        "endpoint_dist": 0.9842348342592065
+      }
+    },
+    "no-semigroup": {
+      "forward": {
+        "mse": 0.00195918270780586,
+        "de_corr": 0.8041895085826833,
+        "mmd": 0.35951861984482014,
+        "pearson": 0.721258614925628
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.09090909090909091,
+        "ndcg": 0.07627732654967648,
+        "endpoint_dist": 1.1208079132166775
+      }
+    },
+    "PIVOT-full": {
+      "forward": {
+        "mse": 0.0018839154465550042,
+        "de_corr": 0.8260913965232829,
+        "mmd": 0.2779389477818921,
+        "pearson": 0.7429934785641888
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0741331706168103,
+        "endpoint_dist": 1.10553578896956
+      }
+    },
+    "EndpointMLP": {
+      "forward": {
+        "mse": 0.0011545048934962362,
+        "de_corr": 0.6582605464661375,
+        "mmd": 0.048875581526963176,
+        "pearson": 0.6028436505572593
+      },
+      "inverse": {
+        "top1": 0.13636363636363635,
+        "top5": 0.22727272727272727,
+        "ndcg": 0.21410929644360008,
+        "endpoint_dist": 1.0358923240141436
+      }
+    }
+  }
+}

experiments/results/norman_ablation_cost.json

@@ -0,0 +1,9 @@
+{
+  "split": "perturbation",
+  "PIVOT": {
+    "params": 892658,
+    "train_time_s": 102.39015054702759,
+    "rank_time_per_query_s": 0.14307117462158203,
+    "candidate_evals": 105
+  }
+}

experiments/results/norman_ablation_datascale.json

@@ -0,0 +1,75 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "0.1": {
+      "forward": {
+        "mse": 0.0026412838559097393,
+        "de_corr": 0.7419204415476068,
+        "mmd": 0.0816293043824765,
+        "pearson": 0.6466187456761427
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.0,
+        "ndcg": 0.04355759901977392,
+        "endpoint_dist": 1.2526228725910187
+      }
+    },
+    "0.25": {
+      "forward": {
+        "mse": 0.0025015621919165106,
+        "de_corr": 0.7626386221418989,
+        "mmd": 0.10973129114722474,
+        "pearson": 0.6617828040046895
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.0,
+        "ndcg": 0.015151515151499999,
+        "endpoint_dist": 1.2300338365814902
+      }
+    },
+    "0.5": {
+      "forward": {
+        "mse": 0.0020840686227026534,
+        "de_corr": 0.806516168916479,
+        "mmd": 0.167632126198037,
+        "pearson": 0.7073832176467205
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.045454545454545456,
+        "ndcg": 0.056701119364541815,
+        "endpoint_dist": 1.1461743387308987
+      }
+    },
+    "0.75": {
+      "forward": {
+        "mse": 0.002003367400128751,
+        "de_corr": 0.8197314882616294,
+        "mmd": 0.23520435415096233,
+        "pearson": 0.721792117633084
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.07098210507469495,
+        "endpoint_dist": 1.1467380632053723
+      }
+    },
+    "1.0": {
+      "forward": {
+        "mse": 0.0018839154465550042,
+        "de_corr": 0.8260913965232829,
+        "mmd": 0.2779389477818921,
+        "pearson": 0.7429934785641888
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0741331706168103,
+        "endpoint_dist": 1.10553578896956
+      }
+    }
+  }
+}

experiments/results/norman_ablation_guidance_steps.json

@@ -0,0 +1,41 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "0": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0741331706168103,
+      "endpoint_dist": 1.10553578896956
+    },
+    "5": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.08466528669577132,
+      "endpoint_dist": 1.1852125579660588
+    },
+    "10": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0932238333390342,
+      "endpoint_dist": 1.1176448301835493
+    },
+    "25": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.09061663913083162,
+      "endpoint_dist": 1.2133627290075475
+    },
+    "50": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.11054362496513667,
+      "endpoint_dist": 1.146642354401675
+    },
+    "100": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0968604433440603,
+      "endpoint_dist": 1.168973827903921
+    }
+  }
+}

experiments/results/norman_ablation_inverse_search.json

@@ -0,0 +1,41 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "ranking-only": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0741331706168103,
+      "endpoint_dist": 1.10553578896956
+    },
+    "random-init-opt": {
+      "top1": 0.09090909090909091,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.13006150527926927,
+      "endpoint_dist": 1.916646886955608
+    },
+    "mean-top-init": {
+      "top1": 0.09090909090909091,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.11048529809413464,
+      "endpoint_dist": 1.633959797295657
+    },
+    "guidance-no-norm": {
+      "top1": 0.0,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.10475755936295951,
+      "endpoint_dist": 2.176489320668307
+    },
+    "guidance-norm": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.10540057074853007,
+      "endpoint_dist": 1.6980433003468947
+    },
+    "guidance+rerank": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.09061663913083162,
+      "endpoint_dist": 1.2133627290075475
+    }
+  }
+}

experiments/results/norman_ablation_matching.json

@@ -0,0 +1,75 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "random": {
+      "forward": {
+        "mse": 0.0020935234107186105,
+        "de_corr": 0.8198527663629106,
+        "mmd": 0.286064224443859,
+        "pearson": 0.7269562229197076
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0689901164002037,
+        "endpoint_dist": 1.1582550325176932
+      }
+    },
+    "batch": {
+      "forward": {
+        "mse": 0.0018839154465550042,
+        "de_corr": 0.8260913965232829,
+        "mmd": 0.2779389477818921,
+        "pearson": 0.7429934785641888
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0741331706168103,
+        "endpoint_dist": 1.10553578896956
+      }
+    },
+    "celltype": {
+      "forward": {
+        "mse": 0.0020935234107186105,
+        "de_corr": 0.8198527663629106,
+        "mmd": 0.286064224443859,
+        "pearson": 0.7269562229197076
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0689901164002037,
+        "endpoint_dist": 1.1582550325176932
+      }
+    },
+    "batch_celltype": {
+      "forward": {
+        "mse": 0.0018839154465550042,
+        "de_corr": 0.8260913965232829,
+        "mmd": 0.2779389477818921,
+        "pearson": 0.7429934785641888
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0741331706168103,
+        "endpoint_dist": 1.10553578896956
+      }
+    },
+    "nearest": {
+      "forward": {
+        "mse": 0.0034554351654963846,
+        "de_corr": 0.6926855278459001,
+        "mmd": 0.08932766919879936,
+        "pearson": 0.5472117423060092
+      },
+      "inverse": {
+        "top1": 0.13636363636363635,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.1363636363635,
+        "endpoint_dist": 1.462783862243999
+      }
+    }
+  }
+}

experiments/results/norman_ablation_representation.json

@@ -0,0 +1,75 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "gene_op": {
+      "forward": {
+        "mse": 0.0018839154465550042,
+        "de_corr": 0.8260913965232829,
+        "mmd": 0.2779389477818921,
+        "pearson": 0.7429934785641888
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.0741331706168103,
+        "endpoint_dist": 1.10553578896956
+      }
+    },
+    "op_only": {
+      "forward": {
+        "mse": 0.0036388238541474765,
+        "de_corr": 0.4861906795425618,
+        "mmd": 0.2997179725547505,
+        "pearson": 0.4692922436850185
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.18181818181818182,
+        "ndcg": 0.10475755936295951,
+        "endpoint_dist": 1.5618621165102178
+      }
+    },
+    "gene_only": {
+      "forward": {
+        "mse": 0.0019814494465963243,
+        "de_corr": 0.8269104006442618,
+        "mmd": 0.2864381163653205,
+        "pearson": 0.7344451238341788
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.045454545454545456,
+        "ndcg": 0.059209173521086636,
+        "endpoint_dist": 1.128328177061948
+      }
+    },
+    "random_id": {
+      "forward": {
+        "mse": 0.003925378121029427,
+        "de_corr": 0.4371013550999317,
+        "mmd": 0.3452091584109617,
+        "pearson": 0.4009652477887241
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.0,
+        "ndcg": 0.0,
+        "endpoint_dist": 1.181209458546205
+      }
+    },
+    "gene_pathway_op": {
+      "forward": {
+        "mse": 0.002060087917228923,
+        "de_corr": 0.8255276011818267,
+        "mmd": 0.2896023761246202,
+        "pearson": 0.7541268989443779
+      },
+      "inverse": {
+        "top1": 0.0,
+        "top5": 0.13636363636363635,
+        "ndcg": 0.09442383563302575,
+        "endpoint_dist": 1.1720866127447649
+      }
+    }
+  }
+}

experiments/results/norman_ablation_reward.json

@@ -0,0 +1,35 @@
+{
+  "split": "perturbation",
+  "rows": {
+    "centroid": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0741331706168103,
+      "endpoint_dist": 1.10553578896956
+    },
+    "cosine": {
+      "top1": 0.22727272727272727,
+      "top5": 0.2727272727272727,
+      "ndcg": 0.24485694578314335,
+      "endpoint_dist": 3.251927175305106
+    },
+    "nn_target": {
+      "top1": 0.0,
+      "top5": 0.045454545454545456,
+      "ndcg": 0.03273573366214339,
+      "endpoint_dist": 1.7553648054599762
+    },
+    "mmd": {
+      "top1": 0.0,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.1380222478334554,
+      "endpoint_dist": 1.3466052819382062
+    },
+    "wasserstein": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.1533429541697297,
+      "endpoint_dist": 1.222495680505579
+    }
+  }
+}

experiments/results/norman_benchmark.json

@@ -0,0 +1,231 @@
+{
+  "dataset": "norman",
+  "n_targets": 22,
+  "n_candidates": 105,
+  "random_top5": 0.047619047619047616,
+  "forward": {
+    "PIVOT": {
+      "mse": 0.0017491921988697326,
+      "r2": -0.8161821530220357,
+      "pearson": 0.7693309462133874,
+      "spearman": 0.5815323970363715,
+      "de_corr": 0.8388409530545803,
+      "mmd": 0.3034484875795543,
+      "wasserstein": 0.43272001688923617
+    },
+    "PIVOT-OT": {
+      "mse": 0.0025015840371278054,
+      "r2": -2.4559457745957882,
+      "pearson": 0.6559229388516,
+      "spearman": 0.46955744461871557,
+      "de_corr": 0.7519170356557724,
+      "mmd": 0.1500954119146127,
+      "wasserstein": 0.6666752559372957
+    },
+    "MeanControl": {
+      "mse": 0.004455934176339144,
+      "r2": -0.05741366300177067,
+      "pearson": 0.06247465056862603,
+      "spearman": 0.019984302966237155,
+      "de_corr": 0.10322069715797663,
+      "mmd": 0.10706937820897701,
+      "wasserstein": 0.323807051495741
+    },
+    "AvgPerturbationEffect": {
+      "mse": 0.003602388566310339,
+      "r2": -1.6519504455809897,
+      "pearson": 0.4787262177213709,
+      "spearman": 0.3930456408465337,
+      "de_corr": 0.5168158494015324,
+      "mmd": 0.09934840288274557,
+      "wasserstein": 0.30081602373086835
+    },
+    "Additive": {
+      "mse": 0.0021094072577939865,
+      "r2": -1.448790396781678,
+      "pearson": 0.6326670402542074,
+      "spearman": 0.5108886765427764,
+      "de_corr": 0.7134229213474913,
+      "mmd": 0.07343746422286138,
+      "wasserstein": 0.25771137196436716
+    },
+    "LinearResponse": {
+      "mse": 0.0011306649979273293,
+      "r2": -0.253476581674941,
+      "pearson": 0.7451306042519021,
+      "spearman": 0.5799033801846452,
+      "de_corr": 0.8105884356067535,
+      "mmd": 0.047508293556133514,
+      "wasserstein": 0.20167587385465413
+    },
+    "NearestPerturbationCentroid": {
+      "mse": 0.003305152804774054,
+      "r2": -4.189406145126261,
+      "pearson": 0.6359215529358133,
+      "spearman": 0.505686299719842,
+      "de_corr": 0.7316346312773988,
+      "mmd": 0.10450971258729616,
+      "wasserstein": 0.30843307594994734
+    },
+    "EndpointMLP": {
+      "mse": 0.0011545048891616232,
+      "r2": 0.04933880491459623,
+      "pearson": 0.6028436519839662,
+      "spearman": 0.5096878649785176,
+      "de_corr": 0.6582605471002295,
+      "mmd": 0.048875581526963176,
+      "wasserstein": 0.19931416156968693
+    },
+    "kNN-latent": {
+      "mse": 0.0021908569667968857,
+      "r2": -1.5670528272365003,
+      "pearson": 0.6744848155674148,
+      "spearman": 0.5199807748887847,
+      "de_corr": 0.7707986322806236,
+      "mmd": 0.07314315796243004,
+      "wasserstein": 0.2532922264836747
+    },
+    "ConditionalMLP": {
+      "mse": 0.0010686125573788848,
+      "r2": 0.30849315257782634,
+      "pearson": 0.619310863127813,
+      "spearman": 0.5438978223783673,
+      "de_corr": 0.6658019708033572,
+      "mmd": 0.42855737667365007,
+      "wasserstein": 0.5315667629719005
+    },
+    "Random": {
+      "mse": 0.00673537037057921,
+      "r2": -6.333347964794077,
+      "pearson": 0.27716910228767294,
+      "spearman": 0.2415044605055355,
+      "de_corr": 0.28529975959595216,
+      "mmd": 0.16910403534817559,
+      "wasserstein": 0.3895185511116175
+    },
+    "PIVOT+dist": {
+      "mse": null,
+      "r2": -1.557,
+      "pearson": 0.759,
+      "spearman": null,
+      "de_corr": 0.844,
+      "mmd": 0.0507,
+      "wasserstein": 0.235
+    }
+  },
+  "inverse": {
+    "PIVOT": {
+      "top1": 0.18181818181818182,
+      "top5": 0.22727272727272727,
+      "ndcg": 0.237863170222463,
+      "func_top5": 0.8636363636363636,
+      "top5_ci": [
+        0.22727272727272727,
+        0.045454545454545456,
+        0.4090909090909091
+      ]
+    },
+    "PIVOT-guidance": {
+      "top1": 0.13636363636363635,
+      "top5": 0.22727272727272727,
+      "ndcg": 0.20036677015736315,
+      "func_top5": 0.8181818181818182
+    },
+    "MeanControl": {
+      "top1": 0.0,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.10475755936295951,
+      "func_top5": 0.8181818181818182
+    },
+    "AvgPerturbationEffect": {
+      "top1": 0.0,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.10475755936295951,
+      "func_top5": 0.8181818181818182
+    },
+    "Additive": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.9545454545454546
+    },
+    "LinearResponse": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.1028117957791206,
+      "func_top5": 0.8181818181818182
+    },
+    "NearestPerturbationCentroid": {
+      "top1": 0.18181818181818182,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.181818181818,
+      "func_top5": 0.9090909090909091
+    },
+    "EndpointMLP": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.13489139848118523,
+      "func_top5": 0.9545454545454546
+    },
+    "kNN-latent": {
+      "top1": 0.045454545454545456,
+      "top5": 0.09090909090909091,
+      "ndcg": 0.09305324684186617,
+      "func_top5": 0.7727272727272727
+    },
+    "ConditionalMLP": {
+      "top1": 0.18181818181818182,
+      "top5": 0.2727272727272727,
+      "ndcg": 0.2535147447237758,
+      "func_top5": 0.9090909090909091
+    },
+    "Random": {
+      "top1": 0.0,
+      "top5": 0.045454545454545456,
+      "ndcg": 0.03472772233663465,
+      "func_top5": 0.7727272727272727
+    }
+  },
+  "significance": {
+    "PIVOT_vs_PIVOT-guidance": {
+      "p": 1.0,
+      "dTop5": 0.0
+    },
+    "PIVOT_vs_MeanControl": {
+      "p": 0.744127936031984,
+      "dTop5": 0.045454545454545456
+    },
+    "PIVOT_vs_AvgPerturbationEffect": {
+      "p": 0.744127936031984,
+      "dTop5": 0.045454545454545456
+    },
+    "PIVOT_vs_Additive": {
+      "p": 0.010994502748625687,
+      "dTop5": 0.22727272727272727
+    },
+    "PIVOT_vs_LinearResponse": {
+      "p": 0.12643678160919541,
+      "dTop5": 0.09090909090909091
+    },
+    "PIVOT_vs_NearestPerturbationCentroid": {
+      "p": 0.25287356321839083,
+      "dTop5": 0.045454545454545456
+    },
+    "PIVOT_vs_EndpointMLP": {
+      "p": 0.3268365817091454,
+      "dTop5": 0.09090909090909091
+    },
+    "PIVOT_vs_kNN-latent": {
+      "p": 0.06346826586706647,
+      "dTop5": 0.13636363636363635
+    },
+    "PIVOT_vs_ConditionalMLP": {
+      "p": 0.2588705647176412,
+      "dTop5": -0.045454545454545456
+    },
+    "PIVOT_vs_Random": {
+      "p": 0.1054472763618191,
+      "dTop5": 0.18181818181818182
+    }
+  }
+}

experiments/results/norman_combinatorial.json

@@ -0,0 +1,271 @@
+{
+  "split": "combination",
+  "n_targets": 26,
+  "n_combo_candidates": 131,
+  "result": {
+    "greedy_exact1": 0.15384615384615385,
+    "greedy_exact5": 0.15384615384615385,
+    "greedy_overlap": 0.34615384615384615,
+    "rank_exact1": 0.5,
+    "rank_exact5": 0.7692307692307693,
+    "rank_ndcg": 0.6432095355574537,
+    "rank_overlap": 0.6153846153846154,
+    "endpoint_dist": 1.181089229308642,
+    "synergy_corr": 0.39749566236367595,
+    "_per": {
+      "greedy_exact1": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "greedy_exact5": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "greedy_overlap": [
+        0.5,
+        0.0,
+        0.5,
+        0.5,
+        0.0,
+        0.0,
+        0.0,
+        0.5,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.5,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        0.5,
+        0.0,
+        0.0,
+        0.0,
+        0.5,
+        0.0,
+        0.5,
+        0.5,
+        0.5
+      ],
+      "rank_exact1": [
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0
+      ],
+      "rank_exact5": [
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0
+      ],
+      "rank_ndcg": [
+        0.49999999999949996,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.0,
+        0.0,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.3868528072341547,
+        0.3868528072341547,
+        0.9999999999989999,
+        0.49999999999949996,
+        0.6309297535708266,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.9999999999989999,
+        0.0,
+        0.30102999566368016,
+        0.3868528072341547,
+        0.6309297535708266,
+        0.0,
+        0.0,
+        0.9999999999989999,
+        0.9999999999989999
+      ],
+      "rank_overlap": [
+        0.5,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.5,
+        0.5,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0,
+        0.0,
+        0.5,
+        0.5,
+        0.5,
+        0.0,
+        1.0,
+        1.0
+      ],
+      "endpoint_dist": [
+        0.6689271330833435,
+        0.8036714196205139,
+        1.0161041021347046,
+        1.1899185180664062,
+        0.9528195858001709,
+        1.0340110063552856,
+        1.1323903799057007,
+        1.7547415494918823,
+        0.5572233200073242,
+        1.2502464056015015,
+        1.7361279726028442,
+        1.0626763105392456,
+        1.6163440942764282,
+        1.1465466022491455,
+        1.046038031578064,
+        1.9452886581420898,
+        0.5758848190307617,
+        0.550717830657959,
+        1.5649155378341675,
+        1.068113088607788,
+        1.2296886444091797,
+        0.5714886784553528,
+        0.9187361001968384,
+        1.9091839790344238,
+        2.291757822036743,
+        1.1147583723068237
+      ],
+      "synergy_corr": [
+        -0.36697694659233093,
+        0.7867258191108704,
+        0.7642111778259277,
+        0.8102744817733765,
+        0.3918057978153229,
+        -0.5012184977531433,
+        -0.44898518919944763,
+        0.9819259643554688,
+        0.9748311042785645,
+        0.5728657245635986,
+        0.6892925500869751,
+        0.9163436889648438,
+        -0.3066551089286804,
+        0.8415080308914185,
+        0.32007336616516113,
+        0.4687807857990265,
+        0.6908244490623474,
+        0.7702524065971375,
+        0.6121890544891357,
+        0.5949224829673767,
+        -0.3006000816822052,
+        0.43497174978256226,
+        -0.37792277336120605,
+        -0.18205173313617706,
+        0.2413092851638794,
+        0.9561896324157715
+      ]
+    },
+    "n_targets": 26
+  }
+}

experiments/results/norman_distloss.json

@@ -0,0 +1,27 @@
+{
+  "split": "perturbation",
+  "desc": "distributional flow loss (population MMD) sweep, Norman held-out perturbation",
+  "rows": {
+    "0.0": {
+      "de_corr": 0.839,
+      "r2": -0.816,
+      "pearson": 0.769,
+      "mmd": 0.3034,
+      "wass": 0.433
+    },
+    "2.0": {
+      "de_corr": 0.835,
+      "r2": -1.118,
+      "pearson": 0.757,
+      "mmd": 0.0507,
+      "wass": 0.231
+    },
+    "10.0": {
+      "de_corr": 0.844,
+      "r2": -1.557,
+      "pearson": 0.759,
+      "mmd": 0.0507,
+      "wass": 0.235
+    }
+  }
+}

experiments/results/norman_extra_ablations.json

@@ -0,0 +1,44 @@
+{
+  "velocity_only": {
+    "mse": 0.003250642179293518,
+    "de_corr": 0.6322063905424735,
+    "mmd": 0.08446152519280205,
+    "endpoint_dist": 2.4770562919703396,
+    "top5": 0.09090909090909091,
+    "ndcg": 0.12225184183812116
+  },
+  "ranking_only": {
+    "mse": 0.0017491921988697326,
+    "de_corr": 0.8388409530545803,
+    "mmd": 0.3034484875795543,
+    "endpoint_dist": 1.068045285615054,
+    "top5": 0.13636363636363635,
+    "ndcg": 0.08727248090397387
+  },
+  "guidance_no_rerank": {
+    "mse": 0.0017491921988697326,
+    "de_corr": 0.8388409530545803,
+    "mmd": 0.3034484875795543,
+    "endpoint_dist": 1.5734560652212664,
+    "top5": 0.13636363636363635,
+    "ndcg": 0.0968604433440603
+  },
+  "avg_effect_ranking": {
+    "top1": 0.0,
+    "top5": 0.18181818181818182,
+    "ndcg": 0.10475755936295951,
+    "func_top5": 0.8181818181818182,
+    "med_rank": 33.5
+  },
+  "gpu_mem_mb": 170.1,
+  "combo_additive": {
+    "exact1": 0.5769230769230769,
+    "exact5": 0.8846153846153846,
+    "overlap": 0.7115384615384616
+  },
+  "combo_random": {
+    "exact1": 0.0,
+    "exact5": 0.038461538461538464,
+    "overlap": 0.038461538461538464
+  }
+}

experiments/results/norman_forward_combination.json

@@ -0,0 +1,1669 @@
+{
+  "split": "combination",
+  "models": {
+    "PIVOT": {
+      "mse": 0.0011608018271195202,
+      "r2": 0.7034275646393116,
+      "pearson": 0.9094234865445358,
+      "spearman": 0.7422449962055115,
+      "de_corr": 0.9502354585207425,
+      "mmd": 0.30546490356834105,
+      "wasserstein": 0.4372436159231492,
+      "_per_pert": {
+        "mse": [
+          0.0004416794399730861,
+          0.00033747247653082013,
+          0.0004790037346538156,
+          0.0006593593861907721,
+          0.0004620901891030371,
+          0.0019741933792829514,
+          0.0037908623926341534,
+          0.001514257164672017,
+          0.00016269889601971954,
+          0.0012954982230439782,
+          0.0012997586745768785,
+          0.0005281568737700582,
+          0.0017190042417496443,
+          0.0003682815295178443,
+          0.0005863922997377813,
+          0.0018724900437518954,
+          0.00015000069106463343,
+          0.00014376378385350108,
+          0.0016738612903282046,
+          0.0009087433572858572,
+          0.001752177020534873,
+          0.0003636993351392448,
+          0.0020773408468812704,
+          0.0024218547623604536,
+          0.0026488068979233503,
+          0.0005494005745276809
+        ],
+        "r2": [
+          0.3020176291465759,
+          0.9467234015464783,
+          0.9203470945358276,
+          0.8184603452682495,
+          0.9147472381591797,
+          0.5887033939361572,
+          0.6299736499786377,
+          0.938554048538208,
+          0.9600407481193542,
+          0.4582985043525696,
+          0.6972436904907227,
+          0.9535616636276245,
+          0.6415849924087524,
+          0.7591631412506104,
+          0.8760426044464111,
+          0.7594628930091858,
+          0.9689065217971802,
+          0.7848913073539734,
+          0.18005633354187012,
+          -0.09197616577148438,
+          0.5074808597564697,
+          0.9264187812805176,
+          0.7165902853012085,
+          0.644216001033783,
+          0.7914367914199829,
+          0.6961709260940552
+        ],
+        "pearson": [
+          0.6286964416503906,
+          0.9733661413192749,
+          0.9625992774963379,
+          0.9056428670883179,
+          0.9792198538780212,
+          0.9477250576019287,
+          0.9555656313896179,
+          0.9880457520484924,
+          0.9798969030380249,
+          0.7223176956176758,
+          0.9317120909690857,
+          0.9887338280677795,
+          0.8971215486526489,
+          0.9108695983886719,
+          0.9660959243774414,
+          0.9386588931083679,
+          0.9844915866851807,
+          0.9293400049209595,
+          0.7490290999412537,
+          0.701932430267334,
+          0.9207367897033691,
+          0.9665771126747131,
+          0.9677329659461975,
+          0.8525486588478088,
+          0.9600315690040588,
+          0.9363229274749756
+        ],
+        "spearman": [
+          0.40532229870442155,
+          0.7695981981365192,
+          0.7168365810557625,
+          0.8223511142479002,
+          0.7619216037663427,
+          0.683857225754397,
+          0.6145115214042277,
+          0.9030108665319926,
+          0.8265676943402535,
+          0.671723228500967,
+          0.6946859011442998,
+          0.8733894077156299,
+          0.7313993757164831,
+          0.7732291276055234,
+          0.89247673992973,
+          0.8519675614383306,
+          0.8543754041910453,
+          0.7109876899782194,
+          0.5988971571019566,
+          0.571165396466269,
+          0.5966738634873373,
+          0.8095972592400602,
+          0.7516057058029206,
+          0.7712747612701276,
+          0.919446070868832,
+          0.7214981469437507
+        ],
+        "de_corr": [
+          0.8499228954315186,
+          0.9913710951805115,
+          0.9797207117080688,
+          0.9553540349006653,
+          0.9917449355125427,
+          0.9736267328262329,
+          0.9834495186805725,
+          0.9824050664901733,
+          0.9921419620513916,
+          0.8035094738006592,
+          0.9676457047462463,
+          0.9952547550201416,
+          0.9743742942810059,
+          0.9676927924156189,
+          0.9835843443870544,
+          0.9907316565513611,
+          0.9951936602592468,
+          0.9855618476867676,
+          0.8434659838676453,
+          0.6738124489784241,
+          0.9695654511451721,
+          0.9861806631088257,
+          0.9885909557342529,
+          0.9232415556907654,
+          0.9831457734107971,
+          0.974833607673645
+        ],
+        "mmd": [
+          0.328915026572134,
+          0.2825841063486447,
+          0.294318340899585,
+          0.3250979304211864,
+          0.2985747716434175,
+          0.3383447859580423,
+          0.3599959851521064,
+          0.2681938456592685,
+          0.25684975999905224,
+          0.2912402456812845,
+          0.2955167755485727,
+          0.3050651947717583,
+          0.33313845325313596,
+          0.26395905100391637,
+          0.3135891655418249,
+          0.32101054965927234,
+          0.2707002241951575,
+          0.26835427290232927,
+          0.262577515584691,
+          0.27464787193905105,
+          0.3378149350365237,
+          0.32967945988472336,
+          0.3673476519636928,
+          0.3334991821371782,
+          0.34538214127359135,
+          0.2756902497467274
+        ],
+        "wasserstein": [
+          0.38160696400595895,
+          0.4137511200401532,
+          0.42309166187224867,
+          0.4809079089532095,
+          0.43214627874244366,
+          0.46716996259261256,
+          0.5414571257715426,
+          0.5464605045876217,
+          0.3775509286352024,
+          0.41904336459342434,
+          0.4467201824337285,
+          0.4392330632078002,
+          0.4642640756035464,
+          0.3592642491196178,
+          0.4572561755616178,
+          0.45067202461840195,
+          0.3663159121643666,
+          0.3444241465740951,
+          0.3829438513983993,
+          0.3762235401968839,
+          0.4657051158337028,
+          0.4276944648936299,
+          0.48315019492358374,
+          0.509800186190512,
+          0.5455922505392994,
+          0.36588876094827727
+        ]
+      },
+      "n_perts": 26
+    },
+    "MeanControl": {
+      "mse": 0.0057471211117584835,
+      "r2": -0.011610170969596276,
+      "pearson": 0.03853007166001659,
+      "spearman": 0.01234826030847139,
+      "de_corr": 0.0049495832779659675,
+      "mmd": 0.13235756195115472,
+      "wasserstein": 0.3782550852396613,
+      "_per_pert": {
+        "mse": [
+          0.0007026545936241746,
+          0.00680745393037796,
+          0.006161879748106003,
+          0.0036344523541629314,
+          0.005524597596377134,
+          0.004370624199509621,
+          0.010359177365899086,
+          0.02515377104282379,
+          0.004014274105429649,
+          0.0023135184310376644,
+          0.004061258863657713,
+          0.011318846605718136,
+          0.004867125768214464,
+          0.0015870737843215466,
+          0.004860573913902044,
+          0.007731732912361622,
+          0.004952032584697008,
+          0.0006888253847137094,
+          0.001931409235112369,
+          0.0008697910816408694,
+          0.0035772169940173626,
+          0.004746480844914913,
+          0.007469563279300928,
+          0.00716841034591198,
+          0.012592131271958351,
+          0.00196027266792953
+        ],
+        "r2": [
+          -0.11039936542510986,
+          -0.07468903064727783,
+          -0.0246506929397583,
+          -0.0006638765335083008,
+          -0.019253849983215332,
+          0.08943921327590942,
+          -0.011159896850585938,
+          -0.020696282386779785,
+          0.01408451795578003,
+          0.03262203931808472,
+          0.05399984121322632,
+          0.004787087440490723,
+          -0.014803051948547363,
+          -0.03786289691925049,
+          -0.027476072311401367,
+          0.006793856620788574,
+          -0.02650141716003418,
+          -0.030665278434753418,
+          0.05389589071273804,
+          -0.045169830322265625,
+          -0.005519390106201172,
+          0.039723873138427734,
+          -0.0190657377243042,
+          -0.05307948589324951,
+          0.008513808250427246,
+          -0.0840684175491333
+        ],
+        "pearson": [
+          -0.06629537045955658,
+          -0.3966469168663025,
+          -0.12479717284440994,
+          0.05818164348602295,
+          -0.07915422320365906,
+          0.4844202995300293,
+          -0.050288137048482895,
+          0.02881428971886635,
+          0.17225602269172668,
+          0.21480095386505127,
+          0.30700379610061646,
+          0.08251570165157318,
+          0.0075753843411803246,
+          -0.06009141728281975,
+          -0.041397422552108765,
+          0.11025799065828323,
+          -0.029640095308423042,
+          0.032515063881874084,
+          0.27604448795318604,
+          0.041267454624176025,
+          0.045245762914419174,
+          0.3109055161476135,
+          -0.07629498839378357,
+          -0.18719244003295898,
+          0.14141690731048584,
+          -0.19964122772216797
+        ],
+        "spearman": [
+          0.04353725647206709,
+          -0.09233009961360504,
+          -0.17818829737082245,
+          -0.12565003140590206,
+          -0.11508361946294095,
+          0.3707466312197973,
+          0.16069679863595812,
+          0.00032152283484183805,
+          -0.06183391666424053,
+          0.025292726933603096,
+          0.14768816927659872,
+          -0.05732592687687896,
+          0.1341233180441605,
+          -0.07679896723652788,
+          0.03204796381481832,
+          0.11720680407351838,
+          -0.18430154293547427,
+          0.01611448405055293,
+          0.28831915469517444,
+          0.10102569790815605,
+          -0.17818684883817582,
+          0.08296864864116403,
+          -0.12180483509557406,
+          -0.21667246809319538,
+          0.24106632652097706,
+          -0.03192418150779438
+        ],
+        "de_corr": [
+          -0.22437791526317596,
+          -0.5656072497367859,
+          0.1137598380446434,
+          0.23108525574207306,
+          -0.14381539821624756,
+          0.7256581783294678,
+          -0.21843844652175903,
+          -0.5244619846343994,
+          0.6701590418815613,
+          0.3335712254047394,
+          0.7878367304801941,
+          0.3845817446708679,
+          -0.5479661822319031,
+          -0.3156905472278595,
+          -0.11130276322364807,
+          0.4411935806274414,
+          -0.19360119104385376,
+          0.0018395064398646355,
+          0.42608770728111267,
+          -0.3652479946613312,
+          0.27641230821609497,
+          0.6019760966300964,
+          -0.43252789974212646,
+          -0.5774248242378235,
+          -0.24419526755809784,
+          -0.40081438422203064
+        ],
+        "mmd": [
+          0.028212891729032075,
+          0.17668493966754995,
+          0.1551985733846034,
+          0.09521743587710563,
+          0.14369480219229347,
+          0.11806902970722166,
+          0.2255878636555937,
+          0.31327826181329843,
+          0.08330115947701278,
+          0.07780785262113332,
+          0.12215313731187416,
+          0.26980985180555317,
+          0.12880152264843836,
+          0.056574219217009114,
+          0.10613605760518652,
+          0.2244930029396509,
+          0.1676700115849149,
+          0.024761279159099425,
+          0.06210051842261466,
+          0.029301764564502952,
+          0.10289429100830971,
+          0.13306709379342874,
+          0.19283435202935384,
+          0.14094517867540513,
+          0.21403328006563993,
+          0.04866823977419643
+        ],
+        "wasserstein": [
+          0.15157757597968538,
+          0.44020539156686445,
+          0.4400235845707495,
+          0.3607232100291323,
+          0.3959694677861648,
+          0.3512882785085011,
+          0.5483459678380452,
+          0.7275385804687302,
+          0.29424641074921526,
+          0.31549036084220616,
+          0.395544556183622,
+          0.6093649157257086,
+          0.37510882071090257,
+          0.22370848710780677,
+          0.38830919854877083,
+          0.5176736723988615,
+          0.4112948989728112,
+          0.1353916600626855,
+          0.2595916480410795,
+          0.16596665988480586,
+          0.3249977138314772,
+          0.3761359704317928,
+          0.4527599415596157,
+          0.4134669910667745,
+          0.5471746097544797,
+          0.21273364361070476
+        ]
+      },
+      "n_perts": 26
+    },
+    "AvgPerturbationEffect": {
+      "mse": 0.0044067181834879406,
+      "r2": 0.0399630069732666,
+      "pearson": 0.4885468087159097,
+      "spearman": 0.4042017919807703,
+      "de_corr": 0.46895806663311446,
+      "mmd": 0.11475870164747928,
+      "wasserstein": 0.327025581030484,
+      "_per_pert": {
+        "mse": [
+          0.001267462968826294,
+          0.004218101501464844,
+          0.003647373989224434,
+          0.0017841884400695562,
+          0.003240479389205575,
+          0.0029262227471917868,
+          0.006986001040786505,
+          0.021046752110123634,
+          0.004038598854094744,
+          0.0022836565040051937,
+          0.0026010992005467415,
+          0.00744654843583703,
+          0.005217214580625296,
+          0.0011601600563153625,
+          0.0025345073081552982,
+          0.007477449253201485,
+          0.004120431840419769,
+          0.002499110298231244,
+          0.0016265985323116183,
+          0.0012811636552214622,
+          0.0017465396085754037,
+          0.0027206726372241974,
+          0.004879365675151348,
+          0.004667452536523342,
+          0.009905749000608921,
+          0.0032517726067453623
+        ],
+        "r2": [
+          -1.0029613971710205,
+          0.3340906500816345,
+          0.3934832215309143,
+          0.5087642669677734,
+          0.4021516442298889,
+          0.39036089181900024,
+          0.3180961012840271,
+          0.14595919847488403,
+          0.0081101655960083,
+          0.04510855674743652,
+          0.3941187858581543,
+          0.34526002407073975,
+          -0.08779716491699219,
+          0.24131619930267334,
+          0.46423083543777466,
+          0.03945869207382202,
+          0.14588022232055664,
+          -2.739330291748047,
+          0.2032080888748169,
+          -0.539488673210144,
+          0.5090654492378235,
+          0.44957178831100464,
+          0.3343126177787781,
+          0.3143252730369568,
+          0.22003579139709473,
+          -0.7982927560806274
+        ],
+        "pearson": [
+          0.3604811429977417,
+          0.6058692932128906,
+          0.6697478294372559,
+          0.7310587167739868,
+          0.6734141111373901,
+          0.6347340941429138,
+          0.6224637031555176,
+          0.4709996283054352,
+          0.28053054213523865,
+          0.40062958002090454,
+          0.6288027167320251,
+          0.687199592590332,
+          0.1971733123064041,
+          0.5907549262046814,
+          0.7059608697891235,
+          0.24624839425086975,
+          0.3904935121536255,
+          -0.28574952483177185,
+          0.5447945594787598,
+          0.44264665246009827,
+          0.7273120880126953,
+          0.6910309195518494,
+          0.6204204559326172,
+          0.5835672616958618,
+          0.49274808168411255,
+          -0.01111543271690607
+        ],
+        "spearman": [
+          -0.005572146720350216,
+          0.4049987336190409,
+          0.48805908653659147,
+          0.7226606894210917,
+          0.3415249398944683,
+          0.34360029244832807,
+          0.41841257715861885,
+          0.5685465523120167,
+          0.45542969537201383,
+          0.5155350259801306,
+          0.4223747608625305,
+          0.4883545880796307,
+          0.46737821638369603,
+          0.4138993150624696,
+          0.7499677014287711,
+          0.3411502579224064,
+          0.07087614538849275,
+          0.012027375959805959,
+          0.398113071496426,
+          0.4623981108419023,
+          0.4823332341005564,
+          0.4203638150922295,
+          0.24832150593271687,
+          0.5368074709487687,
+          0.5605109316946306,
+          0.18117464428304272
+        ],
+        "de_corr": [
+          0.7283901572227478,
+          0.7522621154785156,
+          0.8509905934333801,
+          0.7136895656585693,
+          0.9145503640174866,
+          0.8487303256988525,
+          0.8070111274719238,
+          -0.658985435962677,
+          -0.14219823479652405,
+          0.4001394510269165,
+          0.8591786026954651,
+          0.7900312542915344,
+          -0.12902699410915375,
+          0.7461597919464111,
+          0.5489757061004639,
+          0.3416403830051422,
+          0.4684438407421112,
+          -0.4527846872806549,
+          0.5128503441810608,
+          0.27120205760002136,
+          0.838915228843689,
+          0.858460009098053,
+          0.8084298372268677,
+          0.5075000524520874,
+          0.2661384046077728,
+          -0.2577841281890869
+        ],
+        "mmd": [
+          0.05001841963404674,
+          0.1243999716578329,
+          0.10577308021160592,
+          0.06989012677420037,
+          0.10020200287508596,
+          0.08902702227400527,
+          0.1724092508063214,
+          0.2911497647074187,
+          0.10117398552149726,
+          0.07722481336758369,
+          0.08886264226803986,
+          0.20220013329751296,
+          0.14174360407776765,
+          0.04268554686292081,
+          0.07242410458620052,
+          0.22036877414645706,
+          0.15004829617682658,
+          0.08273019820860938,
+          0.054775785790464826,
+          0.046795093836730195,
+          0.06360700855966939,
+          0.09240974717766504,
+          0.1414321989080809,
+          0.11456708778898483,
+          0.1887216315143322,
+          0.09908595180460111
+        ],
+        "wasserstein": [
+          0.19751347198629352,
+          0.32000839841210266,
+          0.3027903984557414,
+          0.28081650843299333,
+          0.29404827368705755,
+          0.27698321473773685,
+          0.42078244654825653,
+          0.6576770736997155,
+          0.31715481247827343,
+          0.2870991900162981,
+          0.31031270858544463,
+          0.4710812588113828,
+          0.3964049639929377,
+          0.17318106225096142,
+          0.2922675958777241,
+          0.46861032907271394,
+          0.362768244503924,
+          0.2712758174607812,
+          0.22606261821831544,
+          0.21748638476438,
+          0.22856126139946967,
+          0.2629585976820611,
+          0.34642352983307817,
+          0.32831178931941957,
+          0.48006523217092956,
+          0.31201992439459203
+        ]
+      },
+      "n_perts": 26
+    },
+    "Additive": {
+      "mse": 0.0016535368653981446,
+      "r2": 0.573949531866954,
+      "pearson": 0.9300772272623502,
+      "spearman": 0.7897218029307761,
+      "de_corr": 0.9704181047586294,
+      "mmd": 0.059320614216664985,
+      "wasserstein": 0.2123106030642661,
+      "_per_pert": {
+        "mse": [
+          0.00015997662558220327,
+          0.0005177605198696256,
+          0.001115658087655902,
+          0.002104917075484991,
+          0.0004899363848380744,
+          0.000492266786750406,
+          0.001916507724672556,
+          0.013743879273533821,
+          0.0027093931566923857,
+          0.0013348587090149522,
+          0.0014990427298471332,
+          0.0015931000234559178,
+          0.0004326222406234592,
+          0.0010920573258772492,
+          0.000530772958882153,
+          0.0008865304989740252,
+          0.0003558291064109653,
+          0.0004061798972543329,
+          0.0022346421610563993,
+          0.0011294002179056406,
+          0.0008639924344606698,
+          0.0004070299328304827,
+          0.000788895704317838,
+          0.0006908114883117378,
+          0.0008649550727568567,
+          0.004630942363291979
+        ],
+        "r2": [
+          0.7471902370452881,
+          0.9182614088058472,
+          0.8144786953926086,
+          0.42045891284942627,
+          0.9096097946166992,
+          0.8974428176879883,
+          0.8129295706748962,
+          0.442297101020813,
+          0.33456629514694214,
+          0.4418402910232544,
+          0.6508238315582275,
+          0.8599262237548828,
+          0.9097976088523865,
+          0.2858518362045288,
+          0.8877999782562256,
+          0.886117696762085,
+          0.9262405633926392,
+          0.3922472596168518,
+          -0.094643235206604,
+          -0.35712456703186035,
+          0.7571404576301575,
+          0.917652428150177,
+          0.892371654510498,
+          0.8985159397125244,
+          0.9318946599960327,
+          -1.5609996318817139
+        ],
+        "pearson": [
+          0.8696218132972717,
+          0.980620265007019,
+          0.9799976944923401,
+          0.89031982421875,
+          0.954730212688446,
+          0.959190845489502,
+          0.933533251285553,
+          0.9897459149360657,
+          0.9803408980369568,
+          0.8745859265327454,
+          0.8584630489349365,
+          0.9879471659660339,
+          0.9558621048927307,
+          0.7636538743972778,
+          0.9562665820121765,
+          0.9557057023048401,
+          0.9704350233078003,
+          0.9516004920005798,
+          0.8845965266227722,
+          0.8278399109840393,
+          0.8941793441772461,
+          0.9762647747993469,
+          0.9550750851631165,
+          0.9652448892593384,
+          0.9663643836975098,
+          0.8998223543167114
+        ],
+        "spearman": [
+          0.6874229546359486,
+          0.7854118225005448,
+          0.8183332800064441,
+          0.8041241358867101,
+          0.7646113842689339,
+          0.7176599546225205,
+          0.688587194942726,
+          0.8911605604049929,
+          0.9065612036885211,
+          0.8493612223751926,
+          0.6984521228159735,
+          0.8898582004776359,
+          0.8367760643784031,
+          0.6926421240891262,
+          0.8416687892109902,
+          0.8655633996685086,
+          0.744568430661192,
+          0.779303185064447,
+          0.8037996979768682,
+          0.6723784886738736,
+          0.6205288290863296,
+          0.8576187371998566,
+          0.753541923090875,
+          0.9028960344131053,
+          0.9308670020760561,
+          0.7290701339844011
+        ],
+        "de_corr": [
+          0.965126633644104,
+          0.9906924366950989,
+          0.9944162964820862,
+          0.9439982771873474,
+          0.9877852201461792,
+          0.9841868877410889,
+          0.965343713760376,
+          0.9861399531364441,
+          0.9939308166503906,
+          0.9483758211135864,
+          0.9121333360671997,
+          0.9934125542640686,
+          0.9876887798309326,
+          0.9351226091384888,
+          0.9721375107765198,
+          0.9926180243492126,
+          0.9922538995742798,
+          0.9765479564666748,
+          0.9229952692985535,
+          0.8940964341163635,
+          0.9726075530052185,
+          0.9875404834747314,
+          0.9810596108436584,
+          0.9823089838027954,
+          0.9831912517547607,
+          0.9851604104042053
+        ],
+        "mmd": [
+          0.00869688189204465,
+          0.027503277129409254,
+          0.043831266173979144,
+          0.11028029639922188,
+          0.02509863159265069,
+          0.022246669756055937,
+          0.06793987733277973,
+          0.26240850873556854,
+          0.11436110231190588,
+          0.047902975380531565,
+          0.058675209843414455,
+          0.05252364628086359,
+          0.024561532527645036,
+          0.04192828122725545,
+          0.03601482869048189,
+          0.03251179337071386,
+          0.015227312009062022,
+          0.01838109460109527,
+          0.06996368945967801,
+          0.04406407510763066,
+          0.031062906260774903,
+          0.022564580166961834,
+          0.03708039271306218,
+          0.062240337885000274,
+          0.08853296664586607,
+          0.17673383613963678
+        ],
+        "wasserstein": [
+          0.10009500007567787,
+          0.1606439628824183,
+          0.20915482549802097,
+          0.30069878232411873,
+          0.15155494231937924,
+          0.15263269444013755,
+          0.25857470502279023,
+          0.5519766869965191,
+          0.2578558580673624,
+          0.22031762226370546,
+          0.22408907335770042,
+          0.24397260914311938,
+          0.1512310505937172,
+          0.17671375569923664,
+          0.16866251349216616,
+          0.16318874698223187,
+          0.14428973430566278,
+          0.12879515475904585,
+          0.25087982720080154,
+          0.205900811513284,
+          0.17909877213727576,
+          0.14794281224682468,
+          0.1865181168327959,
+          0.21668269487861277,
+          0.2571256575227321,
+          0.31147926911558194
+        ]
+      },
+      "n_perts": 26
+    },
+    "LinearResponse": {
+      "mse": 0.0007943772249676001,
+      "r2": 0.7440571349400741,
+      "pearson": 0.9199725710428678,
+      "spearman": 0.7693219288655411,
+      "de_corr": 0.9511560132870307,
+      "mmd": 0.04490069678916941,
+      "wasserstein": 0.1850864126089564,
+      "_per_pert": {
+        "mse": [
+          0.0002673232520464808,
+          0.0004400323086883873,
+          0.0005671681137755513,
+          0.0006946227513253689,
+          0.00020604993915185332,
+          0.0004807524965144694,
+          0.0011033053742721677,
+          0.0013256630627438426,
+          0.00016457772289868444,
+          0.0014632371021434665,
+          0.0010216188384220004,
+          0.00042015797225758433,
+          0.0011820258805528283,
+          0.0008684452041052282,
+          0.0002365340042160824,
+          0.001679806038737297,
+          0.00034780616988427937,
+          0.0003253179893363267,
+          0.0013649225002154708,
+          0.0006130867404863238,
+          0.0014941217377781868,
+          0.00046094926074147224,
+          0.00033666467061266303,
+          0.0011592293158173561,
+          0.000986011466011405,
+          0.0014443779364228249
+        ],
+        "r2": [
+          0.5775511264801025,
+          0.9305323958396912,
+          0.9056863784790039,
+          0.8087513446807861,
+          0.9619851112365723,
+          0.8998416662216187,
+          0.8923063278198242,
+          0.9462068676948547,
+          0.9595792889595032,
+          0.38815993070602417,
+          0.7620314955711365,
+          0.9630575180053711,
+          0.7535457611083984,
+          0.4320824146270752,
+          0.9499990940093994,
+          0.7842147946357727,
+          0.9279036521911621,
+          0.5132381916046143,
+          0.3313904404640198,
+          0.263294517993927,
+          0.5800174474716187,
+          0.9067438244819641,
+          0.9540691375732422,
+          0.8297027349472046,
+          0.9223629236221313,
+          0.20123112201690674
+        ],
+        "pearson": [
+          0.7870759963989258,
+          0.9787894487380981,
+          0.9782218933105469,
+          0.9218029975891113,
+          0.9844422340393066,
+          0.9751392006874084,
+          0.988003671169281,
+          0.9864732623100281,
+          0.9834932684898376,
+          0.7187127470970154,
+          0.8731866478919983,
+          0.9890651702880859,
+          0.938484787940979,
+          0.781520664691925,
+          0.9848747849464417,
+          0.9313053488731384,
+          0.9756849408149719,
+          0.8324065804481506,
+          0.8292630314826965,
+          0.7894912958145142,
+          0.9063573479652405,
+          0.9756512641906738,
+          0.9891635775566101,
+          0.9665358066558838,
+          0.9667638540267944,
+          0.8873770236968994
+        ],
+        "spearman": [
+          0.551747035385344,
+          0.7370737997453489,
+          0.803836848773959,
+          0.8292184235600115,
+          0.7613014349223204,
+          0.7675304115846785,
+          0.7789766806394016,
+          0.8832378448143863,
+          0.8493184084598053,
+          0.7242364798579012,
+          0.6666711304178595,
+          0.8587951388645674,
+          0.8209920731601313,
+          0.6643807105643174,
+          0.94185369178596,
+          0.8130837590466344,
+          0.7475316814597881,
+          0.7205471445258141,
+          0.7121962959539753,
+          0.7126376637972636,
+          0.47444913454653115,
+          0.8554636495518614,
+          0.79350591958881,
+          0.9104760445018664,
+          0.9143626239345516,
+          0.7089461210609797
+        ],
+        "de_corr": [
+          0.9269323945045471,
+          0.9903609752655029,
+          0.9926550984382629,
+          0.969302773475647,
+          0.9927334785461426,
+          0.9951089024543762,
+          0.9945881366729736,
+          0.9869873523712158,
+          0.9948418736457825,
+          0.8022546768188477,
+          0.9485812187194824,
+          0.9966453909873962,
+          0.9563841819763184,
+          0.9618679881095886,
+          0.9917111396789551,
+          0.9792109131813049,
+          0.9953674077987671,
+          0.7746251821517944,
+          0.8504496216773987,
+          0.73276686668396,
+          0.9748659133911133,
+          0.9905813932418823,
+          0.9938662052154541,
+          0.9811793565750122,
+          0.9787013530731201,
+          0.9774865508079529
+        ],
+        "mmd": [
+          0.013021821112970566,
+          0.02689526231028827,
+          0.02820178421391406,
+          0.06906585385531339,
+          0.020487769660918698,
+          0.023150655193067582,
+          0.050137506492634265,
+          0.16251804350257926,
+          0.0258187441534784,
+          0.049896416209351147,
+          0.045910940850522386,
+          0.035428211191668835,
+          0.04205538802670716,
+          0.03411345341440386,
+          0.03430092272019736,
+          0.06699429318588046,
+          0.022095179005238497,
+          0.014287880451578472,
+          0.04600966690867225,
+          0.02442238698581134,
+          0.04895633861652493,
+          0.025632502027059134,
+          0.026054079367394012,
+          0.0659416347612718,
+          0.08807082656125143,
+          0.07795055573970688
+        ],
+        "wasserstein": [
+          0.11233875102879669,
+          0.1382577469116224,
+          0.16257367499728104,
+          0.22345839197149164,
+          0.13100025399928808,
+          0.1473105598750808,
+          0.21374490149100697,
+          0.378033049583956,
+          0.12087458622020454,
+          0.21157240183543238,
+          0.19642191220685326,
+          0.1745468951781186,
+          0.1953054238124035,
+          0.15487670067352413,
+          0.15586411645888687,
+          0.2547957024119093,
+          0.14254234915454567,
+          0.10974410507796854,
+          0.19968812475020362,
+          0.1647749943071228,
+          0.21860856110177715,
+          0.15151733043827764,
+          0.14455454923654443,
+          0.23581756988551764,
+          0.26925133427962333,
+          0.2047727409454289
+        ]
+      },
+      "n_perts": 26
+    },
+    "NearestPerturbationCentroid": {
+      "mse": 0.002478486358505996,
+      "r2": 0.23386846138880804,
+      "pearson": 0.7247861949010537,
+      "spearman": 0.5799124729318662,
+      "de_corr": 0.7635814500614427,
+      "mmd": 0.08080473789892954,
+      "wasserstein": 0.2582930905262011,
+      "_per_pert": {
+        "mse": [
+          0.000533635844476521,
+          0.0013795195845887065,
+          0.0013664873549714684,
+          0.0007153262849897146,
+          0.0016368427313864231,
+          0.0010332483798265457,
+          0.004374958109110594,
+          0.0006444199243560433,
+          0.001957371598109603,
+          0.006869337521493435,
+          0.0009074644767679274,
+          0.0017874609911814332,
+          0.0010570818558335304,
+          0.0036049378104507923,
+          0.003051120787858963,
+          0.0032517763320356607,
+          0.0004848494427278638,
+          0.0002504873846191913,
+          0.002362786792218685,
+          0.0025162675883620977,
+          0.001352915191091597,
+          0.00018224639643449336,
+          0.003386517520993948,
+          0.00904778577387333,
+          0.005635064095258713,
+          0.0050507355481386185
+        ],
+        "r2": [
+          0.1566997766494751,
+          0.7822160124778748,
+          0.7727686762809753,
+          0.8030511736869812,
+          0.6980127096176147,
+          0.7847366333007812,
+          0.5729600787162781,
+          0.9738505482673645,
+          0.5192646980285645,
+          -1.8723547458648682,
+          0.7886219024658203,
+          0.8428369760513306,
+          0.7795968055725098,
+          -1.3574402332305908,
+          0.35502398014068604,
+          0.5822820067405701,
+          0.8994961380958557,
+          0.6252045631408691,
+          -0.15741503238677979,
+          -2.0236306190490723,
+          0.6197092533111572,
+          0.9631291031837463,
+          0.5379804372787476,
+          -0.3291703462600708,
+          0.5563032627105713,
+          -1.7931537628173828
+        ],
+        "pearson": [
+          0.40049001574516296,
+          0.929872989654541,
+          0.9087874293327332,
+          0.9011523723602295,
+          0.9123130440711975,
+          0.9401535987854004,
+          0.9237536787986755,
+          0.9924522042274475,
+          0.9656147956848145,
+          -0.08230501413345337,
+          0.9518076181411743,
+          0.9306371808052063,
+          0.8863422870635986,
+          0.09355726093053818,
+          0.7735655307769775,
+          0.778367280960083,
+          0.948794424533844,
+          0.8171080946922302,
+          0.05523582175374031,
+          0.627107560634613,
+          0.8792233467102051,
+          0.9813959002494812,
+          0.9108834266662598,
+          -0.29144471883773804,
+          0.8878174424171448,
+          0.8217574954032898
+        ],
+        "spearman": [
+          0.3810437789050358,
+          0.6065274617974341,
+          0.5486168655162754,
+          0.7831034787407409,
+          0.5216215074472673,
+          0.6799291849212872,
+          0.6649632422127042,
+          0.9339662157003352,
+          0.8787702436922701,
+          0.014809978657983117,
+          0.7728194244675329,
+          0.7233591536263967,
+          0.7574225069434324,
+          0.3961334579681166,
+          0.6307828225901657,
+          0.7217318566328387,
+          0.6870010005857667,
+          0.5758524374306192,
+          0.08045635799333074,
+          0.5319892656791716,
+          0.5764579665922199,
+          0.8967433429046583,
+          0.620134526905448,
+          -0.3594185817850521,
+          0.836594005006243,
+          0.6163127950962983
+        ],
+        "de_corr": [
+          0.5721431970596313,
+          0.9719492197036743,
+          0.939814567565918,
+          0.9490275382995605,
+          0.9564734101295471,
+          0.9784294366836548,
+          0.957977294921875,
+          0.9853857755661011,
+          0.9904815554618835,
+          -0.3254048228263855,
+          0.9855234622955322,
+          0.9800997972488403,
+          0.9723494648933411,
+          0.010173377580940723,
+          0.856938362121582,
+          0.9231246709823608,
+          0.9796337485313416,
+          0.855949342250824,
+          0.5607375502586365,
+          0.6474562287330627,
+          0.9740811586380005,
+          0.9901545643806458,
+          0.9644842743873596,
+          -0.7426186800003052,
+          0.9561570286750793,
+          0.9625961780548096
+        ],
+        "mmd": [
+          0.02204469496328021,
+          0.04624911196675574,
+          0.04746513316097101,
+          0.05850050663460027,
+          0.05384147191517019,
+          0.0361109265783357,
+          0.11663521270909372,
+          0.15518572284447507,
+          0.09348214564328561,
+          0.1778393454296321,
+          0.029855567470220667,
+          0.06980892692864027,
+          0.04091339127139526,
+          0.11479473976791221,
+          0.07370843238077363,
+          0.11546226816321503,
+          0.022746616132742092,
+          0.011514855898313026,
+          0.07266941986915876,
+          0.0854665723722755,
+          0.045003382837002004,
+          0.018198857259373757,
+          0.10215822264566865,
+          0.1671924776037782,
+          0.13503237873100937,
+          0.18904280419509
+        ],
+        "wasserstein": [
+          0.1402848677767436,
+          0.22384115974220262,
+          0.22574395983493822,
+          0.22839451319865708,
+          0.23123111355692186,
+          0.18999337168742156,
+          0.3651302607002648,
+          0.33065099107446744,
+          0.21426254452904867,
+          0.44905922384330976,
+          0.20683663195158197,
+          0.2605474445830111,
+          0.20645871223374182,
+          0.28734785374330934,
+          0.3018896834928422,
+          0.2903402758335888,
+          0.15180851176513208,
+          0.09403153041927144,
+          0.2821338821418673,
+          0.2188255783476873,
+          0.20926775740598483,
+          0.11903806205432738,
+          0.31814383882848457,
+          0.4467690433555681,
+          0.3799488600241454,
+          0.3436406815567083
+        ]
+      },
+      "n_perts": 26
+    },
+    "EndpointMLP": {
+      "mse": 0.0008020812652270811,
+      "r2": 0.7639783162337083,
+      "pearson": 0.943373576952861,
+      "spearman": 0.7928224143378853,
+      "de_corr": 0.9737307039590982,
+      "mmd": 0.044485534927875284,
+      "wasserstein": 0.17595379308630796,
+      "_per_pert": {
+        "mse": [
+          0.0001764298212947324,
+          0.000341860024491325,
+          0.0006514676497317851,
+          0.0016977416817098856,
+          0.00020893856708426028,
+          0.0003437062550801784,
+          0.0006340914987958968,
+          0.0032982639968395233,
+          0.001114899292588234,
+          0.0005150196957401931,
+          0.0002106309839291498,
+          0.001079932670108974,
+          0.0007576587377116084,
+          0.0005042431876063347,
+          0.00016221677651628852,
+          0.00108602293767035,
+          0.00026029604487121105,
+          0.0003001794684678316,
+          0.0018586951773613691,
+          0.0004277301486581564,
+          0.0008145751780830324,
+          0.00017303899221587926,
+          0.000495580374263227,
+          0.000787517346907407,
+          0.0008026810828596354,
+          0.0021506953053176403
+        ],
+        "r2": [
+          0.7211893200874329,
+          0.9460307359695435,
+          0.8916683197021484,
+          0.5325654745101929,
+          0.9614521265029907,
+          0.9283934235572815,
+          0.9381062984466553,
+          0.866162121295929,
+          0.7261780500411987,
+          0.7846488952636719,
+          0.9509371519088745,
+          0.9050465822219849,
+          0.8420270085334778,
+          0.6702514290809631,
+          0.9657090306282043,
+          0.8604912161827087,
+          0.9460435509681702,
+          0.5508520007133484,
+          0.08951520919799805,
+          0.4860253930091858,
+          0.7710312008857727,
+          0.9649919271469116,
+          0.932388424873352,
+          0.8843092918395996,
+          0.9367980360984802,
+          -0.18937599658966064
+        ],
+        "pearson": [
+          0.8538834452629089,
+          0.9834492802619934,
+          0.9840121269226074,
+          0.9230494499206543,
+          0.9823115468025208,
+          0.977793276309967,
+          0.9880353212356567,
+          0.9933536052703857,
+          0.9654573202133179,
+          0.8889068961143494,
+          0.9755520224571228,
+          0.9825723171234131,
+          0.9482463002204895,
+          0.8223302364349365,
+          0.9833102822303772,
+          0.929058849811554,
+          0.9731715321540833,
+          0.9565269351005554,
+          0.875830888748169,
+          0.855214536190033,
+          0.8972867727279663,
+          0.9853354692459106,
+          0.9754910469055176,
+          0.952223539352417,
+          0.967992901802063,
+          0.9073171019554138
+        ],
+        "spearman": [
+          0.527013020367931,
+          0.7515835285741037,
+          0.8457713827450878,
+          0.8362780284228648,
+          0.7574566203962355,
+          0.8095630693734549,
+          0.8643096912456038,
+          0.8981804707573718,
+          0.8548378028787879,
+          0.7521596037261155,
+          0.8067627081345097,
+          0.8789945026946745,
+          0.8338661967880412,
+          0.7013842708277744,
+          0.9425145511111144,
+          0.8359182966491646,
+          0.6965080796194033,
+          0.761916213550671,
+          0.757682484740644,
+          0.7902816421564133,
+          0.5547945336832226,
+          0.8580117556276767,
+          0.806298656459696,
+          0.8274195420853317,
+          0.9179559408813905,
+          0.7459201792877279
+        ],
+        "de_corr": [
+          0.9722740650177002,
+          0.9957163333892822,
+          0.9919744729995728,
+          0.9635945558547974,
+          0.9935960173606873,
+          0.9915443658828735,
+          0.99666827917099,
+          0.9915331602096558,
+          0.9873785376548767,
+          0.9613246917724609,
+          0.9909056425094604,
+          0.9955531358718872,
+          0.9637845158576965,
+          0.9094120860099792,
+          0.9885944128036499,
+          0.9769296646118164,
+          0.9854501485824585,
+          0.9806379675865173,
+          0.9181355834007263,
+          0.8516497611999512,
+          0.969434380531311,
+          0.9968209266662598,
+          0.9937450885772705,
+          0.979896605014801,
+          0.9899572730064392,
+          0.9804866313934326
+        ],
+        "mmd": [
+          0.00939074502599535,
+          0.023945967306793103,
+          0.032513114908470864,
+          0.10172367153089756,
+          0.022291654025017982,
+          0.019033009974969928,
+          0.04078682089177177,
+          0.17810661065121425,
+          0.06605911452199564,
+          0.023305841300289343,
+          0.0179010890322433,
+          0.04698349858770068,
+          0.0320253415100229,
+          0.02243332577424806,
+          0.036360509155239695,
+          0.04310328437344857,
+          0.014676688852410136,
+          0.014393517162749836,
+          0.058695578526053116,
+          0.017454215915711502,
+          0.03021586689117839,
+          0.018672551217580713,
+          0.028699622044615958,
+          0.06686701296800857,
+          0.08779529986539847,
+          0.10318995611073167
+        ],
+        "wasserstein": [
+          0.09634433725809048,
+          0.14457362822844394,
+          0.17813129015134715,
+          0.26963856392159335,
+          0.1434556965459493,
+          0.12590953731785043,
+          0.1738429979290879,
+          0.38652253694458105,
+          0.19249031864272617,
+          0.13392930952241766,
+          0.1319038161119049,
+          0.19968582500395804,
+          0.16901401790296622,
+          0.13802216887287094,
+          0.16001044192442296,
+          0.1879722848449178,
+          0.1285070085429668,
+          0.1189616944563749,
+          0.22085639480056357,
+          0.1295554748901658,
+          0.16637893428918307,
+          0.1253171778105432,
+          0.1512992796521846,
+          0.22283154374680045,
+          0.2648075181248954,
+          0.21483682280720123
+        ]
+      },
+      "n_perts": 26
+    },
+    "Random": {
+      "mse": 0.006530506056151353,
+      "r2": -0.6242899436217088,
+      "pearson": 0.33471211712234294,
+      "spearman": 0.30173074557310325,
+      "de_corr": 0.2762379719553372,
+      "mmd": 0.16339670384709448,
+      "wasserstein": 0.3892757439227088,
+      "_per_pert": {
+        "mse": [
+          0.0036576436832547188,
+          0.0056915609166026115,
+          0.0060289716348052025,
+          0.004404988139867783,
+          0.007452930323779583,
+          0.004236144013702869,
+          0.001192743075080216,
+          0.024693042039871216,
+          0.0013728041667491198,
+          0.009539428167045116,
+          0.002810685196891427,
+          0.00863055419176817,
+          0.005879695527255535,
+          0.0028781192377209663,
+          0.0029862020164728165,
+          0.013793515972793102,
+          0.015004010871052742,
+          0.0038161741103976965,
+          0.0020486859139055014,
+          0.000368712906492874,
+          0.007352271117269993,
+          0.0023348368704319,
+          0.017345810309052467,
+          0.004993144888430834,
+          0.007239057682454586,
+          0.004041424486786127
+        ],
+        "r2": [
+          -4.780143737792969,
+          0.10147649049758911,
+          -0.002549409866333008,
+          -0.2128133773803711,
+          -0.3750194311141968,
+          0.11745631694793701,
+          0.8835762739181519,
+          -0.0020008087158203125,
+          0.6628358960151672,
+          -2.988830089569092,
+          0.34529948234558105,
+          0.24115610122680664,
+          -0.22592544555664062,
+          -0.8821392059326172,
+          0.36874711513519287,
+          -0.7718935012817383,
+          -2.1101648807525635,
+          -4.710006237030029,
+          -0.0035523176193237305,
+          0.5569422245025635,
+          -1.0666491985321045,
+          0.5276314616203308,
+          -1.3664729595184326,
+          0.2664791941642761,
+          0.43000710010528564,
+          -1.234985589981079
+        ],
+        "pearson": [
+          -0.32098689675331116,
+          0.6315445899963379,
+          0.06893187761306763,
+          0.17515534162521362,
+          0.4494899809360504,
+          0.376334011554718,
+          0.9715424180030823,
+          0.212151899933815,
+          0.8740909099578857,
+          -0.14101824164390564,
+          0.6391974091529846,
+          0.696272611618042,
+          -0.16628801822662354,
+          0.019388029351830482,
+          0.6841511726379395,
+          0.18468309938907623,
+          -0.35539254546165466,
+          0.2860647141933441,
+          0.5686036348342896,
+          0.8090764880180359,
+          0.717863917350769,
+          0.7279661893844604,
+          -0.5468238592147827,
+          0.520050585269928,
+          0.7499146461486816,
+          -0.12944892048835754
+        ],
+        "spearman": [
+          -0.09682912051447327,
+          0.22879759978816758,
+          0.26566111845095053,
+          0.19210605561994568,
+          0.3072106109499961,
+          0.18366238676489946,
+          0.7787123272327153,
+          0.38650271534346553,
+          0.5770389367808767,
+          0.109951854134633,
+          0.41707697971382285,
+          0.43196796653145103,
+          0.12069515886786666,
+          0.048372930159525734,
+          0.5196168931176147,
+          0.22551694443483453,
+          -0.05681824908382497,
+          0.4219306056185222,
+          0.485013799143099,
+          0.680594747336716,
+          0.44971425543613464,
+          0.48711372039248285,
+          -0.23501064413033665,
+          0.5138413323971912,
+          0.6161601103392491,
+          -0.2136016499248397
+        ],
+        "de_corr": [
+          -0.7161344885826111,
+          0.8492564558982849,
+          -0.1793336272239685,
+          0.13276320695877075,
+          0.776057243347168,
+          0.5899722576141357,
+          0.9700168371200562,
+          -0.3755188286304474,
+          0.9809064269065857,
+          -0.4573112726211548,
+          0.60968416929245,
+          0.840336263179779,
+          -0.4690474271774292,
+          -0.006662833970040083,
+          0.6662706732749939,
+          0.1331259310245514,
+          -0.7366300225257874,
+          0.07454127073287964,
+          0.7414866089820862,
+          0.8588231205940247,
+          0.8517552614212036,
+          0.7582389116287231,
+          -0.727645754814148,
+          0.4333813190460205,
+          0.8468313813209534,
+          -0.262975811958313
+        ],
+        "mmd": [
+          0.1259100149447856,
+          0.16946875132238604,
+          0.15328670939502576,
+          0.11925395481013612,
+          0.21227452919845446,
+          0.11639224942562032,
+          0.05440252783775834,
+          0.333752679237582,
+          0.03177710284945923,
+          0.24049995578142902,
+          0.09254898518275667,
+          0.2386030471134899,
+          0.15347768850526877,
+          0.09591408351052366,
+          0.1002594879705514,
+          0.3399638335367665,
+          0.39018998027496965,
+          0.1259245276964478,
+          0.06979720367977793,
+          0.015508065578698216,
+          0.2094256725394087,
+          0.08344996485088807,
+          0.3747955517551732,
+          0.12558276191202788,
+          0.1565787000513006,
+          0.11927627106377081
+        ],
+        "wasserstein": [
+          0.32759463319893284,
+          0.4425174979876795,
+          0.4252578782643919,
+          0.38039587629106897,
+          0.4211703004133938,
+          0.3463849288268218,
+          0.20747240099971048,
+          0.7083644932209476,
+          0.18970051912524752,
+          0.5432435331111674,
+          0.32828753055254845,
+          0.4381020627120035,
+          0.3929922959591078,
+          0.2821122568067752,
+          0.34254322073800736,
+          0.5808119012996298,
+          0.6576707281875873,
+          0.35540729146222566,
+          0.25355683532358997,
+          0.11177937542410157,
+          0.48529021725106586,
+          0.2419028528459981,
+          0.5954052939235607,
+          0.3152188587606063,
+          0.4070771722904518,
+          0.3409093870138086
+        ]
+      },
+      "n_perts": 26
+    }
+  }
+}

experiments/results/norman_forward_perturbation.json

@@ -0,0 +1,2845 @@
+{
+  "split": "perturbation",
+  "models": {
+    "PIVOT": {
+      "mse": 0.0017456959687208043,
+      "r2": -0.8285323470196826,
+      "pearson": 0.76253557141791,
+      "spearman": 0.5823030614288411,
+      "de_corr": 0.8314210008433525,
+      "mmd": 0.3021512914899839,
+      "wasserstein": 0.43160508252391033,
+      "_per_pert": {
+        "mse": [
+          0.0015962913166731596,
+          0.0008809863356873393,
+          0.0019998853094875813,
+          0.0003412741352804005,
+          0.0008116157841868699,
+          0.0006022303132340312,
+          0.0007515754550695419,
+          0.0007353638648055494,
+          0.0009762277477420866,
+          0.004156765062361956,
+          0.0056117321364581585,
+          0.0009224773966707289,
+          0.0014235102571547031,
+          0.000321702565997839,
+          0.001231062808074057,
+          0.0017586882458999753,
+          0.002621252555400133,
+          0.00047866388922557235,
+          0.0004518006753642112,
+          0.0009143200586549938,
+          0.00048078721738420427,
+          0.0035726698115468025,
+          0.0006342866108752787,
+          0.0007832234841771424,
+          0.0029461875092238188,
+          0.0012722138781100512,
+          0.00037073460407555103,
+          0.005331906024366617,
+          0.007546290289610624,
+          0.001551676308736205,
+          0.0010973499156534672,
+          0.0006320606917142868,
+          0.00013778671564068645,
+          0.0014407264534384012,
+          0.0016012798296287656,
+          0.001502867671661079,
+          0.0015092769172042608,
+          0.004813094157725573,
+          0.0007678758120164275,
+          0.002616616664454341,
+          0.0011721006594598293,
+          0.0018023933516815305,
+          0.0007824173080734909,
+          0.0014333735452964902,
+          0.0012945494381710887,
+          0.005306630861014128,
+          0.0010599088855087757
+        ],
+        "r2": [
+          0.5472016930580139,
+          -1.842552900314331,
+          0.6095108389854431,
+          -4.876230716705322,
+          -20.437246322631836,
+          -2.7194979190826416,
+          -0.18770825862884521,
+          0.8777173161506653,
+          0.7312178611755371,
+          -1.9020304679870605,
+          -2.7078135013580322,
+          0.8078142404556274,
+          0.8908155560493469,
+          0.8376991748809814,
+          0.42308592796325684,
+          0.1281682848930359,
+          0.6018931865692139,
+          0.9521409869194031,
+          0.6058731079101562,
+          0.9128751754760742,
+          0.867173433303833,
+          -7.576755523681641,
+          0.8659182786941528,
+          -0.182220458984375,
+          0.27227669954299927,
+          0.5839238166809082,
+          0.9288119077682495,
+          0.31507188081741333,
+          0.601916491985321,
+          0.6740221977233887,
+          -1.2246947288513184,
+          0.9510142803192139,
+          0.7938345074653625,
+          0.7671914100646973,
+          -4.445379257202148,
+          0.31845784187316895,
+          0.8332926034927368,
+          0.31394362449645996,
+          0.4389669895172119,
+          0.6129298210144043,
+          0.8421118855476379,
+          -7.5136003494262695,
+          0.4907867908477783,
+          -3.4536638259887695,
+          0.8398243188858032,
+          0.6412551403045654,
+          -1.7503633499145508
+        ],
+        "pearson": [
+          0.7536920309066772,
+          0.5233908891677856,
+          0.8592501878738403,
+          0.22083774209022522,
+          0.2833574116230011,
+          0.039258554577827454,
+          0.7498257756233215,
+          0.9401399493217468,
+          0.8797203898429871,
+          0.6033342480659485,
+          0.9358171224594116,
+          0.9723519682884216,
+          0.9687368869781494,
+          0.9374346137046814,
+          0.6731510758399963,
+          0.47313177585601807,
+          0.9422309994697571,
+          0.9758107662200928,
+          0.803334653377533,
+          0.965327799320221,
+          0.9316220283508301,
+          0.8100623488426208,
+          0.9581425189971924,
+          0.6218380331993103,
+          0.8873342275619507,
+          0.8127406239509583,
+          0.9650146961212158,
+          0.5918835997581482,
+          0.846104085445404,
+          0.8276202082633972,
+          0.6287824511528015,
+          0.9759727716445923,
+          0.8941470384597778,
+          0.983452558517456,
+          0.18894581496715546,
+          0.7274587154388428,
+          0.9846226572990417,
+          0.6012974977493286,
+          0.7263070344924927,
+          0.871539294719696,
+          0.9778054356575012,
+          0.8281512260437012,
+          0.7819417715072632,
+          0.39418646693229675,
+          0.9719808101654053,
+          0.9259516596794128,
+          0.624131441116333
+        ],
+        "spearman": [
+          0.6555050882255908,
+          0.40542519904581553,
+          0.4964159787310143,
+          0.09228475010163843,
+          -0.15748521125132137,
+          -0.11657048285974761,
+          0.21862650960808533,
+          0.605807510152057,
+          0.769920244183745,
+          0.5469354385725889,
+          0.5626566869480071,
+          0.6832155535335042,
+          0.9070905775510695,
+          0.8487869280419458,
+          0.45339023110868865,
+          0.2868166739366436,
+          0.7518973609126082,
+          0.8643221348230034,
+          0.6194693370885784,
+          0.7732447700697314,
+          0.8543534221221268,
+          0.6644175705672826,
+          0.9216471532145842,
+          0.5517182833777677,
+          0.7707817801400423,
+          0.6253514731561677,
+          0.85791655113273,
+          0.5972728738711756,
+          0.6745397902412349,
+          0.680843485444789,
+          0.45267624472772633,
+          0.8136672703384545,
+          0.6424177851315931,
+          0.8843047114275199,
+          0.11669099443566826,
+          0.5227399671880087,
+          0.7238119692458834,
+          0.5503434385154962,
+          0.46469522726553875,
+          0.717613957110383,
+          0.8244220387943655,
+          0.39631782092974593,
+          0.6393433529567272,
+          0.24420713551435697,
+          0.704496413148991,
+          0.8780034344331563,
+          0.3258944642007685
+        ],
+        "de_corr": [
+          0.9333991408348083,
+          0.7491828203201294,
+          0.9241585731506348,
+          0.24465131759643555,
+          0.4710516333580017,
+          0.03841133415699005,
+          0.9465099573135376,
+          0.9783341288566589,
+          0.9323738813400269,
+          0.7223357558250427,
+          0.9700398445129395,
+          0.9917811155319214,
+          0.9845224022865295,
+          0.95426344871521,
+          0.8516639471054077,
+          0.539853572845459,
+          0.9640456438064575,
+          0.9839518070220947,
+          0.92417311668396,
+          0.979576051235199,
+          0.9622992277145386,
+          0.9482614398002625,
+          0.9775505661964417,
+          0.8960217237472534,
+          0.9404195547103882,
+          0.9081787467002869,
+          0.988193929195404,
+          0.698651909828186,
+          0.7528803944587708,
+          0.7510294318199158,
+          0.5872046947479248,
+          0.9912537336349487,
+          0.9690126776695251,
+          0.997524082660675,
+          0.25434747338294983,
+          0.8284298181533813,
+          0.9940236806869507,
+          0.6034884452819824,
+          0.9004725813865662,
+          0.8926483392715454,
+          0.9886642098426819,
+          0.959450364112854,
+          0.8362482190132141,
+          0.6785160303115845,
+          0.9831262230873108,
+          0.9680206775665283,
+          0.7365893721580505
+        ],
+        "mmd": [
+          0.31358329356625925,
+          0.2829083437860266,
+          0.3268910257886851,
+          0.3034275433088548,
+          0.26561077532549326,
+          0.27375241880245993,
+          0.2764328706803504,
+          0.30042128408200697,
+          0.3065089420728595,
+          0.3461828757735623,
+          0.36859794732202045,
+          0.2710819231170011,
+          0.2931891493477259,
+          0.2320166530035307,
+          0.2678655791247254,
+          0.3100671682066971,
+          0.2947334531122461,
+          0.27877327345627356,
+          0.26099842907739523,
+          0.2544545857509505,
+          0.3034685334443069,
+          0.34142556532541024,
+          0.2865900872098376,
+          0.2621079957448894,
+          0.33341331852705725,
+          0.2986212856731577,
+          0.3046254001186489,
+          0.4049631989245549,
+          0.29687186899197815,
+          0.2892757718681802,
+          0.2832749104171687,
+          0.2963613126384349,
+          0.262936228194131,
+          0.28750202458966534,
+          0.31853469099007437,
+          0.3254136872440071,
+          0.3229344827643106,
+          0.394373763081333,
+          0.27312924977788144,
+          0.3664557058728112,
+          0.2771543305879254,
+          0.32460494136859086,
+          0.2727312048226932,
+          0.27398726155845665,
+          0.3571783400626488,
+          0.3426648703072518,
+          0.27301313521871284
+        ],
+        "wasserstein": [
+          0.4821041697967327,
+          0.37169123004200516,
+          0.46441134658798594,
+          0.37558352939015793,
+          0.35424735165224336,
+          0.34667920523386314,
+          0.36154464792525415,
+          0.42276941312530963,
+          0.46618256614801645,
+          0.46860028466716114,
+          0.5220581034096756,
+          0.4019005545926686,
+          0.5063314914221153,
+          0.3512468343961775,
+          0.37955504571508614,
+          0.40661420355491534,
+          0.4442106491816762,
+          0.44726791691187456,
+          0.3572826731205464,
+          0.4066724517513526,
+          0.42381805290126495,
+          0.43329532088579326,
+          0.43324303879231296,
+          0.36331708800584595,
+          0.5153061605365753,
+          0.4254621708617453,
+          0.4270177856138994,
+          0.5573708062094781,
+          0.5803057499360839,
+          0.4368545237685248,
+          0.38260245122802966,
+          0.4435057205256277,
+          0.3411261893862858,
+          0.39030869652872263,
+          0.4198432339215978,
+          0.5112260641620169,
+          0.4723184218096585,
+          0.5083531046129299,
+          0.3921637349952938,
+          0.5017335441630584,
+          0.4175469696081974,
+          0.4156747659203838,
+          0.3698995877111973,
+          0.3685795259443726,
+          0.4717643181925033,
+          0.573698411676562,
+          0.3721497721010105
+        ]
+      },
+      "n_perts": 47
+    },
+    "MeanControl": {
+      "mse": 0.004514833508050713,
+      "r2": -0.10789477064254437,
+      "pearson": -0.022738046513157004,
+      "spearman": -0.04880332349662557,
+      "de_corr": -0.029544741906067158,
+      "mmd": 0.10804938097705778,
+      "wasserstein": 0.3255573552861375,
+      "_per_pert": {
+        "mse": [
+          0.003526095300912857,
+          0.00047110902960412204,
+          0.005013410467654467,
+          0.00011254555283812806,
+          8.388989954255521e-05,
+          0.0002046082663582638,
+          0.0007276904652826488,
+          0.006099091377109289,
+          0.0038916850462555885,
+          0.0013538715429604053,
+          0.0013269009068608284,
+          0.00479864701628685,
+          0.013018874451518059,
+          0.0020068504381924868,
+          0.0022634572815150023,
+          0.002172177890315652,
+          0.006917996797710657,
+          0.01052918191999197,
+          0.0010125469416379929,
+          0.010958677157759666,
+          0.003653470892459154,
+          0.0005656962166540325,
+          0.0047027855180203915,
+          0.0007785499328747392,
+          0.0039011535700410604,
+          0.003163485322147608,
+          0.005225673783570528,
+          0.0077210054732859135,
+          0.020093467086553574,
+          0.00497997272759676,
+          0.00054371845908463,
+          0.013299396261572838,
+          0.0007702374714426696,
+          0.006292510777711868,
+          0.00042728715925477445,
+          0.002570908050984144,
+          0.008804382756352425,
+          0.007052035070955753,
+          0.0014501134864985943,
+          0.0073930020444095135,
+          0.007186289876699448,
+          0.0002132457448169589,
+          0.0015514662954956293,
+          0.00042830329039134085,
+          0.00801896583288908,
+          0.014559644274413586,
+          0.0003610997518990189
+        ],
+        "r2": [
+          -0.0001995563507080078,
+          -0.5200603008270264,
+          0.021102607250213623,
+          -0.9378657341003418,
+          -1.2157888412475586,
+          -0.26370251178741455,
+          -0.1499631404876709,
+          -0.014209747314453125,
+          -0.07148706912994385,
+          0.054799675941467285,
+          0.12328284978866577,
+          0.0002665519714355469,
+          0.0014414191246032715,
+          -0.012468218803405762,
+          -0.060726165771484375,
+          -0.07681047916412354,
+          -0.0506819486618042,
+          -0.05275595188140869,
+          0.11670804023742676,
+          -0.044243812561035156,
+          -0.009341120719909668,
+          -0.35804271697998047,
+          0.005878746509552002,
+          -0.17516624927520752,
+          0.03639495372772217,
+          -0.03461432456970215,
+          -0.0034291744232177734,
+          0.00817185640335083,
+          -0.05997490882873535,
+          -0.04619753360748291,
+          -0.10229921340942383,
+          -0.030724048614501953,
+          -0.15247929096221924,
+          -0.01681363582611084,
+          -0.45305097103118896,
+          -0.16589272022247314,
+          0.027510643005371094,
+          -0.005194187164306641,
+          -0.059496164321899414,
+          -0.09362983703613281,
+          0.03196924924850464,
+          -0.0072650909423828125,
+          -0.00972592830657959,
+          -0.3307896852493286,
+          0.007806599140167236,
+          0.01572251319885254,
+          0.06298035383224487
+        ],
+        "pearson": [
+          0.0591709278523922,
+          -0.40775421261787415,
+          0.15382766723632812,
+          0.0747164785861969,
+          -0.1517912894487381,
+          0.05104249715805054,
+          -0.1530401110649109,
+          -0.05691283568739891,
+          -0.2880525290966034,
+          0.2644456923007965,
+          0.3980618417263031,
+          0.06142786517739296,
+          0.2099093198776245,
+          0.09384554624557495,
+          -0.09174783527851105,
+          -0.22845198214054108,
+          -0.2895209491252899,
+          -0.370250940322876,
+          0.39623144268989563,
+          -0.19269369542598724,
+          0.05708274617791176,
+          -0.14504733681678772,
+          0.15338094532489777,
+          -0.22136132419109344,
+          0.2530163526535034,
+          -0.08538380265235901,
+          0.051351871341466904,
+          0.11927977204322815,
+          -0.27650654315948486,
+          -0.08860388398170471,
+          -0.06261344999074936,
+          -0.1181451827287674,
+          -0.15346801280975342,
+          -0.0022375716362148523,
+          -0.3507479429244995,
+          -0.5506516695022583,
+          0.2766454517841339,
+          0.07293259352445602,
+          -0.049403443932533264,
+          -0.5229116082191467,
+          0.24387092888355255,
+          0.3320770561695099,
+          0.07020393013954163,
+          -0.13814716041088104,
+          0.09130217880010605,
+          0.1804584115743637,
+          0.2624756097793579
+        ],
+        "spearman": [
+          0.19216284713491025,
+          -0.19284188589236262,
+          -0.11938483946898354,
+          0.08759452212678809,
+          -0.14483278589106036,
+          -0.23891023950917015,
+          -0.09278604301227522,
+          -0.07776816930023873,
+          -0.34219772716910113,
+          0.08997476599209138,
+          0.1571429676875295,
+          -0.009334050216522768,
+          0.30276350005000907,
+          0.11369565713516436,
+          0.026765146902985506,
+          -0.11985193137592841,
+          -0.0676444480055454,
+          -0.26492315410202055,
+          -0.022761245256123594,
+          -0.17927895359507073,
+          0.12467714385242742,
+          -0.06812438063506722,
+          0.14233023016137084,
+          -0.13192931206284725,
+          0.2202828745445375,
+          -0.04506938627789246,
+          0.1044222268044151,
+          0.05574005331374939,
+          -0.13509633039854418,
+          -0.08472786457997593,
+          0.06745851827828901,
+          -0.02750546917303849,
+          -0.14517513649361455,
+          -0.11361011361230283,
+          -0.3363138725037647,
+          -0.30150052130845006,
+          0.03761056068859066,
+          -0.11917355169692095,
+          0.040746374501949574,
+          -0.33333291383322283,
+          0.0914415356877504,
+          0.06168918202823291,
+          -0.1441669084879804,
+          -0.21287281650164175,
+          -0.11814292463066067,
+          -0.039590558196958545,
+          0.01859322195509365
+        ],
+        "de_corr": [
+          -0.300130695104599,
+          -0.5648529529571533,
+          0.3370738923549652,
+          0.3860224485397339,
+          -0.1955641210079193,
+          0.02822083979845047,
+          -0.48389118909835815,
+          -0.009495597332715988,
+          -0.05394879728555679,
+          0.36357206106185913,
+          0.667620062828064,
+          0.43334126472473145,
+          -0.4320434629917145,
+          0.14437812566757202,
+          -0.32033661007881165,
+          -0.3400384485721588,
+          -0.2038392722606659,
+          -0.5935828685760498,
+          0.541037917137146,
+          -0.2471340298652649,
+          -0.10499667376279831,
+          -0.17938752472400665,
+          0.077311210334301,
+          -0.3787427544593811,
+          0.4311549961566925,
+          -0.1292337328195572,
+          -0.12586845457553864,
+          0.415390282869339,
+          -0.5604172945022583,
+          -0.011644965037703514,
+          -0.18901412189006805,
+          -0.47924914956092834,
+          -0.5018155574798584,
+          0.33002686500549316,
+          -0.3823208510875702,
+          -0.8062375783920288,
+          0.5056836009025574,
+          0.12952904403209686,
+          -0.22419269382953644,
+          -0.4344070255756378,
+          0.3809707760810852,
+          0.8074125647544861,
+          0.16500452160835266,
+          -0.31204333901405334,
+          0.142561137676239,
+          0.25846385955810547,
+          0.6310514211654663
+        ],
+        "mmd": [
+          0.09349272787495821,
+          0.017079164586266127,
+          0.14250329469241785,
+          0.00759453203977023,
+          0.006482170008059729,
+          0.011063732850183694,
+          0.026731542733261282,
+          0.1470853808210062,
+          0.09653127548836427,
+          0.04204111040549552,
+          0.04216490529019157,
+          0.1312443358607276,
+          0.21005125605035002,
+          0.05309926180032265,
+          0.06683692811031039,
+          0.0707329364249959,
+          0.19890545534435256,
+          0.26540914314875486,
+          0.03516672967215262,
+          0.24847255678051916,
+          0.0821541846284356,
+          0.025634852829435117,
+          0.1032416212736893,
+          0.02939766576232139,
+          0.10044225887905478,
+          0.09454968542461961,
+          0.144971177320813,
+          0.2158981101013302,
+          0.27529296477662657,
+          0.13301308819063173,
+          0.022128561707196348,
+          0.2808152270527752,
+          0.033273759910667056,
+          0.18338728148350747,
+          0.016317289022899972,
+          0.06928072801802743,
+          0.2114590654919788,
+          0.18688661728037947,
+          0.04694983392849206,
+          0.15847260958398424,
+          0.18867237244125679,
+          0.009225637311758073,
+          0.051785343083803004,
+          0.017391647303436564,
+          0.20626368571513176,
+          0.26286667188565194,
+          0.01586052553135231
+        ],
+        "wasserstein": [
+          0.3465150925077128,
+          0.14262462636439255,
+          0.3924961748765501,
+          0.0884045304954344,
+          0.07540012800379445,
+          0.11246360100660709,
+          0.14956275073881106,
+          0.4246831496977201,
+          0.36262283630451864,
+          0.20913232861350933,
+          0.2027074007927328,
+          0.37186873605567927,
+          0.5165355363724826,
+          0.21696055667376182,
+          0.23970034947649663,
+          0.2468377396448319,
+          0.47336123278609643,
+          0.6069115027749394,
+          0.1988019244084769,
+          0.5360744081970711,
+          0.32131733750831143,
+          0.15920445814820125,
+          0.37406449459488234,
+          0.1906735867984297,
+          0.3921992392111027,
+          0.3386888100758347,
+          0.4107306759792381,
+          0.5186042173434964,
+          0.648326146552633,
+          0.4215655626262298,
+          0.1460777869166568,
+          0.6562208847516053,
+          0.1565663488804904,
+          0.4559409539989952,
+          0.14973875231061093,
+          0.29716214661285395,
+          0.5065901284321037,
+          0.46066045826130547,
+          0.20351251804672624,
+          0.44478038092501976,
+          0.4880487648741447,
+          0.08884758015177945,
+          0.23486717757267328,
+          0.11477118067833636,
+          0.4918381143601012,
+          0.5930188602971916,
+          0.1235145267478904
+        ]
+      },
+      "n_perts": 47
+    },
+    "AvgPerturbationEffect": {
+      "mse": 0.0036481301476089085,
+      "r2": -1.6605322170764842,
+      "pearson": 0.4622683938830457,
+      "spearman": 0.39142583024932337,
+      "de_corr": 0.4883508604971018,
+      "mmd": 0.10014484937386135,
+      "wasserstein": 0.3018542549345404,
+      "_per_pert": {
+        "mse": [
+          0.00167435547336936,
+          0.0009507954237051308,
+          0.0027328734286129475,
+          0.0014651004457846284,
+          0.0011944141006097198,
+          0.0018723756074905396,
+          0.0013803355395793915,
+          0.0035241846926510334,
+          0.002143725287169218,
+          0.0022780015133321285,
+          0.001303242170251906,
+          0.003029764164239168,
+          0.010649318806827068,
+          0.0014153380179777741,
+          0.001925644464790821,
+          0.001829093205742538,
+          0.0048235333524644375,
+          0.006736383307725191,
+          0.0006359592662192881,
+          0.010419798083603382,
+          0.0021609030663967133,
+          0.0011679689632728696,
+          0.0025179528165608644,
+          0.0008555786334909499,
+          0.001846952480264008,
+          0.0012071691453456879,
+          0.0025123802479356527,
+          0.0076608723029494286,
+          0.016836199909448624,
+          0.003412527497857809,
+          0.0010437192395329475,
+          0.00940390769392252,
+          0.0026134364306926727,
+          0.007003629580140114,
+          0.0014786251122131944,
+          0.0009856611723080277,
+          0.005891507025808096,
+          0.004652815405279398,
+          0.0011924020946025848,
+          0.0059386445209383965,
+          0.006750002969056368,
+          0.0009650677675381303,
+          0.0024069820065051317,
+          0.0010960380313917994,
+          0.005309190601110458,
+          0.01162755023688078,
+          0.0009401956340298057
+        ],
+        "r2": [
+          0.5250583291053772,
+          -2.067795991897583,
+          0.4663906693458557,
+          -24.22682762145996,
+          -30.548120498657227,
+          -10.564176559448242,
+          -1.181333065032959,
+          0.41396796703338623,
+          0.40977394580841064,
+          -0.5903782844543457,
+          0.13891476392745972,
+          0.3687893748283386,
+          0.1831883192062378,
+          0.2859535217285156,
+          0.09758329391479492,
+          0.0932665467262268,
+          0.26741814613342285,
+          0.3264654874801636,
+          0.44522279500961304,
+          0.007105767726898193,
+          0.4030093550682068,
+          -1.8038930892944336,
+          0.46773022413253784,
+          -0.29143571853637695,
+          0.5437933206558228,
+          0.6051965951919556,
+          0.5175750255584717,
+          0.01589643955230713,
+          0.11185312271118164,
+          0.28309279680252075,
+          -1.1159677505493164,
+          0.2711823582649231,
+          -2.910392999649048,
+          -0.13172423839569092,
+          -4.0282745361328125,
+          0.5530080795288086,
+          0.34925276041030884,
+          0.33678966760635376,
+          0.12879538536071777,
+          0.12150990962982178,
+          0.09073930978775024,
+          -3.5584936141967773,
+          -0.5665128231048584,
+          -2.405522108078003,
+          0.34308922290802,
+          0.21394115686416626,
+          -1.4397189617156982
+        ],
+        "pearson": [
+          0.737379252910614,
+          0.4586031138896942,
+          0.7093303799629211,
+          -0.25234323740005493,
+          0.1738933026790619,
+          -0.15050499141216278,
+          0.3571605682373047,
+          0.679621696472168,
+          0.6435927152633667,
+          0.19227777421474457,
+          0.5869457721710205,
+          0.6202855110168457,
+          0.45253705978393555,
+          0.5960014462471008,
+          0.4905829429626465,
+          0.44439244270324707,
+          0.5275831818580627,
+          0.6465961337089539,
+          0.7708851099014282,
+          0.1986197978258133,
+          0.6349406838417053,
+          0.35338643193244934,
+          0.7023636102676392,
+          0.6127575039863586,
+          0.7472073435783386,
+          0.7900751829147339,
+          0.7679491639137268,
+          0.22588463127613068,
+          0.36706048250198364,
+          0.538213849067688,
+          0.5286682844161987,
+          0.5769003033638,
+          -0.30524545907974243,
+          0.09802873432636261,
+          0.11854038387537003,
+          0.7439865469932556,
+          0.6508612036705017,
+          0.6041733622550964,
+          0.5806814432144165,
+          0.35568973422050476,
+          0.3291589915752411,
+          0.5059716701507568,
+          0.11591579020023346,
+          0.4430306851863861,
+          0.641900360584259,
+          0.4907100796699524,
+          0.6243635416030884
+        ],
+        "spearman": [
+          0.7083933706773007,
+          0.3079713100159294,
+          0.4046327512363655,
+          -0.041920830546138856,
+          -0.22250455279841613,
+          -0.37446157279400916,
+          -0.04115601111004243,
+          0.4428028882606632,
+          0.7038113920796653,
+          0.5201007455465465,
+          0.1764553940253625,
+          0.307479988808722,
+          0.6482464654386071,
+          0.6252430897360204,
+          0.3229249814306044,
+          0.3245433173888679,
+          0.2560018710704993,
+          0.4606582793062035,
+          0.5833880686915266,
+          0.45562647697843667,
+          0.6309124784820304,
+          0.4308533873224278,
+          0.767816622161061,
+          0.5198515634258314,
+          0.6736379330035954,
+          0.5083028549033244,
+          0.6489180099832399,
+          0.346059653642849,
+          0.48145509380778256,
+          0.4494448301253817,
+          0.4366564155926501,
+          0.4280707717755995,
+          -0.024840797120072432,
+          0.396796492645064,
+          0.0785348149733743,
+          0.705523408801131,
+          0.299715903337534,
+          0.5683499054652498,
+          0.46037392128493043,
+          0.6424110492753116,
+          0.25206411135794143,
+          0.15070971116413803,
+          0.3665107979630368,
+          0.3772544999395016,
+          0.35032410981040296,
+          0.5469437560747836,
+          0.33612529907739225
+        ],
+        "de_corr": [
+          0.8540282845497131,
+          0.6691981554031372,
+          0.8922524452209473,
+          -0.49096253514289856,
+          0.3278610110282898,
+          -0.16388878226280212,
+          0.7121587991714478,
+          0.8541043996810913,
+          0.6396335363388062,
+          0.0007307860068976879,
+          0.8476625680923462,
+          0.8329525589942932,
+          -0.3111203908920288,
+          0.7493950724601746,
+          0.7377703785896301,
+          0.5266879200935364,
+          0.8043926358222961,
+          0.7881635427474976,
+          0.9095393419265747,
+          -0.4482191503047943,
+          0.5458834767341614,
+          0.16728194057941437,
+          0.5472290515899658,
+          0.9184627532958984,
+          0.9280699491500854,
+          0.8740137219429016,
+          0.8739995360374451,
+          0.3085290193557739,
+          -0.3118397295475006,
+          0.36631032824516296,
+          0.5417255163192749,
+          0.7066925168037415,
+          -0.5113534331321716,
+          -0.17085237801074982,
+          0.16569198668003082,
+          0.872644305229187,
+          0.8713144063949585,
+          0.6834990978240967,
+          0.8228168487548828,
+          0.15404105186462402,
+          0.19959162175655365,
+          0.8399051427841187,
+          -0.00472659757360816,
+          0.7255598306655884,
+          0.9051570296287537,
+          0.33812928199768066,
+          0.8623735904693604
+        ],
+        "mmd": [
+          0.06291116145751907,
+          0.03737636644246678,
+          0.09266723026850843,
+          0.049567228025216625,
+          0.044329834391393486,
+          0.06264545216692041,
+          0.05133777735970935,
+          0.09621605241284636,
+          0.07391450792826637,
+          0.07196999240626378,
+          0.046156650204655536,
+          0.09682136583351053,
+          0.19489274801003653,
+          0.044353686639593715,
+          0.0586054047280552,
+          0.06642907820275523,
+          0.15141614292000194,
+          0.19550864448962824,
+          0.028198343313148078,
+          0.24949023789575764,
+          0.06256788804255775,
+          0.04464470220389738,
+          0.07173329658399596,
+          0.03284182947036085,
+          0.06417759206185203,
+          0.043663762429208886,
+          0.08845362106376908,
+          0.21566233510805466,
+          0.25596716829163724,
+          0.11002364394215791,
+          0.040609433315430654,
+          0.22463460265151292,
+          0.09658945849103306,
+          0.1956928415381134,
+          0.057826830020943465,
+          0.059529163109169336,
+          0.16081938583966682,
+          0.1424068295197991,
+          0.04660008390879111,
+          0.14559845988534936,
+          0.18544416480316483,
+          0.039382917173956145,
+          0.07997849287093939,
+          0.043427717128807,
+          0.15654692956076366,
+          0.23068043688661477,
+          0.03649642957368271
+        ],
+        "wasserstein": [
+          0.25153837911396004,
+          0.16893252381456805,
+          0.2676053666094325,
+          0.20834732128994043,
+          0.20906201319240444,
+          0.2623580543810338,
+          0.2082874359997082,
+          0.29089588786363046,
+          0.2932813172795775,
+          0.28382051087695415,
+          0.17286301008204027,
+          0.2768469751638227,
+          0.47183194502653286,
+          0.1827742416065256,
+          0.20242473866032043,
+          0.23531629986454738,
+          0.3880936894164569,
+          0.4650012666968979,
+          0.13480157794115927,
+          0.5289359817374468,
+          0.2794908138904269,
+          0.22313267381781063,
+          0.28506071168270364,
+          0.17023104447518222,
+          0.2860487361756974,
+          0.19420078673364963,
+          0.2737223791372607,
+          0.467083203653169,
+          0.6013220910540901,
+          0.3559960638660447,
+          0.19122034399507742,
+          0.5544341593328391,
+          0.2896746096005146,
+          0.456163411743442,
+          0.23632673230202164,
+          0.24449735425823316,
+          0.3823542699831418,
+          0.33566691132404364,
+          0.20076542659868996,
+          0.4174490782135223,
+          0.4888898951437967,
+          0.17955707688635514,
+          0.2753026664949679,
+          0.1872511026375303,
+          0.3795474592512466,
+          0.5413690636213984,
+          0.18737337943358137
+        ]
+      },
+      "n_perts": 47
+    },
+    "Additive": {
+      "mse": 0.002130915904962557,
+      "r2": -1.452756440385859,
+      "pearson": 0.6206745877069362,
+      "spearman": 0.5072803608982661,
+      "de_corr": 0.6938813036862523,
+      "mmd": 0.07430983260676954,
+      "wasserstein": 0.2578024455719882,
+      "_per_pert": {
+        "mse": [
+          0.00167435547336936,
+          0.0009507954237051308,
+          0.0027328734286129475,
+          0.0014651004457846284,
+          0.0011944141006097198,
+          0.0018723756074905396,
+          0.0013803355395793915,
+          0.0011990240309387445,
+          0.0011195228435099125,
+          0.0022780015133321285,
+          0.001303242170251906,
+          0.003029764164239168,
+          0.0012679604114964604,
+          0.0037433153484016657,
+          0.001925644464790821,
+          0.001829093205742538,
+          0.0048235333524644375,
+          0.0035266857594251633,
+          0.0006359592662192881,
+          0.002117374911904335,
+          0.0016054949956014752,
+          0.0011679689632728696,
+          0.0025179528165608644,
+          0.0008555786334909499,
+          0.001846952480264008,
+          0.0012071691453456879,
+          0.002857340034097433,
+          0.0031821466982364655,
+          0.003948783501982689,
+          0.0011508818715810776,
+          0.0010437192395329475,
+          0.013433117419481277,
+          0.00039268890395760536,
+          0.007003629580140114,
+          0.0014786251122131944,
+          0.0007619631942361593,
+          0.002539267996326089,
+          0.001459421357139945,
+          0.0011924020946025848,
+          0.0012287272838875651,
+          0.000827369571197778,
+          0.0009650677675381303,
+          0.0024069820065051317,
+          0.0010960380313917994,
+          0.0009213167359121144,
+          0.002052875002846122,
+          0.0009401956340298057
+        ],
+        "r2": [
+          0.5250583291053772,
+          -2.067795991897583,
+          0.4663906693458557,
+          -24.22682762145996,
+          -30.548120498657227,
+          -10.564176559448242,
+          -1.181333065032959,
+          0.8006158471107483,
+          0.6917648315429688,
+          -0.5903782844543457,
+          0.13891476392745972,
+          0.3687893748283386,
+          0.9027464389801025,
+          -0.8885250091552734,
+          0.09758329391479492,
+          0.0932665467262268,
+          0.26741814613342285,
+          0.6473857760429382,
+          0.44522279500961304,
+          0.7982370257377625,
+          0.5564513206481934,
+          -1.8038930892944336,
+          0.46773022413253784,
+          -0.29143571853637695,
+          0.5437933206558228,
+          0.6051965951919556,
+          0.4513360857963562,
+          0.5912264585494995,
+          0.7916929721832275,
+          0.7582215666770935,
+          -1.1159677505493164,
+          -0.04108762741088867,
+          0.41243356466293335,
+          -0.13172423839569092,
+          -4.0282745361328125,
+          0.6544538736343384,
+          0.7195248007774353,
+          0.791974663734436,
+          0.12879538536071777,
+          0.8182371854782104,
+          0.8885490298271179,
+          -3.5584936141967773,
+          -0.5665128231048584,
+          -2.405522108078003,
+          0.8860046863555908,
+          0.8612192273139954,
+          -1.4397189617156982
+        ],
+        "pearson": [
+          0.737379252910614,
+          0.4586031138896942,
+          0.7093303799629211,
+          -0.25234323740005493,
+          0.1738933026790619,
+          -0.15050499141216278,
+          0.3571605682373047,
+          0.977320671081543,
+          0.8485552668571472,
+          0.19227777421474457,
+          0.5869457721710205,
+          0.6202855110168457,
+          0.9806216955184937,
+          0.907036304473877,
+          0.4905829429626465,
+          0.44439244270324707,
+          0.5275831818580627,
+          0.8898630142211914,
+          0.7708851099014282,
+          0.897230327129364,
+          0.7903505563735962,
+          0.35338643193244934,
+          0.7023636102676392,
+          0.6127575039863586,
+          0.7472073435783386,
+          0.7900751829147339,
+          0.9672414660453796,
+          0.7713102698326111,
+          0.9467152953147888,
+          0.8849393725395203,
+          0.5286682844161987,
+          0.018820714205503464,
+          0.943000853061676,
+          0.09802873432636261,
+          0.11854038387537003,
+          0.8176720142364502,
+          0.9195569157600403,
+          0.9476909041404724,
+          0.5806814432144165,
+          0.905232310295105,
+          0.9894416332244873,
+          0.5059716701507568,
+          0.11591579020023346,
+          0.4430306851863861,
+          0.9485238790512085,
+          0.9331204295158386,
+          0.6243635416030884
+        ],
+        "spearman": [
+          0.7083933706773007,
+          0.3079713100159294,
+          0.4046327512363655,
+          -0.041920830546138856,
+          -0.22250455279841613,
+          -0.37446157279400916,
+          -0.04115601111004243,
+          0.8302420688301261,
+          0.758864041237267,
+          0.5201007455465465,
+          0.1764553940253625,
+          0.307479988808722,
+          0.9300612434600944,
+          0.8013651787102366,
+          0.3229249814306044,
+          0.3245433173888679,
+          0.2560018710704993,
+          0.6411851049522752,
+          0.5833880686915266,
+          0.758357528861593,
+          0.6538860797606403,
+          0.4308533873224278,
+          0.767816622161061,
+          0.5198515634258314,
+          0.6736379330035954,
+          0.5083028549033244,
+          0.8299614037560654,
+          0.63705683850344,
+          0.8853876420378269,
+          0.7477887642857057,
+          0.4366564155926501,
+          0.04160304135281166,
+          0.750667553052903,
+          0.396796492645064,
+          0.0785348149733743,
+          0.8229722542204181,
+          0.7481139514391116,
+          0.9005541928837376,
+          0.46037392128493043,
+          0.8153443278432305,
+          0.8967476883792962,
+          0.15070971116413803,
+          0.3665107979630368,
+          0.3772544999395016,
+          0.7829543318084666,
+          0.8737905817438184,
+          0.33612529907739225
+        ],
+        "de_corr": [
+          0.8540282845497131,
+          0.6691981554031372,
+          0.8922524452209473,
+          -0.49096253514289856,
+          0.3278610110282898,
+          -0.16388878226280212,
+          0.7121587991714478,
+          0.9931616187095642,
+          0.9208729863166809,
+          0.0007307860068976879,
+          0.8476625680923462,
+          0.8329525589942932,
+          0.9927356243133545,
+          0.9374369978904724,
+          0.7377703785896301,
+          0.5266879200935364,
+          0.8043926358222961,
+          0.9638471007347107,
+          0.9095393419265747,
+          0.9674869179725647,
+          0.8717171549797058,
+          0.16728194057941437,
+          0.5472290515899658,
+          0.9184627532958984,
+          0.9280699491500854,
+          0.8740137219429016,
+          0.9877463579177856,
+          0.929778516292572,
+          0.9343132972717285,
+          0.7840830087661743,
+          0.5417255163192749,
+          0.014756222255527973,
+          0.9695466160774231,
+          -0.17085237801074982,
+          0.16569198668003082,
+          0.8839684724807739,
+          0.9486202597618103,
+          0.973825991153717,
+          0.8228168487548828,
+          0.918163537979126,
+          0.9966879487037659,
+          0.8399051427841187,
+          -0.00472659757360816,
+          0.7255598306655884,
+          0.9777360558509827,
+          0.968001663684845,
+          0.8623735904693604
+        ],
+        "mmd": [
+          0.06291116145751907,
+          0.03737636644246678,
+          0.09266723026850843,
+          0.049567228025216625,
+          0.044329834391393486,
+          0.06264545216692041,
+          0.05133777735970935,
+          0.05021868456888656,
+          0.061060278690874825,
+          0.07196999240626378,
+          0.046156650204655536,
+          0.09682136583351053,
+          0.10767813138675597,
+          0.13701120582615078,
+          0.0586054047280552,
+          0.06642907820275523,
+          0.15141614292000194,
+          0.12208139388607153,
+          0.028198343313148078,
+          0.09468476215576449,
+          0.042705668881369396,
+          0.04464470220389738,
+          0.07173329658399596,
+          0.03284182947036085,
+          0.06417759206185203,
+          0.043663762429208886,
+          0.08901107561180033,
+          0.10592565083802308,
+          0.19750044167399128,
+          0.03350597964301849,
+          0.040609433315430654,
+          0.2806191491336122,
+          0.015222367381501956,
+          0.1956928415381134,
+          0.057826830020943465,
+          0.05387107038760042,
+          0.08132990830542441,
+          0.059559122262586306,
+          0.04660008390879111,
+          0.06415637938945573,
+          0.035576516207533215,
+          0.039382917173956145,
+          0.07997849287093939,
+          0.043427717128807,
+          0.045977810793240925,
+          0.09735857949440396,
+          0.03649642957368271
+        ],
+        "wasserstein": [
+          0.25153837911396004,
+          0.16893252381456805,
+          0.2676053666094325,
+          0.20834732128994043,
+          0.20906201319240444,
+          0.2623580543810338,
+          0.2082874359997082,
+          0.208885073570133,
+          0.23475329927753966,
+          0.28382051087695415,
+          0.17286301008204027,
+          0.2768469751638227,
+          0.29428721718537315,
+          0.30712842512790184,
+          0.20242473866032043,
+          0.23531629986454738,
+          0.3880936894164569,
+          0.3733510470160322,
+          0.13480157794115927,
+          0.29645300056511287,
+          0.21274011479809748,
+          0.22313267381781063,
+          0.28506071168270364,
+          0.17023104447518222,
+          0.2860487361756974,
+          0.19420078673364963,
+          0.29555100498454245,
+          0.3762041377967754,
+          0.4745980007385362,
+          0.18575707833168842,
+          0.19122034399507742,
+          0.6938497058349854,
+          0.12119612743338175,
+          0.456163411743442,
+          0.23632673230202164,
+          0.236194979054538,
+          0.2834715800082889,
+          0.2572407973833032,
+          0.20076542659868996,
+          0.21113540905871203,
+          0.19100342704708737,
+          0.17955707688635514,
+          0.2753026664949679,
+          0.1872511026375303,
+          0.2113030277750908,
+          0.3086794995132667,
+          0.18737337943358137
+        ]
+      },
+      "n_perts": 47
+    },
+    "LinearResponse": {
+      "mse": 0.0011560855510129415,
+      "r2": -0.2554005422490708,
+      "pearson": 0.7328052140296774,
+      "spearman": 0.5781660967292104,
+      "de_corr": 0.7979922754333374,
+      "mmd": 0.04799841794265553,
+      "wasserstein": 0.20276722761569496,
+      "_per_pert": {
+        "mse": [
+          0.001815351890400052,
+          0.0005269621615298092,
+          0.0030712697189301252,
+          0.000511285150423646,
+          0.000643533596303314,
+          0.0007793476688675582,
+          0.0008428554865531623,
+          0.00044451531721279025,
+          0.0006145142251625657,
+          0.0024074052926152945,
+          0.0012071673991158605,
+          0.00037343817530199885,
+          0.000748060061596334,
+          0.00042762543307617307,
+          0.0017397834453731775,
+          0.0017684008926153183,
+          0.0010283575393259525,
+          0.0005933258798904717,
+          0.0004704850143752992,
+          0.0010955922771245241,
+          0.00029082701075822115,
+          0.0007098543574102223,
+          0.0003000711731147021,
+          0.00047978051588870585,
+          0.001113002304919064,
+          0.001506912987679243,
+          0.0008496627560816705,
+          0.004119977355003357,
+          0.00357564864680171,
+          0.0008324038935825229,
+          0.0006465500337071717,
+          0.0017127461032941937,
+          0.00030324782710522413,
+          0.0010518929921090603,
+          0.0008361907093785703,
+          0.0013509067939594388,
+          0.0015559562016278505,
+          0.0013129838043823838,
+          0.0009852976072579622,
+          0.0016814854461699724,
+          0.0006546838558278978,
+          0.000686608487740159,
+          0.0010450928239151835,
+          0.0005867264699190855,
+          0.0008671642863191664,
+          0.003665788099169731,
+          0.0005052817286923528
+        ],
+        "r2": [
+          0.48506367206573486,
+          -0.7002733945846558,
+          0.40031683444976807,
+          -7.803562164306641,
+          -15.997682571411133,
+          -3.813410758972168,
+          -0.33195745944976807,
+          0.9260821342468262,
+          0.8308074474334717,
+          -0.6807211637496948,
+          0.20239382982254028,
+          0.9221991896629333,
+          0.9426231980323792,
+          0.7842604517936707,
+          0.18468362092971802,
+          0.12335318326950073,
+          0.8438165187835693,
+          0.9406765699386597,
+          0.5895739793777466,
+          0.8956018686294556,
+          0.9196535348892212,
+          -0.704116702079773,
+          0.9365679621696472,
+          0.27580517530441284,
+          0.7250826358795166,
+          0.507165789604187,
+          0.8368484973907471,
+          0.47075414657592773,
+          0.811376690864563,
+          0.8251277804374695,
+          -0.31077301502227783,
+          0.8672595024108887,
+          0.5462608337402344,
+          0.830023467540741,
+          -1.8435852527618408,
+          0.38737112283706665,
+          0.8281366229057312,
+          0.8128478527069092,
+          0.280112087726593,
+          0.7512617111206055,
+          0.9118106365203857,
+          -2.243192195892334,
+          0.3198322653770447,
+          -0.823029637336731,
+          0.892704963684082,
+          0.7521812319755554,
+          -0.3111581802368164
+        ],
+        "pearson": [
+          0.7579284310340881,
+          0.47559016942977905,
+          0.6993327140808105,
+          -0.16080106794834137,
+          0.11650417000055313,
+          -0.12152967602014542,
+          0.5724650621414185,
+          0.9785683155059814,
+          0.9157998561859131,
+          0.4514901638031006,
+          0.8907166123390198,
+          0.9634212851524353,
+          0.9832399487495422,
+          0.9335516691207886,
+          0.4725910425186157,
+          0.38926422595977783,
+          0.9449183940887451,
+          0.9823147058486938,
+          0.7734150886535645,
+          0.9583873152732849,
+          0.9650200605392456,
+          0.6997309923171997,
+          0.9710015654563904,
+          0.659519374370575,
+          0.891148567199707,
+          0.7662409543991089,
+          0.9433495402336121,
+          0.7203361392021179,
+          0.9030676484107971,
+          0.910010576248169,
+          0.5172854661941528,
+          0.9742609858512878,
+          0.8469299674034119,
+          0.9616471529006958,
+          0.17740903794765472,
+          0.7472275495529175,
+          0.982086718082428,
+          0.9204921722412109,
+          0.5689943432807922,
+          0.9189297556877136,
+          0.9800853133201599,
+          0.6968837976455688,
+          0.6960882544517517,
+          0.4919852316379547,
+          0.9659656286239624,
+          0.9531323313713074,
+          0.6358475089073181
+        ],
+        "spearman": [
+          0.7267308305160979,
+          0.31736740644742656,
+          0.4216959907901176,
+          0.09782925927800983,
+          -0.24098214056210873,
+          -0.3451894804929966,
+          0.07573042623336623,
+          0.8199193010659636,
+          0.8175295774756172,
+          0.5655961559966857,
+          0.3984229661797205,
+          0.6557342397818725,
+          0.9402929366906774,
+          0.8674973284861689,
+          0.3116749904282631,
+          0.2980009207789957,
+          0.6792444185233542,
+          0.8402760939127509,
+          0.5834434652061444,
+          0.8207432128236153,
+          0.8749760061811058,
+          0.573012032926743,
+          0.9297289170092395,
+          0.5504182864916254,
+          0.8073274992307962,
+          0.5025536668690098,
+          0.6564830794832915,
+          0.657349432398684,
+          0.8396238562408863,
+          0.8014734601432296,
+          0.4272860214421892,
+          0.7321032153263551,
+          0.6238218472324555,
+          0.8380126318805834,
+          0.11462644599705345,
+          0.794577246419071,
+          0.7311134626725649,
+          0.7269334100695536,
+          0.4811152217042791,
+          0.8358231282393944,
+          0.8521301566622356,
+          0.2780840219636733,
+          0.645078593970114,
+          0.39831319285489375,
+          0.598538622214927,
+          0.8899792725932806,
+          0.3617659184959151
+        ],
+        "de_corr": [
+          0.8566803932189941,
+          0.6914318203926086,
+          0.8387675881385803,
+          -0.32153546810150146,
+          0.26463887095451355,
+          -0.16012905538082123,
+          0.8890593647956848,
+          0.9942091107368469,
+          0.9700576663017273,
+          0.4726371765136719,
+          0.9673143029212952,
+          0.9935668706893921,
+          0.9963659644126892,
+          0.9523894190788269,
+          0.7570591568946838,
+          0.49530255794525146,
+          0.9838354587554932,
+          0.9930711984634399,
+          0.920901894569397,
+          0.9879890084266663,
+          0.9676618576049805,
+          0.714035153388977,
+          0.967126727104187,
+          0.8958656787872314,
+          0.9540151953697205,
+          0.8432382941246033,
+          0.9835994839668274,
+          0.8748316764831543,
+          0.8785965442657471,
+          0.7334020733833313,
+          0.5192469358444214,
+          0.977804958820343,
+          0.820503294467926,
+          0.9924158453941345,
+          0.18588325381278992,
+          0.8143969774246216,
+          0.9902132153511047,
+          0.9613940119743347,
+          0.754327654838562,
+          0.9321264624595642,
+          0.9903842210769653,
+          0.9279942512512207,
+          0.7527039051055908,
+          0.7175959348678589,
+          0.9930542707443237,
+          0.983783483505249,
+          0.8358222842216492
+        ],
+        "mmd": [
+          0.0629487132017087,
+          0.022456076923069057,
+          0.09968608333981799,
+          0.019345235876682798,
+          0.025701186718797286,
+          0.028099133125839115,
+          0.03359164574618889,
+          0.022820628018136446,
+          0.054755285262679254,
+          0.08060966045136686,
+          0.04327910366022525,
+          0.02597645571513474,
+          0.07836909986412388,
+          0.026435072263026216,
+          0.052843560263926115,
+          0.06282847958579918,
+          0.04689447736599561,
+          0.04571262204721249,
+          0.021987667452409987,
+          0.05925346373925522,
+          0.03021151442244563,
+          0.029661855849053675,
+          0.03843123932103254,
+          0.0208589208184653,
+          0.045885523907673864,
+          0.05085836046271952,
+          0.036318519873277166,
+          0.13032579795668975,
+          0.16314296583544763,
+          0.030986573677355023,
+          0.027234458699533115,
+          0.07404027342328645,
+          0.012739657915447844,
+          0.04155824406909081,
+          0.036207635881859934,
+          0.05428925728897305,
+          0.06009881585171151,
+          0.0534873625868717,
+          0.038367872652006696,
+          0.0633094090432128,
+          0.029446081378519118,
+          0.028128208862154613,
+          0.03978929112492802,
+          0.025372501140748516,
+          0.04561530973775052,
+          0.11377756668985495,
+          0.022188774213305895
+        ],
+        "wasserstein": [
+          0.2566886884580639,
+          0.12852174171606898,
+          0.28999688116767314,
+          0.13220750878374615,
+          0.15637176144463816,
+          0.1822112176075181,
+          0.1794160322713854,
+          0.1422491776338578,
+          0.2094718904535178,
+          0.27663250696543273,
+          0.20272781105528417,
+          0.14300537525789722,
+          0.25418636412861717,
+          0.1418415124534268,
+          0.18949076792692993,
+          0.22958166388786372,
+          0.19895359836274593,
+          0.21322797355669365,
+          0.12038133791801668,
+          0.21999133516795236,
+          0.1506127703629451,
+          0.17365526534875048,
+          0.1664007394540281,
+          0.13894913695800695,
+          0.2369690471481899,
+          0.2270707735174547,
+          0.1809110964820951,
+          0.4044117331140134,
+          0.4124102136696584,
+          0.18091131274702593,
+          0.15158340322557998,
+          0.2896965232193937,
+          0.10218128314947407,
+          0.19067208907009128,
+          0.18465850656604055,
+          0.2487455741256824,
+          0.2383106791602504,
+          0.21613429193883057,
+          0.17311263449183453,
+          0.21913956834357698,
+          0.17800456359408123,
+          0.15993518777288698,
+          0.19096271242439403,
+          0.13678648921740166,
+          0.20984909332081356,
+          0.3547931218713146,
+          0.14603674142651785
+        ]
+      },
+      "n_perts": 47
+    },
+    "NearestPerturbationCentroid": {
+      "mse": 0.0033090435918371017,
+      "r2": -4.196858470744275,
+      "pearson": 0.6274464218381871,
+      "spearman": 0.503322270605923,
+      "de_corr": 0.7151544857572051,
+      "mmd": 0.1042386035709379,
+      "wasserstein": 0.307973293298589,
+      "_per_pert": {
+        "mse": [
+          0.0023796367458999157,
+          0.001801011385396123,
+          0.0033724126406013966,
+          0.0035735154524445534,
+          0.002037221100181341,
+          0.0038693780079483986,
+          0.0034680429380387068,
+          0.0013926805695518851,
+          0.0011195228435099125,
+          0.0152303921058774,
+          0.008728939108550549,
+          0.004362241830676794,
+          0.0018618378089740872,
+          0.005347156431525946,
+          0.0028414963744580746,
+          0.002719290554523468,
+          0.003768590511754155,
+          0.0035266857594251633,
+          0.001328550628386438,
+          0.002117374911904335,
+          0.0016054949956014752,
+          0.00422050291672349,
+          0.0006051911623217165,
+          0.0016853106208145618,
+          0.001079079112969339,
+          0.0017575172241777182,
+          0.0007794360863044858,
+          0.0031821466982364655,
+          0.002119982149451971,
+          0.004216709174215794,
+          0.0020710572134703398,
+          0.013433117419481277,
+          0.00028920292970724404,
+          0.00193432392552495,
+          0.002310030395165086,
+          0.0016017381567507982,
+          0.0030285550747066736,
+          0.0034309562761336565,
+          0.002545519033446908,
+          0.003481273539364338,
+          0.0024392546620219946,
+          0.00397268682718277,
+          0.0022698568645864725,
+          0.0019501668866723776,
+          0.0026432841550558805,
+          0.009829113259911537,
+          0.002197564346715808
+        ],
+        "r2": [
+          0.32500070333480835,
+          -4.8110671043396,
+          0.3415168523788452,
+          -60.53059387207031,
+          -52.80923843383789,
+          -22.8980712890625,
+          -4.480518341064453,
+          0.7684130668640137,
+          0.6917648315429688,
+          -9.633040428161621,
+          -4.767431735992432,
+          0.0911855697631836,
+          0.8571954965591431,
+          -1.6976723670959473,
+          -0.33161354064941406,
+          -0.34802961349487305,
+          0.4276392459869385,
+          0.6473857760429382,
+          -0.1589568853378296,
+          0.7982370257377625,
+          0.5564513206481934,
+          -9.131980895996094,
+          0.8720687031745911,
+          -1.5438568592071533,
+          0.7334619164466858,
+          0.4252058267593384,
+          0.8503333926200867,
+          0.5912264585494995,
+          0.8881662487983704,
+          0.11414945125579834,
+          -3.1987247467041016,
+          -0.04108762741088867,
+          0.5672757625579834,
+          0.6874305009841919,
+          -6.85558557510376,
+          0.2736205458641052,
+          0.6654804348945618,
+          0.5109529495239258,
+          -0.8598324060440063,
+          0.4850231409072876,
+          0.671419620513916,
+          -17.764972686767578,
+          -0.4772692918777466,
+          -5.059403419494629,
+          0.672944188117981,
+          0.33552122116088867,
+          -4.7024712562561035
+        ],
+        "pearson": [
+          0.6041916608810425,
+          0.4075838625431061,
+          0.5931739807128906,
+          -0.13428668677806854,
+          0.24753394722938538,
+          0.12699802219867706,
+          0.561170756816864,
+          0.9018423557281494,
+          0.8485552668571472,
+          0.19876433908939362,
+          0.9028801321983337,
+          0.9268650412559509,
+          0.9389623403549194,
+          0.7554796934127808,
+          0.40001431107521057,
+          0.3590351641178131,
+          0.8593475818634033,
+          0.8898630142211914,
+          0.6816719174385071,
+          0.897230327129364,
+          0.7903505563735962,
+          0.8397958278656006,
+          0.952285647392273,
+          0.5144321322441101,
+          0.8642792701721191,
+          0.6805150508880615,
+          0.9423225522041321,
+          0.7713102698326111,
+          0.9771626591682434,
+          0.3689722418785095,
+          0.42257338762283325,
+          0.018820714205503464,
+          0.7846940755844116,
+          0.9069423079490662,
+          0.1806703507900238,
+          0.5792919993400574,
+          0.934908390045166,
+          0.7277368307113647,
+          0.36097484827041626,
+          0.7531649470329285,
+          0.8293632864952087,
+          0.6741036772727966,
+          0.28217679262161255,
+          0.4049389064311981,
+          0.9165318012237549,
+          0.6722699403762817,
+          0.37251633405685425
+        ],
+        "spearman": [
+          0.6203252538166802,
+          0.34102383585978086,
+          0.3837774128219293,
+          -0.08538268075448442,
+          -0.15712193365328184,
+          -0.08690643216056418,
+          0.12051850525662616,
+          0.5728070483326835,
+          0.758864041237267,
+          0.4063638602959689,
+          0.46510206544970045,
+          0.6587072622895157,
+          0.8623349338856354,
+          0.6791235634911009,
+          0.24173554726317414,
+          0.3196232471795954,
+          0.67926463806182,
+          0.6411851049522752,
+          0.5718394343952093,
+          0.758357528861593,
+          0.6538860797606403,
+          0.6796146821610123,
+          0.8869795504626674,
+          0.5045848809718708,
+          0.7226048945742045,
+          0.5395662119130941,
+          0.8068636251611392,
+          0.63705683850344,
+          0.8851632168224438,
+          0.4674815784115909,
+          0.37613841940140774,
+          0.04160304135281166,
+          0.5045291544570197,
+          0.7242173938528648,
+          0.19138985182716672,
+          0.7090978056290951,
+          0.7526828356464393,
+          0.7120370146217535,
+          0.3447675147598305,
+          0.703218443421422,
+          0.6379013429899208,
+          0.23381386660995387,
+          0.3517490441858732,
+          0.3820320385587042,
+          0.6665511931362395,
+          0.53366749407151,
+          0.2554064683320432
+        ],
+        "de_corr": [
+          0.7687333226203918,
+          0.7302901148796082,
+          0.8511683940887451,
+          -0.4226294755935669,
+          0.40809565782546997,
+          0.12300767749547958,
+          0.712519109249115,
+          0.9314944744110107,
+          0.9208729863166809,
+          0.1709519922733307,
+          0.9414488077163696,
+          0.9556061625480652,
+          0.9622368812561035,
+          0.6859473586082458,
+          0.7796323895454407,
+          0.34363195300102234,
+          0.887298583984375,
+          0.9638471007347107,
+          0.901648223400116,
+          0.9674869179725647,
+          0.8717171549797058,
+          0.9460306763648987,
+          0.9477066397666931,
+          0.860784649848938,
+          0.9132860898971558,
+          0.866118311882019,
+          0.9768178462982178,
+          0.929778516292572,
+          0.9468079209327698,
+          -0.11106303334236145,
+          0.5682790279388428,
+          0.014756222255527973,
+          0.8187680244445801,
+          0.9813927412033081,
+          0.2747507095336914,
+          0.649765133857727,
+          0.9683998227119446,
+          0.7584554553031921,
+          0.6796631217002869,
+          0.8098202347755432,
+          0.8685473799705505,
+          0.8888605833053589,
+          0.33092421293258667,
+          0.7125753164291382,
+          0.9640641808509827,
+          0.8201625943183899,
+          0.7718026638031006
+        ],
+        "mmd": [
+          0.08293167912239963,
+          0.06567023866872024,
+          0.10981149427315096,
+          0.11360836991853707,
+          0.07356138077538543,
+          0.13257402776857463,
+          0.12208309135675088,
+          0.045923174426250934,
+          0.061060278690874825,
+          0.37495171939497807,
+          0.23522206539320523,
+          0.1299205046141153,
+          0.09585080507822352,
+          0.17690753395243897,
+          0.08517639343377525,
+          0.09609675296496789,
+          0.11969412337570717,
+          0.12208139388607153,
+          0.05038438010190227,
+          0.09468476215576449,
+          0.042705668881369396,
+          0.1344707578483153,
+          0.052099511988847436,
+          0.06088613102868312,
+          0.04568076626253059,
+          0.06267252242363996,
+          0.03607533595689161,
+          0.10592565083802308,
+          0.15829487043475998,
+          0.127898550949471,
+          0.07343368933446792,
+          0.2806191491336122,
+          0.01344139054898863,
+          0.06217937882347979,
+          0.08381984764257366,
+          0.05800418421911402,
+          0.09118554255760081,
+          0.11214983798306177,
+          0.08993828549492211,
+          0.09231938033546683,
+          0.07981969305115844,
+          0.13099192564222595,
+          0.07517288418616264,
+          0.072506674779657,
+          0.08886136637627329,
+          0.2019740941021999,
+          0.07789310765879187
+        ],
+        "wasserstein": [
+          0.27050395334556,
+          0.24584534168150252,
+          0.3082609425935583,
+          0.3408674339484363,
+          0.2866592989331655,
+          0.37534452611573565,
+          0.3489738084317255,
+          0.21457563399528223,
+          0.23475329927753966,
+          0.6088018751232922,
+          0.47906488228095306,
+          0.3323204870813563,
+          0.27302225057812024,
+          0.31507033117063266,
+          0.2563207208449249,
+          0.3075557417708538,
+          0.36553640982299584,
+          0.3733510470160322,
+          0.1946241162930515,
+          0.29645300056511287,
+          0.21274011479809748,
+          0.3394227367247307,
+          0.20071874020825067,
+          0.2262191027235575,
+          0.23233900694510506,
+          0.22184497430691644,
+          0.18679896885264202,
+          0.3762041377967754,
+          0.3768296638704943,
+          0.39835683509802455,
+          0.28071070301355494,
+          0.6938497058349854,
+          0.10078415560965251,
+          0.20926892369380642,
+          0.29832451347657224,
+          0.256509956026794,
+          0.31141232467383717,
+          0.31393154163828746,
+          0.27114600673178285,
+          0.28744571184910006,
+          0.3105869786357878,
+          0.3670474614914913,
+          0.23118492268390065,
+          0.27029257202798895,
+          0.3050042117050033,
+          0.4946549211788823,
+          0.27321079256782954
+        ]
+      },
+      "n_perts": 47
+    },
+    "EndpointMLP": {
+      "mse": 0.0011665141643421289,
+      "r2": 0.05987304575899814,
+      "pearson": 0.5941063728223138,
+      "spearman": 0.5126150765634386,
+      "de_corr": 0.6435572355589334,
+      "mmd": 0.04903882547947358,
+      "wasserstein": 0.2003736154331534,
+      "_per_pert": {
+        "mse": [
+          0.0036682309582829475,
+          0.0005103750736452639,
+          0.005402616690844297,
+          0.00031863481854088604,
+          0.00032014859607443213,
+          0.00048404073459096253,
+          0.0007227485184557736,
+          0.0007629882893525064,
+          0.0011065156431868672,
+          0.001901013427414,
+          0.0007759711588732898,
+          0.0003969112003687769,
+          0.0005158786661922932,
+          0.00045928655890747905,
+          0.0023647623602300882,
+          0.002461922587826848,
+          0.0005208771326579154,
+          0.0002854724589269608,
+          0.0010409415699541569,
+          0.0006360114784911275,
+          0.0003024112666025758,
+          0.0009925079066306353,
+          0.0013474463485181332,
+          0.0006413684459403157,
+          0.0006882536690682173,
+          0.0035410255659371614,
+          0.0013137771748006344,
+          0.0031697379890829325,
+          0.003913164138793945,
+          0.0009889702778309584,
+          0.0007571351598016918,
+          0.0007665202720090747,
+          0.0002901284606195986,
+          0.00045556839904747903,
+          0.0004817750887013972,
+          0.0006223590462468565,
+          0.0015162485651671886,
+          0.0010664068395271897,
+          0.001473718206398189,
+          0.0011742303613573313,
+          0.0003918787115253508,
+          0.00031622310052625835,
+          0.0003426342736929655,
+          0.0003908651415258646,
+          0.0005885711871087551,
+          0.0020876340568065643,
+          0.000550258147995919
+        ],
+        "r2": [
+          -0.040517330169677734,
+          -0.646754264831543,
+          -0.05489206314086914,
+          -4.486414432525635,
+          -7.45610237121582,
+          -1.989534854888916,
+          -0.14215362071990967,
+          0.8731237053871155,
+          0.6953460574150085,
+          -0.3271852731704712,
+          0.4872961640357971,
+          0.9173088669776917,
+          0.960431694984436,
+          0.768287181854248,
+          -0.10820090770721436,
+          -0.2204451560974121,
+          0.9208909273147583,
+          0.9714571237564087,
+          0.0919378399848938,
+          0.9393949508666992,
+          0.9164531230926514,
+          -1.3826713562011719,
+          0.7151634693145752,
+          0.03189957141876221,
+          0.8299977779388428,
+          -0.1580885648727417,
+          0.7477296590805054,
+          0.5928204655647278,
+          0.793571949005127,
+          0.7922360897064209,
+          -0.534965991973877,
+          0.9405934810638428,
+          0.565890908241272,
+          0.9263842105865479,
+          -0.6383448839187622,
+          0.7177635431289673,
+          0.8325225114822388,
+          0.8479948043823242,
+          -0.07674252986907959,
+          0.8262987732887268,
+          0.9472119212150574,
+          -0.49367833137512207,
+          0.7770066261291504,
+          -0.21446502208709717,
+          0.9271755814552307,
+          0.8588693737983704,
+          -0.4278682470321655
+        ],
+        "pearson": [
+          0.05270738527178764,
+          0.03536360338330269,
+          -0.010150463320314884,
+          0.29591673612594604,
+          -0.5409607291221619,
+          -0.2308097779750824,
+          0.4392741024494171,
+          0.9674931764602661,
+          0.851591944694519,
+          0.7005103826522827,
+          0.929423987865448,
+          0.9622913002967834,
+          0.9821222424507141,
+          0.939292311668396,
+          0.030473805963993073,
+          -0.2070801705121994,
+          0.9726182222366333,
+          0.9865130186080933,
+          0.3205333650112152,
+          0.9783572554588318,
+          0.9621458649635315,
+          0.7361637353897095,
+          0.9669621586799622,
+          0.28028619289398193,
+          0.9313540458679199,
+          -0.20505286753177643,
+          0.9817032814025879,
+          0.7723673582077026,
+          0.8908286094665527,
+          0.8989760875701904,
+          -0.08951489627361298,
+          0.975121021270752,
+          0.912742018699646,
+          0.980613648891449,
+          -0.0990881398320198,
+          0.8734300136566162,
+          0.9732058644294739,
+          0.9366763830184937,
+          0.07404083758592606,
+          0.9202995896339417,
+          0.986939549446106,
+          0.6866685152053833,
+          0.9149501919746399,
+          0.2164500653743744,
+          0.9744678735733032,
+          0.946538507938385,
+          0.06824231147766113
+        ],
+        "spearman": [
+          0.18519241717322635,
+          0.10822405351703812,
+          0.0099834334442791,
+          0.32957079734190514,
+          -0.38282363941605874,
+          -0.19851410140400802,
+          0.06293497227217011,
+          0.684183466545481,
+          0.7633651219730669,
+          0.6090042402780066,
+          0.5550195144127006,
+          0.6680776607895218,
+          0.9226776819896063,
+          0.8633905096673422,
+          0.038745875090305415,
+          0.03024947597073692,
+          0.7993649769288612,
+          0.8616925520559876,
+          0.2007840903342541,
+          0.8679031971924926,
+          0.8521447921661414,
+          0.5938497877609684,
+          0.9097774986173688,
+          0.1485962736009069,
+          0.8294586601117799,
+          -0.06403498575742866,
+          0.9133512546680557,
+          0.6379728270089983,
+          0.8564610276774808,
+          0.7783468319713108,
+          0.15165596073502338,
+          0.8064451534176335,
+          0.742932961749033,
+          0.9156179799489049,
+          -0.19875691269698034,
+          0.8384051489841586,
+          0.8271939265073791,
+          0.7626775754655413,
+          0.19578851391369415,
+          0.7616423377283719,
+          0.9087424025444016,
+          0.32851076506258664,
+          0.7594768336569494,
+          0.18198143633859454,
+          0.7260797373442908,
+          0.8718316408045742,
+          0.07773287299496047
+        ],
+        "de_corr": [
+          -0.047485753893852234,
+          0.08712006360292435,
+          0.020298732444643974,
+          0.6543987393379211,
+          -0.7934816479682922,
+          -0.44739097356796265,
+          0.7694762945175171,
+          0.9939884543418884,
+          0.9243724346160889,
+          0.7755008339881897,
+          0.976364016532898,
+          0.9942675828933716,
+          0.9900578260421753,
+          0.9717514514923096,
+          0.2907801866531372,
+          -0.23932424187660217,
+          0.9863185882568359,
+          0.9952530264854431,
+          0.6756848096847534,
+          0.9973934292793274,
+          0.9712522625923157,
+          0.9157187342643738,
+          0.9714534282684326,
+          0.5391358733177185,
+          0.9711427092552185,
+          -0.18987883627414703,
+          0.9913809299468994,
+          0.9305169582366943,
+          0.856482207775116,
+          0.7723619341850281,
+          -0.1443070024251938,
+          0.9835894703865051,
+          0.9258934855461121,
+          0.9954570531845093,
+          -0.10771752148866653,
+          0.9215916395187378,
+          0.9929366707801819,
+          0.9812349677085876,
+          0.09604117274284363,
+          0.9337543845176697,
+          0.9960479140281677,
+          0.9483177661895752,
+          0.9571083784103394,
+          0.3764186203479767,
+          0.9903275370597839,
+          0.9790841937065125,
+          0.11650128662586212
+        ],
+        "mmd": [
+          0.09884149292600153,
+          0.01946630129346827,
+          0.15171916808125996,
+          0.01677440707837663,
+          0.015576968228695032,
+          0.022819728212541968,
+          0.028901073634166274,
+          0.03629310212660142,
+          0.060784358970313646,
+          0.06759542769611904,
+          0.029645911324162944,
+          0.026939055914784404,
+          0.08743045767279123,
+          0.028307923566447735,
+          0.06905576237887412,
+          0.0797458918350391,
+          0.030019970001558538,
+          0.034842742497081414,
+          0.03638062109455609,
+          0.05321159242782125,
+          0.037715922203221064,
+          0.03972560217839849,
+          0.08046538346937837,
+          0.026080472603711158,
+          0.03700166800847815,
+          0.10378621415481315,
+          0.051635032985873575,
+          0.10556357239049574,
+          0.17011389888447925,
+          0.02957054053553898,
+          0.029268021363971553,
+          0.05678986274996556,
+          0.011811875833486196,
+          0.022807423737228105,
+          0.020770561428410272,
+          0.04979912365457795,
+          0.0582438261768079,
+          0.04700165123440936,
+          0.046960950863845774,
+          0.05820412759803262,
+          0.022515691973779672,
+          0.015578848498585263,
+          0.016934224570258505,
+          0.016198152371246666,
+          0.03738652011899268,
+          0.09475845081580081,
+          0.023785220170810417
+        ],
+        "wasserstein": [
+          0.34860048423544365,
+          0.12762712183614455,
+          0.399214916609018,
+          0.1257260244166984,
+          0.10169126561146175,
+          0.13718531649965465,
+          0.1631261575339426,
+          0.17884611453706611,
+          0.2338028239218351,
+          0.24464420344797677,
+          0.16695945350802588,
+          0.13668125218964633,
+          0.2636733644328416,
+          0.13194942639478316,
+          0.2295172279942127,
+          0.2662629058790926,
+          0.15312860954232194,
+          0.17961483128945008,
+          0.20226673508647297,
+          0.1821283675702408,
+          0.15910928740642663,
+          0.15789000268005446,
+          0.23752961041334839,
+          0.1657625071861199,
+          0.1998985368209983,
+          0.3469739391319118,
+          0.2211719215833392,
+          0.3753918946984671,
+          0.4377202495175173,
+          0.18825635686929837,
+          0.15371424038611559,
+          0.24215011123070174,
+          0.09309381660995744,
+          0.1430783438146134,
+          0.14201223355852183,
+          0.22237483417978077,
+          0.2216233619866504,
+          0.21794066446238272,
+          0.19986639207741955,
+          0.19920056674123057,
+          0.15563173184666473,
+          0.10774408592838654,
+          0.11859925892772133,
+          0.10275211762736548,
+          0.17996026728308198,
+          0.31309201486930155,
+          0.14237497498450202
+        ]
+      },
+      "n_perts": 47
+    },
+    "Random": {
+      "mse": 0.006797554055625137,
+      "r2": -6.335227725353647,
+      "pearson": 0.2715044714351918,
+      "spearman": 0.2418118923013008,
+      "de_corr": 0.285452477891553,
+      "mmd": 0.16974875210115886,
+      "wasserstein": 0.3916861021971288,
+      "_per_pert": {
+        "mse": [
+          0.002127312822267413,
+          0.002602711785584688,
+          0.004285301081836224,
+          0.00466095469892025,
+          0.00258551724255085,
+          0.0011789415730163455,
+          0.0020540133118629456,
+          0.005653779953718185,
+          0.00194260582793504,
+          0.004209674894809723,
+          0.008063239976763725,
+          0.009813507087528706,
+          0.019128642976284027,
+          0.0022924779914319515,
+          0.015366286039352417,
+          0.0021281992085278034,
+          0.005382271949201822,
+          0.012778623029589653,
+          0.0021838033571839333,
+          0.024319134652614594,
+          0.004210384096950293,
+          0.0021094647236168385,
+          0.0014473391929641366,
+          0.0016853106208145618,
+          0.002427374944090843,
+          0.005390360951423645,
+          0.006245764903724194,
+          0.012452896684408188,
+          0.02416934259235859,
+          0.004013997968286276,
+          0.003113203914836049,
+          0.012817734852433205,
+          0.006395484320819378,
+          0.0062553430907428265,
+          0.0007524136453866959,
+          0.0029931014869362116,
+          0.004285393748432398,
+          0.01057181041687727,
+          0.0013701937859877944,
+          0.004784707445651293,
+          0.008943222463130951,
+          0.002139439107850194,
+          0.003122832393273711,
+          0.025033049285411835,
+          0.015380092896521091,
+          0.010265622287988663,
+          0.004352159332484007
+        ],
+        "r2": [
+          0.3965739607810974,
+          -7.397797584533691,
+          0.1632699966430664,
+          -79.25465393066406,
+          -67.2914047241211,
+          -6.281386375427246,
+          -2.2459397315979004,
+          0.05984055995941162,
+          0.46514761447906494,
+          -1.9389686584472656,
+          -4.327587604522705,
+          -1.0445117950439453,
+          -0.467182993888855,
+          -0.1565690040588379,
+          -6.201117515563965,
+          -0.055008888244628906,
+          0.1825588345527649,
+          -0.27766525745391846,
+          -0.9050338268280029,
+          -1.3173508644104004,
+          -0.1631988286972046,
+          -4.06410026550293,
+          0.694047212600708,
+          -1.5438568592071533,
+          0.4004262089729309,
+          -0.7629115581512451,
+          -0.19930613040924072,
+          -0.5996795892715454,
+          -0.27498626708984375,
+          0.15673530101776123,
+          -5.311505317687988,
+          0.006605386734008789,
+          -8.56933879852295,
+          -0.010807633399963379,
+          -1.558689832687378,
+          -0.35735511779785156,
+          0.5266561508178711,
+          -0.5069013833999634,
+          -0.0011047124862670898,
+          0.2922091484069824,
+          -0.20469868183135986,
+          -9.105634689331055,
+          -1.0324029922485352,
+          -76.78069305419922,
+          -0.9029921293258667,
+          0.3060118556022644,
+          -10.293442726135254
+        ],
+        "pearson": [
+          0.6368151903152466,
+          0.3726290762424469,
+          0.7447395920753479,
+          0.315721333026886,
+          -0.38576221466064453,
+          0.03760609030723572,
+          0.5410704016685486,
+          0.5128589272499084,
+          0.6846206784248352,
+          -0.1384371668100357,
+          -0.5415480732917786,
+          0.6354108452796936,
+          -0.14010487496852875,
+          0.09047293663024902,
+          0.041458435356616974,
+          0.3128522038459778,
+          0.481003075838089,
+          0.11361993104219437,
+          -0.208737313747406,
+          -0.398562490940094,
+          0.3834177851676941,
+          0.19152754545211792,
+          0.8739404082298279,
+          0.5144321322441101,
+          0.8524073362350464,
+          0.4876151978969574,
+          -0.09158381074666977,
+          0.1261187642812729,
+          -0.09059347212314606,
+          0.4108235836029053,
+          -0.15390656888484955,
+          0.2239699810743332,
+          -0.30447566509246826,
+          0.14265793561935425,
+          0.126038059592247,
+          0.36229193210601807,
+          0.7881902456283569,
+          0.42433008551597595,
+          0.3949683904647827,
+          0.544305682182312,
+          0.25457534193992615,
+          0.7993372678756714,
+          0.16079279780387878,
+          0.24126645922660828,
+          0.40922462940216064,
+          0.5611183047294617,
+          0.42019322514533997
+        ],
+        "spearman": [
+          0.5310529498815971,
+          0.29072441046490377,
+          0.43642513838461816,
+          0.17381459232158886,
+          -0.3283585753806948,
+          -0.22055620227822795,
+          0.2055008514765372,
+          0.05186900211199081,
+          0.6403380137191934,
+          0.21481422410354808,
+          -0.21637657866435825,
+          0.3798212132572192,
+          -0.03674943755750615,
+          -0.014436720064903277,
+          0.18363924632758857,
+          0.2092071265821519,
+          0.361022396795536,
+          0.20171339015147832,
+          -0.219319054448528,
+          -0.08815865322980515,
+          0.43652040983268925,
+          0.2756451397863652,
+          0.8317345702191635,
+          0.5045848809718708,
+          0.7590550197881629,
+          0.36163291987517315,
+          -0.016590193371515953,
+          0.253648292442388,
+          -0.11070730267137427,
+          0.40015038854013496,
+          0.17864666796226952,
+          0.34988195724354165,
+          -0.018705086741920025,
+          0.24596091498305447,
+          0.08549474959847182,
+          0.3821888723899761,
+          0.4397286239482889,
+          0.36892227620613244,
+          0.3143309932549579,
+          0.6068896338930021,
+          0.23259526412607534,
+          0.3464865572186,
+          0.2943930949557598,
+          0.2548029642505411,
+          0.18254030716782355,
+          0.4849836537250617,
+          0.16435603461251672
+        ],
+        "de_corr": [
+          0.6353400945663452,
+          0.6368319392204285,
+          0.883208155632019,
+          0.6014145016670227,
+          -0.7253379225730896,
+          0.06181507557630539,
+          0.8785457015037537,
+          0.5250102281570435,
+          0.781560480594635,
+          -0.31299322843551636,
+          -0.7867901921272278,
+          0.8307375907897949,
+          -0.5707265734672546,
+          0.26218822598457336,
+          0.12379666417837143,
+          0.569264829158783,
+          0.4927721321582794,
+          0.025338536128401756,
+          -0.27684250473976135,
+          -0.8801300525665283,
+          0.1953013688325882,
+          0.0615214928984642,
+          0.9277873039245605,
+          0.860784649848938,
+          0.9239442944526672,
+          0.6461829543113708,
+          0.07855163514614105,
+          0.08836401253938675,
+          -0.41076961159706116,
+          0.17942115664482117,
+          -0.3638710081577301,
+          -0.07429542392492294,
+          -0.5127584934234619,
+          0.43892529606819153,
+          0.1457577645778656,
+          0.3521004617214203,
+          0.9495858550071716,
+          0.6043353080749512,
+          0.5511282086372375,
+          0.40757182240486145,
+          0.2117714136838913,
+          0.9158341884613037,
+          -0.010485046543180943,
+          0.5453157424926758,
+          0.663602352142334,
+          0.6413303017616272,
+          0.644324779510498
+        ],
+        "mmd": [
+          0.07582423070000577,
+          0.09066030471428443,
+          0.13440939834259258,
+          0.15669938595753508,
+          0.09433798250725844,
+          0.04083013235400712,
+          0.07541903320460452,
+          0.16280151297449774,
+          0.07923091625328016,
+          0.12258262992934121,
+          0.21741939603791371,
+          0.2719747586976766,
+          0.2989449496887061,
+          0.05989172056223657,
+          0.34906586657617056,
+          0.07681751019169347,
+          0.16409425708629988,
+          0.3096559604587591,
+          0.06942023545417886,
+          0.4613971666419753,
+          0.10886920379904819,
+          0.0753278417420915,
+          0.07004345768384057,
+          0.06088613102868312,
+          0.09035913843038934,
+          0.15901598591416166,
+          0.1676486447176112,
+          0.31342991469262405,
+          0.3228716332238504,
+          0.12478109170540197,
+          0.10392395112691877,
+          0.28367725308136127,
+          0.20236387850499327,
+          0.18093193948842523,
+          0.0314208226916749,
+          0.10333577865562027,
+          0.12568169023156728,
+          0.2704420928984306,
+          0.050206667875308475,
+          0.1331245515640146,
+          0.2390971665902627,
+          0.06865022393785758,
+          0.10264098900531793,
+          0.5416579725882449,
+          0.3738631913442223,
+          0.2291659573858711,
+          0.13329683051365715
+        ],
+        "wasserstein": [
+          0.2642483104273426,
+          0.22755625862254336,
+          0.3795218265785536,
+          0.36722831406082085,
+          0.21921312521240682,
+          0.2055172542355999,
+          0.24338368856814907,
+          0.3743023131000601,
+          0.2866021605998024,
+          0.36364981599250074,
+          0.3876888131099121,
+          0.45845710413365504,
+          0.6548866648130107,
+          0.22070921078712658,
+          0.5413695945606194,
+          0.24476616319496178,
+          0.4192243219339754,
+          0.6093798144485473,
+          0.28708298785192027,
+          0.7928895882157252,
+          0.35499459780271536,
+          0.29159671334039333,
+          0.24400529341732097,
+          0.2262191027235575,
+          0.2968901788601761,
+          0.41169307158882873,
+          0.46124014647731826,
+          0.5479934202071135,
+          0.7132902073724848,
+          0.36284336809629314,
+          0.30780573662145594,
+          0.6427933537474344,
+          0.4184564232621895,
+          0.44394911221452515,
+          0.19738676814739847,
+          0.31440434152639307,
+          0.3352521747253045,
+          0.4557445354499354,
+          0.20604127144019885,
+          0.38820798613802365,
+          0.501121909574595,
+          0.2250441501159834,
+          0.3195775018898895,
+          0.7079157195686178,
+          0.5903799193185655,
+          0.5201768708734071,
+          0.37654559831770007
+        ]
+      },
+      "n_perts": 47
+    }
+  }
+}

experiments/results/norman_nom_cell.json

@@ -0,0 +1,2773 @@
+{
+  "split": "cell",
+  "reward": "centroid",
+  "n_candidates": 105,
+  "n_targets": 40,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.55,
+      "top5": 0.9,
+      "ndcg": 0.7454003731910971,
+      "func_top5": 1.0,
+      "func_ndcg": 0.7362368327199266,
+      "rank": 2.6,
+      "endpoint_dist": 0.9490216635167599,
+      "spec_margin": 1.730977939069271,
+      "off_dist": 2.6799996107816697,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.0,
+          0.3868528072341547,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.49999999999949996,
+          0.9999999999989999,
+          0.3868528072341547,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.30102999566368016,
+          0.9999999999989999,
+          0.49999999999949996,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.43067655807296235,
+          0.9999999999989999,
+          0.49999999999949996,
+          0.49999999999949996,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.35620718710766597,
+          0.0,
+          0.49999999999949996,
+          0.49999999999949996,
+          0.43067655807296235,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.7719858642580791,
+          0.5452792644002579,
+          0.4622969259439367,
+          0.9999999999996392,
+          0.7601875334312905,
+          0.6309297535708266,
+          0.49999999999949996,
+          0.9999999999996392,
+          0.6585618697746796,
+          0.9999999999996392,
+          0.5679946984137687,
+          0.761639033082953,
+          0.9430934427597557,
+          0.7985960275052596,
+          0.7492069087978183,
+          0.865959765373318,
+          0.7723904789459194,
+          0.8814413010323419,
+          0.5679946984137687,
+          0.7729141359198264,
+          0.7209064905476281,
+          0.909805240729282,
+          0.909805240729282,
+          0.9478557351531002,
+          0.7101872472819487,
+          0.7229846523326642,
+          0.9540734464802791,
+          0.629042953814712,
+          0.9999999999996392,
+          0.5449154948697388,
+          0.7601875334312905,
+          0.803358319898604,
+          0.6980754691453294,
+          0.5905990152147871,
+          0.305935146935109,
+          0.7209064905476281,
+          0.4384685740390607,
+          0.7239931533469753,
+          0.6792194063292641,
+          0.6686819963485252
+        ],
+        "rank": [
+          1,
+          12,
+          5,
+          1,
+          1,
+          2,
+          3,
+          1,
+          5,
+          1,
+          2,
+          9,
+          1,
+          3,
+          2,
+          1,
+          1,
+          1,
+          2,
+          4,
+          1,
+          3,
+          3,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          6,
+          11,
+          3,
+          3,
+          4,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          1.2624379396438599,
+          1.987898588180542,
+          0.6130089163780212,
+          0.7837377190589905,
+          0.3801593482494354,
+          0.5316212177276611,
+          1.1274526119232178,
+          0.9317931532859802,
+          0.8158573508262634,
+          0.5724427103996277,
+          1.043930172920227,
+          0.9461696147918701,
+          1.0851073265075684,
+          0.9249292612075806,
+          1.3508139848709106,
+          1.8004765510559082,
+          0.8150457739830017,
+          0.7767999768257141,
+          1.0714013576507568,
+          1.3535343408584595,
+          1.0559062957763672,
+          1.2047194242477417,
+          1.0925180912017822,
+          1.2413640022277832,
+          1.438982605934143,
+          0.7611419558525085,
+          0.5610932111740112,
+          0.6496677398681641,
+          0.5188019871711731,
+          0.8329150676727295,
+          0.8286756277084351,
+          0.6340292692184448,
+          0.37346237897872925,
+          0.7785728573799133,
+          1.0523473024368286,
+          1.1261812448501587,
+          0.8302287459373474,
+          0.9894613027572632,
+          1.143169641494751,
+          0.6730098724365234
+        ],
+        "spec_margin": [
+          0.923587441444397,
+          -0.014548420906066895,
+          1.6584539413452148,
+          1.815046787261963,
+          1.5790126323699951,
+          1.7291734218597412,
+          1.253880262374878,
+          2.382495403289795,
+          1.2740118503570557,
+          1.9212472438812256,
+          1.310372233390808,
+          0.9761004447937012,
+          1.247694969177246,
+          1.690231442451477,
+          1.1252025365829468,
+          4.583049297332764,
+          3.685917377471924,
+          3.4045183658599854,
+          1.1300947666168213,
+          0.2355557680130005,
+          1.5858967304229736,
+          2.10831880569458,
+          1.798520803451538,
+          1.1007182598114014,
+          1.2903398275375366,
+          2.7685585021972656,
+          1.9255796670913696,
+          2.0334508419036865,
+          2.7833263874053955,
+          1.098803997039795,
+          2.4604926109313965,
+          3.0710902214050293,
+          3.5961661338806152,
+          2.221989154815674,
+          1.0602291822433472,
+          0.5479577779769897,
+          0.813274085521698,
+          0.7865604162216187,
+          1.0803568363189697,
+          1.1963895559310913
+        ],
+        "off_dist": [
+          2.186025381088257,
+          1.973350167274475,
+          2.271462917327881,
+          2.5987844467163086,
+          1.959172010421753,
+          2.2607946395874023,
+          2.3813328742980957,
+          3.31428861618042,
+          2.089869260787964,
+          2.493690013885498,
+          2.354302406311035,
+          1.9222700595855713,
+          2.3328022956848145,
+          2.6151607036590576,
+          2.4760165214538574,
+          6.383525848388672,
+          4.50096321105957,
+          4.181318283081055,
+          2.201496124267578,
+          1.58909010887146,
+          2.641803026199341,
+          3.3130383491516113,
+          2.8910388946533203,
+          2.3420822620391846,
+          2.7293224334716797,
+          3.529700517654419,
+          2.486672878265381,
+          2.6831185817718506,
+          3.302128314971924,
+          1.9317190647125244,
+          3.289168119430542,
+          3.7051196098327637,
+          3.9696285724639893,
+          3.0005619525909424,
+          2.112576484680176,
+          1.6741390228271484,
+          1.6435028314590454,
+          1.7760217189788818,
+          2.2235264778137207,
+          1.8693994283676147
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "PIVOT-guidance": {
+      "top1": 0.525,
+      "top5": 0.875,
+      "ndcg": 0.7150776971677019,
+      "func_top5": 0.975,
+      "func_ndcg": 0.6276044333475127,
+      "rank": 2.925,
+      "endpoint_dist": 0.982820986956358,
+      "spec_margin": 1.6728850707411766,
+      "off_dist": 2.655706062912941,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.0,
+          0.3868528072341547,
+          0.49999999999949996,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.49999999999949996,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.49999999999949996,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.0,
+          0.9999999999989999,
+          0.49999999999949996,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.43067655807296235,
+          0.49999999999949996,
+          0.49999999999949996,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.3949701603078018,
+          0.5088566520086942,
+          0.4851649195781592,
+          0.9478557351531002,
+          0.7601875334312905,
+          0.6309297535708266,
+          0.49999999999949996,
+          0.642476447789185,
+          0.3843973922908389,
+          0.642476447789185,
+          0.5583403462841774,
+          0.9999999999996392,
+          0.8814413010323419,
+          0.6766675710155576,
+          0.7058140748849717,
+          0.75951284454264,
+          0.5647869108315535,
+          0.7122606394323479,
+          0.5713692255708102,
+          0.3718921699781131,
+          0.6920970407325434,
+          0.909805240729282,
+          0.7700701178681301,
+          0.642476447789185,
+          0.3607790370814292,
+          0.5267364164077353,
+          0.9540734464802791,
+          0.6169311756780926,
+          0.9478557351531002,
+          0.5916118404088373,
+          0.7601875334312905,
+          0.0,
+          0.5347219910747678,
+          0.4080157475002485,
+          0.604636395515731,
+          0.6220658801619905,
+          0.5241325151519908,
+          0.6562570033883632,
+          0.5957547517842381,
+          0.686568892072539
+        ],
+        "rank": [
+          11,
+          5,
+          3,
+          1,
+          1,
+          2,
+          3,
+          1,
+          12,
+          1,
+          2,
+          1,
+          1,
+          3,
+          2,
+          1,
+          1,
+          1,
+          2,
+          12,
+          1,
+          3,
+          2,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          11,
+          1,
+          2,
+          4,
+          3,
+          3,
+          11,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          1.626929521560669,
+          1.987898588180542,
+          0.6130089163780212,
+          0.7837377190589905,
+          0.3801593482494354,
+          0.5316212177276611,
+          1.1274526119232178,
+          0.9317931532859802,
+          0.8158573508262634,
+          0.5724427103996277,
+          1.043930172920227,
+          0.8083254098892212,
+          1.0851073265075684,
+          0.9249292612075806,
+          1.3508139848709106,
+          1.8004765510559082,
+          0.8150457739830017,
+          0.7767999768257141,
+          1.0714013576507568,
+          1.3535343408584595,
+          1.0559062957763672,
+          1.2047194242477417,
+          1.0925180912017822,
+          1.2413640022277832,
+          1.438982605934143,
+          0.7611419558525085,
+          0.5610932111740112,
+          0.6496677398681641,
+          0.5188019871711731,
+          0.8329150676727295,
+          0.8286756277084351,
+          1.759354829788208,
+          0.37346237897872925,
+          0.7785728573799133,
+          1.0523473024368286,
+          1.1261812448501587,
+          0.8302287459373474,
+          0.9894613027572632,
+          1.143169641494751,
+          0.6730098724365234
+        ],
+        "spec_margin": [
+          0.13825678825378418,
+          -0.014548420906066895,
+          1.6584539413452148,
+          1.815046787261963,
+          1.5790126323699951,
+          1.7291734218597412,
+          1.253880262374878,
+          2.382495403289795,
+          1.2740118503570557,
+          1.9212472438812256,
+          1.310372233390808,
+          1.7914849519729614,
+          1.247694969177246,
+          1.690231442451477,
+          1.1252025365829468,
+          4.583049297332764,
+          3.685917377471924,
+          3.4045183658599854,
+          1.1300947666168213,
+          0.2355557680130005,
+          1.5858967304229736,
+          2.10831880569458,
+          1.798520803451538,
+          1.1007182598114014,
+          1.2903398275375366,
+          2.7685585021972656,
+          1.9255796670913696,
+          2.0334508419036865,
+          2.7833263874053955,
+          1.098803997039795,
+          2.4604926109313965,
+          0.7173216342926025,
+          3.5961661338806152,
+          2.221989154815674,
+          1.0602291822433472,
+          0.5479577779769897,
+          0.813274085521698,
+          0.7865604162216187,
+          1.0803568363189697,
+          1.1963895559310913
+        ],
+        "off_dist": [
+          1.7651863098144531,
+          1.973350167274475,
+          2.271462917327881,
+          2.5987844467163086,
+          1.959172010421753,
+          2.2607946395874023,
+          2.3813328742980957,
+          3.31428861618042,
+          2.089869260787964,
+          2.493690013885498,
+          2.354302406311035,
+          2.5998103618621826,
+          2.3328022956848145,
+          2.6151607036590576,
+          2.4760165214538574,
+          6.383525848388672,
+          4.50096321105957,
+          4.181318283081055,
+          2.201496124267578,
+          1.58909010887146,
+          2.641803026199341,
+          3.3130383491516113,
+          2.8910388946533203,
+          2.3420822620391846,
+          2.7293224334716797,
+          3.529700517654419,
+          2.486672878265381,
+          2.6831185817718506,
+          3.302128314971924,
+          1.9317190647125244,
+          3.289168119430542,
+          2.4766764640808105,
+          3.9696285724639893,
+          3.0005619525909424,
+          2.112576484680176,
+          1.6741390228271484,
+          1.6435028314590454,
+          1.7760217189788818,
+          2.2235264778137207,
+          1.8693994283676147
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "Additive+ranking": {
+      "top1": 1.0,
+      "top5": 1.0,
+      "ndcg": 0.9999999999989999,
+      "func_top5": 1.0,
+      "func_ndcg": 0.8479507485118546,
+      "rank": 1.0,
+      "endpoint_dist": 0.42116108611226083,
+      "spec_margin": 2.4543789327144623,
+      "off_dist": 2.8755400240421296,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.7644049042788869,
+          0.7623315121284877,
+          0.7691671966722313,
+          0.9999999999996392,
+          0.7601875334312905,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999996392,
+          0.7209064905476281,
+          0.9999999999996392,
+          0.7122606394323479,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.8525262713988451,
+          0.7490035979724269,
+          0.8580544200588494,
+          0.7122606394323479,
+          0.9223104630420079,
+          0.8700859688429049,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9478557351531002,
+          0.6588088387020696,
+          0.6655389434274009,
+          0.8460101051759085,
+          0.6734442887418696,
+          0.9999999999996392,
+          0.3607790370814292,
+          0.7601875334312905,
+          0.8935530791689612,
+          0.6319672012189719,
+          0.8366465219290776,
+          0.7923964318852734,
+          0.9478557351531002,
+          0.6719052786222132,
+          0.8887907867756167,
+          0.9456973440155002,
+          0.9430934427597557
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.4336645305156708,
+          0.4492749273777008,
+          0.36839205026626587,
+          0.5400750041007996,
+          0.4056673049926758,
+          0.5053470730781555,
+          0.3870347738265991,
+          0.46958184242248535,
+          0.3555386960506439,
+          0.49682140350341797,
+          0.3571249842643738,
+          0.448895126581192,
+          0.416703999042511,
+          0.43106698989868164,
+          0.44865232706069946,
+          0.44056838750839233,
+          0.38007888197898865,
+          0.38654324412345886,
+          0.33925434947013855,
+          0.4254252016544342,
+          0.7614416480064392,
+          0.44820210337638855,
+          0.3869432210922241,
+          0.4388890266418457,
+          0.370621919631958,
+          0.49089911580085754,
+          0.3214934170246124,
+          0.42563191056251526,
+          0.3777088224887848,
+          0.4537658989429474,
+          0.3759283721446991,
+          0.34280818700790405,
+          0.4167364537715912,
+          0.43289661407470703,
+          0.37268662452697754,
+          0.39969366788864136,
+          0.3186255097389221,
+          0.38318541646003723,
+          0.4199433922767639,
+          0.42263102531433105
+        ],
+        "spec_margin": [
+          1.8226473331451416,
+          2.429075002670288,
+          2.0126900672912598,
+          2.4777753353118896,
+          1.6256582736968994,
+          1.7526733875274658,
+          2.1037702560424805,
+          2.804387331008911,
+          1.832645058631897,
+          2.120689868927002,
+          2.2210195064544678,
+          1.8804901838302612,
+          1.9542350769042969,
+          1.9912099838256836,
+          2.1478829383850098,
+          7.4463677406311035,
+          3.817469596862793,
+          4.136026859283447,
+          2.065373182296753,
+          1.8025559186935425,
+          1.9463722705841064,
+          2.922027587890625,
+          2.956066131591797,
+          2.1518471240997314,
+          2.761521577835083,
+          3.247483968734741,
+          2.3110668659210205,
+          2.3323137760162354,
+          3.1342692375183105,
+          1.6014208793640137,
+          2.703058958053589,
+          2.9958693981170654,
+          3.7247540950775146,
+          2.5874547958374023,
+          1.8483490943908691,
+          1.554269552230835,
+          1.318281650543213,
+          1.786970853805542,
+          2.3373780250549316,
+          1.509738564491272
+        ],
+        "off_dist": [
+          2.2563118934631348,
+          2.878350019454956,
+          2.381082057952881,
+          3.017850399017334,
+          2.031325578689575,
+          2.2580204010009766,
+          2.49080491065979,
+          3.2739691734313965,
+          2.1881837844848633,
+          2.61751127243042,
+          2.5781445503234863,
+          2.329385280609131,
+          2.370939016342163,
+          2.4222769737243652,
+          2.5965352058410645,
+          7.886936187744141,
+          4.1975483894348145,
+          4.5225701332092285,
+          2.404627561569214,
+          2.2279810905456543,
+          2.7078139781951904,
+          3.370229721069336,
+          3.3430094718933105,
+          2.590736150741577,
+          3.132143497467041,
+          3.7383830547332764,
+          2.6325602531433105,
+          2.7579457759857178,
+          3.5119781494140625,
+          2.0551867485046387,
+          3.0789873600006104,
+          3.3386776447296143,
+          4.141490459442139,
+          3.0203514099121094,
+          2.2210357189178467,
+          1.953963279724121,
+          1.6369072198867798,
+          2.170156240463257,
+          2.757321357727051,
+          1.932369589805603
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.8,
+      "top5": 0.95,
+      "ndcg": 0.9022189725330112,
+      "func_top5": 1.0,
+      "func_ndcg": 0.7964563634648597,
+      "rank": 1.575,
+      "endpoint_dist": 0.9708886995911599,
+      "spec_margin": 1.5014912456274032,
+      "off_dist": 2.4723799377679825,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.33333333333299997,
+          0.30102999566368016,
+          0.9999999999989999,
+          0.49999999999949996,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.43067655807296235,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.7778130477875116,
+          0.7390855690096315,
+          0.6888737584588553,
+          0.9999999999996392,
+          0.7601875334312905,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999996392,
+          0.6988271454578748,
+          0.821149491578581,
+          0.691397430362036,
+          0.8137832979294921,
+          0.9223104630420079,
+          0.8321157037716506,
+          0.9223104630420079,
+          0.8011478628189659,
+          0.743621924561139,
+          0.8131794116030213,
+          0.7402521670387343,
+          0.9398701604860678,
+          0.8814413010323419,
+          0.9999999999996392,
+          0.9478557351531002,
+          0.8877258956395286,
+          0.6587286093495057,
+          0.6500574077683771,
+          0.9467676761260954,
+          0.6165377315019861,
+          0.9999999999996392,
+          0.48270749357113113,
+          0.47962493313589827,
+          0.8269766751080135,
+          0.6478990166307772,
+          0.8935530791689612,
+          0.5916118404088373,
+          0.9334235959386915,
+          0.20752333876180207,
+          0.873309251116593,
+          0.7509714103044139,
+          0.8756141175029094
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          2,
+          7,
+          9,
+          1,
+          3,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          2,
+          2,
+          1,
+          1,
+          1,
+          2,
+          4,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.6907550096511841,
+          1.0634472370147705,
+          0.5679367184638977,
+          1.165585994720459,
+          0.6991990804672241,
+          0.5411589741706848,
+          0.699982225894928,
+          0.6999955773353577,
+          0.7555252313613892,
+          0.8050605654716492,
+          1.4271050691604614,
+          0.8213156461715698,
+          0.7616903781890869,
+          0.6038821339607239,
+          0.862602710723877,
+          2.178295850753784,
+          1.4125244617462158,
+          1.9672163724899292,
+          0.8852179646492004,
+          0.7257881760597229,
+          1.1350444555282593,
+          1.1267881393432617,
+          0.8013559579849243,
+          1.0217680931091309,
+          1.4548118114471436,
+          1.2549763917922974,
+          0.7250621318817139,
+          1.434755802154541,
+          1.083292007446289,
+          0.6659277081489563,
+          0.8551198244094849,
+          0.6937877535820007,
+          1.2051607370376587,
+          1.2373899221420288,
+          0.8727580308914185,
+          0.6869524121284485,
+          0.7910900115966797,
+          1.0167807340621948,
+          0.83255934715271,
+          0.6058813333511353
+        ],
+        "spec_margin": [
+          1.4733346700668335,
+          1.1693987846374512,
+          1.6650640964508057,
+          1.156996726989746,
+          1.1377700567245483,
+          1.5223705768585205,
+          1.5267906188964844,
+          2.104612350463867,
+          1.323801875114441,
+          1.8257782459259033,
+          0.5402255058288574,
+          1.453631043434143,
+          1.3117954730987549,
+          1.7336227893829346,
+          1.2966454029083252,
+          3.671785593032837,
+          1.9243266582489014,
+          1.1463512182235718,
+          1.1231210231781006,
+          1.0982592105865479,
+          1.1551765203475952,
+          1.6141057014465332,
+          2.186246395111084,
+          0.9928333759307861,
+          0.7203879356384277,
+          1.6999324560165405,
+          1.4218502044677734,
+          1.0667059421539307,
+          1.8538975715637207,
+          1.1988515853881836,
+          1.804688572883606,
+          2.4775850772857666,
+          2.8387041091918945,
+          1.6696089506149292,
+          1.3214129209518433,
+          1.2888331413269043,
+          0.7697153091430664,
+          1.1673275232315063,
+          1.4035670757293701,
+          1.2025375366210938
+        ],
+        "off_dist": [
+          2.1640896797180176,
+          2.2328460216522217,
+          2.2330007553100586,
+          2.322582721710205,
+          1.8369691371917725,
+          2.0635294914245605,
+          2.2267727851867676,
+          2.80460786819458,
+          2.07932710647583,
+          2.6308388710021973,
+          1.9673305749893188,
+          2.274946689605713,
+          2.073485851287842,
+          2.3375048637390137,
+          2.159248113632202,
+          5.850081443786621,
+          3.336851119995117,
+          3.113567590713501,
+          2.0083389282226562,
+          1.8240474462509155,
+          2.2902209758758545,
+          2.740893840789795,
+          2.987602472305298,
+          2.014601469039917,
+          2.1751997470855713,
+          2.954908847808838,
+          2.1469123363494873,
+          2.5014617443084717,
+          2.9371895790100098,
+          1.8647792339324951,
+          2.659808397293091,
+          3.171372890472412,
+          4.043864727020264,
+          2.906998872756958,
+          2.1941709518432617,
+          1.975785493850708,
+          1.560805320739746,
+          2.184108257293701,
+          2.23612642288208,
+          1.808418869972229
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 1.0,
+      "top5": 1.0,
+      "ndcg": 0.9999999999989999,
+      "func_top5": 1.0,
+      "func_ndcg": 0.8479507485118546,
+      "rank": 1.0,
+      "endpoint_dist": 0.42116108611226083,
+      "spec_margin": 2.4543789327144623,
+      "off_dist": 2.8755400240421296,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.7644049042788869,
+          0.7623315121284877,
+          0.7691671966722313,
+          0.9999999999996392,
+          0.7601875334312905,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999996392,
+          0.7209064905476281,
+          0.9999999999996392,
+          0.7122606394323479,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.8525262713988451,
+          0.7490035979724269,
+          0.8580544200588494,
+          0.7122606394323479,
+          0.9223104630420079,
+          0.8700859688429049,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9478557351531002,
+          0.6588088387020696,
+          0.6655389434274009,
+          0.8460101051759085,
+          0.6734442887418696,
+          0.9999999999996392,
+          0.3607790370814292,
+          0.7601875334312905,
+          0.8935530791689612,
+          0.6319672012189719,
+          0.8366465219290776,
+          0.7923964318852734,
+          0.9478557351531002,
+          0.6719052786222132,
+          0.8887907867756167,
+          0.9456973440155002,
+          0.9430934427597557
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.4336645305156708,
+          0.4492749273777008,
+          0.36839205026626587,
+          0.5400750041007996,
+          0.4056673049926758,
+          0.5053470730781555,
+          0.3870347738265991,
+          0.46958184242248535,
+          0.3555386960506439,
+          0.49682140350341797,
+          0.3571249842643738,
+          0.448895126581192,
+          0.416703999042511,
+          0.43106698989868164,
+          0.44865232706069946,
+          0.44056838750839233,
+          0.38007888197898865,
+          0.38654324412345886,
+          0.33925434947013855,
+          0.4254252016544342,
+          0.7614416480064392,
+          0.44820210337638855,
+          0.3869432210922241,
+          0.4388890266418457,
+          0.370621919631958,
+          0.49089911580085754,
+          0.3214934170246124,
+          0.42563191056251526,
+          0.3777088224887848,
+          0.4537658989429474,
+          0.3759283721446991,
+          0.34280818700790405,
+          0.4167364537715912,
+          0.43289661407470703,
+          0.37268662452697754,
+          0.39969366788864136,
+          0.3186255097389221,
+          0.38318541646003723,
+          0.4199433922767639,
+          0.42263102531433105
+        ],
+        "spec_margin": [
+          1.8226473331451416,
+          2.429075002670288,
+          2.0126900672912598,
+          2.4777753353118896,
+          1.6256582736968994,
+          1.7526733875274658,
+          2.1037702560424805,
+          2.804387331008911,
+          1.832645058631897,
+          2.120689868927002,
+          2.2210195064544678,
+          1.8804901838302612,
+          1.9542350769042969,
+          1.9912099838256836,
+          2.1478829383850098,
+          7.4463677406311035,
+          3.817469596862793,
+          4.136026859283447,
+          2.065373182296753,
+          1.8025559186935425,
+          1.9463722705841064,
+          2.922027587890625,
+          2.956066131591797,
+          2.1518471240997314,
+          2.761521577835083,
+          3.247483968734741,
+          2.3110668659210205,
+          2.3323137760162354,
+          3.1342692375183105,
+          1.6014208793640137,
+          2.703058958053589,
+          2.9958693981170654,
+          3.7247540950775146,
+          2.5874547958374023,
+          1.8483490943908691,
+          1.554269552230835,
+          1.318281650543213,
+          1.786970853805542,
+          2.3373780250549316,
+          1.509738564491272
+        ],
+        "off_dist": [
+          2.2563118934631348,
+          2.878350019454956,
+          2.381082057952881,
+          3.017850399017334,
+          2.031325578689575,
+          2.2580204010009766,
+          2.49080491065979,
+          3.2739691734313965,
+          2.1881837844848633,
+          2.61751127243042,
+          2.5781445503234863,
+          2.329385280609131,
+          2.370939016342163,
+          2.4222769737243652,
+          2.5965352058410645,
+          7.886936187744141,
+          4.1975483894348145,
+          4.5225701332092285,
+          2.404627561569214,
+          2.2279810905456543,
+          2.7078139781951904,
+          3.370229721069336,
+          3.3430094718933105,
+          2.590736150741577,
+          3.132143497467041,
+          3.7383830547332764,
+          2.6325602531433105,
+          2.7579457759857178,
+          3.5119781494140625,
+          2.0551867485046387,
+          3.0789873600006104,
+          3.3386776447296143,
+          4.141490459442139,
+          3.0203514099121094,
+          2.2210357189178467,
+          1.953963279724121,
+          1.6369072198867798,
+          2.170156240463257,
+          2.757321357727051,
+          1.932369589805603
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "EndpointMLP+ranking": {
+      "top1": 1.0,
+      "top5": 1.0,
+      "ndcg": 0.9999999999989999,
+      "func_top5": 1.0,
+      "func_ndcg": 0.844723911438565,
+      "rank": 1.0,
+      "endpoint_dist": 0.42520923763513563,
+      "spec_margin": 2.4454854100942613,
+      "off_dist": 2.8706946551799772,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.7644049042788869,
+          0.7358622867359436,
+          0.7691671966722313,
+          0.9999999999996392,
+          0.7601875334312905,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999996392,
+          0.7209064905476281,
+          0.9999999999996392,
+          0.7122606394323479,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.8525262713988451,
+          0.7490035979724269,
+          0.8580544200588494,
+          0.7122606394323479,
+          0.9223104630420079,
+          0.8700859688429049,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9478557351531002,
+          0.6045061827179307,
+          0.6629350421716564,
+          0.8460101051759085,
+          0.6734442887418696,
+          0.9999999999996392,
+          0.3607790370814292,
+          0.7601875334312905,
+          0.8935530791689612,
+          0.6351904834926599,
+          0.8366465219290776,
+          0.7434754493124225,
+          0.9478557351531002,
+          0.6719052786222132,
+          0.8887907867756167,
+          0.9456973440155002,
+          0.9430934427597557
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.42910057306289673,
+          0.4497443437576294,
+          0.3689153492450714,
+          0.5424296259880066,
+          0.4387088418006897,
+          0.4989120066165924,
+          0.38678574562072754,
+          0.44920316338539124,
+          0.37661030888557434,
+          0.48695677518844604,
+          0.3633248805999756,
+          0.4482707679271698,
+          0.44536110758781433,
+          0.45165324211120605,
+          0.456876277923584,
+          0.4695214033126831,
+          0.38211068511009216,
+          0.3952172100543976,
+          0.34323543310165405,
+          0.4283789098262787,
+          0.7605032920837402,
+          0.4499318599700928,
+          0.4084526002407074,
+          0.43640345335006714,
+          0.3723512589931488,
+          0.4967805743217468,
+          0.3198273777961731,
+          0.43172505497932434,
+          0.3519124686717987,
+          0.4496127665042877,
+          0.38622480630874634,
+          0.33023756742477417,
+          0.4086756408214569,
+          0.44142019748687744,
+          0.3617817163467407,
+          0.4287257194519043,
+          0.3391049802303314,
+          0.37814775109291077,
+          0.41947412490844727,
+          0.4257596433162689
+        ],
+        "spec_margin": [
+          1.8297321796417236,
+          2.4234132766723633,
+          2.0120315551757812,
+          2.474533796310425,
+          1.5998640060424805,
+          1.7592231035232544,
+          2.1033191680908203,
+          2.8135390281677246,
+          1.8096857070922852,
+          2.1264402866363525,
+          2.2138192653656006,
+          1.861198902130127,
+          1.9200901985168457,
+          1.95857834815979,
+          2.123638391494751,
+          7.403108596801758,
+          3.7975454330444336,
+          4.105343341827393,
+          2.063859701156616,
+          1.7980142831802368,
+          1.9481453895568848,
+          2.9155890941619873,
+          2.932025671005249,
+          2.1543331146240234,
+          2.7645161151885986,
+          3.2405927181243896,
+          2.310844659805298,
+          2.3246302604675293,
+          3.131648063659668,
+          1.6016348600387573,
+          2.693023920059204,
+          2.9935696125030518,
+          3.7314066886901855,
+          2.577932357788086,
+          1.844446063041687,
+          1.5249234437942505,
+          1.2980587482452393,
+          1.7848142385482788,
+          2.3416037559509277,
+          1.5086990594863892
+        ],
+        "off_dist": [
+          2.2588326930999756,
+          2.873157501220703,
+          2.3809468746185303,
+          3.016963481903076,
+          2.0385727882385254,
+          2.2581350803375244,
+          2.490104913711548,
+          3.262742280960083,
+          2.186295986175537,
+          2.6133971214294434,
+          2.577144145965576,
+          2.309469699859619,
+          2.3654513359069824,
+          2.410231590270996,
+          2.580514669418335,
+          7.8726301193237305,
+          4.179656028747559,
+          4.500560760498047,
+          2.407095193862915,
+          2.226393222808838,
+          2.708648681640625,
+          3.36552095413208,
+          3.3404781818389893,
+          2.5907366275787354,
+          3.1368672847747803,
+          3.7373733520507812,
+          2.630671977996826,
+          2.7563552856445312,
+          3.483560562133789,
+          2.0512475967407227,
+          3.0792486667633057,
+          3.3238072395324707,
+          4.140082359313965,
+          3.019352674484253,
+          2.2062277793884277,
+          1.9536491632461548,
+          1.637163758277893,
+          2.162961959838867,
+          2.761077880859375,
+          1.9344587326049805
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    },
+    "Random+ranking": {
+      "top1": 0.025,
+      "top5": 0.025,
+      "ndcg": 0.0484465759108753,
+      "func_top5": 0.625,
+      "func_ndcg": 0.20469424490855914,
+      "rank": 57.925,
+      "endpoint_dist": 3.470254364609718,
+      "spec_margin": -0.21521338820457458,
+      "off_dist": 3.2550409764051436,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.33333333333299997,
+          0.2890648263175988,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.3154648767854133,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.43262589581815525,
+          0.37924165572138796,
+          0.3718921699781131,
+          0.05430265598413898,
+          0.0,
+          0.0,
+          0.0,
+          0.1678842962279888,
+          0.3144297531670358,
+          0.3225665727174216,
+          0.11381311447976694,
+          0.23214033921114718,
+          0.27853764988081753,
+          0.3198776724744765,
+          0.4919092042576051,
+          0.11640030782969744,
+          0.0,
+          0.0,
+          0.25017371018387746,
+          0.0,
+          0.1298338018041706,
+          0.4378169904238303,
+          0.19408984478732896,
+          0.4615310696772767,
+          0.05430265598413898,
+          0.05430265598413898,
+          0.0,
+          0.052144264846539085,
+          0.33384167555429484,
+          0.20246886363046793,
+          0.0,
+          0.23819899697649688,
+          0.2645101253542494,
+          0.0,
+          0.1841364577883096,
+          0.18889875018165395,
+          0.14194557994078985,
+          0.32749083505095544,
+          0.527340302734206,
+          0.5491219276918895
+        ],
+        "rank": [
+          14,
+          74,
+          102,
+          28,
+          89,
+          23,
+          94,
+          92,
+          56,
+          30,
+          43,
+          93,
+          48,
+          7,
+          10,
+          14,
+          78,
+          59,
+          59,
+          97,
+          67,
+          56,
+          57,
+          74,
+          63,
+          67,
+          105,
+          28,
+          74,
+          38,
+          102,
+          8,
+          68,
+          72,
+          87,
+          51,
+          66,
+          94,
+          29,
+          1
+        ],
+        "endpoint_dist": [
+          2.2824044227600098,
+          4.309011936187744,
+          3.309451103210449,
+          3.564979076385498,
+          1.453060507774353,
+          2.8975343704223633,
+          1.6756479740142822,
+          2.8101394176483154,
+          3.0139412879943848,
+          3.0206949710845947,
+          5.2664995193481445,
+          2.2033324241638184,
+          2.293858766555786,
+          2.880345344543457,
+          2.1053731441497803,
+          8.31590461730957,
+          6.106352806091309,
+          4.284556865692139,
+          2.4269845485687256,
+          2.013439178466797,
+          3.7241005897521973,
+          2.720334053039551,
+          4.638664722442627,
+          2.371800422668457,
+          4.18781852722168,
+          5.791450500488281,
+          5.960268497467041,
+          4.852285861968994,
+          5.4412407875061035,
+          2.308344602584839,
+          2.7184574604034424,
+          2.7296082973480225,
+          2.9228410720825195,
+          4.18679141998291,
+          2.3668174743652344,
+          1.5269440412521362,
+          1.1631391048431396,
+          5.09120512008667,
+          6.774564743041992,
+          1.0999850034713745
+        ],
+        "spec_margin": [
+          -0.20399761199951172,
+          0.09865903854370117,
+          0.7027626037597656,
+          -0.2298269271850586,
+          0.6164559125900269,
+          0.7414908409118652,
+          0.9587249755859375,
+          0.38802433013916016,
+          0.4304380416870117,
+          0.06131625175476074,
+          -1.3636457920074463,
+          0.02685999870300293,
+          0.9651339054107666,
+          -0.0017824172973632812,
+          0.412534236907959,
+          -5.367891311645508,
+          -2.511678457260132,
+          -1.9479377269744873,
+          -0.026017427444458008,
+          -0.077117919921875,
+          -0.0017910003662109375,
+          1.7640347480773926,
+          -1.2810420989990234,
+          0.16104459762573242,
+          -1.0742332935333252,
+          -1.1297025680541992,
+          -0.7748074531555176,
+          -0.8083209991455078,
+          -1.7950336933135986,
+          0.09963798522949219,
+          0.4017605781555176,
+          -0.3908803462982178,
+          -0.15664148330688477,
+          -0.5637521743774414,
+          0.6465599536895752,
+          0.42621946334838867,
+          0.5508947372436523,
+          0.8817577362060547,
+          -0.12642383575439453,
+          0.8896790742874146
+        ],
+        "off_dist": [
+          2.078406810760498,
+          4.407670974731445,
+          4.012213706970215,
+          3.3351521492004395,
+          2.06951642036438,
+          3.6390252113342285,
+          2.6343729496002197,
+          3.1981637477874756,
+          3.4443793296813965,
+          3.0820112228393555,
+          3.9028537273406982,
+          2.2301924228668213,
+          3.2589926719665527,
+          2.8785629272460938,
+          2.5179073810577393,
+          2.9480133056640625,
+          3.5946743488311768,
+          2.3366191387176514,
+          2.4009671211242676,
+          1.9363212585449219,
+          3.7223095893859863,
+          4.484368801116943,
+          3.3576226234436035,
+          2.5328450202941895,
+          3.1135852336883545,
+          4.661747932434082,
+          5.185461044311523,
+          4.043964862823486,
+          3.646207094192505,
+          2.407982587814331,
+          3.12021803855896,
+          2.3387279510498047,
+          2.7661995887756348,
+          3.6230392456054688,
+          3.0133774280548096,
+          1.953163504600525,
+          1.714033842086792,
+          5.972962856292725,
+          6.648140907287598,
+          1.989664077758789
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.047619047619047616
+    }
+  }
+}

experiments/results/norman_nom_improved.json

@@ -0,0 +1,295 @@
+{
+  "dataset": "norman",
+  "split": "perturbation",
+  "reward": "cosine",
+  "n_candidates": 105,
+  "n_targets": 22,
+  "random_top5": 0.047619047619047616,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.18181818181818182,
+      "top5": 0.22727272727272727,
+      "ndcg": 0.237863170222463,
+      "func_top5": 0.8636363636363636,
+      "func_ndcg": 0.6379857109126424,
+      "top5_ci": [
+        0.22727272727272727,
+        0.045454545454545456,
+        0.4090909090909091
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 26.0
+    },
+    "PIVOT-guidance(random-init)": {
+      "top1": 0.13636363636363635,
+      "top5": 0.22727272727272727,
+      "ndcg": 0.20036677015736315,
+      "func_top5": 0.8181818181818182,
+      "func_ndcg": 0.5717396066885763,
+      "top5_ci": [
+        0.22727272727272727,
+        0.045454545454545456,
+        0.4090909090909091
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 27.0
+    },
+    "EndpointMLP+ranking": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.13489139848118523,
+      "func_top5": 0.9545454545454546,
+      "func_ndcg": 0.5888636133223321,
+      "top5_ci": [
+        0.13636363636363635,
+        0.0,
+        0.2727272727272727
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 74.5
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.045454545454545456,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.1028117957791206,
+      "func_top5": 0.8181818181818182,
+      "func_ndcg": 0.5703934469254858,
+      "top5_ci": [
+        0.13636363636363635,
+        0.0,
+        0.2727272727272727
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 53.0
+    },
+    "Additive+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.9545454545454546,
+      "func_ndcg": 0.5342699870208798,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 40.5
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 0.18181818181818182,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.181818181818,
+      "func_top5": 0.9090909090909091,
+      "func_ndcg": 0.5876837637320901,
+      "top5_ci": [
+        0.18181818181818182,
+        0.045454545454545456,
+        0.36363636363636365
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 35.5
+    },
+    "Random+ranking": {
+      "top1": 0.0,
+      "top5": 0.045454545454545456,
+      "ndcg": 0.030723528797806977,
+      "func_top5": 0.7727272727272727,
+      "func_ndcg": 0.24888537092571922,
+      "top5_ci": [
+        0.045454545454545456,
+        0.0,
+        0.13636363636363635
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 57.5
+    }
+  },
+  "significance": {
+    "PIVOT-ranking_vs_PIVOT-guidance(random-init)": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_EndpointMLP+ranking": {
+      "p": 0.3268365817091454,
+      "mean_diff_top5": 0.09090909090909091
+    },
+    "PIVOT-ranking_vs_LinearResponse+ranking": {
+      "p": 0.12643678160919541,
+      "mean_diff_top5": 0.09090909090909091
+    },
+    "PIVOT-ranking_vs_Additive+ranking": {
+      "p": 0.010994502748625687,
+      "mean_diff_top5": 0.22727272727272727
+    },
+    "PIVOT-ranking_vs_NearestPerturbationCentroid+ranking": {
+      "p": 0.25287356321839083,
+      "mean_diff_top5": 0.045454545454545456
+    },
+    "PIVOT-ranking_vs_Random+ranking": {
+      "p": 0.03298350824587706,
+      "mean_diff_top5": 0.18181818181818182
+    }
+  }
+}

experiments/results/norman_nom_perturbation.json

@@ -0,0 +1,1639 @@
+{
+  "split": "perturbation",
+  "reward": "centroid",
+  "n_candidates": 105,
+  "n_targets": 22,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.08727248090397387,
+      "func_top5": 0.9090909090909091,
+      "func_ndcg": 0.4865406670801178,
+      "rank": 35.40909090909091,
+      "endpoint_dist": 1.068045285615054,
+      "spec_margin": 1.3363434076309204,
+      "off_dist": 2.4043886986645786,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.49999999999949996,
+          0.0,
+          0.0,
+          0.49999999999949996,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.2890648263175988,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.733275093386013,
+          0.37033766516698385,
+          0.8196104814589247,
+          0.38009376671564526,
+          0.0,
+          0.4387000482749779,
+          0.7553544384757664,
+          0.4749645636517497,
+          0.558712758374731,
+          0.8196104814589247,
+          0.4960354077271919,
+          0.15032459878392884,
+          0.4567687772839323,
+          0.8196104814589247,
+          0.5059324046044343,
+          0.30215600509022716,
+          0.7910845718217951,
+          0.0,
+          0.6430803341156557,
+          0.40257647075284236,
+          0.34696627888496423,
+          0.4387000482749779
+        ],
+        "rank": [
+          3,
+          35,
+          17,
+          3,
+          25,
+          74,
+          84,
+          26,
+          23,
+          19,
+          10,
+          75,
+          36,
+          63,
+          31,
+          35,
+          2,
+          48,
+          23,
+          52,
+          48,
+          47
+        ],
+        "endpoint_dist": [
+          1.9384753704071045,
+          0.7429381012916565,
+          1.1510169506072998,
+          0.7578966021537781,
+          0.6557507514953613,
+          0.9810452461242676,
+          0.819040060043335,
+          1.5743571519851685,
+          1.7966794967651367,
+          2.0639142990112305,
+          1.1475152969360352,
+          0.840554416179657,
+          0.8537219762802124,
+          0.8637474179267883,
+          1.4352035522460938,
+          0.8421792984008789,
+          1.0563677549362183,
+          0.7560780048370361,
+          1.2075574398040771,
+          0.6115195155143738,
+          0.691246747970581,
+          0.7101908326148987
+        ],
+        "spec_margin": [
+          -0.18687379360198975,
+          1.2044005393981934,
+          2.3071529865264893,
+          1.7771611213684082,
+          1.122977375984192,
+          1.3516638278961182,
+          1.9459350109100342,
+          0.6711388826370239,
+          0.42350029945373535,
+          0.5177159309387207,
+          1.2125356197357178,
+          1.326828956604004,
+          0.9825109243392944,
+          2.5846638679504395,
+          1.4229609966278076,
+          0.9427473545074463,
+          3.5186824798583984,
+          1.3512330055236816,
+          0.6580355167388916,
+          1.3769090175628662,
+          1.4731967449188232,
+          1.4144783020019531
+        ],
+        "off_dist": [
+          1.7516015768051147,
+          1.9473387002944946,
+          3.458169937133789,
+          2.535057783126831,
+          1.7787281274795532,
+          2.3327090740203857,
+          2.764975070953369,
+          2.2454960346221924,
+          2.220179796218872,
+          2.581630229949951,
+          2.360050916671753,
+          2.1673834323883057,
+          1.8362329006195068,
+          3.448411226272583,
+          2.8581645488739014,
+          1.7849266529083252,
+          4.575050354003906,
+          2.1073110103607178,
+          1.8655929565429688,
+          1.9884284734725952,
+          2.1644434928894043,
+          2.124669075012207
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "PIVOT-guidance": {
+      "top1": 0.0,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0741331706168103,
+      "func_top5": 0.9090909090909091,
+      "func_ndcg": 0.40010813422906144,
+      "rank": 37.72727272727273,
+      "endpoint_dist": 1.0795677466826006,
+      "spec_margin": 1.2815920818935742,
+      "off_dist": 2.361159833994779,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.49999999999949996,
+          0.0,
+          0.0,
+          0.49999999999949996,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.6220658801619905,
+          0.3259363302398262,
+          0.8196104814589247,
+          0.38009376671564526,
+          0.0,
+          0.34634689786702066,
+          0.4620869292484704,
+          0.24464556152387298,
+          0.5791066180961114,
+          0.5920009604052048,
+          0.34036846945423477,
+          0.2714871990069707,
+          0.46683251373600076,
+          0.6441452252517439,
+          0.1835973061229298,
+          0.3181934003204448,
+          0.5499613274410384,
+          0.0,
+          0.4445272318044105,
+          0.5189767785825399,
+          0.3047754010374445,
+          0.3876206745645269
+        ],
+        "rank": [
+          3,
+          35,
+          20,
+          3,
+          28,
+          75,
+          84,
+          32,
+          28,
+          26,
+          14,
+          75,
+          38,
+          64,
+          37,
+          37,
+          2,
+          49,
+          29,
+          52,
+          50,
+          49
+        ],
+        "endpoint_dist": [
+          1.9384753704071045,
+          0.7429381012916565,
+          1.1510169506072998,
+          0.7578966021537781,
+          0.6557507514953613,
+          0.9810452461242676,
+          0.819040060043335,
+          1.5743571519851685,
+          1.8661444187164307,
+          2.0639142990112305,
+          1.1475152969360352,
+          0.840554416179657,
+          0.8537219762802124,
+          0.8637474179267883,
+          1.6192327737808228,
+          0.8421792984008789,
+          1.0563677549362183,
+          0.7560780048370361,
+          1.2075574398040771,
+          0.6115195155143738,
+          0.691246747970581,
+          0.7101908326148987
+        ],
+        "spec_margin": [
+          -0.18687379360198975,
+          1.2044005393981934,
+          2.3071529865264893,
+          1.7771611213684082,
+          1.122977375984192,
+          1.3516638278961182,
+          1.9459350109100342,
+          0.6711388826370239,
+          0.15226531028747559,
+          0.5177159309387207,
+          1.2125356197357178,
+          1.326828956604004,
+          0.9825109243392944,
+          2.5846638679504395,
+          0.48966681957244873,
+          0.9427473545074463,
+          3.5186824798583984,
+          1.3512330055236816,
+          0.6580355167388916,
+          1.3769090175628662,
+          1.4731967449188232,
+          1.4144783020019531
+        ],
+        "off_dist": [
+          1.7516015768051147,
+          1.9473387002944946,
+          3.458169937133789,
+          2.535057783126831,
+          1.7787281274795532,
+          2.3327090740203857,
+          2.764975070953369,
+          2.2454960346221924,
+          2.0184097290039062,
+          2.581630229949951,
+          2.360050916671753,
+          2.1673834323883057,
+          1.8362329006195068,
+          3.448411226272583,
+          2.1088995933532715,
+          1.7849266529083252,
+          4.575050354003906,
+          2.1073110103607178,
+          1.8655929565429688,
+          1.9884284734725952,
+          2.1644434928894043,
+          2.124669075012207
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "Additive+ranking": {
+      "top1": 0.045454545454545456,
+      "top5": 0.045454545454545456,
+      "ndcg": 0.0454545454545,
+      "func_top5": 0.8636363636363636,
+      "func_ndcg": 0.4675706692782015,
+      "rank": 31.227272727272727,
+      "endpoint_dist": 1.154505336826498,
+      "spec_margin": 1.2376486469398846,
+      "off_dist": 2.3921539837663826,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.5314987088010797,
+          0.5055599925138808,
+          0.8196104814589247,
+          0.0,
+          0.0,
+          0.4620869292484704,
+          0.8196104814589247,
+          0.5263429722316287,
+          0.5765194247461809,
+          0.8196104814589247,
+          0.15445080225351568,
+          0.4524003745215921,
+          0.5806456282157678,
+          0.49305477020115496,
+          0.3348403207942303,
+          0.4988298877885495,
+          0.6207583442782515,
+          0.0,
+          0.7553544384757664,
+          0.5059324046044343,
+          0.5477508703614005,
+          0.2816974107077558
+        ],
+        "rank": [
+          1,
+          40,
+          31,
+          58,
+          31,
+          47,
+          29,
+          29,
+          14,
+          25,
+          11,
+          37,
+          22,
+          19,
+          22,
+          22,
+          45,
+          52,
+          38,
+          44,
+          34,
+          36
+        ],
+        "endpoint_dist": [
+          1.9496577978134155,
+          0.5574938654899597,
+          1.2164744138717651,
+          0.7687604427337646,
+          0.7320241332054138,
+          0.9937659502029419,
+          0.7439819574356079,
+          1.4131525754928589,
+          1.8808938264846802,
+          1.99337899684906,
+          1.1153759956359863,
+          0.9441820979118347,
+          0.8986158967018127,
+          1.046272873878479,
+          1.3573639392852783,
+          0.8964778184890747,
+          2.548943281173706,
+          0.8479581475257874,
+          1.0793635845184326,
+          0.8028191328048706,
+          0.87811279296875,
+          0.7340478897094727
+        ],
+        "spec_margin": [
+          -0.06023609638214111,
+          1.4850876331329346,
+          2.124490261077881,
+          1.9016368389129639,
+          1.1379446983337402,
+          1.4654606580734253,
+          1.7133015394210815,
+          0.9593864679336548,
+          0.3259471654891968,
+          0.7593246698379517,
+          1.059178352355957,
+          1.2604901790618896,
+          0.9792718291282654,
+          2.7629809379577637,
+          1.251596450805664,
+          1.0095385313034058,
+          0.5528774261474609,
+          1.3290481567382812,
+          1.105086326599121,
+          1.2595843076705933,
+          1.3820161819458008,
+          1.4642577171325684
+        ],
+        "off_dist": [
+          1.8894217014312744,
+          2.042581558227539,
+          3.3409647941589355,
+          2.6703972816467285,
+          1.8699687719345093,
+          2.459226608276367,
+          2.4572834968566895,
+          2.3725390434265137,
+          2.206840991973877,
+          2.7527036666870117,
+          2.1745543479919434,
+          2.204672336578369,
+          1.8778877258300781,
+          3.809253692626953,
+          2.6089603900909424,
+          1.9060163497924805,
+          3.101820707321167,
+          2.177006244659424,
+          2.1844499111175537,
+          2.062403440475464,
+          2.260128974914551,
+          2.198305606842041
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.09090909090909091,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.12667653387175581,
+      "func_top5": 0.8636363636363636,
+      "func_ndcg": 0.465389360745805,
+      "rank": 24.5,
+      "endpoint_dist": 1.0359658057039434,
+      "spec_margin": 1.2246569259600206,
+      "off_dist": 2.260622728954662,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.35620718710766597,
+          0.0,
+          0.0,
+          0.43067655807296235,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.33860396794763925,
+          0.3796350998974945,
+          0.8196104814589247,
+          0.0,
+          0.0,
+          0.3843973922908389,
+          0.7594806419453533,
+          0.5727402829012992,
+          0.6976820249692228,
+          0.9999999999996392,
+          0.3926497992300126,
+          0.18205829600327347,
+          0.3214166825941328,
+          0.7130833312756829,
+          0.7498262898157618,
+          0.34634689786702066,
+          0.9478557351531002,
+          0.0,
+          0.6976820249692228,
+          0.26128684308056127,
+          0.3023179750304166,
+          0.3718921699781131
+        ],
+        "rank": [
+          32,
+          16,
+          43,
+          11,
+          31,
+          68,
+          16,
+          23,
+          20,
+          1,
+          6,
+          35,
+          19,
+          4,
+          41,
+          21,
+          1,
+          37,
+          25,
+          38,
+          25,
+          26
+        ],
+        "endpoint_dist": [
+          1.8301076889038086,
+          0.5907474160194397,
+          1.1353018283843994,
+          0.9802330136299133,
+          0.7048063278198242,
+          1.0184342861175537,
+          0.7584463357925415,
+          1.399523377418518,
+          1.6685357093811035,
+          1.4839903116226196,
+          1.0153692960739136,
+          0.670589029788971,
+          0.7878819704055786,
+          0.9112664461135864,
+          1.1367274522781372,
+          0.9118342995643616,
+          1.3724935054779053,
+          0.9561153650283813,
+          0.9823485016822815,
+          0.8708151578903198,
+          0.8896368741989136,
+          0.7160435318946838
+        ],
+        "spec_margin": [
+          0.11864185333251953,
+          1.3417978286743164,
+          1.9002373218536377,
+          1.2397401332855225,
+          1.1664302349090576,
+          1.25345778465271,
+          1.6654146909713745,
+          0.9994257688522339,
+          0.6466996669769287,
+          1.0209730863571167,
+          1.0659247636795044,
+          1.268491506576538,
+          1.0920313596725464,
+          2.0039072036743164,
+          1.3454638719558716,
+          0.9079654812812805,
+          1.8100285530090332,
+          1.0981038808822632,
+          1.0993499755859375,
+          1.188269019126892,
+          1.3159099817276,
+          1.394188404083252
+        ],
+        "off_dist": [
+          1.9487495422363281,
+          1.9325453042984009,
+          3.035539150238037,
+          2.219973087310791,
+          1.8712365627288818,
+          2.2718920707702637,
+          2.423861026763916,
+          2.398949146270752,
+          2.3152353763580322,
+          2.5049633979797363,
+          2.081294059753418,
+          1.9390804767608643,
+          1.879913330078125,
+          2.9151737689971924,
+          2.482191324234009,
+          1.819799780845642,
+          3.1825220584869385,
+          2.0542192459106445,
+          2.081698417663574,
+          2.059084177017212,
+          2.2055468559265137,
+          2.110231876373291
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 0.045454545454545456,
+      "top5": 0.09090909090909091,
+      "ndcg": 0.0741331706168103,
+      "func_top5": 0.8636363636363636,
+      "func_ndcg": 0.5080102766031381,
+      "rank": 46.5,
+      "endpoint_dist": 1.1265862611207096,
+      "spec_margin": 1.356340909546072,
+      "off_dist": 2.4829271652481775,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.2718596110975242,
+          0.5055599925138808,
+          0.8196104814589247,
+          0.0,
+          0.0,
+          0.4620869292484704,
+          0.8196104814589247,
+          0.5806456282157678,
+          0.7032101736292273,
+          0.8196104814589247,
+          0.2565245506872169,
+          0.4524003745215921,
+          0.5806456282157678,
+          0.7571924834648507,
+          0.7419209445012933,
+          0.4988298877885495,
+          0.8157833128587658,
+          0.0,
+          0.7553544384757664,
+          0.5059324046044343,
+          0.5477508703614005,
+          0.2816974107077558
+        ],
+        "rank": [
+          24,
+          55,
+          30,
+          75,
+          50,
+          100,
+          93,
+          53,
+          27,
+          17,
+          35,
+          77,
+          50,
+          2,
+          18,
+          50,
+          1,
+          47,
+          47,
+          79,
+          47,
+          46
+        ],
+        "endpoint_dist": [
+          2.040769577026367,
+          0.5574938654899597,
+          1.2164744138717651,
+          0.7687604427337646,
+          0.7320241332054138,
+          0.9937659502029419,
+          0.7439819574356079,
+          1.4131525754928589,
+          1.8808938264846802,
+          1.99337899684906,
+          1.1153759956359863,
+          0.9441820979118347,
+          0.8986158967018127,
+          1.046272873878479,
+          1.3573639392852783,
+          0.8964778184890747,
+          1.8436118364334106,
+          0.8479581475257874,
+          1.0793635845184326,
+          0.8028191328048706,
+          0.87811279296875,
+          0.7340478897094727
+        ],
+        "spec_margin": [
+          -0.03342318534851074,
+          1.4850876331329346,
+          2.124490261077881,
+          1.9016368389129639,
+          1.1379446983337402,
+          1.4654606580734253,
+          1.7133015394210815,
+          0.9593864679336548,
+          0.3259471654891968,
+          0.7593246698379517,
+          1.059178352355957,
+          1.2604901790618896,
+          0.9792718291282654,
+          2.7629809379577637,
+          1.251596450805664,
+          1.0095385313034058,
+          3.137294292449951,
+          1.3290481567382812,
+          1.105086326599121,
+          1.2595843076705933,
+          1.3820161819458008,
+          1.4642577171325684
+        ],
+        "off_dist": [
+          2.0073463916778564,
+          2.042581558227539,
+          3.3409647941589355,
+          2.6703972816467285,
+          1.8699687719345093,
+          2.459226608276367,
+          2.4572834968566895,
+          2.3725390434265137,
+          2.206840991973877,
+          2.7527036666870117,
+          2.1745543479919434,
+          2.204672336578369,
+          1.8778877258300781,
+          3.809253692626953,
+          2.6089603900909424,
+          1.9060163497924805,
+          4.980906009674072,
+          2.177006244659424,
+          2.1844499111175537,
+          2.062403440475464,
+          2.260128974914551,
+          2.198305606842041
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "EndpointMLP+ranking": {
+      "top1": 0.13636363636363635,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.2107243250360866,
+      "func_top5": 0.9090909090909091,
+      "func_ndcg": 0.5992360271056622,
+      "rank": 23.40909090909091,
+      "endpoint_dist": 1.027024125510996,
+      "spec_margin": 1.4001903235912323,
+      "off_dist": 2.4272144491022285,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.35620718710766597,
+          0.0,
+          0.9999999999989999,
+          0.3154648767854133,
+          0.0,
+          0.33333333333299997,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.48576880590462984,
+          0.6917444919867412,
+          0.8196104814589247,
+          0.7601875334312905,
+          0.3154648767854133,
+          0.4620869292484704,
+          0.8797403209724962,
+          0.6407754677293394,
+          0.7653078254747857,
+          0.9999999999996392,
+          0.27853764988081753,
+          0.33392190490685864,
+          0.5263429722316287,
+          0.8636394042955287,
+          0.7498262898157618,
+          0.4445272318044105,
+          0.8266931349300584,
+          0.0,
+          0.7594806419453533,
+          0.618206508964545,
+          0.55770425736042,
+          0.40362586719745763
+        ],
+        "rank": [
+          55,
+          6,
+          64,
+          1,
+          8,
+          58,
+          7,
+          31,
+          24,
+          1,
+          24,
+          29,
+          21,
+          2,
+          51,
+          18,
+          1,
+          19,
+          34,
+          24,
+          18,
+          19
+        ],
+        "endpoint_dist": [
+          1.9789191484451294,
+          0.5568957924842834,
+          1.1287992000579834,
+          0.7257286906242371,
+          0.7324892282485962,
+          0.9938557147979736,
+          0.7250792980194092,
+          1.4069081544876099,
+          1.8390849828720093,
+          0.8711382150650024,
+          1.1161667108535767,
+          0.9455131888389587,
+          0.8955139517784119,
+          1.0437345504760742,
+          1.3547587394714355,
+          0.8966653943061829,
+          1.0293209552764893,
+          0.8490437269210815,
+          1.0827422142028809,
+          0.8027507066726685,
+          0.8848638534545898,
+          0.7345583438873291
+        ],
+        "spec_margin": [
+          0.19798123836517334,
+          1.48545503616333,
+          2.4005796909332275,
+          1.6699566841125488,
+          1.137984037399292,
+          1.465325117111206,
+          1.727329969406128,
+          0.9635328054428101,
+          0.24142420291900635,
+          2.109982967376709,
+          1.0585142374038696,
+          1.2584810256958008,
+          0.9801564812660217,
+          2.764212131500244,
+          1.2506382465362549,
+          1.0092215538024902,
+          2.556349992752075,
+          1.3282328844070435,
+          1.1001033782958984,
+          1.2599939107894897,
+          1.3763890266418457,
+          1.4623425006866455
+        ],
+        "off_dist": [
+          2.1769003868103027,
+          2.0423507690429688,
+          3.529378890991211,
+          2.3956854343414307,
+          1.8704732656478882,
+          2.4591808319091797,
+          2.452409267425537,
+          2.37044095993042,
+          2.0805091857910156,
+          2.981121063232422,
+          2.1746809482574463,
+          2.2039942741394043,
+          1.8756704330444336,
+          3.8079466819763184,
+          2.6053969860076904,
+          1.9058870077133179,
+          3.5856709480285645,
+          2.177276611328125,
+          2.1828455924987793,
+          2.062744617462158,
+          2.2612528800964355,
+          2.1969008445739746
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    },
+    "Random+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.6818181818181818,
+      "func_ndcg": 0.26643206726700686,
+      "rank": 57.54545454545455,
+      "endpoint_dist": 2.9179966937411916,
+      "spec_margin": 0.07963230935010043,
+      "off_dist": 2.997629003091292,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.4749645636517497,
+          0.06425604298315837,
+          0.6082531219655256,
+          0.0,
+          0.0,
+          0.23253378338725372,
+          0.12116260022304186,
+          0.5791066180961114,
+          0.4809429920645356,
+          0.41382622312386713,
+          0.13459609419751495,
+          0.1444974152544963,
+          0.4985662109553454,
+          0.17070296381383643,
+          0.18682058839661783,
+          0.4131337440783325,
+          0.4387000482749779,
+          0.0,
+          0.06978419164316285,
+          0.15997895091352016,
+          0.10644692083067808,
+          0.5632324060204243
+        ],
+        "rank": [
+          46,
+          70,
+          78,
+          77,
+          49,
+          56,
+          27,
+          99,
+          69,
+          25,
+          19,
+          52,
+          48,
+          104,
+          25,
+          102,
+          95,
+          86,
+          42,
+          17,
+          35,
+          45
+        ],
+        "endpoint_dist": [
+          3.9145050048828125,
+          2.1377127170562744,
+          3.022123336791992,
+          2.1810457706451416,
+          3.4453420639038086,
+          2.9303030967712402,
+          2.341578483581543,
+          2.6385629177093506,
+          3.595100164413452,
+          3.311495065689087,
+          2.3130178451538086,
+          1.4427813291549683,
+          2.7334203720092773,
+          4.076140880584717,
+          2.273077964782715,
+          3.58622670173645,
+          4.741064548492432,
+          2.7654285430908203,
+          5.706614971160889,
+          1.9952198266983032,
+          1.2292784452438354,
+          1.815887212753296
+        ],
+        "spec_margin": [
+          -0.2119739055633545,
+          0.597123384475708,
+          -0.8162951469421387,
+          0.6589181423187256,
+          -0.03424239158630371,
+          -0.1356971263885498,
+          0.1988682746887207,
+          0.0703420639038086,
+          -0.07148480415344238,
+          -1.5779609680175781,
+          0.6732721328735352,
+          0.686278223991394,
+          0.288144588470459,
+          -0.2951502799987793,
+          1.7428030967712402,
+          0.08529400825500488,
+          -2.770522356033325,
+          0.42055368423461914,
+          -0.3490447998046875,
+          0.4473620653152466,
+          1.1670209169387817,
+          0.978302001953125
+        ],
+        "off_dist": [
+          3.702531099319458,
+          2.7348361015319824,
+          2.2058281898498535,
+          2.839963912963867,
+          3.411099672317505,
+          2.7946059703826904,
+          2.5404467582702637,
+          2.708904981613159,
+          3.5236153602600098,
+          1.7335340976715088,
+          2.9862899780273438,
+          2.1290595531463623,
+          3.0215649604797363,
+          3.7809906005859375,
+          4.015881061553955,
+          3.671520709991455,
+          1.9705421924591064,
+          3.1859822273254395,
+          5.357570171356201,
+          2.44258189201355,
+          2.396299362182617,
+          2.794189214706421
+        ],
+        "clf": []
+      },
+      "n_targets": 22,
+      "random_top5": 0.047619047619047616
+    }
+  }
+}

experiments/results/norman_timing_scaling.json

@@ -0,0 +1,89 @@
+{
+  "combo_guidance": {
+    "top1": 0.5384615384615384,
+    "top5": 0.7307692307692307,
+    "ndcg": 0.6529941534069046,
+    "endpoint_dist": 1.2043300362733693
+  },
+  "search_timing": {
+    "ranking_only": {
+      "sec_per_target": 0.01,
+      "queries": 105,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.08727248090397387,
+      "endpoint_dist": 1.068045285615054
+    },
+    "random_opt": {
+      "sec_per_target": 0.058,
+      "queries": 25,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.11048529809413464,
+      "endpoint_dist": 1.6047562198205427
+    },
+    "mean_top_init": {
+      "sec_per_target": 0.058,
+      "queries": 35,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.15070294894465516,
+      "endpoint_dist": 1.5476418218829415
+    },
+    "guidance_no_norm": {
+      "sec_per_target": 0.061,
+      "queries": 25,
+      "top5": 0.18181818181818182,
+      "ndcg": 0.10475755936295951,
+      "endpoint_dist": 2.317273969000036
+    },
+    "guidance_norm": {
+      "sec_per_target": 0.064,
+      "queries": 25,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0968604433440603,
+      "endpoint_dist": 1.5734560652212664
+    },
+    "guidance_rerank": {
+      "sec_per_target": 0.065,
+      "queries": 35,
+      "top5": 0.13636363636363635,
+      "ndcg": 0.0741331706168103,
+      "endpoint_dist": 1.0795677466826006
+    }
+  },
+  "guidance_step_time": {
+    "0": 0.01,
+    "5": 0.024,
+    "10": 0.034,
+    "25": 0.065,
+    "50": 0.112,
+    "100": 0.201
+  },
+  "data_scaling_counts": {
+    "0.1": {
+      "n_perts": 16,
+      "cells_per_pert": 354
+    },
+    "0.25": {
+      "n_perts": 41,
+      "cells_per_pert": 354
+    },
+    "0.5": {
+      "n_perts": 83,
+      "cells_per_pert": 354
+    },
+    "0.75": {
+      "n_perts": 124,
+      "cells_per_pert": 354
+    },
+    "1.0": {
+      "n_perts": 166,
+      "cells_per_pert": 354
+    }
+  },
+  "heldout_gene_mse": {
+    "op_only": 0.0034,
+    "gene_only": 0.0035,
+    "random_id": 0.0034,
+    "gene_op": 0.0035,
+    "gene_pathway_op": 0.0032
+  }
+}

experiments/results/pivot_vs_gears.json

@@ -0,0 +1,7 @@
+{
+  "n_common": 45,
+  "pivot_pearson_de_expr": 0.9713062047958374,
+  "pivot_delta_de_corr": 0.8417869806289673,
+  "gears_pearson_de_expr": 0.8981071008576287,
+  "gears_pearson_all": 0.9916384220123291
+}

experiments/results/replogle_k562_nom_cell.json

@@ -0,0 +1,2773 @@
+{
+  "split": "cell",
+  "reward": "centroid",
+  "n_candidates": 1805,
+  "n_targets": 40,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.025,
+      "top5": 0.15,
+      "ndcg": 0.13208262047549235,
+      "func_top5": 0.725,
+      "func_ndcg": 0.3600167442071178,
+      "rank": 194.2,
+      "endpoint_dist": 1.2301450669765472,
+      "spec_margin": 1.2941597983241082,
+      "off_dist": 2.5243048638105394,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.3154648767854133,
+          0.0,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.0,
+          0.43067655807296235,
+          0.0,
+          0.0,
+          0.30102999566368016,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.6309297535708266,
+          0.6309297535708266,
+          0.0,
+          0.0,
+          0.0,
+          0.35620718710766597,
+          0.0,
+          0.0,
+          0.0,
+          0.35620718710766597,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.0,
+          0.9334235959386915,
+          0.0,
+          0.8196104814589247,
+          0.8765170386988083,
+          0.672509164948684,
+          0.9999999999996392,
+          0.7863222794284508,
+          0.06978419164316285,
+          0.05430265598413898,
+          0.0,
+          0.8973000184165562,
+          0.11381311447976694,
+          0.0,
+          0.10860531196827795,
+          0.5254078484384431,
+          0.408403654282275,
+          0.1803895185407146,
+          0.0,
+          0.3821659031256479,
+          0.06012983951357154,
+          0.5499613274410384,
+          0.7988275017411766,
+          0.6053288745612656,
+          0.3123817114824851,
+          0.3155894990647003,
+          0.12851208596631675,
+          0.39817794789001704,
+          0.4170340107060822,
+          0.4057842583350576,
+          0.883866524442083,
+          0.20400787375012425,
+          0.5065517856223779,
+          0.11381311447976694,
+          0.0,
+          0.13459609419751495,
+          0.0,
+          0.30055347352382106,
+          0.29250165296063585,
+          0.1444974152544963
+        ],
+        "rank": [
+          410,
+          2,
+          134,
+          23,
+          8,
+          49,
+          1,
+          2,
+          25,
+          108,
+          290,
+          4,
+          41,
+          561,
+          9,
+          77,
+          151,
+          185,
+          103,
+          59,
+          232,
+          2,
+          2,
+          758,
+          121,
+          346,
+          6,
+          20,
+          28,
+          606,
+          6,
+          145,
+          651,
+          281,
+          789,
+          92,
+          332,
+          739,
+          105,
+          265
+        ],
+        "endpoint_dist": [
+          0.7788619995117188,
+          1.0574302673339844,
+          0.8981291055679321,
+          1.1791802644729614,
+          1.9431819915771484,
+          1.2274514436721802,
+          1.570068120956421,
+          0.8371641039848328,
+          1.1645748615264893,
+          1.684700846672058,
+          1.1419439315795898,
+          1.1942020654678345,
+          1.262868881225586,
+          0.9169500470161438,
+          0.9980612397193909,
+          1.5394939184188843,
+          1.5928115844726562,
+          0.8636164665222168,
+          0.5946463346481323,
+          0.6364714503288269,
+          0.9466665983200073,
+          1.3823901414871216,
+          1.350879430770874,
+          1.2104741334915161,
+          1.2438565492630005,
+          1.172344446182251,
+          1.0656836032867432,
+          0.8919244408607483,
+          1.4183169603347778,
+          1.4194594621658325,
+          2.675273895263672,
+          1.5831266641616821,
+          1.1956759691238403,
+          1.3301384449005127,
+          1.29361891746521,
+          0.6645997166633606,
+          1.49713134765625,
+          1.0711721181869507,
+          1.2048929929733276,
+          1.5063679218292236
+        ],
+        "spec_margin": [
+          1.6204698085784912,
+          1.539564609527588,
+          1.7510727643966675,
+          3.4558401107788086,
+          0.8276746273040771,
+          1.0009196996688843,
+          1.8262240886688232,
+          1.4334993362426758,
+          1.2165818214416504,
+          0.6880141496658325,
+          1.535628318786621,
+          2.1577844619750977,
+          1.1765739917755127,
+          1.50803542137146,
+          0.6875573992729187,
+          0.7675319910049438,
+          1.07802152633667,
+          2.0859181880950928,
+          1.3635112047195435,
+          1.8178761005401611,
+          1.6484087705612183,
+          0.6289695501327515,
+          1.1417169570922852,
+          1.1009031534194946,
+          1.310752272605896,
+          1.564943552017212,
+          1.851792812347412,
+          1.88722562789917,
+          1.2785838842391968,
+          0.3919602632522583,
+          0.6825873851776123,
+          0.3784511089324951,
+          1.6347113847732544,
+          1.420490026473999,
+          0.4672729969024658,
+          1.5788979530334473,
+          0.6988112926483154,
+          0.9903422594070435,
+          1.238538384437561,
+          0.3327326774597168
+        ],
+        "off_dist": [
+          2.39933180809021,
+          2.5969948768615723,
+          2.6492018699645996,
+          4.6350202560424805,
+          2.7708566188812256,
+          2.2283711433410645,
+          3.396292209625244,
+          2.2706634998321533,
+          2.3811566829681396,
+          2.3727149963378906,
+          2.677572250366211,
+          3.3519866466522217,
+          2.4394428730010986,
+          2.424985408782959,
+          1.6856186389923096,
+          2.307025909423828,
+          2.670833110809326,
+          2.9495346546173096,
+          1.9581575393676758,
+          2.454347610473633,
+          2.5950753688812256,
+          2.011359691619873,
+          2.492596387863159,
+          2.3113772869110107,
+          2.5546088218688965,
+          2.737287998199463,
+          2.9174764156341553,
+          2.7791500091552734,
+          2.6969008445739746,
+          1.8114197254180908,
+          3.357861280441284,
+          1.9615777730941772,
+          2.8303873538970947,
+          2.7506284713745117,
+          1.7608919143676758,
+          2.243497610092163,
+          2.1959426403045654,
+          2.061514377593994,
+          2.4434313774108887,
+          1.8391005992889404
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "PIVOT-guidance": {
+      "top1": 0.05,
+      "top5": 0.075,
+      "ndcg": 0.06076691395177405,
+      "func_top5": 0.675,
+      "func_ndcg": 0.28302129846659285,
+      "rank": 197.55,
+      "endpoint_dist": 1.3085480779409409,
+      "spec_margin": 1.1785019636154175,
+      "off_dist": 2.4870500385761263,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.43067655807296235,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.0,
+          0.5490262036478528,
+          0.17394295399333848,
+          0.8196104814589247,
+          0.6976820249692228,
+          0.6536531021326187,
+          0.8196104814589247,
+          0.05690655723988347,
+          0.052144264846539085,
+          0.0,
+          0.0,
+          0.8196104814589247,
+          0.24557383318562032,
+          0.0,
+          0.11381311447976694,
+          0.3731885353501183,
+          0.43146614992049087,
+          0.1803895185407146,
+          0.0,
+          0.42886224866474637,
+          0.06978419164316285,
+          0.5548688818687583,
+          0.5031180611983256,
+          0.5838689104894559,
+          0.1788350137295854,
+          0.3934156554966727,
+          0.0,
+          0.2900442269536079,
+          0.0,
+          0.16918066159999412,
+          0.7032101736292273,
+          0.1298338018041706,
+          0.4492728162919408,
+          0.0,
+          0.0,
+          0.06978419164316285,
+          0.0,
+          0.24048715545699928,
+          0.4015524750470583,
+          0.16811577046390594
+        ],
+        "rank": [
+          411,
+          12,
+          135,
+          26,
+          17,
+          56,
+          11,
+          12,
+          33,
+          113,
+          291,
+          14,
+          4,
+          561,
+          18,
+          84,
+          156,
+          188,
+          106,
+          62,
+          234,
+          1,
+          1,
+          758,
+          128,
+          349,
+          15,
+          28,
+          38,
+          607,
+          14,
+          147,
+          651,
+          283,
+          790,
+          97,
+          336,
+          739,
+          111,
+          265
+        ],
+        "endpoint_dist": [
+          0.7788619995117188,
+          2.117431879043579,
+          0.8981291055679321,
+          1.1791802644729614,
+          2.233966827392578,
+          1.2716954946517944,
+          2.434670925140381,
+          1.1498734951019287,
+          1.1645748615264893,
+          1.684700846672058,
+          1.1419439315795898,
+          1.521671175956726,
+          1.262868881225586,
+          0.9169500470161438,
+          1.0522229671478271,
+          1.5394939184188843,
+          1.5928115844726562,
+          0.8636164665222168,
+          0.5946463346481323,
+          0.6364714503288269,
+          0.9466665983200073,
+          1.3372395038604736,
+          1.2468385696411133,
+          1.2104741334915161,
+          1.4099704027175903,
+          1.172344446182251,
+          1.0656836032867432,
+          0.8919244408607483,
+          1.6904374361038208,
+          1.4194594621658325,
+          2.675273895263672,
+          1.5831266641616821,
+          1.1956759691238403,
+          1.3301384449005127,
+          1.29361891746521,
+          0.6645997166633606,
+          1.390235424041748,
+          1.0711721181869507,
+          1.2048929929733276,
+          1.5063679218292236
+        ],
+        "spec_margin": [
+          1.6204698085784912,
+          -0.10165095329284668,
+          1.7510727643966675,
+          3.4558401107788086,
+          -0.023256778717041016,
+          1.0417529344558716,
+          0.268704891204834,
+          1.0524473190307617,
+          1.2165818214416504,
+          0.6880141496658325,
+          1.535628318786621,
+          2.293555736541748,
+          1.1765739917755127,
+          1.50803542137146,
+          0.6877552270889282,
+          0.7675319910049438,
+          1.07802152633667,
+          2.0859181880950928,
+          1.3635112047195435,
+          1.8178761005401611,
+          1.6484087705612183,
+          0.7179131507873535,
+          1.3804106712341309,
+          1.1009031534194946,
+          0.9676400423049927,
+          1.564943552017212,
+          1.851792812347412,
+          1.88722562789917,
+          0.8419169187545776,
+          0.3919602632522583,
+          0.6825873851776123,
+          0.3784511089324951,
+          1.6347113847732544,
+          1.420490026473999,
+          0.4672729969024658,
+          1.5788979530334473,
+          0.7785556316375732,
+          0.9903422594070435,
+          1.238538384437561,
+          0.3327326774597168
+        ],
+        "off_dist": [
+          2.39933180809021,
+          2.0157809257507324,
+          2.6492018699645996,
+          4.6350202560424805,
+          2.210710048675537,
+          2.313448429107666,
+          2.703375816345215,
+          2.2023208141326904,
+          2.3811566829681396,
+          2.3727149963378906,
+          2.677572250366211,
+          3.8152270317077637,
+          2.4394428730010986,
+          2.424985408782959,
+          1.7399781942367554,
+          2.307025909423828,
+          2.670833110809326,
+          2.9495346546173096,
+          1.9581575393676758,
+          2.454347610473633,
+          2.5950753688812256,
+          2.055152654647827,
+          2.627249240875244,
+          2.3113772869110107,
+          2.377610445022583,
+          2.737287998199463,
+          2.9174764156341553,
+          2.7791500091552734,
+          2.5323543548583984,
+          1.8114197254180908,
+          3.357861280441284,
+          1.9615777730941772,
+          2.8303873538970947,
+          2.7506284713745117,
+          1.7608919143676758,
+          2.243497610092163,
+          2.1687910556793213,
+          2.061514377593994,
+          2.4434313774108887,
+          1.8391005992889404
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "Additive+ranking": {
+      "top1": 0.95,
+      "top5": 0.975,
+      "ndcg": 0.9657732438383206,
+      "func_top5": 1.0,
+      "func_ndcg": 0.6652048246826138,
+      "rank": 1.675,
+      "endpoint_dist": 0.8411503508687019,
+      "spec_margin": 2.1007207214832304,
+      "off_dist": 2.9418710708618163,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.3607790370814292,
+          0.9999999999996392,
+          0.4129233019279683,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9456973440155002,
+          0.4305632287245921,
+          0.4745921515611962,
+          0.5426118416977634,
+          0.9999999999996392,
+          0.4305632287245921,
+          0.4646120878678205,
+          0.5570272730063581,
+          0.873309251116593,
+          0.687334748339199,
+          0.562183009575809,
+          0.3607790370814292,
+          0.5082527656822236,
+          0.11381311447976694,
+          0.6146996865129016,
+          0.8654039058021243,
+          0.7236207412564216,
+          0.759496136636826,
+          0.8909491779132166,
+          0.4927712300231997,
+          0.3607790370814292,
+          0.7878767842395802,
+          0.9223104630420079,
+          0.9999999999996392,
+          0.6553540821924644,
+          0.5299596986814235,
+          0.6734442887418696,
+          0.3607790370814292,
+          0.3607790370814292,
+          0.5721363965748284,
+          0.8455491977461236,
+          0.9357439570164808,
+          0.5314987088010797
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          2,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          27,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.7173737287521362,
+          0.7461453080177307,
+          0.785219132900238,
+          0.538507878780365,
+          1.097732424736023,
+          0.6946150660514832,
+          0.7801607251167297,
+          0.7369425296783447,
+          1.015824794769287,
+          1.3046245574951172,
+          1.0173161029815674,
+          0.9735252857208252,
+          1.092432975769043,
+          0.8244563341140747,
+          0.674659013748169,
+          0.9909878373146057,
+          0.9689846038818359,
+          0.732428252696991,
+          0.5732553005218506,
+          0.6204644441604614,
+          0.9914161562919617,
+          0.7275379300117493,
+          0.7922949194908142,
+          0.8339179158210754,
+          0.9220225214958191,
+          0.8180479407310486,
+          0.8987788558006287,
+          0.7535209059715271,
+          0.7320271730422974,
+          1.037440299987793,
+          1.0151782035827637,
+          0.7500125169754028,
+          0.8864849805831909,
+          0.7386977672576904,
+          0.9256577491760254,
+          0.8188286423683167,
+          0.6080302000045776,
+          0.6138859391212463,
+          0.9211081266403198,
+          0.9754689931869507
+        ],
+        "spec_margin": [
+          1.9674495458602905,
+          2.103719711303711,
+          2.245748996734619,
+          3.573702335357666,
+          2.0890731811523438,
+          1.7624471187591553,
+          3.303356885910034,
+          1.8128483295440674,
+          2.0443527698516846,
+          2.065767288208008,
+          1.6571438312530518,
+          2.6439476013183594,
+          2.032092571258545,
+          1.785965085029602,
+          1.3168447017669678,
+          2.013680934906006,
+          2.1790592670440674,
+          2.5351104736328125,
+          1.5227258205413818,
+          2.132510185241699,
+          1.9073677062988281,
+          1.880821704864502,
+          2.3598074913024902,
+          1.7640490531921387,
+          1.7445905208587646,
+          2.3762378692626953,
+          2.4540412425994873,
+          2.107139825820923,
+          2.3813529014587402,
+          1.7228913307189941,
+          3.270918846130371,
+          1.9885872602462769,
+          2.2718491554260254,
+          2.49912691116333,
+          1.3993134498596191,
+          1.4822916984558105,
+          2.1268625259399414,
+          1.731029987335205,
+          2.133510112762451,
+          1.6394926309585571
+        ],
+        "off_dist": [
+          2.6848232746124268,
+          2.849864959716797,
+          3.030968189239502,
+          4.112210273742676,
+          3.1868057250976562,
+          2.457062244415283,
+          4.083517551422119,
+          2.549790859222412,
+          3.0601775646209717,
+          3.370391845703125,
+          2.674459934234619,
+          3.6174728870391846,
+          3.124525547027588,
+          2.6104214191436768,
+          1.9915037155151367,
+          3.004668712615967,
+          3.1480438709259033,
+          3.2675387859344482,
+          2.0959811210632324,
+          2.752974510192871,
+          2.8987839221954346,
+          2.6083595752716064,
+          3.152102470397949,
+          2.5979669094085693,
+          2.6666131019592285,
+          3.1942858695983887,
+          3.3528201580047607,
+          2.8606607913970947,
+          3.113379955291748,
+          2.760331630706787,
+          4.286097049713135,
+          2.7385997772216797,
+          3.158334255218506,
+          3.2378246784210205,
+          2.3249711990356445,
+          2.3011202812194824,
+          2.7348926067352295,
+          2.3449158668518066,
+          3.0546181201934814,
+          2.614961624145508
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.9,
+      "top5": 0.95,
+      "ndcg": 0.9193426403608076,
+      "func_top5": 0.975,
+      "func_ndcg": 0.6613311093652607,
+      "rank": 2.2,
+      "endpoint_dist": 1.0196396365761757,
+      "spec_margin": 1.4613386183977126,
+      "off_dist": 2.4809782683849333,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.3868528072341547,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.3868528072341547,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.4874697859644756,
+          0.9999999999996392,
+          0.5286633333094181,
+          0.8196104814589247,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.8893946731020875,
+          0.4209088765950008,
+          0.4745921515611962,
+          0.3607790370814292,
+          0.9999999999996392,
+          0.6947009419882879,
+          0.17973649046676787,
+          0.5394675755622982,
+          0.8266931349300584,
+          0.689545205418837,
+          0.7122606394323479,
+          0.5721363965748284,
+          0.6038700937941731,
+          0.5776645452348329,
+          0.7489233686198631,
+          0.8286609472620452,
+          0.5082527656822236,
+          0.873309251116593,
+          0.4948192717077505,
+          0.4384685740390607,
+          0.3607790370814292,
+          0.7054249548886042,
+          0.7515272698756076,
+          0.9999999999996392,
+          0.5887931728229896,
+          0.4209088765950008,
+          0.5929193762925764,
+          0.580887827508521,
+          0.17868853848086888,
+          0.7589569849714464,
+          0.6821957196755621,
+          0.9456973440155002,
+          0.6165377315019861
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          29,
+          1,
+          1,
+          1,
+          5,
+          1,
+          1,
+          1,
+          1,
+          1,
+          5,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          13,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.6616038680076599,
+          1.2548247575759888,
+          0.7813584804534912,
+          1.158394694328308,
+          1.5520241260528564,
+          0.923303484916687,
+          1.76686429977417,
+          0.8745992183685303,
+          1.070665955543518,
+          1.2548863887786865,
+          0.9371170401573181,
+          1.1795976161956787,
+          1.0225927829742432,
+          0.7508274912834167,
+          1.0226606130599976,
+          1.131973385810852,
+          1.378308653831482,
+          0.7616466879844666,
+          0.7265413999557495,
+          0.8877161741256714,
+          0.6296344995498657,
+          1.0646913051605225,
+          1.0984606742858887,
+          0.8197231292724609,
+          1.1387754678726196,
+          0.7523069381713867,
+          0.8117942214012146,
+          1.0146288871765137,
+          1.2389315366744995,
+          0.7988754510879517,
+          1.6972215175628662,
+          1.050825834274292,
+          1.1287428140640259,
+          1.0291059017181396,
+          0.812923789024353,
+          1.0367337465286255,
+          0.8561542630195618,
+          0.7039358019828796,
+          0.7540229558944702,
+          1.2505896091461182
+        ],
+        "spec_margin": [
+          1.7379565238952637,
+          0.9655307531356812,
+          1.852024793624878,
+          2.293393135070801,
+          0.8486089706420898,
+          1.3431891202926636,
+          1.0528581142425537,
+          1.4652132987976074,
+          1.4513288736343384,
+          1.3505439758300781,
+          1.6941022872924805,
+          1.6614453792572021,
+          1.5319406986236572,
+          1.5675017833709717,
+          0.7009553909301758,
+          1.4009870290756226,
+          1.2738226652145386,
+          2.199557304382324,
+          1.1724765300750732,
+          1.6043356657028198,
+          2.161515235900879,
+          1.0891995429992676,
+          1.498476505279541,
+          1.4074206352233887,
+          1.2639492750167847,
+          2.045914888381958,
+          2.110194206237793,
+          1.5556697845458984,
+          1.3904625177383423,
+          1.3211067914962769,
+          1.302931785583496,
+          0.9997358322143555,
+          1.7146793603897095,
+          1.830432653427124,
+          1.124277949333191,
+          1.097406268119812,
+          1.4047913551330566,
+          1.422738790512085,
+          1.796676754951477,
+          0.748192310333252
+        ],
+        "off_dist": [
+          2.3995604515075684,
+          2.22035551071167,
+          2.633383274078369,
+          3.4517879486083984,
+          2.4006330966949463,
+          2.2664926052093506,
+          2.8197224140167236,
+          2.3398125171661377,
+          2.5219948291778564,
+          2.6054303646087646,
+          2.6312193870544434,
+          2.841042995452881,
+          2.5545334815979004,
+          2.318329334259033,
+          1.7236160039901733,
+          2.5329604148864746,
+          2.6521313190460205,
+          2.9612040519714355,
+          1.8990179300308228,
+          2.492051839828491,
+          2.791149854660034,
+          2.15389084815979,
+          2.5969371795654297,
+          2.2271437644958496,
+          2.4027247428894043,
+          2.7982218265533447,
+          2.9219884872436523,
+          2.570298671722412,
+          2.629394054412842,
+          2.1199822425842285,
+          3.0001533031463623,
+          2.0505616664886475,
+          2.8434221744537354,
+          2.8595385551452637,
+          1.937201738357544,
+          2.1341400146484375,
+          2.2609455585479736,
+          2.1266746520996094,
+          2.5506997108459473,
+          1.9987819194793701
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 0.95,
+      "top5": 0.975,
+      "ndcg": 0.9657732438383206,
+      "func_top5": 1.0,
+      "func_ndcg": 0.6652048246826138,
+      "rank": 1.675,
+      "endpoint_dist": 0.8411503508687019,
+      "spec_margin": 2.1007207214832304,
+      "off_dist": 2.9418710708618163,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.3607790370814292,
+          0.9999999999996392,
+          0.4129233019279683,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9456973440155002,
+          0.4305632287245921,
+          0.4745921515611962,
+          0.5426118416977634,
+          0.9999999999996392,
+          0.4305632287245921,
+          0.4646120878678205,
+          0.5570272730063581,
+          0.873309251116593,
+          0.687334748339199,
+          0.562183009575809,
+          0.3607790370814292,
+          0.5082527656822236,
+          0.11381311447976694,
+          0.6146996865129016,
+          0.8654039058021243,
+          0.7236207412564216,
+          0.759496136636826,
+          0.8909491779132166,
+          0.4927712300231997,
+          0.3607790370814292,
+          0.7878767842395802,
+          0.9223104630420079,
+          0.9999999999996392,
+          0.6553540821924644,
+          0.5299596986814235,
+          0.6734442887418696,
+          0.3607790370814292,
+          0.3607790370814292,
+          0.5721363965748284,
+          0.8455491977461236,
+          0.9357439570164808,
+          0.5314987088010797
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          2,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          27,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.7173737287521362,
+          0.7461453080177307,
+          0.785219132900238,
+          0.538507878780365,
+          1.097732424736023,
+          0.6946150660514832,
+          0.7801607251167297,
+          0.7369425296783447,
+          1.015824794769287,
+          1.3046245574951172,
+          1.0173161029815674,
+          0.9735252857208252,
+          1.092432975769043,
+          0.8244563341140747,
+          0.674659013748169,
+          0.9909878373146057,
+          0.9689846038818359,
+          0.732428252696991,
+          0.5732553005218506,
+          0.6204644441604614,
+          0.9914161562919617,
+          0.7275379300117493,
+          0.7922949194908142,
+          0.8339179158210754,
+          0.9220225214958191,
+          0.8180479407310486,
+          0.8987788558006287,
+          0.7535209059715271,
+          0.7320271730422974,
+          1.037440299987793,
+          1.0151782035827637,
+          0.7500125169754028,
+          0.8864849805831909,
+          0.7386977672576904,
+          0.9256577491760254,
+          0.8188286423683167,
+          0.6080302000045776,
+          0.6138859391212463,
+          0.9211081266403198,
+          0.9754689931869507
+        ],
+        "spec_margin": [
+          1.9674495458602905,
+          2.103719711303711,
+          2.245748996734619,
+          3.573702335357666,
+          2.0890731811523438,
+          1.7624471187591553,
+          3.303356885910034,
+          1.8128483295440674,
+          2.0443527698516846,
+          2.065767288208008,
+          1.6571438312530518,
+          2.6439476013183594,
+          2.032092571258545,
+          1.785965085029602,
+          1.3168447017669678,
+          2.013680934906006,
+          2.1790592670440674,
+          2.5351104736328125,
+          1.5227258205413818,
+          2.132510185241699,
+          1.9073677062988281,
+          1.880821704864502,
+          2.3598074913024902,
+          1.7640490531921387,
+          1.7445905208587646,
+          2.3762378692626953,
+          2.4540412425994873,
+          2.107139825820923,
+          2.3813529014587402,
+          1.7228913307189941,
+          3.270918846130371,
+          1.9885872602462769,
+          2.2718491554260254,
+          2.49912691116333,
+          1.3993134498596191,
+          1.4822916984558105,
+          2.1268625259399414,
+          1.731029987335205,
+          2.133510112762451,
+          1.6394926309585571
+        ],
+        "off_dist": [
+          2.6848232746124268,
+          2.849864959716797,
+          3.030968189239502,
+          4.112210273742676,
+          3.1868057250976562,
+          2.457062244415283,
+          4.083517551422119,
+          2.549790859222412,
+          3.0601775646209717,
+          3.370391845703125,
+          2.674459934234619,
+          3.6174728870391846,
+          3.124525547027588,
+          2.6104214191436768,
+          1.9915037155151367,
+          3.004668712615967,
+          3.1480438709259033,
+          3.2675387859344482,
+          2.0959811210632324,
+          2.752974510192871,
+          2.8987839221954346,
+          2.6083595752716064,
+          3.152102470397949,
+          2.5979669094085693,
+          2.6666131019592285,
+          3.1942858695983887,
+          3.3528201580047607,
+          2.8606607913970947,
+          3.113379955291748,
+          2.760331630706787,
+          4.286097049713135,
+          2.7385997772216797,
+          3.158334255218506,
+          3.2378246784210205,
+          2.3249711990356445,
+          2.3011202812194824,
+          2.7348926067352295,
+          2.3449158668518066,
+          3.0546181201934814,
+          2.614961624145508
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "EndpointMLP+ranking": {
+      "top1": 0.95,
+      "top5": 0.975,
+      "ndcg": 0.9657732438383206,
+      "func_top5": 1.0,
+      "func_ndcg": 0.6643281414589618,
+      "rank": 1.6,
+      "endpoint_dist": 0.8406605675816536,
+      "spec_margin": 2.099722996354103,
+      "off_dist": 2.9403835624456405,
+      "clf": null,
+      "_per": {
+        "top1": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "ndcg": [
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.6309297535708266,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.0,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999,
+          0.9999999999989999
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.3607790370814292,
+          0.9999999999996392,
+          0.4129233019279683,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9999999999996392,
+          0.9456973440155002,
+          0.4305632287245921,
+          0.4745921515611962,
+          0.5426118416977634,
+          0.9999999999996392,
+          0.4305632287245921,
+          0.4646120878678205,
+          0.5570272730063581,
+          0.873309251116593,
+          0.691460951808786,
+          0.5654062918494971,
+          0.3607790370814292,
+          0.5082527656822236,
+          0.11381311447976694,
+          0.5548688818687583,
+          0.8621806235284363,
+          0.7277469447260085,
+          0.7650242852968305,
+          0.8909491779132166,
+          0.50788035359167,
+          0.3607790370814292,
+          0.7878767842395802,
+          0.9223104630420079,
+          0.9999999999996392,
+          0.6553540821924644,
+          0.5299596986814235,
+          0.6693180852722828,
+          0.3607790370814292,
+          0.3607790370814292,
+          0.5721363965748284,
+          0.8455491977461236,
+          0.9357439570164808,
+          0.5314987088010797
+        ],
+        "rank": [
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          2,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          24,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1,
+          1
+        ],
+        "endpoint_dist": [
+          0.7115392684936523,
+          0.7434130311012268,
+          0.782884955406189,
+          0.5349298119544983,
+          1.1008118391036987,
+          0.6904085278511047,
+          0.7770178914070129,
+          0.7384058237075806,
+          1.0147595405578613,
+          1.302067756652832,
+          1.015263557434082,
+          0.9738177061080933,
+          1.0930898189544678,
+          0.8314464092254639,
+          0.6720605492591858,
+          0.9890942573547363,
+          0.9730018973350525,
+          0.7305974960327148,
+          0.57706218957901,
+          0.6231805682182312,
+          0.989425539970398,
+          0.7258337736129761,
+          0.789780855178833,
+          0.8334861993789673,
+          0.9197162985801697,
+          0.8218328356742859,
+          0.9016136527061462,
+          0.7549859285354614,
+          0.7306838631629944,
+          1.038749098777771,
+          1.0174500942230225,
+          0.74358731508255,
+          0.8858193159103394,
+          0.7361005544662476,
+          0.9223771095275879,
+          0.824489951133728,
+          0.6082383394241333,
+          0.6169683933258057,
+          0.9182528257369995,
+          0.9721778631210327
+        ],
+        "spec_margin": [
+          1.9711594581604004,
+          2.0922412872314453,
+          2.2475204467773438,
+          3.5705361366271973,
+          2.0842247009277344,
+          1.7688353061676025,
+          3.3032357692718506,
+          1.8071125745773315,
+          2.0441155433654785,
+          2.0665674209594727,
+          1.657548427581787,
+          2.640410900115967,
+          2.03220272064209,
+          1.7836847305297852,
+          1.3169708251953125,
+          2.016561508178711,
+          2.173065185546875,
+          2.5372564792633057,
+          1.520702600479126,
+          2.132035732269287,
+          1.905267596244812,
+          1.8801466226577759,
+          2.3641176223754883,
+          1.7644177675247192,
+          1.7468843460083008,
+          2.3724145889282227,
+          2.451648235321045,
+          2.107424259185791,
+          2.3798720836639404,
+          1.7198468446731567,
+          3.2619473934173584,
+          1.9915785789489746,
+          2.274409770965576,
+          2.5012764930725098,
+          1.3998422622680664,
+          1.4798988103866577,
+          2.1273646354675293,
+          1.7228996753692627,
+          2.1340837478637695,
+          1.637590765953064
+        ],
+        "off_dist": [
+          2.6826987266540527,
+          2.8356542587280273,
+          3.030405282974243,
+          4.105465888977051,
+          3.1850366592407227,
+          2.4592437744140625,
+          4.080253601074219,
+          2.545518398284912,
+          3.05887508392334,
+          3.3686351776123047,
+          2.672811985015869,
+          3.6142284870147705,
+          3.1252925395965576,
+          2.615131139755249,
+          1.989031434059143,
+          3.0056557655334473,
+          3.1460671424865723,
+          3.2678539752960205,
+          2.097764730453491,
+          2.755216360092163,
+          2.89469313621521,
+          2.605980396270752,
+          3.1538984775543213,
+          2.5979039669036865,
+          2.6666007041931152,
+          3.1942474842071533,
+          3.353261947631836,
+          2.862410068511963,
+          3.11055588722229,
+          2.7585959434509277,
+          4.279397487640381,
+          2.73516583442688,
+          3.160229206085205,
+          3.237377166748047,
+          2.3222193717956543,
+          2.3043887615203857,
+          2.735602855682373,
+          2.3398680686950684,
+          3.0523366928100586,
+          2.6097686290740967
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "Random+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.6,
+      "func_ndcg": 0.31792464420168004,
+      "rank": 820.225,
+      "endpoint_dist": 3.069971165060997,
+      "spec_margin": 0.3528890937566757,
+      "off_dist": 3.4228602588176726,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0
+        ],
+        "func_ndcg": [
+          0.0,
+          0.7419209445012933,
+          0.16595737932630603,
+          0.18682058839661783,
+          0.6409219429780558,
+          0.6433793689850839,
+          0.5679779905079548,
+          0.6817911049877218,
+          0.0,
+          0.0,
+          0.0,
+          0.4637557067110293,
+          0.0,
+          0.0,
+          0.0,
+          0.44771831148081165,
+          0.7653078254747857,
+          0.06978419164316285,
+          0.0,
+          0.672509164948684,
+          0.0,
+          0.4524003745215921,
+          0.5817105193518559,
+          0.5790911234046386,
+          0.6488908097392743,
+          0.6409219429780558,
+          0.0,
+          0.6929197325758784,
+          0.0,
+          0.6850143872614097,
+          0.3901923732229845,
+          0.0,
+          0.6772714573420283,
+          0.05430265598413898,
+          0.07768953695763149,
+          0.0,
+          0.0,
+          0.5606285047646798,
+          0.6281078300215263,
+          0.0
+        ],
+        "rank": [
+          1484,
+          86,
+          12,
+          728,
+          1567,
+          428,
+          72,
+          141,
+          744,
+          1494,
+          1327,
+          147,
+          994,
+          366,
+          1309,
+          215,
+          65,
+          372,
+          1661,
+          1425,
+          759,
+          62,
+          241,
+          1139,
+          702,
+          1572,
+          727,
+          790,
+          1133,
+          1070,
+          991,
+          811,
+          156,
+          1589,
+          927,
+          578,
+          1158,
+          910,
+          1428,
+          1429
+        ],
+        "endpoint_dist": [
+          1.2471455335617065,
+          3.780824661254883,
+          2.3386223316192627,
+          3.5947601795196533,
+          3.435229778289795,
+          5.462389945983887,
+          4.583520412445068,
+          1.6975603103637695,
+          6.2813029289245605,
+          3.2201132774353027,
+          1.954548954963684,
+          2.568264961242676,
+          4.240303993225098,
+          1.9021812677383423,
+          1.581223964691162,
+          1.8886135816574097,
+          2.0643303394317627,
+          3.607733726501465,
+          2.0478649139404297,
+          2.4884705543518066,
+          4.466367721557617,
+          1.8336158990859985,
+          3.0728282928466797,
+          2.878474712371826,
+          4.979918003082275,
+          2.1764323711395264,
+          8.584672927856445,
+          1.425441026687622,
+          3.742549419403076,
+          1.836504340171814,
+          3.9608829021453857,
+          2.16227388381958,
+          2.613037586212158,
+          2.0466930866241455,
+          2.479177236557007,
+          4.7175116539001465,
+          2.2013516426086426,
+          1.6452608108520508,
+          1.8277415037155151,
+          4.1631059646606445
+        ],
+        "spec_margin": [
+          1.2742022275924683,
+          -0.42791128158569336,
+          1.129060983657837,
+          -1.119408369064331,
+          -0.7760045528411865,
+          0.183258056640625,
+          -1.3876416683197021,
+          0.876413106918335,
+          -0.03194618225097656,
+          -0.21651339530944824,
+          1.0298165082931519,
+          0.21366572380065918,
+          0.578819751739502,
+          1.1407452821731567,
+          0.3463408946990967,
+          0.5529183149337769,
+          0.9577670097351074,
+          1.0641913414001465,
+          0.35391855239868164,
+          0.5692179203033447,
+          0.011257171630859375,
+          0.668890118598938,
+          0.20898771286010742,
+          0.37965893745422363,
+          -0.10495424270629883,
+          1.282132863998413,
+          -0.4226036071777344,
+          1.4416706562042236,
+          0.32228803634643555,
+          0.3516949415206909,
+          -1.1902787685394287,
+          0.18331551551818848,
+          0.9950697422027588,
+          1.1751482486724854,
+          0.2742455005645752,
+          -0.3204960823059082,
+          0.6251316070556641,
+          0.9052252769470215,
+          1.1004918813705444,
+          -0.08222198486328125
+        ],
+        "off_dist": [
+          2.521347761154175,
+          3.3529133796691895,
+          3.4676833152770996,
+          2.4753518104553223,
+          2.6592252254486084,
+          5.645648002624512,
+          3.195878744125366,
+          2.5739734172821045,
+          6.249356746673584,
+          3.0035998821258545,
+          2.984365463256836,
+          2.781930685043335,
+          4.8191237449646,
+          3.042926549911499,
+          1.9275648593902588,
+          2.4415318965911865,
+          3.02209734916687,
+          4.671925067901611,
+          2.4017834663391113,
+          3.0576884746551514,
+          4.477624893188477,
+          2.5025060176849365,
+          3.281816005706787,
+          3.25813364982605,
+          4.874963760375977,
+          3.4585652351379395,
+          8.162069320678711,
+          2.8671116828918457,
+          4.064837455749512,
+          2.188199281692505,
+          2.770604133605957,
+          2.3455893993377686,
+          3.608107328414917,
+          3.221841335296631,
+          2.753422737121582,
+          4.397015571594238,
+          2.8264832496643066,
+          2.5504860877990723,
+          2.9282333850860596,
+          4.080883979797363
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    }
+  }
+}

experiments/results/replogle_k562_nom_improved.json

@@ -0,0 +1,981 @@
+{
+  "dataset": "replogle_k562",
+  "split": "perturbation",
+  "reward": "cosine",
+  "n_candidates": 1805,
+  "n_targets": 120,
+  "random_top5": 0.002770083102493075,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7416666666666667,
+      "func_ndcg": 0.5503647310194125,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 857.0
+    },
+    "PIVOT-guidance(random-init)": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7916666666666666,
+      "func_ndcg": 0.5487322808181787,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 866.0
+    },
+    "EndpointMLP+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7416666666666667,
+      "func_ndcg": 0.5599157820889673,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 761.0
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7416666666666667,
+      "func_ndcg": 0.558927740279396,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 839.0
+    },
+    "Additive+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7416666666666667,
+      "func_ndcg": 0.5599157820889673,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 844.5
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7416666666666667,
+      "func_ndcg": 0.5599157820889673,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 790.0
+    },
+    "Random+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7083333333333334,
+      "func_ndcg": 0.3706553528381431,
+      "top5_ci": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "_top5_per": [
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "med_rank": 955.0
+    }
+  },
+  "significance": {
+    "PIVOT-ranking_vs_PIVOT-guidance(random-init)": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_EndpointMLP+ranking": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_LinearResponse+ranking": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_Additive+ranking": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_NearestPerturbationCentroid+ranking": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    },
+    "PIVOT-ranking_vs_Random+ranking": {
+      "p": 1.0,
+      "mean_diff_top5": 0.0
+    }
+  }
+}

experiments/results/replogle_k562_nom_perturbation.json

@@ -0,0 +1,2773 @@
+{
+  "split": "perturbation",
+  "reward": "centroid",
+  "n_candidates": 1805,
+  "n_targets": 40,
+  "methods": {
+    "PIVOT-ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.75,
+      "func_ndcg": 0.43398773723290346,
+      "rank": 477.075,
+      "endpoint_dist": 1.6277449652552605,
+      "spec_margin": 0.9393299654126167,
+      "off_dist": 2.5670749247074127,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.0,
+          0.8196104814589247,
+          0.25898197669424483,
+          0.25807905549834614,
+          0.7084012682349022,
+          0.4144301094503377,
+          0.39174687803411373,
+          0.439764939411066,
+          0.10905082208642256,
+          0.8196104814589247,
+          0.11443249549771052,
+          0.5247884674204997,
+          0.2974259152941696,
+          0.3821659031256479,
+          0.0,
+          0.20400787375012425,
+          0.8196104814589247,
+          0.5221845661647552,
+          0.8196104814589247,
+          0.22479085346787228,
+          0.8196104814589247,
+          0.05690655723988347,
+          0.07768953695763149,
+          0.8196104814589247,
+          0.06012983951357154,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.05430265598413898,
+          0.5513465647256421,
+          0.8196104814589247,
+          0.0,
+          0.29420263302048155,
+          0.21688550815340366,
+          0.8196104814589247,
+          0.11703639675345502,
+          0.0,
+          0.6896964503021904,
+          0.555736445028433
+        ],
+        "rank": [
+          606,
+          798,
+          925,
+          529,
+          357,
+          169,
+          692,
+          726,
+          354,
+          511,
+          163,
+          226,
+          243,
+          107,
+          209,
+          101,
+          862,
+          52,
+          504,
+          606,
+          620,
+          950,
+          845,
+          567,
+          516,
+          783,
+          330,
+          393,
+          329,
+          442,
+          306,
+          441,
+          982,
+          522,
+          145,
+          1016,
+          656,
+          370,
+          92,
+          38
+        ],
+        "endpoint_dist": [
+          0.7434885501861572,
+          2.4635772705078125,
+          1.6023441553115845,
+          0.8692498803138733,
+          1.1865061521530151,
+          1.5722774267196655,
+          0.7362974286079407,
+          0.9789136052131653,
+          0.9780240654945374,
+          6.6848225593566895,
+          0.9765823483467102,
+          1.9768712520599365,
+          1.1871503591537476,
+          1.01445472240448,
+          1.8242679834365845,
+          0.8309136033058167,
+          2.4996466636657715,
+          1.4547226428985596,
+          1.2289146184921265,
+          0.8584837317466736,
+          2.255495071411133,
+          0.9234660267829895,
+          0.626628041267395,
+          2.0583951473236084,
+          0.983538806438446,
+          1.82752525806427,
+          1.7855417728424072,
+          2.05389404296875,
+          1.4342339038848877,
+          1.1439824104309082,
+          1.2706406116485596,
+          2.8937065601348877,
+          0.7645078301429749,
+          1.3121113777160645,
+          1.0582448244094849,
+          5.560652256011963,
+          0.9708915948867798,
+          0.6884087920188904,
+          1.975685715675354,
+          1.8547395467758179
+        ],
+        "spec_margin": [
+          1.672365427017212,
+          0.7733314037322998,
+          1.019189715385437,
+          1.3633439540863037,
+          0.8521960973739624,
+          0.6960502862930298,
+          1.4837749004364014,
+          1.2703609466552734,
+          1.397662878036499,
+          -2.9303510189056396,
+          1.6186654567718506,
+          0.47718214988708496,
+          1.130790114402771,
+          1.2825955152511597,
+          -0.15734422206878662,
+          1.480198860168457,
+          1.9295220375061035,
+          1.5434458255767822,
+          0.9457141160964966,
+          1.6729440689086914,
+          0.6493873596191406,
+          0.8838562369346619,
+          1.9253515005111694,
+          0.3431985378265381,
+          1.3820078372955322,
+          0.7749754190444946,
+          1.9326670169830322,
+          0.5550122261047363,
+          1.195478916168213,
+          0.6238242387771606,
+          1.2171826362609863,
+          0.5463707447052002,
+          1.9847569465637207,
+          1.4454312324523926,
+          0.9364702701568604,
+          -1.686753273010254,
+          1.1488052606582642,
+          1.5929045677185059,
+          0.35722815990448,
+          0.24340426921844482
+        ],
+        "off_dist": [
+          2.415853977203369,
+          3.2369086742401123,
+          2.6215338706970215,
+          2.2325937747955322,
+          2.0387022495269775,
+          2.2683277130126953,
+          2.2200722694396973,
+          2.249274492263794,
+          2.3756868839263916,
+          3.75447154045105,
+          2.595247745513916,
+          2.4540534019470215,
+          2.3179404735565186,
+          2.2970502376556396,
+          1.6669237613677979,
+          2.311112403869629,
+          4.429168701171875,
+          2.998168468475342,
+          2.174628734588623,
+          2.5314278602600098,
+          2.9048824310302734,
+          1.8073222637176514,
+          2.5519795417785645,
+          2.4015936851501465,
+          2.365546703338623,
+          2.6025006771087646,
+          3.7182087898254395,
+          2.6089062690734863,
+          2.6297128200531006,
+          1.7678066492080688,
+          2.487823247909546,
+          3.440077304840088,
+          2.749264717102051,
+          2.757542610168457,
+          1.9947150945663452,
+          3.873898983001709,
+          2.119696855545044,
+          2.281313419342041,
+          2.332913875579834,
+          2.0981438159942627
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "PIVOT-guidance": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.85,
+      "func_ndcg": 0.346193945477374,
+      "rank": 479.3,
+      "endpoint_dist": 1.6929722726345062,
+      "spec_margin": 0.8205313935875893,
+      "off_dist": 2.513503661751747,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.06012983951357154,
+          0.7594806419453533,
+          0.1423390241168964,
+          0.1803895185407146,
+          0.4371610381553215,
+          0.6392209629182101,
+          0.1803895185407146,
+          0.18996364131774213,
+          0.2557586944205568,
+          0.8196104814589247,
+          0.1298338018041706,
+          0.3673613518206858,
+          0.1803895185407146,
+          0.2372960757805981,
+          0.0,
+          0.052144264846539085,
+          0.8196104814589247,
+          0.24364691628393753,
+          0.4482741710520053,
+          0.33392190490685864,
+          0.8196104814589247,
+          0.0,
+          0.07768953695763149,
+          0.46531021152215846,
+          0.20659506710005476,
+          0.6721367528581305,
+          0.5759635651749873,
+          0.8196104814589247,
+          0.6229688013578893,
+          0.12991403115673436,
+          0.2302134223094644,
+          0.7553544384757664,
+          0.18421668714087336,
+          0.1803895185407146,
+          0.3130586958365468,
+          0.8196104814589247,
+          0.0,
+          0.0,
+          0.2346921745248536,
+          0.26350162433993823
+        ],
+        "rank": [
+          608,
+          800,
+          926,
+          531,
+          360,
+          169,
+          694,
+          727,
+          357,
+          511,
+          166,
+          234,
+          245,
+          111,
+          212,
+          102,
+          862,
+          57,
+          509,
+          606,
+          621,
+          950,
+          845,
+          569,
+          518,
+          783,
+          331,
+          397,
+          334,
+          443,
+          307,
+          448,
+          982,
+          522,
+          149,
+          1016,
+          657,
+          370,
+          99,
+          44
+        ],
+        "endpoint_dist": [
+          0.7434885501861572,
+          2.4635772705078125,
+          1.6023441553115845,
+          0.8692498803138733,
+          1.1865061521530151,
+          1.5722774267196655,
+          0.7362974286079407,
+          1.1201238632202148,
+          0.9780240654945374,
+          6.6848225593566895,
+          0.9765823483467102,
+          1.9768712520599365,
+          1.1871503591537476,
+          1.01445472240448,
+          1.8242679834365845,
+          0.8309136033058167,
+          2.4996466636657715,
+          1.4547226428985596,
+          1.7815905809402466,
+          0.8584837317466736,
+          2.255495071411133,
+          0.9234660267829895,
+          0.626628041267395,
+          2.4905507564544678,
+          0.983538806438446,
+          1.82752525806427,
+          2.1815366744995117,
+          2.05389404296875,
+          1.4342339038848877,
+          1.1439824104309082,
+          1.4812977313995361,
+          3.3611063957214355,
+          0.7645078301429749,
+          1.4217694997787476,
+          1.0582448244094849,
+          5.560652256011963,
+          0.9708915948867798,
+          0.6884087920188904,
+          1.9726204872131348,
+          2.1571452617645264
+        ],
+        "spec_margin": [
+          1.672365427017212,
+          0.7733314037322998,
+          1.019189715385437,
+          1.3633439540863037,
+          0.8521960973739624,
+          0.6960502862930298,
+          1.4837749004364014,
+          1.1176819801330566,
+          1.397662878036499,
+          -2.9303510189056396,
+          1.6186654567718506,
+          0.47718214988708496,
+          1.130790114402771,
+          1.2825955152511597,
+          -0.15734422206878662,
+          1.480198860168457,
+          1.9295220375061035,
+          1.5434458255767822,
+          0.0906752347946167,
+          1.6729440689086914,
+          0.6493873596191406,
+          0.8838562369346619,
+          1.9253515005111694,
+          -0.4390888214111328,
+          1.3820078372955322,
+          0.7749754190444946,
+          1.1829278469085693,
+          0.5550122261047363,
+          1.195478916168213,
+          0.6238242387771606,
+          0.9619626998901367,
+          -0.876856803894043,
+          1.9847569465637207,
+          1.38202965259552,
+          0.9364702701568604,
+          -1.686753273010254,
+          1.1488052606582642,
+          1.5929045677185059,
+          0.423114538192749,
+          -0.29283154010772705
+        ],
+        "off_dist": [
+          2.415853977203369,
+          3.2369086742401123,
+          2.6215338706970215,
+          2.2325937747955322,
+          2.0387022495269775,
+          2.2683277130126953,
+          2.2200722694396973,
+          2.2378058433532715,
+          2.3756868839263916,
+          3.75447154045105,
+          2.595247745513916,
+          2.4540534019470215,
+          2.3179404735565186,
+          2.2970502376556396,
+          1.6669237613677979,
+          2.311112403869629,
+          4.429168701171875,
+          2.998168468475342,
+          1.8722658157348633,
+          2.5314278602600098,
+          2.9048824310302734,
+          1.8073222637176514,
+          2.5519795417785645,
+          2.051461935043335,
+          2.365546703338623,
+          2.6025006771087646,
+          3.364464521408081,
+          2.6089062690734863,
+          2.6297128200531006,
+          1.7678066492080688,
+          2.443260431289673,
+          2.4842495918273926,
+          2.749264717102051,
+          2.8037991523742676,
+          1.9947150945663452,
+          3.873898983001709,
+          2.119696855545044,
+          2.281313419342041,
+          2.395735025405884,
+          1.8643137216567993
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "Additive+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.925,
+      "func_ndcg": 0.5289588692976046,
+      "rank": 449.3,
+      "endpoint_dist": 1.4835666730999946,
+      "spec_margin": 1.5093856036663056,
+      "off_dist": 2.9929522782564164,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.18618449947286028,
+          0.8196104814589247,
+          0.5093871611140395,
+          0.11381311447976694,
+          0.6896964503021904,
+          0.499202299879103,
+          0.05430265598413898,
+          0.8196104814589247,
+          0.17623111247384093,
+          0.8196104814589247,
+          0.6149832266908569,
+          0.7294157221885674,
+          0.501433789044294,
+          0.5791233260019254,
+          0.25807905549834614,
+          0.0,
+          0.8196104814589247,
+          0.7419209445012933,
+          0.7594806419453533,
+          0.6053288745612656,
+          0.8196104814589247,
+          0.07768953695763149,
+          0.29420263302048155,
+          0.7627039242190412,
+          0.4883727071603744,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.3102233203448852,
+          0.6353937943180514,
+          0.8196104814589247,
+          0.4889118588257541,
+          0.16811577046390594,
+          0.1841364577883096,
+          0.8196104814589247,
+          0.05430265598413898,
+          0.07768953695763149,
+          0.8196104814589247,
+          0.7627039242190412
+        ],
+        "rank": [
+          464,
+          436,
+          562,
+          558,
+          459,
+          559,
+          230,
+          487,
+          547,
+          610,
+          471,
+          436,
+          277,
+          225,
+          343,
+          327,
+          609,
+          382,
+          505,
+          798,
+          192,
+          419,
+          423,
+          571,
+          474,
+          338,
+          357,
+          314,
+          355,
+          454,
+          362,
+          375,
+          649,
+          794,
+          404,
+          681,
+          300,
+          307,
+          556,
+          362
+        ],
+        "endpoint_dist": [
+          0.7910816073417664,
+          2.325538396835327,
+          1.4062681198120117,
+          0.8322772979736328,
+          1.2257609367370605,
+          1.4010910987854004,
+          0.7616842985153198,
+          0.8932631611824036,
+          0.8577340245246887,
+          4.744374752044678,
+          0.7894836068153381,
+          1.8425379991531372,
+          1.1874315738677979,
+          0.978216826915741,
+          1.4649451971054077,
+          0.8955826163291931,
+          2.891011953353882,
+          1.1721434593200684,
+          1.2098002433776855,
+          0.8302912712097168,
+          1.8107973337173462,
+          0.8613649606704712,
+          0.878169596195221,
+          1.9043508768081665,
+          0.9688758850097656,
+          1.9366815090179443,
+          2.2544405460357666,
+          1.667983889579773,
+          1.1924028396606445,
+          1.242406964302063,
+          1.270032525062561,
+          2.7028005123138428,
+          0.6736534237861633,
+          1.0901234149932861,
+          0.9097232222557068,
+          3.980757474899292,
+          0.8556053638458252,
+          0.8449692726135254,
+          1.8862968683242798,
+          1.9107120037078857
+        ],
+        "spec_margin": [
+          1.7232837677001953,
+          1.9759433269500732,
+          1.599470615386963,
+          1.453108549118042,
+          1.0649609565734863,
+          1.2389452457427979,
+          1.5532299280166626,
+          1.4933724403381348,
+          1.6919171810150146,
+          1.1277027130126953,
+          1.839672565460205,
+          0.97779381275177,
+          1.3402292728424072,
+          1.3308873176574707,
+          0.6002246141433716,
+          1.4666364192962646,
+          1.1187450885772705,
+          1.9536728858947754,
+          1.3608942031860352,
+          1.896925926208496,
+          2.1419949531555176,
+          1.0486434698104858,
+          1.7888805866241455,
+          0.994531512260437,
+          1.54184889793396,
+          2.2699358463287354,
+          1.7643601894378662,
+          1.9719387292861938,
+          1.732537031173706,
+          0.6122345924377441,
+          1.593953013420105,
+          2.815749406814575,
+          2.2328741550445557,
+          1.8885915279388428,
+          1.112678050994873,
+          1.611879587173462,
+          1.4536590576171875,
+          1.4404144287109375,
+          1.1916362047195435,
+          0.3594660758972168
+        ],
+        "off_dist": [
+          2.5143654346466064,
+          4.3014817237854,
+          3.0057387351989746,
+          2.285385847091675,
+          2.290721893310547,
+          2.6400363445281982,
+          2.3149142265319824,
+          2.3866355419158936,
+          2.5496511459350586,
+          5.872077465057373,
+          2.6291561126708984,
+          2.8203318119049072,
+          2.527660846710205,
+          2.3091042041778564,
+          2.0651698112487793,
+          2.3622190952301025,
+          4.009757041931152,
+          3.1258163452148438,
+          2.5706944465637207,
+          2.727217197418213,
+          3.9527924060821533,
+          1.910008430480957,
+          2.6670501232147217,
+          2.8988823890686035,
+          2.5107247829437256,
+          4.20661735534668,
+          4.018800735473633,
+          3.639922618865967,
+          2.9249398708343506,
+          1.8546415567398071,
+          2.863985538482666,
+          5.518549919128418,
+          2.906527519226074,
+          2.978714942932129,
+          2.0224013328552246,
+          5.592637062072754,
+          2.3092644214630127,
+          2.285383701324463,
+          3.0779330730438232,
+          2.2701780796051025
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "LinearResponse+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.925,
+      "func_ndcg": 0.5275197617991798,
+      "rank": 594.75,
+      "endpoint_dist": 1.5074330180883408,
+      "spec_margin": 1.0903448686003685,
+      "off_dist": 2.597777882218361,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.4361482129612715,
+          0.8196104814589247,
+          0.5051807282918888,
+          0.23253378338725372,
+          0.7594806419453533,
+          0.5967465448927352,
+          0.32749083505095544,
+          0.5749649199350518,
+          0.16335347807056153,
+          0.8196104814589247,
+          0.17394295399333848,
+          0.8196104814589247,
+          0.23497571470280879,
+          0.5222167687620419,
+          0.24464556152387298,
+          0.0,
+          0.8196104814589247,
+          0.7627039242190412,
+          0.8196104814589247,
+          0.5586960504689169,
+          0.8196104814589247,
+          0.24464556152387298,
+          0.22608721883987756,
+          0.8196104814589247,
+          0.11443249549771052,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.496278052474843,
+          0.6850143872614097,
+          0.7553544384757664,
+          0.3225665727174216,
+          0.2372960757805981,
+          0.1707196717196504,
+          0.8196104814589247,
+          0.11381311447976694,
+          0.24464556152387298,
+          0.8196104814589247,
+          0.7419209445012933
+        ],
+        "rank": [
+          819,
+          613,
+          620,
+          584,
+          560,
+          880,
+          718,
+          315,
+          622,
+          761,
+          684,
+          471,
+          801,
+          768,
+          740,
+          776,
+          526,
+          228,
+          111,
+          798,
+          497,
+          970,
+          893,
+          548,
+          626,
+          196,
+          270,
+          534,
+          586,
+          457,
+          264,
+          361,
+          782,
+          819,
+          443,
+          553,
+          767,
+          905,
+          357,
+          567
+        ],
+        "endpoint_dist": [
+          0.7317129373550415,
+          1.920383334159851,
+          1.4073760509490967,
+          0.7570603489875793,
+          1.17207670211792,
+          1.4544312953948975,
+          0.6235144734382629,
+          0.8932536244392395,
+          0.8369724750518799,
+          6.552571773529053,
+          0.6789464950561523,
+          1.666611671447754,
+          1.142364263534546,
+          0.9602575898170471,
+          1.5338935852050781,
+          0.9362837076187134,
+          3.332012891769409,
+          1.1075212955474854,
+          1.127482533454895,
+          1.1178308725357056,
+          2.0792980194091797,
+          0.9287006258964539,
+          0.8524771332740784,
+          2.000643253326416,
+          0.7920212149620056,
+          1.2070013284683228,
+          2.0071728229522705,
+          1.4477404356002808,
+          1.1261117458343506,
+          1.2170510292053223,
+          1.068366527557373,
+          2.0998857021331787,
+          0.9043663740158081,
+          1.2781434059143066,
+          0.8842405676841736,
+          4.939247131347656,
+          0.9376744031906128,
+          0.9116864204406738,
+          1.769436240196228,
+          1.893498420715332
+        ],
+        "spec_margin": [
+          1.6089192628860474,
+          1.5517617464065552,
+          1.3225722312927246,
+          1.4850592613220215,
+          0.9671440124511719,
+          0.9703366756439209,
+          1.6035497188568115,
+          1.4796998500823975,
+          1.617544412612915,
+          -3.017054319381714,
+          1.8496487140655518,
+          0.9044928550720215,
+          1.2502381801605225,
+          1.2019782066345215,
+          0.25848865509033203,
+          1.374677062034607,
+          0.3797588348388672,
+          1.9601185321807861,
+          0.9472290277481079,
+          1.3767625093460083,
+          0.8319146633148193,
+          0.9684036374092102,
+          1.6929290294647217,
+          0.2231457233428955,
+          1.6079893112182617,
+          1.8344088792800903,
+          1.7327463626861572,
+          1.6003540754318237,
+          1.855177640914917,
+          0.546280026435852,
+          1.5585684776306152,
+          1.4500024318695068,
+          1.7879267930984497,
+          1.571855068206787,
+          1.071916103363037,
+          -1.3561031818389893,
+          1.2547610998153687,
+          1.2583446502685547,
+          0.7246888875961304,
+          0.3055596351623535
+        ],
+        "off_dist": [
+          2.340632200241089,
+          3.4721450805664062,
+          2.7299482822418213,
+          2.242119550704956,
+          2.139220714569092,
+          2.4247679710388184,
+          2.2270641326904297,
+          2.372953414916992,
+          2.454516887664795,
+          3.535517454147339,
+          2.528595209121704,
+          2.5711045265197754,
+          2.3926024436950684,
+          2.162235736846924,
+          1.7923822402954102,
+          2.3109607696533203,
+          3.7117717266082764,
+          3.0676398277282715,
+          2.074711561203003,
+          2.494593381881714,
+          2.911212682723999,
+          1.897104263305664,
+          2.5454061031341553,
+          2.2237889766693115,
+          2.400010585784912,
+          3.041410207748413,
+          3.7399191856384277,
+          3.0480945110321045,
+          2.9812893867492676,
+          1.7633310556411743,
+          2.6269350051879883,
+          3.5498881340026855,
+          2.692293167114258,
+          2.8499984741210938,
+          1.9561567306518555,
+          3.583143949508667,
+          2.1924355030059814,
+          2.1700310707092285,
+          2.4941251277923584,
+          2.1990580558776855
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "NearestPerturbationCentroid+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.925,
+      "func_ndcg": 0.526194541190098,
+      "rank": 1014.875,
+      "endpoint_dist": 1.4835666730999946,
+      "spec_margin": 1.5093856036663056,
+      "off_dist": 2.9929522782564164,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.18618449947286028,
+          0.8196104814589247,
+          0.5093871611140395,
+          0.11381311447976694,
+          0.6896964503021904,
+          0.499202299879103,
+          0.05430265598413898,
+          0.6489075176450882,
+          0.17623111247384093,
+          0.8196104814589247,
+          0.6149832266908569,
+          0.7294157221885674,
+          0.501433789044294,
+          0.5791233260019254,
+          0.25807905549834614,
+          0.0,
+          0.8196104814589247,
+          0.7419209445012933,
+          0.8196104814589247,
+          0.6053288745612656,
+          0.8196104814589247,
+          0.07768953695763149,
+          0.29420263302048155,
+          0.7627039242190412,
+          0.4883727071603744,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.3102233203448852,
+          0.6353937943180514,
+          0.8196104814589247,
+          0.4889118588257541,
+          0.16811577046390594,
+          0.1841364577883096,
+          0.8196104814589247,
+          0.05430265598413898,
+          0.07768953695763149,
+          0.8196104814589247,
+          0.7627039242190412
+        ],
+        "rank": [
+          955,
+          433,
+          951,
+          1202,
+          995,
+          1204,
+          1163,
+          890,
+          934,
+          1064,
+          1213,
+          844,
+          965,
+          1303,
+          1114,
+          1310,
+          453,
+          1008,
+          1363,
+          1025,
+          593,
+          1043,
+          902,
+          1424,
+          798,
+          1000,
+          1059,
+          977,
+          923,
+          908,
+          868,
+          1331,
+          829,
+          975,
+          925,
+          1431,
+          1012,
+          1303,
+          899,
+          1006
+        ],
+        "endpoint_dist": [
+          0.7910816073417664,
+          2.325538396835327,
+          1.4062681198120117,
+          0.8322772979736328,
+          1.2257609367370605,
+          1.4010910987854004,
+          0.7616842985153198,
+          0.8932631611824036,
+          0.8577340245246887,
+          4.744374752044678,
+          0.7894836068153381,
+          1.8425379991531372,
+          1.1874315738677979,
+          0.978216826915741,
+          1.4649451971054077,
+          0.8955826163291931,
+          2.891011953353882,
+          1.1721434593200684,
+          1.2098002433776855,
+          0.8302912712097168,
+          1.8107973337173462,
+          0.8613649606704712,
+          0.878169596195221,
+          1.9043508768081665,
+          0.9688758850097656,
+          1.9366815090179443,
+          2.2544405460357666,
+          1.667983889579773,
+          1.1924028396606445,
+          1.242406964302063,
+          1.270032525062561,
+          2.7028005123138428,
+          0.6736534237861633,
+          1.0901234149932861,
+          0.9097232222557068,
+          3.980757474899292,
+          0.8556053638458252,
+          0.8449692726135254,
+          1.8862968683242798,
+          1.9107120037078857
+        ],
+        "spec_margin": [
+          1.7232837677001953,
+          1.9759433269500732,
+          1.599470615386963,
+          1.453108549118042,
+          1.0649609565734863,
+          1.2389452457427979,
+          1.5532299280166626,
+          1.4933724403381348,
+          1.6919171810150146,
+          1.1277027130126953,
+          1.839672565460205,
+          0.97779381275177,
+          1.3402292728424072,
+          1.3308873176574707,
+          0.6002246141433716,
+          1.4666364192962646,
+          1.1187450885772705,
+          1.9536728858947754,
+          1.3608942031860352,
+          1.896925926208496,
+          2.1419949531555176,
+          1.0486434698104858,
+          1.7888805866241455,
+          0.994531512260437,
+          1.54184889793396,
+          2.2699358463287354,
+          1.7643601894378662,
+          1.9719387292861938,
+          1.732537031173706,
+          0.6122345924377441,
+          1.593953013420105,
+          2.815749406814575,
+          2.2328741550445557,
+          1.8885915279388428,
+          1.112678050994873,
+          1.611879587173462,
+          1.4536590576171875,
+          1.4404144287109375,
+          1.1916362047195435,
+          0.3594660758972168
+        ],
+        "off_dist": [
+          2.5143654346466064,
+          4.3014817237854,
+          3.0057387351989746,
+          2.285385847091675,
+          2.290721893310547,
+          2.6400363445281982,
+          2.3149142265319824,
+          2.3866355419158936,
+          2.5496511459350586,
+          5.872077465057373,
+          2.6291561126708984,
+          2.8203318119049072,
+          2.527660846710205,
+          2.3091042041778564,
+          2.0651698112487793,
+          2.3622190952301025,
+          4.009757041931152,
+          3.1258163452148438,
+          2.5706944465637207,
+          2.727217197418213,
+          3.9527924060821533,
+          1.910008430480957,
+          2.6670501232147217,
+          2.8988823890686035,
+          2.5107247829437256,
+          4.20661735534668,
+          4.018800735473633,
+          3.639922618865967,
+          2.9249398708343506,
+          1.8546415567398071,
+          2.863985538482666,
+          5.518549919128418,
+          2.906527519226074,
+          2.978714942932129,
+          2.0224013328552246,
+          5.592637062072754,
+          2.3092644214630127,
+          2.285383701324463,
+          3.0779330730438232,
+          2.2701780796051025
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "EndpointMLP+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.925,
+      "func_ndcg": 0.5220999841830257,
+      "rank": 507.55,
+      "endpoint_dist": 1.4828851819038391,
+      "spec_margin": 1.5101026505231858,
+      "off_dist": 2.992987832427025,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.19408984478732896,
+          0.8196104814589247,
+          0.5093871611140395,
+          0.11381311447976694,
+          0.6896964503021904,
+          0.499202299879103,
+          0.05430265598413898,
+          0.6489075176450882,
+          0.17623111247384093,
+          0.8196104814589247,
+          0.6149832266908569,
+          0.7294157221885674,
+          0.501433789044294,
+          0.5791233260019254,
+          0.25807905549834614,
+          0.0,
+          0.8196104814589247,
+          0.7419209445012933,
+          0.7032101736292273,
+          0.6053288745612656,
+          0.8196104814589247,
+          0.07768953695763149,
+          0.29420263302048155,
+          0.7627039242190412,
+          0.4883727071603744,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.8196104814589247,
+          0.3023179750304166,
+          0.6375521854556513,
+          0.8196104814589247,
+          0.4889118588257541,
+          0.16811577046390594,
+          0.13199219294177048,
+          0.8196104814589247,
+          0.05430265598413898,
+          0.07768953695763149,
+          0.8196104814589247,
+          0.7653078254747857
+        ],
+        "rank": [
+          501,
+          456,
+          793,
+          491,
+          602,
+          722,
+          460,
+          441,
+          761,
+          211,
+          610,
+          410,
+          740,
+          596,
+          519,
+          791,
+          389,
+          395,
+          196,
+          730,
+          459,
+          682,
+          571,
+          239,
+          317,
+          328,
+          438,
+          378,
+          369,
+          330,
+          458,
+          332,
+          616,
+          657,
+          524,
+          246,
+          977,
+          616,
+          596,
+          355
+        ],
+        "endpoint_dist": [
+          0.788127601146698,
+          2.3236305713653564,
+          1.4040254354476929,
+          0.8310549259185791,
+          1.2236676216125488,
+          1.3991789817810059,
+          0.7625831365585327,
+          0.8916113376617432,
+          0.8571329712867737,
+          4.743624210357666,
+          0.7842857241630554,
+          1.8409243822097778,
+          1.1889394521713257,
+          0.9782581925392151,
+          1.4625096321105957,
+          0.898481011390686,
+          2.8906986713409424,
+          1.1717466115951538,
+          1.2101223468780518,
+          0.8264373540878296,
+          1.8072891235351562,
+          0.8618491291999817,
+          0.8796766400337219,
+          1.9048603773117065,
+          0.9663487076759338,
+          1.9356915950775146,
+          2.260815382003784,
+          1.6731778383255005,
+          1.1898635625839233,
+          1.2420670986175537,
+          1.268711805343628,
+          2.697918176651001,
+          0.6752347946166992,
+          1.0929676294326782,
+          0.906967282295227,
+          3.9801554679870605,
+          0.8546260595321655,
+          0.846272885799408,
+          1.8830167055130005,
+          1.91085684299469
+        ],
+        "spec_margin": [
+          1.7244186401367188,
+          1.9791104793548584,
+          1.6007493734359741,
+          1.454641580581665,
+          1.0647914409637451,
+          1.241217851638794,
+          1.5519427061080933,
+          1.4942684173583984,
+          1.6921191215515137,
+          1.132720947265625,
+          1.8430020809173584,
+          0.9792376756668091,
+          1.3373607397079468,
+          1.3277947902679443,
+          0.6020269393920898,
+          1.4632021188735962,
+          1.1198742389678955,
+          1.9535547494888306,
+          1.3624038696289062,
+          1.8991485834121704,
+          2.1501173973083496,
+          1.0468807220458984,
+          1.7869219779968262,
+          0.991362452507019,
+          1.5429887771606445,
+          2.2720224857330322,
+          1.7664763927459717,
+          1.9695860147476196,
+          1.7339993715286255,
+          0.6108479499816895,
+          1.5951225757598877,
+          2.822641134262085,
+          2.2305965423583984,
+          1.8858641386032104,
+          1.115355134010315,
+          1.6147780418395996,
+          1.453623652458191,
+          1.439103603363037,
+          1.1922818422317505,
+          0.3599494695663452
+        ],
+        "off_dist": [
+          2.5125463008880615,
+          4.302741050720215,
+          3.004774808883667,
+          2.285696506500244,
+          2.288459062576294,
+          2.6403968334198,
+          2.314525842666626,
+          2.3858797550201416,
+          2.5492520332336426,
+          5.876345157623291,
+          2.6272878646850586,
+          2.820162057876587,
+          2.5263001918792725,
+          2.3060529232025146,
+          2.0645365715026855,
+          2.3616831302642822,
+          4.010572910308838,
+          3.1253013610839844,
+          2.572526216506958,
+          2.7255859375,
+          3.957406520843506,
+          1.908729910850525,
+          2.6665985584259033,
+          2.8962228298187256,
+          2.5093374252319336,
+          4.207714080810547,
+          4.027291774749756,
+          3.64276385307312,
+          2.923862934112549,
+          1.8529150485992432,
+          2.8638343811035156,
+          5.520559310913086,
+          2.9058313369750977,
+          2.9788317680358887,
+          2.022322416305542,
+          5.59493350982666,
+          2.3082497119903564,
+          2.28537654876709,
+          3.075298547744751,
+          2.270806312561035
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    },
+    "Random+ranking": {
+      "top1": 0.0,
+      "top5": 0.0,
+      "ndcg": 0.0,
+      "func_top5": 0.7,
+      "func_ndcg": 0.3796227424826798,
+      "rank": 835.075,
+      "endpoint_dist": 3.5514531686902044,
+      "spec_margin": 0.06944499611854553,
+      "off_dist": 3.6208981722593307,
+      "clf": null,
+      "_per": {
+        "top1": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "top5": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "ndcg": [
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          0.0
+        ],
+        "func_top5": [
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          1.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0,
+          0.0,
+          0.0,
+          1.0,
+          1.0
+        ],
+        "func_ndcg": [
+          0.0,
+          0.5410728315781073,
+          0.7627039242190412,
+          0.0,
+          0.6415413239959994,
+          0.1803895185407146,
+          0.06425604298315837,
+          0.5887609702257027,
+          0.07768953695763149,
+          0.7057973669791577,
+          0.36068331303739254,
+          0.6504298198589307,
+          0.0,
+          0.6156026077088004,
+          0.05430265598413898,
+          0.0,
+          0.5874701422477776,
+          0.7419209445012933,
+          0.7294157221885674,
+          0.1803895185407146,
+          0.46527800892487164,
+          0.0,
+          0.1803895185407146,
+          0.5935232626190471,
+          0.4685012911985596,
+          0.6897766796547542,
+          0.8196104814589247,
+          0.478419320161355,
+          0.4713366666902212,
+          0.06012983951357154,
+          0.6149832266908569,
+          0.4550042757773366,
+          0.05430265598413898,
+          0.0,
+          0.4482741710520053,
+          0.6692858826749959,
+          0.0,
+          0.0,
+          0.6721367528581305,
+          0.5615314259605786
+        ],
+        "rank": [
+          101,
+          1138,
+          272,
+          1527,
+          577,
+          379,
+          563,
+          341,
+          295,
+          982,
+          752,
+          625,
+          525,
+          126,
+          1447,
+          55,
+          914,
+          1022,
+          396,
+          1657,
+          1275,
+          636,
+          1287,
+          988,
+          1025,
+          1371,
+          1451,
+          1506,
+          114,
+          758,
+          940,
+          1330,
+          1073,
+          1569,
+          1761,
+          328,
+          288,
+          125,
+          1620,
+          264
+        ],
+        "endpoint_dist": [
+          1.4414644241333008,
+          6.693058013916016,
+          2.2693819999694824,
+          0.9652828574180603,
+          3.2507967948913574,
+          4.876590251922607,
+          3.3306970596313477,
+          1.2349841594696045,
+          5.8425092697143555,
+          9.255013465881348,
+          1.2984212636947632,
+          2.5836963653564453,
+          5.906203269958496,
+          3.9347445964813232,
+          1.8527549505233765,
+          2.458263397216797,
+          7.6965436935424805,
+          2.2596096992492676,
+          2.4497601985931396,
+          4.623591423034668,
+          3.3757615089416504,
+          2.8686976432800293,
+          4.354642391204834,
+          2.6507208347320557,
+          1.2674821615219116,
+          3.1807076930999756,
+          4.321929454803467,
+          2.25634765625,
+          2.4679768085479736,
+          1.242406964302063,
+          1.650354266166687,
+          4.492090225219727,
+          5.940750598907471,
+          1.6874139308929443,
+          6.355512619018555,
+          7.02701473236084,
+          1.2605292797088623,
+          5.716012001037598,
+          2.3085837364196777,
+          3.409825086593628
+        ],
+        "spec_margin": [
+          1.4017744064331055,
+          -0.9852714538574219,
+          0.5474181175231934,
+          1.404078483581543,
+          0.287736177444458,
+          -0.2198953628540039,
+          0.43722963333129883,
+          1.1993567943572998,
+          0.1627063751220703,
+          -6.782273292541504,
+          1.448604702949524,
+          0.40788698196411133,
+          -0.2827153205871582,
+          0.13470673561096191,
+          -0.07403314113616943,
+          0.5791707038879395,
+          -2.201404571533203,
+          1.170691967010498,
+          -0.0077054500579833984,
+          -0.23633718490600586,
+          -0.5291824340820312,
+          0.1898641586303711,
+          0.7707839012145996,
+          0.1075143814086914,
+          1.2684568166732788,
+          0.2617514133453369,
+          -0.09739160537719727,
+          0.4169282913208008,
+          0.5450108051300049,
+          0.6122345924377441,
+          1.0239275693893433,
+          -0.9050800800323486,
+          0.17412042617797852,
+          1.30082106590271,
+          0.1179203987121582,
+          -2.387010097503662,
+          1.2541511058807373,
+          -0.34055328369140625,
+          0.47608160972595215,
+          0.12572550773620605
+        ],
+        "off_dist": [
+          2.8432388305664062,
+          5.707786560058594,
+          2.816800117492676,
+          2.369361400604248,
+          3.5385329723358154,
+          4.6566948890686035,
+          3.7679266929626465,
+          2.4343409538269043,
+          6.005215644836426,
+          2.472740411758423,
+          2.747025966644287,
+          2.9915833473205566,
+          5.623487949371338,
+          4.069451332092285,
+          1.778721809387207,
+          3.0374341011047363,
+          5.495139122009277,
+          3.4303016662597656,
+          2.4420547485351562,
+          4.387254238128662,
+          2.846579074859619,
+          3.0585618019104004,
+          5.125426292419434,
+          2.758235216140747,
+          2.5359389781951904,
+          3.4424591064453125,
+          4.2245378494262695,
+          2.673275947570801,
+          3.0129876136779785,
+          1.8546415567398071,
+          2.6742818355560303,
+          3.587010145187378,
+          6.114871025085449,
+          2.9882349967956543,
+          6.473433017730713,
+          4.640004634857178,
+          2.5146803855895996,
+          5.375458717346191,
+          2.78466534614563,
+          3.535550594329834
+        ],
+        "clf": []
+      },
+      "n_targets": 40,
+      "random_top5": 0.002770083102493075
+    }
+  }
+}

models/norman/config.json

@@ -0,0 +1,30 @@
+{
+  "dataset": "norman",
+  "embedding": "pca",
+  "split": "cell",
+  "rep_mode": "gene_op",
+  "match": "batch",
+  "d_pert": 64,
+  "hidden": 512,
+  "depth": 4,
+  "dropout": 0.0,
+  "lr": 0.001,
+  "weight_decay": 1e-05,
+  "batch_size": 1024,
+  "epochs": 60,
+  "lam_tan": 1.0,
+  "lam_semi": 0.5,
+  "lam_reg": 0.0001,
+  "grad_clip": 5.0,
+  "train_frac": 1.0,
+  "lam_dist": 0.0,
+  "n_dist_perts": 4,
+  "dist_n": 64,
+  "seed": 0,
+  "device_index": 3,
+  "components": [
+    "map",
+    "tan",
+    "semi"
+  ]
+}

models/norman/model.ptw

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66033fbe3d3755b025ca2cc9079f9d5796a6a0ad1e5174a917d3a8d29b01280b
+size 3575254

models/norman/train_info.json

@@ -0,0 +1,427 @@
+{
+  "history": [
+    {
+      "map": 9.090681012939005,
+      "tan": 51.53360254624311,
+      "semi": 0.024223575147729756,
+      "reg": 0.054827462553101426,
+      "total": 60.63640072766472
+    },
+    {
+      "map": 8.425921391038333,
+      "tan": 47.94027726790484,
+      "semi": 0.06671591771437842,
+      "reg": 0.08704450069104924,
+      "total": 56.39956412595861
+    },
+    {
+      "map": 8.010584641905393,
+      "tan": 45.6380573721493,
+      "semi": 0.14629346241845803,
+      "reg": 0.09322421428035288,
+      "total": 53.72179743822883
+    },
+    {
+      "map": 7.59055574501262,
+      "tan": 43.34580931944006,
+      "semi": 0.20786436011686044,
+      "reg": 0.10074490922338822,
+      "total": 51.04030850354363
+    },
+    {
+      "map": 7.467282842187321,
+      "tan": 42.08133489945356,
+      "semi": 0.229316008003319,
+      "reg": 0.10917095872847472,
+      "total": 49.66328727497774
+    },
+    {
+      "map": 7.1339511941461,
+      "tan": 41.18273690167595,
+      "semi": 0.23491062793661566,
+      "reg": 0.12083662180777859,
+      "total": 48.43415518367992
+    },
+    {
+      "map": 7.049157009405248,
+      "tan": 40.3212100758272,
+      "semi": 0.2424944858340656,
+      "reg": 0.1320137967739035,
+      "total": 47.49162696389591
+    },
+    {
+      "map": 6.835424668648663,
+      "tan": 39.61966542636647,
+      "semi": 0.24695228631882107,
+      "reg": 0.14088603240602157,
+      "total": 46.578581249012665
+    },
+    {
+      "map": 6.701632667990292,
+      "tan": 38.931431994718665,
+      "semi": 0.25987118329195413,
+      "reg": 0.14920588977196636,
+      "total": 45.763015522676355
+    },
+    {
+      "map": 6.650894136989818,
+      "tan": 38.54156505360323,
+      "semi": 0.2688319908783716,
+      "reg": 0.1569551400402013,
+      "total": 45.326890384449676
+    },
+    {
+      "map": 6.540102075127995,
+      "tan": 37.852282131419464,
+      "semi": 0.2698856874862138,
+      "reg": 0.16361256776487126,
+      "total": 44.527342571931726
+    },
+    {
+      "map": 6.474147845717037,
+      "tan": 37.53386289933149,
+      "semi": 0.2626304008504924,
+      "reg": 0.16940459477550843,
+      "total": 44.13934225194595
+    },
+    {
+      "map": 6.415744248558493,
+      "tan": 37.1947494394639,
+      "semi": 0.26695386714795055,
+      "reg": 0.17525369871188612,
+      "total": 43.743988541995776
+    },
+    {
+      "map": 6.372432673678679,
+      "tan": 36.79617848115809,
+      "semi": 0.2595589763539679,
+      "reg": 0.18128912391908028,
+      "total": 43.29840963027056
+    },
+    {
+      "map": 6.281425553209641,
+      "tan": 36.5531156764311,
+      "semi": 0.2646997798891628,
+      "reg": 0.18509905193658435,
+      "total": 42.966910137849695
+    },
+    {
+      "map": 6.194271781865289,
+      "tan": 36.4991086511051,
+      "semi": 0.25276901047019396,
+      "reg": 0.1893512685509289,
+      "total": 42.81978365954231
+    },
+    {
+      "map": 6.151554794872508,
+      "tan": 36.10174599815817,
+      "semi": 0.25931472944862705,
+      "reg": 0.1925594914047157,
+      "total": 42.3829771490658
+    },
+    {
+      "map": 6.139232712633469,
+      "tan": 35.99210352056167,
+      "semi": 0.255086947889889,
+      "reg": 0.19602525365703247,
+      "total": 42.25889907163732
+    },
+    {
+      "map": 6.0830746117760155,
+      "tan": 35.74959294936236,
+      "semi": 0.25373832552748565,
+      "reg": 0.19860927585293264,
+      "total": 41.95955607470344
+    },
+    {
+      "map": 6.0335288117913635,
+      "tan": 35.30244064331055,
+      "semi": 0.2590217774405199,
+      "reg": 0.20176466751624555,
+      "total": 41.465500102323645
+    },
+    {
+      "map": 6.110965315033408,
+      "tan": 35.13910877003389,
+      "semi": 0.2592454000430949,
+      "reg": 0.20424350789364645,
+      "total": 41.37971681707046
+    },
+    {
+      "map": 5.966023985077353,
+      "tan": 35.05921419929056,
+      "semi": 0.26047960142878923,
+      "reg": 0.2073487919919631,
+      "total": 41.155498280244714
+    },
+    {
+      "map": 5.955339403713451,
+      "tan": 34.779306804432586,
+      "semi": 0.26458901188829365,
+      "reg": 0.21020734945640845,
+      "total": 40.8669615352855
+    },
+    {
+      "map": 5.903381824493408,
+      "tan": 34.6496306587668,
+      "semi": 0.2582090713960283,
+      "reg": 0.2120374574819032,
+      "total": 40.682138835682586
+    },
+    {
+      "map": 5.896596165264354,
+      "tan": 34.57028377757353,
+      "semi": 0.26008522992624955,
+      "reg": 0.21361819347914526,
+      "total": 40.59694492115694
+    },
+    {
+      "map": 5.872959740021649,
+      "tan": 34.36479944341323,
+      "semi": 0.2692155930049279,
+      "reg": 0.2164839766919613,
+      "total": 40.372389400706574
+    },
+    {
+      "map": 5.916246035519769,
+      "tan": 34.28742004843319,
+      "semi": 0.2609161355477922,
+      "reg": 0.21840506331885562,
+      "total": 40.33414661183077
+    },
+    {
+      "map": 5.82154234717874,
+      "tan": 34.11426684435676,
+      "semi": 0.2631807318505119,
+      "reg": 0.21939034212161512,
+      "total": 40.06742224973791
+    },
+    {
+      "map": 5.7701695245855,
+      "tan": 33.894943573895624,
+      "semi": 0.25884204467429833,
+      "reg": 0.22040404467021718,
+      "total": 39.794556617736816
+    },
+    {
+      "map": 5.773050658843097,
+      "tan": 33.97260435889749,
+      "semi": 0.26208172738552094,
+      "reg": 0.22239502431715236,
+      "total": 39.876718521118164
+    },
+    {
+      "map": 5.795562414562001,
+      "tan": 33.92704879536348,
+      "semi": 0.25910024191526804,
+      "reg": 0.22363066476057558,
+      "total": 39.85218395906336
+    },
+    {
+      "map": 5.773452331038082,
+      "tan": 33.81521006191478,
+      "semi": 0.26146621125585895,
+      "reg": 0.22516642160275402,
+      "total": 39.719418357400336
+    },
+    {
+      "map": 5.737005745663362,
+      "tan": 33.67261639763327,
+      "semi": 0.25316396969206195,
+      "reg": 0.2269155650454409,
+      "total": 39.53622688966639
+    },
+    {
+      "map": 5.755465367261102,
+      "tan": 33.821473963120404,
+      "semi": 0.2626038178363267,
+      "reg": 0.22766746010850458,
+      "total": 39.708264126497156
+    },
+    {
+      "map": 5.697806498583625,
+      "tan": 33.55083689970129,
+      "semi": 0.2530074382529539,
+      "reg": 0.22914176521932378,
+      "total": 39.375170202816236
+    },
+    {
+      "map": 5.718713122255662,
+      "tan": 33.486163700328156,
+      "semi": 0.2511062082998893,
+      "reg": 0.23026478904135086,
+      "total": 39.33045269461239
+    },
+    {
+      "map": 5.680959217688617,
+      "tan": 33.43111181259155,
+      "semi": 0.25421492119922356,
+      "reg": 0.23129703542765448,
+      "total": 39.239201489616846
+    },
+    {
+      "map": 5.768596242455875,
+      "tan": 33.45481502308565,
+      "semi": 0.2551145647816798,
+      "reg": 0.23215885328895905,
+      "total": 39.35099158567541
+    },
+    {
+      "map": 5.68937446790583,
+      "tan": 33.34391178804285,
+      "semi": 0.24921904438558748,
+      "reg": 0.2329040185055312,
+      "total": 39.15791881785673
+    },
+    {
+      "map": 5.674329182680915,
+      "tan": 33.517153571633735,
+      "semi": 0.24444453637389577,
+      "reg": 0.23398998576928587,
+      "total": 39.31372793983011
+    },
+    {
+      "map": 5.64740593994365,
+      "tan": 33.40329338522518,
+      "semi": 0.24836034262005022,
+      "reg": 0.23468266953440273,
+      "total": 39.174902467166675
+    },
+    {
+      "map": 5.660573699895074,
+      "tan": 33.342162553001856,
+      "semi": 0.2492706146748627,
+      "reg": 0.23517600753728082,
+      "total": 39.12739445181454
+    },
+    {
+      "map": 5.709663426174837,
+      "tan": 33.25413148543414,
+      "semi": 0.24833782630808213,
+      "reg": 0.23567993259605238,
+      "total": 39.08798677781049
+    },
+    {
+      "map": 5.732073342098909,
+      "tan": 33.28247132020838,
+      "semi": 0.24573037137880044,
+      "reg": 0.23661140352487564,
+      "total": 39.137432659373566
+    },
+    {
+      "map": 5.699795814121471,
+      "tan": 33.2870241613949,
+      "semi": 0.24319663643836975,
+      "reg": 0.23693522328839584,
+      "total": 39.108441408942724
+    },
+    {
+      "map": 5.66639927555533,
+      "tan": 33.22080320470474,
+      "semi": 0.24251559464370503,
+      "reg": 0.2373233597506495,
+      "total": 39.008483325733856
+    },
+    {
+      "map": 5.605263008790858,
+      "tan": 33.22774385003483,
+      "semi": 0.24726183033164809,
+      "reg": 0.23782538315829108,
+      "total": 38.95666128046372
+    },
+    {
+      "map": 5.697606759912827,
+      "tan": 33.32277006261489,
+      "semi": 0.24908233061432838,
+      "reg": 0.23823434114456177,
+      "total": 39.1449411055621
+    },
+    {
+      "map": 5.6367628854863785,
+      "tan": 33.19991524079267,
+      "semi": 0.24523958914420185,
+      "reg": 0.2383270320646903,
+      "total": 38.95932102203369
+    },
+    {
+      "map": 5.643202080446131,
+      "tan": 33.072624010198254,
+      "semi": 0.24845830263460383,
+      "reg": 0.23823890024248293,
+      "total": 38.84007891486673
+    },
+    {
+      "map": 5.637564006973715,
+      "tan": 33.053748214946076,
+      "semi": 0.24669679955524557,
+      "reg": 0.23827644108849413,
+      "total": 38.814683970283056
+    },
+    {
+      "map": 5.623584515908185,
+      "tan": 32.98539599250345,
+      "semi": 0.24680674032253377,
+      "reg": 0.2385626555803944,
+      "total": 38.73240667230942
+    },
+    {
+      "map": 5.637006205670974,
+      "tan": 33.03702702241785,
+      "semi": 0.24454942257965312,
+      "reg": 0.2386796995997429,
+      "total": 38.796331125147205
+    },
+    {
+      "map": 5.6437254442888145,
+      "tan": 33.01595491521499,
+      "semi": 0.24437346265596502,
+      "reg": 0.2387647799709264,
+      "total": 38.781890252057245
+    },
+    {
+      "map": 5.622949656318216,
+      "tan": 33.08323285158943,
+      "semi": 0.2454144970020827,
+      "reg": 0.2388445695533472,
+      "total": 38.828912903280816
+    },
+    {
+      "map": 5.591019700555241,
+      "tan": 33.03575386720545,
+      "semi": 0.2427436273325892,
+      "reg": 0.23893670343300877,
+      "total": 38.748169001410986
+    },
+    {
+      "map": 5.5686474337297325,
+      "tan": 33.10028589473051,
+      "semi": 0.24532319868312163,
+      "reg": 0.2389937122078503,
+      "total": 38.79161789838005
+    },
+    {
+      "map": 5.666492995093851,
+      "tan": 33.08380558911492,
+      "semi": 0.24304543435573578,
+      "reg": 0.23905649868881002,
+      "total": 38.871844572179455
+    },
+    {
+      "map": 5.617153132663054,
+      "tan": 33.01807515761431,
+      "semi": 0.24422780490096876,
+      "reg": 0.2390376346952775,
+      "total": 38.75736561943503
+    },
+    {
+      "map": 5.6478853506200455,
+      "tan": 32.98348696091596,
+      "semi": 0.237635247190209,
+      "reg": 0.23903003269258669,
+      "total": 38.750213005963495
+    }
+  ],
+  "duration_s": 132.3485288619995,
+  "device": "cuda:3",
+  "n_train_cells": 69601
+}

models/replogle_k562/config.json

@@ -0,0 +1,30 @@
+{
+  "dataset": "replogle_k562",
+  "embedding": "pca",
+  "split": "cell",
+  "rep_mode": "gene_op",
+  "match": "batch",
+  "d_pert": 64,
+  "hidden": 512,
+  "depth": 4,
+  "dropout": 0.0,
+  "lr": 0.001,
+  "weight_decay": 1e-05,
+  "batch_size": 1024,
+  "epochs": 60,
+  "lam_tan": 1.0,
+  "lam_semi": 0.5,
+  "lam_reg": 0.0001,
+  "grad_clip": 5.0,
+  "train_frac": 1.0,
+  "lam_dist": 0.0,
+  "n_dist_perts": 4,
+  "dist_n": 64,
+  "seed": 0,
+  "device_index": 3,
+  "components": [
+    "map",
+    "tan",
+    "semi"
+  ]
+}

models/replogle_k562/model.ptw

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86c51f6874ae5ddd8a9db148b643c369bb6849341f1c2a11bd72af560ab92c96
+size 4010454

models/replogle_k562/train_info.json

@@ -0,0 +1,427 @@
+{
+  "history": [
+    {
+      "map": 17.031049418759036,
+      "tan": 100.7652035007229,
+      "semi": 0.0300869325780984,
+      "reg": 0.05529186458556683,
+      "total": 117.81130406763647
+    },
+    {
+      "map": 15.702098115698083,
+      "tan": 92.89242543802632,
+      "semi": 0.23400488679672216,
+      "reg": 0.06300994932845042,
+      "total": 108.71153378176999
+    },
+    {
+      "map": 14.478880721253233,
+      "tan": 86.36426633364195,
+      "semi": 0.44110062215235324,
+      "reg": 0.06825163805639589,
+      "total": 101.06370415625634
+    },
+    {
+      "map": 13.889420348328429,
+      "tan": 82.95308506953252,
+      "semi": 0.5040058114311912,
+      "reg": 0.07468520646745508,
+      "total": 97.09451611011059
+    },
+    {
+      "map": 13.466107021678578,
+      "tan": 80.20504859825233,
+      "semi": 0.543986253924184,
+      "reg": 0.08253169998333051,
+      "total": 93.94315684925427
+    },
+    {
+      "map": 12.98160643391795,
+      "tan": 78.17586279534675,
+      "semi": 0.5785175519330161,
+      "reg": 0.08983203342982701,
+      "total": 91.44673602611988
+    },
+    {
+      "map": 12.554369554891215,
+      "tan": 76.1268295684418,
+      "semi": 0.6103655777194283,
+      "reg": 0.0945982209273747,
+      "total": 88.98638935832234
+    },
+    {
+      "map": 12.333751034427,
+      "tan": 74.71545479514383,
+      "semi": 0.630091528613846,
+      "reg": 0.09966781087122954,
+      "total": 87.36425840699827
+    },
+    {
+      "map": 12.077223864468662,
+      "tan": 73.42789716844435,
+      "semi": 0.6364420017638763,
+      "reg": 0.10332031257740863,
+      "total": 85.82335018801999
+    },
+    {
+      "map": 11.925610579453506,
+      "tan": 72.46371103261973,
+      "semi": 0.6642577833943553,
+      "reg": 0.10804506952499414,
+      "total": 84.7214582368925
+    },
+    {
+      "map": 11.804282176030146,
+      "tan": 71.54561079941787,
+      "semi": 0.6641300312884442,
+      "reg": 0.11268213855755793,
+      "total": 83.68196769813439
+    },
+    {
+      "map": 11.561371233556178,
+      "tan": 70.57861020967557,
+      "semi": 0.6674197308428875,
+      "reg": 0.11685481599786064,
+      "total": 82.47370484587434
+    },
+    {
+      "map": 11.527189329073027,
+      "tan": 69.46824645996094,
+      "semi": 0.6522551836905541,
+      "reg": 0.12121122417511879,
+      "total": 81.3215787813261
+    },
+    {
+      "map": 11.307936358761477,
+      "tan": 68.95758423247895,
+      "semi": 0.6583475516988085,
+      "reg": 0.12639355069244063,
+      "total": 80.59471001563134
+    },
+    {
+      "map": 11.178683095164113,
+      "tan": 68.32847733931108,
+      "semi": 0.6504223911793201,
+      "reg": 0.13096818376284142,
+      "total": 79.83238745354987
+    },
+    {
+      "map": 11.003427319712452,
+      "tan": 67.71086586295785,
+      "semi": 0.6521954520956262,
+      "reg": 0.13524451967957732,
+      "total": 79.04040557068664
+    },
+    {
+      "map": 10.951560813111143,
+      "tan": 67.20007869175502,
+      "semi": 0.6550748394681262,
+      "reg": 0.13996704812947805,
+      "total": 78.47919226312018
+    },
+    {
+      "map": 10.97006556275603,
+      "tan": 66.65089327329164,
+      "semi": 0.6446052549721358,
+      "reg": 0.14411745791311387,
+      "total": 77.94327693790585
+    },
+    {
+      "map": 10.943851285166554,
+      "tan": 66.29637517557515,
+      "semi": 0.6255597565081212,
+      "reg": 0.14808494839575384,
+      "total": 77.55302161674994
+    },
+    {
+      "map": 10.829310615341385,
+      "tan": 65.82614502349458,
+      "semi": 0.6388694264672019,
+      "reg": 0.15234306628827926,
+      "total": 76.97490573239017
+    },
+    {
+      "map": 10.641024490455528,
+      "tan": 65.32731301443917,
+      "semi": 0.6232788764037095,
+      "reg": 0.15583418348392883,
+      "total": 76.27999204164976
+    },
+    {
+      "map": 10.647455141141817,
+      "tan": 64.99982744687563,
+      "semi": 0.6248106956481934,
+      "reg": 0.1595929733344487,
+      "total": 75.95970312341467
+    },
+    {
+      "map": 10.558299708676028,
+      "tan": 64.73885132430436,
+      "semi": 0.6366313130824597,
+      "reg": 0.16249608412965552,
+      "total": 75.61548237986379
+    },
+    {
+      "map": 10.54614951393821,
+      "tan": 64.54226055392971,
+      "semi": 0.6242870850996538,
+      "reg": 0.16610339148478073,
+      "total": 75.4005688753995
+    },
+    {
+      "map": 10.415968498626313,
+      "tan": 64.06486724259017,
+      "semi": 0.6104588338306972,
+      "reg": 0.16973760066094337,
+      "total": 74.78607970398743
+    },
+    {
+      "map": 10.342192488831358,
+      "tan": 63.679309597263085,
+      "semi": 0.6154440424659036,
+      "reg": 0.17334448517142953,
+      "total": 74.32923928793375
+    },
+    {
+      "map": 10.316441857969606,
+      "tan": 63.50818619170746,
+      "semi": 0.6117670683117656,
+      "reg": 0.175885058455653,
+      "total": 74.13052675321505
+    },
+    {
+      "map": 10.33138624414221,
+      "tan": 63.21885601885907,
+      "semi": 0.6094937672862759,
+      "reg": 0.17810528212553495,
+      "total": 73.85500434776405
+    },
+    {
+      "map": 10.27574026429808,
+      "tan": 62.94442481499214,
+      "semi": 0.6109191310870183,
+      "reg": 0.18016438457098874,
+      "total": 73.52563981886034
+    },
+    {
+      "map": 10.174346775203556,
+      "tan": 62.66180484325855,
+      "semi": 0.6059550039179913,
+      "reg": 0.1823571047612599,
+      "total": 73.13914450112875
+    },
+    {
+      "map": 10.115131217163878,
+      "tan": 62.483534082189784,
+      "semi": 0.593998660514881,
+      "reg": 0.1846313884893021,
+      "total": 72.8956800188337
+    },
+    {
+      "map": 10.109223378169073,
+      "tan": 62.23712921142578,
+      "semi": 0.6017299449289,
+      "reg": 0.18623309301865565,
+      "total": 72.64723294741148
+    },
+    {
+      "map": 10.1195849381484,
+      "tan": 62.454900221391156,
+      "semi": 0.5973637127257013,
+      "reg": 0.18793246014551682,
+      "total": 72.87318370868633
+    },
+    {
+      "map": 10.153793297804794,
+      "tan": 62.148185383189805,
+      "semi": 0.5928794482311646,
+      "reg": 0.18917821386417785,
+      "total": 72.59843682623529
+    },
+    {
+      "map": 10.018563270568848,
+      "tan": 62.01780002148121,
+      "semi": 0.5965188106933197,
+      "reg": 0.1902788954895812,
+      "total": 72.33464129559405
+    },
+    {
+      "map": 9.928655525306603,
+      "tan": 61.684608756721794,
+      "semi": 0.5881947108677456,
+      "reg": 0.19194930753150544,
+      "total": 71.907382122882
+    },
+    {
+      "map": 9.94748303797338,
+      "tan": 61.25752768578467,
+      "semi": 0.5875537790261306,
+      "reg": 0.19271386037399243,
+      "total": 71.4988080309583
+    },
+    {
+      "map": 10.026378891684793,
+      "tan": 61.36622505683403,
+      "semi": 0.592558935091093,
+      "reg": 0.1937573047040345,
+      "total": 71.68890479942421
+    },
+    {
+      "map": 10.008671971110555,
+      "tan": 61.23317356852742,
+      "semi": 0.5771013513788,
+      "reg": 0.1943636681739386,
+      "total": 71.53041770241477
+    },
+    {
+      "map": 9.882405454462225,
+      "tan": 61.013533703692545,
+      "semi": 0.5912098389167291,
+      "reg": 0.19557252777861311,
+      "total": 71.19156617003601
+    },
+    {
+      "map": 9.871191408727077,
+      "tan": 60.72403360342051,
+      "semi": 0.5780030558635663,
+      "reg": 0.19602004738597126,
+      "total": 70.88424890691584
+    },
+    {
+      "map": 9.842699013747177,
+      "tan": 60.829766062947066,
+      "semi": 0.5811261499856973,
+      "reg": 0.19663439207262806,
+      "total": 70.96305054503601
+    },
+    {
+      "map": 9.851649445372743,
+      "tan": 60.72009956062614,
+      "semi": 0.5874130930993464,
+      "reg": 0.1971096477725289,
+      "total": 70.86547792112673
+    },
+    {
+      "map": 9.752088125649985,
+      "tan": 60.520079773741884,
+      "semi": 0.5886614949672253,
+      "reg": 0.1976228573105552,
+      "total": 70.56652079000102
+    },
+    {
+      "map": 9.752064197094409,
+      "tan": 60.41498630077808,
+      "semi": 0.5917925590818579,
+      "reg": 0.19813017547130585,
+      "total": 70.4629694951045
+    },
+    {
+      "map": 9.708581379481725,
+      "tan": 60.503518339875455,
+      "semi": 0.5872518009953684,
+      "reg": 0.1987074689044581,
+      "total": 70.50574889740386
+    },
+    {
+      "map": 9.752153607157918,
+      "tan": 60.31089173354112,
+      "semi": 0.5835730073513923,
+      "reg": 0.19888550739783745,
+      "total": 70.3548547323648
+    },
+    {
+      "map": 9.624358759297953,
+      "tan": 60.314821961638216,
+      "semi": 0.5837421378532013,
+      "reg": 0.1992025768215006,
+      "total": 70.2310743455763
+    },
+    {
+      "map": 9.673671264153022,
+      "tan": 60.04714554625672,
+      "semi": 0.5863785871437618,
+      "reg": 0.19950627119510206,
+      "total": 70.01402956479555
+    },
+    {
+      "map": 9.601263975168203,
+      "tan": 59.98841005795962,
+      "semi": 0.582589351898664,
+      "reg": 0.19971654747987722,
+      "total": 69.88099145270013
+    },
+    {
+      "map": 9.659211703709193,
+      "tan": 60.230314973112826,
+      "semi": 0.5836410588258273,
+      "reg": 0.19984227941407787,
+      "total": 70.1813699301187
+    },
+    {
+      "map": 9.695229802812849,
+      "tan": 60.008484233509414,
+      "semi": 0.5898574017859125,
+      "reg": 0.199969258401301,
+      "total": 69.99866515320618
+    },
+    {
+      "map": 9.731371854806875,
+      "tan": 60.117312245554736,
+      "semi": 0.5865367480686733,
+      "reg": 0.20019410569946489,
+      "total": 70.14197520466594
+    },
+    {
+      "map": 9.704108560240114,
+      "tan": 60.16391328093293,
+      "semi": 0.5837481083808007,
+      "reg": 0.20012537322261117,
+      "total": 70.15991874793907
+    },
+    {
+      "map": 9.649337966720779,
+      "tan": 59.84723182777306,
+      "semi": 0.595232380675031,
+      "reg": 0.2003737132657658,
+      "total": 69.7942084770698
+    },
+    {
+      "map": 9.67079952784947,
+      "tan": 59.99238888629071,
+      "semi": 0.5785262170550111,
+      "reg": 0.20029796698650756,
+      "total": 69.95247412347175
+    },
+    {
+      "map": 9.637403655361819,
+      "tan": 59.95936525022829,
+      "semi": 0.5868523740149164,
+      "reg": 0.20019558710711344,
+      "total": 69.89021766959847
+    },
+    {
+      "map": 9.705964162752226,
+      "tan": 59.901117597307476,
+      "semi": 0.5755069797689264,
+      "reg": 0.20033776256945227,
+      "total": 69.8948581249683
+    },
+    {
+      "map": 9.650553046882926,
+      "tan": 60.12031802883396,
+      "semi": 0.5841203432578546,
+      "reg": 0.20030619087931398,
+      "total": 70.06295439484832
+    },
+    {
+      "map": 9.67817362252768,
+      "tan": 60.0385335947012,
+      "semi": 0.5809869928793474,
+      "reg": 0.20038042672268755,
+      "total": 70.00722394670758
+    }
+  ],
+  "duration_s": 134.94324254989624,
+  "device": "cuda:3",
+  "n_train_cells": 78128
+}

requirements.txt

@@ -0,0 +1,12 @@
+torch>=2.0
+numpy
+scipy
+scikit-learn
+pandas
+pyarrow
+scanpy
+anndata
+matplotlib
+
+# gears is only needed for the head-to-head baseline and lives in a separate env
+# (see scripts/setup_gears_env.sh) because it pins older torch/pyg versions.

scripts/extra_ablations.py

@@ -0,0 +1,110 @@
+"""extra experiments to fill in the runnable table cells. external methods we
+can't reproduce stay unreported; only real computed values get emitted."""
+import sys, os, json, time
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import numpy as np, torch
+from src.data.perturb_data import load_dataset
+from src.data.splits import load_split
+from src.training.train import TrainConfig, train
+from src.experiments.predictors import PivotPredictor, BaselinePredictor
+from src.experiments.forward_eval import evaluate_forward
+from src.experiments.nomination_eval import evaluate_nomination
+from src.evaluation.baselines import build_baseline
+from src.utils.common import save_json
+
+gpu = int(os.environ.get("PIVOT_GPU", "3"))
+data = load_dataset("norman")
+out = {}
+
+# ===== core-ablation extras (centroid reward, held-out perturbation) =====
+sp = load_split(data.dir, "perturbation")
+cands = [p for p in data.perturbations if len(data.parse(p)) == 1]
+targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30]
+gc = data.functional_clusters(seed=0)
+
+def fwd_inv(model, reward="centroid"):
+    dev = next(model.parameters()).device
+    pred = PivotPredictor(model, data, dev)
+    f = evaluate_forward(pred, data, list(sp["test_perts"])[:50], data.control_idx, max_perts=50)
+    r = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind=reward,
+                            method="ranking", gene_cluster=gc, device=dev)
+    g = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind=reward,
+                            method="guidance", guidance_init="warm", rerank=False,
+                            gene_cluster=gc, model=model, device=dev)
+    return f, r, g
+
+# velocity-only: train with only the tangent (velocity) loss
+mv, _ = train(TrainConfig(dataset="norman", split="perturbation", epochs=60, device_index=gpu,
+                          components=["tan"]), data=data, verbose=False)
+fv, rv, _ = fwd_inv(mv)
+out["velocity_only"] = {"mse": fv["mse"], "de_corr": fv["de_corr"], "mmd": fv["mmd"],
+                        "endpoint_dist": rv["endpoint_dist"], "top5": rv["top5"], "ndcg": rv["ndcg"]}
+print("velocity_only", out["velocity_only"], flush=True)
+
+# full model: ranking-only and guidance-without-reranking inverse rows
+mf, info = train(TrainConfig(dataset="norman", split="perturbation", epochs=60, device_index=gpu),
+                 data=data, verbose=False)
+dev = next(mf.parameters()).device
+ff, rf, gf = fwd_inv(mf)
+out["ranking_only"] = {"mse": ff["mse"], "de_corr": ff["de_corr"], "mmd": ff["mmd"],
+                       "endpoint_dist": rf["endpoint_dist"], "top5": rf["top5"], "ndcg": rf["ndcg"]}
+out["guidance_no_rerank"] = {"mse": ff["mse"], "de_corr": ff["de_corr"], "mmd": ff["mmd"],
+                             "endpoint_dist": gf["endpoint_dist"], "top5": gf["top5"], "ndcg": gf["ndcg"]}
+print("ranking_only", out["ranking_only"], flush=True)
+print("guidance_no_rerank", out["guidance_no_rerank"], flush=True)
+
+# ===== inverse-table baseline: average perturbation effect + ranking (cosine) =====
+bl = BaselinePredictor(build_baseline("AvgPerturbationEffect").fit(data, sp["train_perts"], sp["train_idx"]))
+ra = evaluate_nomination(bl, data, targets, cands, data.control_idx, reward_kind="cosine",
+                         method="ranking", gene_cluster=gc, device=dev)
+out["avg_effect_ranking"] = {k: ra[k] for k in ["top1", "top5", "ndcg", "func_top5"]}
+out["avg_effect_ranking"]["med_rank"] = float(np.median(ra["_per"]["rank"]))
+print("avg_effect_ranking", out["avg_effect_ranking"], flush=True)
+
+# ===== gpu memory for compute table =====
+torch.cuda.reset_peak_memory_stats(dev)
+c0 = torch.as_tensor(data.emb[data.control_idx[:256]], dtype=torch.float32, device=dev)
+from src.evaluation import inference as inf
+_ = inf.endpoint_ranking(mf, data, cands, c0, __import__("src.evaluation.rewards", fromlist=["Reward"]).Reward(
+    "centroid", target_c=data.emb[data.pert_to_idx[targets[0]]].mean(0), device=dev), device=dev)
+out["gpu_mem_mb"] = round(torch.cuda.max_memory_allocated(dev) / 1e6, 1)
+print("gpu_mem_mb", out["gpu_mem_mb"], flush=True)
+
+# ===== combination table: additive + random + pivot guidance (combination split) =====
+spc = load_split(data.dir, "combination")
+combo_cands = data.combos
+ctgt = [p for p in spc["test_perts"] if len(data.parse(p)) == 2][:26]
+mc, _ = train(TrainConfig(dataset="norman", split="combination", epochs=60, device_index=gpu),
+              data=data, verbose=False)
+devc = next(mc.parameters()).device
+predc = PivotPredictor(mc, data, devc)
+from src.experiments.nomination_eval import rank_candidates
+import src.evaluation.metrics as M
+rng = np.random.default_rng(0)
+# additive baseline ranking over observed combos
+addb = build_baseline("Additive").fit(data, spc["train_perts"], spc["train_idx"])
+addp = BaselinePredictor(addb)
+def combo_eval(predictor):
+    e1=e5=ov=0.0
+    for p in ctgt:
+        cstar = data.emb[data.pert_to_idx[p]].mean(0)
+        c0n = data.emb[rng.choice(data.control_idx, 128, replace=False)]
+        sk = dict(kind="centroid", c_star=cstar, target_sample=data.emb[data.pert_to_idx[p]],
+                  device=devc, control_ref=data.emb[data.control_idx].mean(0))
+        ranked,_ = rank_candidates(predictor, combo_cands, c0n, sk)
+        e1 += M.top_k_accuracy(ranked,p,1); e5 += M.top_k_accuracy(ranked,p,5)
+        ov += M.partial_overlap(data.parse(ranked[0]), set(data.parse(p)))
+    n=len(ctgt); return e1/n, e5/n, ov/n
+ae1, ae5, aov = combo_eval(addp)
+out["combo_additive"] = {"exact1": ae1, "exact5": ae5, "overlap": aov}
+# random combos
+r_e1=r_e5=r_ov=0.0
+for p in ctgt:
+    pick = list(rng.choice(combo_cands, 5, replace=False))
+    r_e1 += float(p==pick[0]); r_e5 += float(p in pick)
+    r_ov += M.partial_overlap(data.parse(pick[0]), set(data.parse(p)))
+n=len(ctgt); out["combo_random"]={"exact1":r_e1/n,"exact5":r_e5/n,"overlap":r_ov/n}
+print("combo_additive", out["combo_additive"], "combo_random", out["combo_random"], flush=True)
+
+save_json(out, "experiments/results/norman_extra_ablations.json")
+print("FILL_DONE", flush=True)

scripts/figures.py

@@ -0,0 +1,172 @@
+"""figures for the paper, helvetica-ish font (nimbus sans).
+figure 1: effect-axis embedding panels (control / observed / pivot-predicted).
+figure 2: quantitative results (forward bars, gears head-to-head, dist-loss, reward)."""
+import sys, os, glob, json
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+from matplotlib import font_manager as fm
+import matplotlib.pyplot as plt
+from matplotlib.patches import Patch
+from matplotlib.lines import Line2D
+
+# helvetica-family font (nimbus sans = urw helvetica clone)
+for patt in ["/usr/share/fonts/opentype/urw-base35/NimbusSans-*.otf",
+             "/usr/share/fonts/truetype/liberation2/LiberationSans-*.ttf"]:
+    for f in glob.glob(patt):
+        try: fm.fontManager.addfont(f)
+        except Exception: pass
+plt.rcParams.update({
+    "font.family": "sans-serif",
+    "font.sans-serif": ["Nimbus Sans", "Helvetica", "Arial", "Liberation Sans", "DejaVu Sans"],
+    "mathtext.fontset": "dejavusans",
+    "font.size": 11, "axes.labelsize": 12, "axes.titlesize": 12.5,
+    "xtick.labelsize": 10, "ytick.labelsize": 10, "legend.fontsize": 10,
+    "axes.linewidth": 0.9, "axes.edgecolor": "#444444",
+    "xtick.color": "#444444", "ytick.color": "#444444",
+    "axes.labelcolor": "#222222", "text.color": "#222222",
+    "figure.dpi": 150, "savefig.dpi": 320, "savefig.bbox": "tight",
+    "axes.spines.top": False, "axes.spines.right": False,
+    "axes.grid": True, "grid.color": "#E6E8EB", "grid.linewidth": 0.8,
+    "axes.axisbelow": True, "legend.frameon": False,
+})
+# palette
+C_CTRL, C_OBS, C_PRED, C_PIVOT, C_BASE, C_ACC = "#B9C2CC", "#2D6FB3", "#E4572E", "#1B9E77", "#9AA6B2", "#6A4C93"
+FIG = "figures"
+RES = "experiments/results"
+os.makedirs(FIG, exist_ok=True)
+def L(n): return json.load(open(f"{RES}/{n}.json"))
+def save(fig, name):
+    fig.savefig(f"{FIG}/{name}.png"); fig.savefig(f"{FIG}/{name}.pdf"); plt.close(fig)
+    print("wrote", name)
+
+
+def kde_contour(ax, X, color, levels=4):
+    try:
+        from scipy.stats import gaussian_kde
+        if len(X) < 10: return
+        k = gaussian_kde(X.T)
+        xmin, ymin = X.min(0); xmax, ymax = X.max(0)
+        xs, ys = np.mgrid[xmin:xmax:80j, ymin:ymax:80j]
+        z = k(np.vstack([xs.ravel(), ys.ravel()])).reshape(xs.shape)
+        ax.contour(xs, ys, z, levels=levels, colors=color, linewidths=1.0, alpha=0.55)
+    except Exception:
+        pass
+
+
+def figure1(model, data, targets, device):
+    """effect-axis panels: x = projection on (target-control) direction, y = orthogonal pc."""
+    from src.evaluation import inference as inf
+    import torch
+    rng = np.random.default_rng(0)
+    ctrl_idx = data.control_idx
+    cmean = data.emb[ctrl_idx].mean(0)
+    c0e = torch.as_tensor(data.emb[rng.choice(ctrl_idx, 400, replace=False)], dtype=torch.float32, device=device)
+    n = len(targets)
+    fig, axes = plt.subplots(1, n, figsize=(4.3 * n, 4.1))
+    if n == 1: axes = [axes]
+    for ax, p in zip(axes, targets):
+        ti = data.pert_to_idx[p]
+        tmean = data.emb[ti].mean(0)
+        d = tmean - cmean; d = d / (np.linalg.norm(d) + 1e-9)          # effect direction
+        # orthogonal axis = top pc of perturbed cells with effect-dir removed
+        Y = data.emb[ti] - data.emb[ti].mean(0)
+        Y = Y - np.outer(Y @ d, d)
+        u, s, vt = np.linalg.svd(Y, full_matrices=False); o = vt[0]
+        def proj(M): return np.c_[(M - cmean) @ d, (M - cmean) @ o]
+        ci = rng.choice(ctrl_idx, 500, replace=False)
+        Pc, Po = proj(data.emb[ci]), proj(data.emb[ti])
+        e = inf.encode_label(model, data, p, device)
+        Ppred = proj(inf.forward_predict(model, c0e, e).cpu().numpy())
+        ax.scatter(Pc[:, 0], Pc[:, 1], s=9, c=C_CTRL, alpha=0.7, linewidths=0, label="control", rasterized=True)
+        ax.scatter(Po[:, 0], Po[:, 1], s=11, c=C_OBS, alpha=0.55, linewidths=0, label="observed perturbed", rasterized=True)
+        ax.scatter(Ppred[:, 0], Ppred[:, 1], s=22, marker="X", c=C_PRED, alpha=0.9,
+                   edgecolors="white", linewidths=0.4, label="PIVOT predicted")
+        kde_contour(ax, Po, C_OBS); kde_contour(ax, Ppred, C_PRED)
+        # arrow control-centroid -> observed-centroid (the transport)
+        cc, oc = proj(cmean[None])[0], proj(tmean[None])[0]
+        ax.annotate("", xy=(oc[0], oc[1]), xytext=(cc[0], cc[1]),
+                    arrowprops=dict(arrowstyle="-|>", color="#333333", lw=1.6, alpha=0.8))
+        ax.set_title(p.replace("_", "+"), fontweight="bold")
+        ax.set_xlabel("effect axis"); ax.set_ylabel("orthogonal axis")
+        ax.tick_params(length=0)
+    handles = [Line2D([], [], marker='o', ls='', mfc=C_CTRL, mec='none', ms=7, label='control'),
+               Line2D([], [], marker='o', ls='', mfc=C_OBS, mec='none', ms=7, label='observed perturbed'),
+               Line2D([], [], marker='X', ls='', mfc=C_PRED, mec='white', ms=8, label='PIVOT predicted')]
+    fig.legend(handles=handles, loc="lower center", ncol=3, bbox_to_anchor=(0.5, -0.04))
+    fig.suptitle("Control cells transported toward the perturbed population", y=1.02,
+                 fontsize=13, fontweight="bold")
+    fig.tight_layout()
+    save(fig, "fig1_embedding_panels")
+
+
+def figure2_results():
+    fig, ax = plt.subplots(2, 2, figsize=(11, 7.4))
+    # (a) forward de-corr across methods (held-out perturbation), from benchmark
+    bf = L("norman_benchmark")["forward"]
+    order = ["PIVOT", "LinearResponse", "kNN-latent", "Additive", "NearestPerturbationCentroid",
+             "ConditionalMLP", "EndpointMLP", "AvgPerturbationEffect", "MeanControl", "Random"]
+    pretty = {"PIVOT": "PIVOT", "LinearResponse": "Linear", "kNN-latent": "kNN-latent",
+              "Additive": "Additive", "NearestPerturbationCentroid": "Nearest centroid",
+              "ConditionalMLP": "Conditional MLP", "EndpointMLP": "Endpoint MLP",
+              "AvgPerturbationEffect": "Avg. effect", "MeanControl": "Mean control", "Random": "Random"}
+    vals = [(pretty[m], bf[m]["de_corr"]) for m in order if m in bf]
+    vals.sort(key=lambda kv: kv[1])
+    names = [v[0] for v in vals]; de = [v[1] for v in vals]
+    cols = [C_PIVOT if n == "PIVOT" else C_BASE for n in names]
+    a = ax[0, 0]; a.barh(names, de, color=cols, edgecolor="white", height=0.74)
+    for i, v in enumerate(de): a.text(v + 0.01, i, f"{v:.2f}", va="center", fontsize=9, color="#333")
+    a.set_xlim(0, 1.0); a.set_xlabel("DE correlation $\\uparrow$"); a.grid(axis="y", visible=False)
+    a.set_title("a  Forward direction, held-out perturbations", loc="left", fontweight="bold")
+
+    # (b) gears head-to-head
+    pg = L("pivot_vs_gears")
+    b = ax[0, 1]
+    bars = b.bar(["PIVOT", "GEARS"], [pg["pivot_pearson_de_expr"], pg["gears_pearson_de_expr"]],
+                 color=[C_PIVOT, C_ACC], edgecolor="white", width=0.55)
+    for r, v in zip(bars, [pg["pivot_pearson_de_expr"], pg["gears_pearson_de_expr"]]):
+        b.text(r.get_x() + r.get_width()/2, v + 0.012, f"{v:.3f}", ha="center", fontsize=11, fontweight="bold")
+    b.set_ylim(0, 1.08); b.set_ylabel("Top-20 DE-gene Pearson $\\uparrow$"); b.grid(axis="x", visible=False)
+    b.set_title("b  Head-to-head vs GEARS (matched perts)", loc="left", fontweight="bold")
+
+    # (c) distributional loss: mmd down, de-corr preserved
+    dl = L("norman_distloss")["rows"]
+    lam = sorted(float(k) for k in dl); mmd = [dl[str(l) if str(l) in dl else f"{l:.1f}"]["mmd"] for l in lam]
+    de2 = [dl[str(l) if str(l) in dl else f"{l:.1f}"]["de_corr"] for l in lam]
+    c = ax[1, 0]
+    c.plot(lam, mmd, "o-", color=C_PRED, lw=2.2, ms=7, label="MMD $\\downarrow$")
+    c.set_xlabel("distributional-loss weight $\\lambda_{\\mathrm{dist}}$"); c.set_ylabel("population MMD $\\downarrow$", color=C_PRED)
+    c.tick_params(axis="y", colors=C_PRED); c.set_ylim(0, max(mmd)*1.15)
+    c2 = c.twinx(); c2.plot(lam, de2, "s--", color=C_OBS, lw=2.0, ms=6, label="DE-corr $\\uparrow$")
+    c2.set_ylabel("DE correlation $\\uparrow$", color=C_OBS); c2.tick_params(axis="y", colors=C_OBS)
+    c2.set_ylim(0.5, 0.95); c2.grid(False); c2.spines["top"].set_visible(False)
+    c.set_title("c  Distributional flow loss: MMD $6\\times$ lower, direction kept", loc="left", fontweight="bold")
+
+    # (d) reward ablation top-5
+    rw = L("norman_ablation_reward")["rows"]
+    rmap = {"centroid": "Centroid", "nn_target": "NN-target", "mmd": "MMD", "wasserstein": "Wasserstein", "cosine": "Cosine"}
+    rn = [rmap.get(k, k) for k in rw]; t5 = [rw[k]["top5"] for k in rw]
+    cols2 = [C_PIVOT if rmap.get(k) == "Cosine" else C_BASE for k in rw]
+    d = ax[1, 1]; bars = d.bar(rn, t5, color=cols2, edgecolor="white", width=0.66)
+    for r, v in zip(bars, t5): d.text(r.get_x()+r.get_width()/2, v+0.006, f"{v:.2f}", ha="center", fontsize=9.5)
+    d.set_ylabel("nomination Top-5 $\\uparrow$"); d.set_ylim(0, max(t5)*1.25); d.grid(axis="x", visible=False)
+    d.tick_params(axis="x", rotation=18)
+    d.set_title("d  Direction-aware reward wins at nomination", loc="left", fontweight="bold")
+    fig.tight_layout()
+    save(fig, "fig2_results")
+
+
+if __name__ == "__main__":
+    from src.data.perturb_data import load_dataset
+    from src.training.train import TrainConfig, train
+    data = load_dataset("norman")
+    gpu = int(os.environ.get("PIVOT_GPU", "3"))
+    cfg = TrainConfig(dataset="norman", split="cell", epochs=60, device_index=gpu)
+    model, info = train(cfg, data=data, verbose=False)
+    dev = next(model.parameters()).device
+    singles = [p for p in data.perturbations if len(data.parse(p)) == 1]
+    combos = [p for p in data.perturbations if len(data.parse(p)) == 2]
+    figure1(model, data, [singles[0], singles[7], combos[0]], dev)
+    figure2_results()
+    print("FIGURES_V2_DONE")

scripts/gears_compare.py

@@ -0,0 +1,58 @@
+"""real gears head-to-head on norman, aligned to our held-out-perturbation test set.
+runs in the isolated pivot_gears env (torch cu118 + pyg + cell-gears, gpu-capable)."""
+import sys, os, json
+import numpy as np
+import torch
+
+# our held-out perturbation test labels (no src import; read the npz directly)
+split = np.load("data/processed/norman/splits/perturbation.npz", allow_pickle=True)
+my_test = [str(p) for p in split["test_perts"]]
+
+def to_gears(p):
+    g = [x for x in p.split("_") if x]
+    return g[0] + "+ctrl" if len(g) == 1 else g[0] + "+" + g[1]
+
+import pickle
+from gears import PertData, GEARS
+from gears.inference import evaluate, compute_metrics
+
+my_train = [str(p) for p in np.load("data/processed/norman/splits/perturbation.npz",
+                                    allow_pickle=True)["train_perts"]]
+dev = "cuda"
+pert_data = PertData("./gears_data")
+pert_data.load(data_name="norman")
+avail = set(map(str, pert_data.adata.obs["condition"].unique()))
+print("example GEARS conditions:", list(avail)[:6], flush=True)
+gears_test = sorted({to_gears(p) for p in my_test} & avail)
+gears_train = sorted(({to_gears(p) for p in my_train} & avail) - set(gears_test))
+if "ctrl" in avail:
+    gears_train = sorted(set(gears_train) | {"ctrl"})
+# carve a small val set from train (gears requires non-empty val)
+rng = np.random.default_rng(0)
+val = sorted(rng.choice([c for c in gears_train if c != "ctrl"],
+                        size=max(1, len(gears_train) // 10), replace=False).tolist())
+gears_train = sorted(set(gears_train) - set(val))
+split_dict = {"train": gears_train, "val": val, "test": gears_test}
+print(f"custom split -> train {len(gears_train)} | val {len(val)} | test {len(gears_test)} "
+      f"(our test {len(my_test)})", flush=True)
+os.makedirs("gears_data", exist_ok=True)
+sd_path = "gears_data/pivot_custom_split.pkl"
+pickle.dump(split_dict, open(sd_path, "wb"))
+pert_data.prepare_split(split="custom", split_dict_path=sd_path)
+pert_data.get_dataloader(batch_size=64, test_batch_size=128)
+
+model = GEARS(pert_data, device=dev)
+model.model_initialize(hidden_size=64)
+model.train(epochs=20)
+
+res = evaluate(pert_data.dataloader["test_loader"], model.model, False, dev)
+out = compute_metrics(res)
+metrics, per_pert = out if isinstance(out, tuple) else (out, {})
+keep = {k: float(v) for k, v in metrics.items()
+        if any(s in k for s in ["pearson", "mse", "r2", "spearman"])}
+print("GEARS held-out-perturbation metrics:", flush=True)
+for k, v in sorted(keep.items()):
+    print(f"  {k:24s} {v:.4f}", flush=True)
+json.dump({"n_test_perts": len(gears_test), "test_perts": gears_test, "metrics": keep},
+          open("experiments/results/gears_norman.json", "w"), indent=2, default=float)
+print("GEARS_RUN_DONE", flush=True)

scripts/gears_ranking.py

@@ -0,0 +1,144 @@
+"""gears + ranking baseline for inverse nomination, done entirely in gears' native
+gene space (fair: top-k recovery is representation-agnostic). trains gears on our
+held-out-perturbation split, predicts post-perturbation expression for every
+candidate, and ranks candidates by l2 distance to each held-out target's true mean
+expression. emits top-1/5, ndcg@10, median rank for single-gene and combination
+targets. runs in the isolated pivot_gears env."""
+import os, json, pickle
+import numpy as np
+
+PROC = "data/processed/norman"
+split = np.load(f"{PROC}/splits/perturbation.npz", allow_pickle=True)
+splitC = np.load(f"{PROC}/splits/combination.npz", allow_pickle=True)
+my_test = [str(p) for p in split["test_perts"]]
+my_train = [str(p) for p in split["train_perts"]]
+combo_test = [str(p) for p in splitC["test_perts"]]
+
+
+def to_gears(p):
+    g = [x for x in p.split("_") if x]
+    return g[0] + "+ctrl" if len(g) == 1 else g[0] + "+" + g[1]
+
+
+def ndcg_at_k(ranked, true, k=10):
+    rel = [1.0 if r == true else 0.0 for r in ranked[:k]]
+    dcg = sum(r / np.log2(i + 2) for i, r in enumerate(rel))
+    return float(dcg)  # ideal dcg = 1 (single relevant) -> ndcg = dcg
+
+
+from gears import PertData, GEARS
+
+dev = "cuda"
+pert_data = PertData("./gears_data")
+pert_data.load(data_name="norman")
+avail = set(map(str, pert_data.adata.obs["condition"].unique()))
+gears_test = sorted({to_gears(p) for p in my_test} & avail)
+gears_train = sorted(({to_gears(p) for p in my_train} & avail) - set(gears_test))
+if "ctrl" in avail:
+    gears_train = sorted(set(gears_train) | {"ctrl"})
+rng = np.random.default_rng(0)
+val = sorted(rng.choice([c for c in gears_train if c != "ctrl"],
+                        size=max(1, len(gears_train) // 10), replace=False).tolist())
+gears_train = sorted(set(gears_train) - set(val))
+split_dict = {"train": gears_train, "val": val, "test": gears_test}
+os.makedirs("gears_data", exist_ok=True)
+pickle.dump(split_dict, open("gears_data/pivot_custom_split.pkl", "wb"))
+pert_data.prepare_split(split="custom", split_dict_path="gears_data/pivot_custom_split.pkl")
+pert_data.get_dataloader(batch_size=64, test_batch_size=128)
+
+model = GEARS(pert_data, device=dev)
+model.model_initialize(hidden_size=64)
+model.train(epochs=20)
+print("GEARS trained", flush=True)
+
+adata = pert_data.adata
+genes = list(adata.var["gene_name"]) if "gene_name" in adata.var else list(adata.var_names)
+cond = adata.obs["condition"].astype(str).values
+X = adata.X
+X = np.asarray(X.todense()) if hasattr(X, "todense") else np.asarray(X)
+
+
+def true_mean(gcond):
+    m = cond == gcond
+    return X[m].mean(0) if m.sum() else None
+
+
+# candidate single genes present in gears as "GENE+ctrl"
+single_conds = sorted([c for c in avail if c.endswith("+ctrl") and c != "ctrl"])
+single_genes = [c.split("+")[0] for c in single_conds]
+# predict every candidate's post-perturbation expression (gene space)
+pred = {}
+B = 64
+for i in range(0, len(single_genes), B):
+    chunk = single_genes[i:i + B]
+    try:
+        out = model.predict([[g] for g in chunk])
+    except Exception as e:
+        print("predict chunk failed", chunk[:3], e, flush=True)
+        out = {}
+    for g in chunk:
+        for key in (g, g + "+ctrl", "_".join([g])):
+            if key in out:
+                pred[g] = np.asarray(out[key]); break
+print(f"predicted {len(pred)}/{len(single_genes)} single candidates", flush=True)
+
+cand_genes = [g for g in single_genes if g in pred]
+P = np.stack([pred[g] for g in cand_genes])  # [n_cand, n_genes]
+
+# single-gene targets (held-out), matched to pivot singles
+single_targets = [t for t in my_test if "_" not in t and to_gears(t).split("+")[0] in pred]
+res = {"single": {"top1": [], "top5": [], "ndcg": [], "rank": []}}
+for t in single_targets:
+    tg = to_gears(t)
+    tm = true_mean(tg)
+    if tm is None:
+        continue
+    d = np.linalg.norm(P - tm[None], axis=1)
+    order = [cand_genes[i] for i in np.argsort(d)]
+    tgene = tg.split("+")[0]
+    res["single"]["top1"].append(float(order[0] == tgene))
+    res["single"]["top5"].append(float(tgene in order[:5]))
+    res["single"]["ndcg"].append(ndcg_at_k(order, tgene, 10))
+    res["single"]["rank"].append(int(order.index(tgene) + 1) if tgene in order else len(order))
+
+# combination targets: predict combos and rank among observed combos
+combo_conds = sorted([c for c in avail if c.count("+") == 1 and "ctrl" not in c])
+cpred = {}
+for i in range(0, len(combo_conds), B):
+    chunk = combo_conds[i:i + B]
+    try:
+        out = model.predict([c.split("+") for c in chunk])
+    except Exception as e:
+        print("combo predict failed", e, flush=True); out = {}
+    for c in chunk:
+        for key in (c, c.replace("+", "_"), "_".join(c.split("+"))):
+            if key in out:
+                cpred[c] = np.asarray(out[key]); break
+print(f"predicted {len(cpred)}/{len(combo_conds)} combo candidates", flush=True)
+cc = [c for c in combo_conds if c in cpred]
+if cc:
+    PC = np.stack([cpred[c] for c in cc])
+    combo_tg = [to_gears(t) for t in combo_test if to_gears(t) in cpred]
+    res["combo"] = {"top1": [], "top5": [], "overlap": [], "ndcg": []}
+    for tg in combo_tg:
+        tm = true_mean(tg)
+        if tm is None:
+            continue
+        d = np.linalg.norm(PC - tm[None], axis=1)
+        order = [cc[i] for i in np.argsort(d)]
+        res["combo"]["top1"].append(float(order[0] == tg))
+        res["combo"]["top5"].append(float(tg in order[:5]))
+        res["combo"]["ndcg"].append(ndcg_at_k(order, tg, 10))
+        s_true = set(tg.split("+"))
+        res["combo"]["overlap"].append(len(set(order[0].split("+")) & s_true) / len(s_true))
+
+agg = {}
+for grp, d in res.items():
+    agg[grp] = {k: (float(np.mean(v)) if v else None) for k, v in d.items()}
+    agg[grp]["n"] = len(next(iter(d.values())))
+    if "rank" in d and d["rank"]:
+        agg[grp]["med_rank"] = float(np.median(d["rank"]))
+agg["n_candidates_single"] = len(cand_genes)
+agg["n_candidates_combo"] = len(cc) if cc else 0
+json.dump(agg, open("experiments/results/gears_ranking.json", "w"), indent=2, default=float)
+print("GEARS_RANK_DONE", json.dumps(agg), flush=True)

scripts/setup_gears_env.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+source /home/bcheng/miniconda3/etc/profile.d/conda.sh
+# fresh env, gears needs py3.10
+echo "=== create fresh env pivot_gears (py3.10) ==="
+conda create -y -n pivot_gears python=3.10 >/tmp/g_env.log 2>&1 && echo "env created"
+conda activate pivot_gears
+# torch cu118 bundles its own cuda 11.8 runtime so it works on the old 470 driver
+echo "=== torch 2.2.0 +cu118 WITH deps (bundles CUDA 11.8 runtime; works on 470 driver) ==="
+pip install --no-input "numpy<2" >/tmp/g_np.log 2>&1
+pip install --no-input torch==2.2.0 torchvision==0.17.0 --index-url https://download.pytorch.org/whl/cu118 >/tmp/g_torch.log 2>&1 && echo "torch+cu118 installed"
+python -c "import torch; print('TORCH', torch.__version__, 'CUDA avail:', torch.cuda.is_available(), 'devices:', torch.cuda.device_count())"
+echo "=== torch_geometric + cell-gears ==="
+pip install --no-input torch_geometric >/tmp/g_pyg.log 2>&1 && echo "pyg ok"
+pip install --no-input cell-gears >/tmp/g_gears.log 2>&1 && echo "gears ok"
+echo "=== final import + GPU test ==="
+python - <<PY
+import torch, torch_geometric, gears, numpy
+print("numpy", numpy.__version__)
+print("torch", torch.__version__, "cuda", torch.cuda.is_available())
+print("PyG", torch_geometric.__version__, "GEARS", getattr(gears,"__version__","?"))
+if torch.cuda.is_available():
+    x=torch.randn(1000,1000,device="cuda"); print("GPU matmul ok:", (x@x).sum().item()!=0)
+PY
+echo REBUILD_DONE

scripts/timing_and_scaling.py

@@ -0,0 +1,115 @@
+"""second round of table fills, all from pivot's own pipeline (no fabrication):
+  a. pivot reward-guidance for combinatorial nomination (Table 7).
+  b. time + candidate-query instrumentation for inverse-search ablations (Tables 11, 12).
+  c. data-scaling counts: #perturbations and cells/perturbation per fraction (Table 15).
+  d. held-out gene mse per perturbation representation (Table 10), trained on Replogle K562
+     gene split.
+writes experiments/results/norman_timing_scaling.json."""
+import sys, os, json, time
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import numpy as np, torch
+from src.data.perturb_data import load_dataset
+from src.data.splits import load_split
+from src.training.train import TrainConfig, train
+from src.experiments.predictors import PivotPredictor
+from src.experiments.forward_eval import evaluate_forward
+from src.experiments.nomination_eval import evaluate_nomination
+from src.utils.common import save_json
+
+gpu = int(os.environ.get("PIVOT_GPU", "3"))
+data = load_dataset("norman")
+gc = data.functional_clusters(seed=0)
+out = {}
+
+# ===== a. pivot reward-guidance for combinatorial nomination (Table 7) =====
+spc = load_split(data.dir, "combination")
+combo_cands = data.combos
+ctgt = [p for p in spc["test_perts"] if len(data.parse(p)) == 2][:26]
+mc, _ = train(TrainConfig(dataset="norman", split="combination", epochs=60, device_index=gpu),
+              data=data, verbose=False)
+devc = next(mc.parameters()).device
+predc = PivotPredictor(mc, data, devc)
+g = evaluate_nomination(predc, data, ctgt, combo_cands, data.control_idx, reward_kind="centroid",
+                        method="guidance", guidance_init="warm", rerank=True, gene_cluster=gc,
+                        model=mc, device=devc)
+out["combo_guidance"] = {"top1": g["top1"], "top5": g["top5"], "ndcg": g["ndcg"],
+                         "endpoint_dist": g["endpoint_dist"]}
+print("combo_guidance", out["combo_guidance"], flush=True)
+
+# ===== b. time + query instrumentation for inverse search (Tables 11, 12) =====
+sp = load_split(data.dir, "perturbation")
+cands = [p for p in data.perturbations if len(data.parse(p)) == 1]
+targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30]
+mf, _ = train(TrainConfig(dataset="norman", split="perturbation", epochs=60, device_index=gpu),
+              data=data, verbose=False)
+dev = next(mf.parameters()).device
+pred = PivotPredictor(mf, data, dev)
+NC = len(cands)
+KN = 10
+
+
+def timed(**kw):
+    t0 = time.perf_counter()
+    r = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind="centroid",
+                            gene_cluster=gc, model=mf, device=dev, **kw)
+    return r, (time.perf_counter() - t0) / max(1, len(targets))
+
+
+# table 11 strategies: (label, kwargs, query_count)
+strat = {
+    "ranking_only":      (dict(method="ranking"), NC),
+    "random_opt":        (dict(method="guidance", guidance_init="random", rerank=False), 25),
+    "mean_top_init":     (dict(method="guidance", guidance_init="mean_top", rerank=False), 25 + KN),
+    "guidance_no_norm":  (dict(method="guidance", guidance_normalize=False, rerank=False), 25),
+    "guidance_norm":     (dict(method="guidance", guidance_normalize=True, rerank=False), 25),
+    "guidance_rerank":   (dict(method="guidance", rerank=True), 25 + KN),
+}
+out["search_timing"] = {}
+for name, (kw, q) in strat.items():
+    r, dt = timed(**kw)
+    out["search_timing"][name] = {"sec_per_target": round(dt, 3), "queries": q,
+                                  "top5": r["top5"], "ndcg": r["ndcg"], "endpoint_dist": r["endpoint_dist"]}
+    print("timing", name, out["search_timing"][name], flush=True)
+
+# table 12: time vs guidance steps
+out["guidance_step_time"] = {}
+for s in [0, 5, 10, 25, 50, 100]:
+    if s == 0:
+        r, dt = timed(method="ranking")
+    else:
+        r, dt = timed(method="guidance", guidance_steps=s, rerank=False, guidance_normalize=True)
+    out["guidance_step_time"][str(s)] = round(dt, 3)
+    print("step_time", s, round(dt, 3), flush=True)
+
+# ===== c. data-scaling counts (Table 15) =====
+# train_frac selects the first int(frac * n_train_perts) training perturbations.
+pert_train = [str(p) for p in sp["train_perts"]]
+n_train = len(pert_train)
+cells_per = [len(data.pert_to_idx[p]) for p in pert_train if p in data.pert_to_idx]
+med_cells = float(np.median(cells_per))
+out["data_scaling_counts"] = {str(f): {"n_perts": max(1, int(f * n_train)),
+                                       "cells_per_pert": int(round(med_cells))}
+                              for f in [0.1, 0.25, 0.5, 0.75, 1.0]}
+print("data_scaling_counts", out["data_scaling_counts"], flush=True)
+save_json(out, "experiments/results/norman_timing_scaling.json")  # checkpoint before slow part
+
+# ===== d. held-out gene mse per representation (Table 10), Replogle K562 gene split =====
+rep_data = load_dataset("replogle_k562")
+spg = load_split(rep_data.dir, "gene")
+gene_targets = list(spg["test_perts"])[:60]
+out["heldout_gene_mse"] = {}
+for rep in ["op_only", "gene_only", "random_id", "gene_op", "gene_pathway_op"]:
+    try:
+        m, _ = train(TrainConfig(dataset="replogle_k562", split="gene", epochs=45,
+                                 rep_mode=rep, device_index=gpu), data=rep_data, verbose=False)
+        d = next(m.parameters()).device
+        p = PivotPredictor(m, rep_data, d)
+        f = evaluate_forward(p, rep_data, gene_targets, rep_data.control_idx, max_perts=60)
+        out["heldout_gene_mse"][rep] = round(float(f["mse"]), 4)
+    except Exception as e:
+        out["heldout_gene_mse"][rep] = None
+        print("rep failed", rep, repr(e), flush=True)
+    print("heldout_gene_mse", rep, out["heldout_gene_mse"][rep], flush=True)
+    save_json(out, "experiments/results/norman_timing_scaling.json")
+
+print("FILL3_DONE", flush=True)

src/data/embeddings.py

@@ -0,0 +1,86 @@
+"""alternative cell-state encoders phi (Table 16).
+
+pca is the default (built in preprocess.py). this adds scvi, a standard deep
+single-cell latent space. to keep cell ordering identical to the cached pca
+embedding, we reproduce the exact filtered cell set from preprocess.py (same
+seed, hvgs, min-cells filter), train scvi on the raw counts, and save
+emb_scvi.npy aligned 1:1.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+
+import numpy as np
+import scipy.sparse as sp
+
+from src.data.preprocess import DATASETS, parse_perturbation
+from src.utils.common import load_json, set_seed
+
+
+def load_filtered_adata(dataset: str, n_hvg: int = 2000):
+    """reproduce preprocess.py's filtered anndata (raw counts kept in .x), in the
+    same cell order as the cached pca embedding."""
+    import scanpy as sc
+
+    cfg = dict(DATASETS[dataset])
+    raw = cfg["raw"]
+    seed = 0
+    set_seed(seed)
+    adata = sc.read_h5ad(raw)
+    max_cells = cfg.get("max_cells")
+    if max_cells is not None and adata.n_obs > max_cells:
+        rng = np.random.default_rng(seed)
+        idx = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False))
+        adata = adata[idx].copy()
+    adata.layers["counts"] = adata.X.copy()
+    # normalize only to pick the same hvgs as preprocess
+    sc.pp.normalize_total(adata, target_sum=1e4)
+    sc.pp.log1p(adata)
+    sc.pp.highly_variable_genes(adata, n_top_genes=n_hvg, flavor="seurat")
+    adata = adata[:, adata.var["highly_variable"]].copy()
+    # min-cells filter identical to preprocess (sep/pert_col use preprocess defaults)
+    control_label = cfg["control_label"]; sep = cfg.get("sep", "_")
+    pert = adata.obs[cfg.get("pert_col", "perturbation")].astype(str).values
+    npg = np.array([len(parse_perturbation(p, control_label, sep)) for p in pert])
+    is_ctrl = npg == 0
+    import pandas as pd
+    vc = pd.Series(pert[~is_ctrl]).value_counts()
+    keep = set(vc[vc >= 20].index)
+    mask = is_ctrl | np.isin(pert, list(keep))
+    adata = adata[mask].copy()
+    # restore raw counts into x
+    adata.X = adata.layers["counts"]
+    return adata
+
+
+def build_scvi(dataset: str, n_latent: int = 50, max_epochs: int = 80, gpu: int = 0):
+    import scvi
+    import torch
+
+    cache = os.path.join("data/processed", dataset)
+    meta = load_json(os.path.join(cache, "meta.json"))
+    adata = load_filtered_adata(dataset)
+    n_cached = np.load(os.path.join(cache, "pca_emb.npy")).shape[0]
+    assert adata.n_obs == n_cached, f"cell count mismatch {adata.n_obs} vs cached {n_cached}"
+
+    scvi.settings.seed = 0
+    scvi.model.SCVI.setup_anndata(adata, layer=None)
+    model = scvi.model.SCVI(adata, n_latent=n_latent)
+    model.to_device(gpu if torch.cuda.is_available() else "cpu")
+    model.train(max_epochs=max_epochs, early_stopping=True,
+                plan_kwargs={"lr": 1e-3}, enable_progress_bar=False)
+    z = model.get_latent_representation().astype(np.float32)
+    np.save(os.path.join(cache, "emb_scvi.npy"), z)
+    print(f"[{dataset}] scVI latent {z.shape} -> {cache}/emb_scvi.npy")
+    return z
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("dataset")
+    ap.add_argument("--gpu", type=int, default=0)
+    ap.add_argument("--n-latent", type=int, default=50)
+    ap.add_argument("--max-epochs", type=int, default=80)
+    args = ap.parse_args()
+    build_scvi(args.dataset, n_latent=args.n_latent, max_epochs=args.max_epochs, gpu=args.gpu)

src/data/perturb_data.py

@@ -0,0 +1,223 @@
+"""perturbdata: in-memory representation of a preprocessed perturb-seq dataset.
+
+holds the data model, perturbation vocabulary, and configurable control-matching
+(Table 18).
+"""
+from __future__ import annotations
+
+import os
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+
+from src.utils.common import load_json
+
+MATCH_STRATEGIES = (
+    "random",
+    "batch",
+    "celltype",
+    "batch_celltype",
+    "nearest",
+    "ot",  # sinkhorn ot coupling control<->perturbed (distribution-preserving)
+)
+
+
+class PerturbData:
+    def __init__(self, cache_dir: str, embedding: str = "pca"):
+        self.dir = cache_dir
+        self.meta = load_json(os.path.join(cache_dir, "meta.json"))
+        self.obs = pd.read_parquet(os.path.join(cache_dir, "obs.parquet"))
+        self.genes = open(os.path.join(cache_dir, "genes_hvg.txt")).read().split("\n")
+        self.Xhvg = sp.load_npz(os.path.join(cache_dir, "Xhvg.npz")).tocsr()
+        self.pca_components = np.load(os.path.join(cache_dir, "pca_components.npy"))
+        self.pca_mean = np.load(os.path.join(cache_dir, "pca_mean.npy"))
+        pb = np.load(os.path.join(cache_dir, "pseudobulk.npz"), allow_pickle=True)
+        self.pb_labels = list(map(str, pb["labels"]))
+        self.pb_vecs = pb["vecs"].astype(np.float32)
+        self.control_mean = pb["control_mean"].astype(np.float32)
+
+        self.embedding = embedding
+        self.emb = self._load_embedding(embedding)
+        self.d = self.emb.shape[1]
+
+        self.sep = self.meta["sep"]
+        self.control_label = self.meta["control_label"]
+        self.operation = self.meta["operation"]
+
+        self.is_control = self.obs["is_control"].values
+        self.control_idx = np.where(self.is_control)[0]
+        self.batch = self.obs["batch"].values
+        self.celltype = self.obs["celltype"].values
+
+        # perturbation -> row indices
+        self.pert_to_idx: dict[str, np.ndarray] = {
+            p: sub.index.values
+            for p, sub in self.obs.groupby("perturbation")
+            if p != self.control_label
+        }
+        self.perturbations = sorted(self.pert_to_idx.keys())
+
+        # vocabulary: genes and operations
+        self.genes_vocab = sorted({g for p in self.perturbations for g in self.parse(p)})
+        self.gene_to_id = {g: i for i, g in enumerate(self.genes_vocab)}
+        # operations: one modality per dataset, plus a 'control'/none slot id 0
+        self.op_vocab = ["none", self.operation]
+        self.op_to_id = {o: i for i, o in enumerate(self.op_vocab)}
+
+        self.singles = [p for p in self.perturbations if len(self.parse(p)) == 1]
+        self.combos = [p for p in self.perturbations if len(self.parse(p)) >= 2]
+
+        self._pb_index = {p: i for i, p in enumerate(self.pb_labels)}
+        self._nn_control_cache: dict[str, np.ndarray] = {}
+
+    # ---- embeddings ----
+    def _load_embedding(self, embedding: str) -> np.ndarray:
+        path = os.path.join(self.dir, f"emb_{embedding}.npy")
+        if embedding == "pca":
+            path = os.path.join(self.dir, "pca_emb.npy")
+        if not os.path.exists(path):
+            raise FileNotFoundError(
+                f"embedding '{embedding}' not found ({path}); build it first"
+            )
+        return np.load(path).astype(np.float32)
+
+    def decode_to_genes(self, emb: np.ndarray) -> np.ndarray:
+        """decode embedding(s) back to hvg gene-space (only exact for pca)."""
+        if self.embedding != "pca":
+            raise NotImplementedError(
+                f"gene-space decode only defined for PCA, not '{self.embedding}'"
+            )
+        return emb @ self.pca_components + self.pca_mean
+
+    # ---- perturbation parsing / encoding ----
+    def parse(self, label: str) -> list[str]:
+        if str(label) == self.control_label:
+            return []
+        return [g for g in str(label).split(self.sep) if g and g != self.control_label]
+
+    def pert_gene_op_ids(self, label: str):
+        """return (gene_ids, op_ids) arrays for a perturbation label."""
+        genes = self.parse(label)
+        gids = np.array([self.gene_to_id[g] for g in genes if g in self.gene_to_id], dtype=np.int64)
+        oids = np.full(len(gids), self.op_to_id[self.operation], dtype=np.int64)
+        return gids, oids
+
+    # ---- pseudobulk / effects (gene space, hvg-log) ----
+    def effect_vector(self, label: str) -> np.ndarray:
+        """true perturbation effect = mean(perturbed) - mean(control), gene space."""
+        return self.pb_vecs[self._pb_index[label]] - self.control_mean
+
+    def all_effects(self) -> tuple[list[str], np.ndarray]:
+        return self.pb_labels, self.pb_vecs - self.control_mean[None, :]
+
+    # ---- control matching (Table 18) ----
+    def sample_controls(self, target_idx: np.ndarray, strategy: str, rng: np.random.Generator):
+        """for each perturbed cell in target_idx return a matched control row index."""
+        if strategy not in MATCH_STRATEGIES:
+            raise ValueError(f"unknown matching strategy {strategy}")
+        cidx = self.control_idx
+        if strategy == "random":
+            return rng.choice(cidx, size=len(target_idx), replace=True)
+
+        if strategy == "nearest":
+            return self._nearest_controls(target_idx)
+
+        if strategy == "ot":
+            if not hasattr(self, "_ot_map"):
+                self.precompute_ot_matching()
+            return np.array([self._ot_map.get(int(i), self.control_idx[rng.integers(len(self.control_idx))])
+                             for i in target_idx], dtype=np.int64)
+
+        # bucketed matching by batch / celltype / both
+        def key(i):
+            if strategy == "batch":
+                return self.batch[i]
+            if strategy == "celltype":
+                return self.celltype[i]
+            return (self.batch[i], self.celltype[i])
+
+        buckets: dict = {}
+        for i in cidx:
+            buckets.setdefault(key(i), []).append(i)
+        buckets = {k: np.asarray(v) for k, v in buckets.items()}
+        out = np.empty(len(target_idx), dtype=np.int64)
+        for j, i in enumerate(target_idx):
+            pool = buckets.get(key(i))
+            if pool is None or len(pool) == 0:
+                pool = cidx  # fall back to any control
+            out[j] = pool[rng.integers(len(pool))]
+        return out
+
+    def _nearest_controls(self, target_idx: np.ndarray) -> np.ndarray:
+        from sklearn.neighbors import NearestNeighbors
+
+        nn = NearestNeighbors(n_neighbors=1).fit(self.emb[self.control_idx])
+        _, j = nn.kneighbors(self.emb[target_idx])
+        return self.control_idx[j.ravel()]
+
+    def precompute_ot_matching(self, max_ctrl: int = 800, max_pert: int = 1200,
+                               eps: float = 0.05, iters: int = 150, seed: int = 0):
+        """for each perturbation, couple its cells to control cells via entropic ot
+        (sinkhorn) on embedding l2 cost, and assign each perturbed cell a control by
+        sampling its coupling row. distribution-preserving alternative to random
+        matching (cf. cellot / ot-cfm). caches self._ot_map (perturbed idx -> control idx)."""
+        import torch
+
+        rng = np.random.default_rng(seed)
+        dev = "cuda" if torch.cuda.is_available() else "cpu"
+        cidx = self.control_idx
+        csamp = cidx if len(cidx) <= max_ctrl else cidx[rng.choice(len(cidx), max_ctrl, replace=False)]
+        C = torch.as_tensor(self.emb[csamp], dtype=torch.float32, device=dev)
+        self._ot_map = {}
+        for p, idx in self.pert_to_idx.items():
+            t_idx = idx if len(idx) <= max_pert else idx[rng.choice(len(idx), max_pert, replace=False)]
+            T = torch.as_tensor(self.emb[t_idx], dtype=torch.float32, device=dev)
+            cost = torch.cdist(T, C).pow(2)
+            cost = cost / (cost.median() + 1e-8)
+            K = torch.exp(-cost / eps)
+            n, m = K.shape
+            u = torch.ones(n, device=dev) / n
+            v = torch.ones(m, device=dev) / m
+            a = torch.full((n,), 1.0 / n, device=dev)
+            b = torch.full((m,), 1.0 / m, device=dev)
+            for _ in range(iters):
+                u = a / (K @ v + 1e-8)
+                v = b / (K.t() @ u + 1e-8)
+            P = (u.unsqueeze(1) * K) * v.unsqueeze(0)  # coupling (n, m)
+            P = P / (P.sum(1, keepdim=True) + 1e-12)
+            # sample a control per perturbed cell from its coupling row
+            choice = torch.multinomial(P, 1, generator=None).squeeze(1).cpu().numpy()
+            for c_local, cell in zip(choice, t_idx):
+                self._ot_map[int(cell)] = int(csamp[c_local])
+        return self._ot_map
+
+    # ---- functional clusters (for pathway/functional recovery) ----
+    def functional_clusters(self, n_clusters: int = 15, seed: int = 0) -> dict[str, int]:
+        """cluster single-gene perturbations by effect-vector correlation.
+
+        data-driven proxy for 'same pathway': perturbations with similar
+        transcriptional effects get grouped. used for functional top-k and
+        pathway-ndcg (Table 6)."""
+        from sklearn.cluster import AgglomerativeClustering
+
+        labels = self.singles
+        E = np.stack([self.effect_vector(p) for p in labels])
+        # correlation distance
+        En = E - E.mean(1, keepdims=True)
+        En = En / (np.linalg.norm(En, axis=1, keepdims=True) + 1e-8)
+        sim = np.clip(En @ En.T, -1, 1)
+        dist = 1 - sim
+        k = min(n_clusters, len(labels))
+        cl = AgglomerativeClustering(n_clusters=k, metric="precomputed", linkage="average")
+        ids = cl.fit_predict(dist)
+        # map by single-gene name
+        out = {}
+        for p, c in zip(labels, ids):
+            g = self.parse(p)[0]
+            out[g] = int(c)
+        return out
+
+
+def load_dataset(name: str, embedding: str = "pca", root: str = "data/processed") -> PerturbData:
+    return PerturbData(os.path.join(root, name), embedding=embedding)

src/data/preprocess.py

@@ -0,0 +1,225 @@
+"""preprocess a raw perturb-seq .h5ad (scperturb format) into a compact,
+reusable cache.
+
+does normalization, hvg selection, and the pca cell-state encoder (the default
+invertible phi used for gene-space metrics).
+
+outputs, under data/processed/<name>/:
+  meta.json            dataset summary (counts, operation, control label, ...)
+  genes_hvg.txt        hvg gene symbols (defines gene-space for de metrics)
+  Xhvg.npz             scipy csr, log1p(cp10k) on hvgs  (n_cells x n_hvg)
+  obs.parquet          per-cell metadata + parsed perturbation
+  pca_emb.npy          pca cell-state embedding          (n_cells x d)
+  pca_components.npy   pca basis                          (d x n_hvg)
+  pca_mean.npy         feature means                      (n_hvg,)
+  pseudobulk.npz       per-perturbation mean hvg-log vectors + control mean
+
+the pca is fit on log1p(cp10k) hvg features (zero-centered, unscaled) so that
+x_hat = emb @ components + mean reconstructs gene-space expression - this is what
+lets us evaluate de-gene correlation on predictions made in embedding space.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+from src.utils.common import save_json, set_seed  # noqa: E402
+
+
+def parse_perturbation(label: str, control_label: str, sep: str = "_"):
+    """return list of perturbed genes for a perturbation label ([] for control)."""
+    if str(label) == control_label:
+        return []
+    return [g for g in str(label).split(sep) if g and g != control_label]
+
+
+def preprocess(
+    raw_path: str,
+    out_dir: str,
+    name: str,
+    operation: str,
+    control_label: str = "control",
+    pert_col: str = "perturbation",
+    batch_col: str | None = "gemgroup",
+    celltype_col: str | None = "celltype",
+    sep: str = "_",
+    n_hvg: int = 2000,
+    n_pca: int = 50,
+    min_cells_per_pert: int = 20,
+    max_cells: int | None = None,
+    seed: int = 0,
+):
+    """preprocess one dataset. operation is the crispr modality, e.g.
+    'activation' (crispra, norman) or 'interference' (crispri, replogle)."""
+    import scanpy as sc
+
+    set_seed(seed)
+    os.makedirs(out_dir, exist_ok=True)
+    print(f"[{name}] reading {raw_path}")
+    adata = sc.read_h5ad(raw_path)
+
+    # optional subsample for tractability (deterministic)
+    if max_cells is not None and adata.n_obs > max_cells:
+        rng = np.random.default_rng(seed)
+        idx = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False))
+        adata = adata[idx].copy()
+        print(f"[{name}] subsampled to {adata.n_obs} cells")
+
+    # resolve columns with graceful fallback
+    if pert_col not in adata.obs:
+        raise KeyError(f"pert_col '{pert_col}' not in obs: {list(adata.obs.columns)}")
+    if batch_col is not None and batch_col not in adata.obs:
+        print(f"[{name}] batch_col '{batch_col}' missing -> single batch")
+        batch_col = None
+    if celltype_col is not None and celltype_col not in adata.obs:
+        celltype_col = None
+
+    # --- normalization: cp10k + log1p ---
+    if not sp.issparse(adata.X):
+        adata.X = sp.csr_matrix(adata.X)
+    # guard: ensure raw-ish counts (integer). if already normalized, skip.
+    x0 = adata.X[:200].toarray()
+    looks_counts = np.allclose(x0, np.round(x0)) and x0.max() > 30
+    if looks_counts:
+        sc.pp.normalize_total(adata, target_sum=1e4)
+        sc.pp.log1p(adata)
+        print(f"[{name}] applied CP10k + log1p")
+    else:
+        print(f"[{name}] data appears pre-normalized (max={x0.max():.2f}); skipping norm")
+
+    # --- hvg selection (on all cells) ---
+    sc.pp.highly_variable_genes(adata, n_top_genes=n_hvg, flavor="seurat")
+    adata = adata[:, adata.var["highly_variable"]].copy()
+    genes = list(map(str, adata.var_names))
+    Xhvg = adata.X.tocsr().astype(np.float32)
+    print(f"[{name}] HVG matrix: {Xhvg.shape}")
+
+    # --- pca (centered, invertible) ---
+    from sklearn.decomposition import PCA
+
+    Xd = Xhvg.toarray()
+    mean = Xd.mean(axis=0).astype(np.float32)
+    pca = PCA(n_components=n_pca, random_state=seed)
+    emb = pca.fit_transform(Xd - mean).astype(np.float32)
+    components = pca.components_.astype(np.float32)  # (d, h)
+    evr = float(pca.explained_variance_ratio_.sum())
+    print(f"[{name}] PCA d={n_pca} explains {evr:.3f} variance")
+
+    # --- parse perturbations ---
+    pert = adata.obs[pert_col].astype(str).values
+    genes_per_cell = [parse_perturbation(p, control_label, sep) for p in pert]
+    nperts = np.array([len(g) for g in genes_per_cell])
+    is_control = nperts == 0
+
+    obs = pd.DataFrame(
+        {
+            "perturbation": pert,
+            "n_pert_genes": nperts,
+            "is_control": is_control,
+            "pert_genes": [";".join(g) for g in genes_per_cell],
+            "batch": (adata.obs[batch_col].astype(str).values if batch_col else "0"),
+            "celltype": (
+                adata.obs[celltype_col].astype(str).values if celltype_col else name
+            ),
+        }
+    )
+
+    # drop perturbations (non-control) with too few cells
+    vc = obs.loc[~obs.is_control, "perturbation"].value_counts()
+    keep_perts = set(vc[vc >= min_cells_per_pert].index)
+    keep_mask = obs.is_control.values | obs.perturbation.isin(keep_perts).values
+    if keep_mask.sum() < len(obs):
+        obs = obs.loc[keep_mask].reset_index(drop=True)
+        Xhvg = Xhvg[keep_mask]
+        emb = emb[keep_mask]
+        print(f"[{name}] dropped low-count perts -> {keep_mask.sum()} cells, "
+              f"{len(keep_perts)} perturbations")
+
+    # --- pseudobulk per perturbation (gene space, hvg-log) ---
+    control_mean = np.asarray(Xhvg[obs.is_control.values].mean(axis=0)).ravel().astype(np.float32)
+    pb_labels, pb_vecs = [], []
+    for p, sub in obs.groupby("perturbation"):
+        if p == control_label:
+            continue
+        idx = sub.index.values
+        pb_labels.append(p)
+        pb_vecs.append(np.asarray(Xhvg[idx].mean(axis=0)).ravel())
+    pb_vecs = np.asarray(pb_vecs, dtype=np.float32)
+
+    # --- write cache ---
+    sp.save_npz(os.path.join(out_dir, "Xhvg.npz"), Xhvg)
+    np.save(os.path.join(out_dir, "pca_emb.npy"), emb)
+    np.save(os.path.join(out_dir, "pca_components.npy"), components)
+    np.save(os.path.join(out_dir, "pca_mean.npy"), mean)
+    np.savez(
+        os.path.join(out_dir, "pseudobulk.npz"),
+        labels=np.array(pb_labels),
+        vecs=pb_vecs,
+        control_mean=control_mean,
+    )
+    obs.to_parquet(os.path.join(out_dir, "obs.parquet"))
+    with open(os.path.join(out_dir, "genes_hvg.txt"), "w") as f:
+        f.write("\n".join(genes))
+
+    singles = [p for p in pb_labels if len(parse_perturbation(p, control_label, sep)) == 1]
+    combos = [p for p in pb_labels if len(parse_perturbation(p, control_label, sep)) >= 2]
+    all_genes = sorted({g for p in pb_labels for g in parse_perturbation(p, control_label, sep)})
+    meta = {
+        "name": name,
+        "operation": operation,
+        "control_label": control_label,
+        "sep": sep,
+        "n_cells": int(obs.shape[0]),
+        "n_control": int(obs.is_control.sum()),
+        "n_hvg": len(genes),
+        "n_pca": n_pca,
+        "pca_explained_var": evr,
+        "n_perturbations": len(pb_labels),
+        "n_singles": len(singles),
+        "n_combos": len(combos),
+        "n_unique_target_genes": len(all_genes),
+        "n_batches": int(obs.batch.nunique()),
+        "n_celltypes": int(obs.celltype.nunique()),
+    }
+    save_json(meta, os.path.join(out_dir, "meta.json"))
+    print(f"[{name}] DONE -> {out_dir}")
+    print(meta)
+    return meta
+
+
+DATASETS = {
+    "norman": dict(
+        raw="data/raw/NormanWeissman2019_filtered.h5ad",
+        name="norman", operation="activation", control_label="control",
+        batch_col="gemgroup", celltype_col="celltype",
+    ),
+    "replogle_k562": dict(
+        raw="data/raw/ReplogleWeissman2022_K562_essential.h5ad",
+        name="replogle_k562", operation="interference", control_label="control",
+        batch_col="gemgroup", celltype_col="celltype", max_cells=120000,
+    ),
+    "replogle_rpe1": dict(
+        raw="data/raw/ReplogleWeissman2022_rpe1.h5ad",
+        name="replogle_rpe1", operation="interference", control_label="control",
+        batch_col="gemgroup", celltype_col="celltype", max_cells=120000,
+    ),
+}
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("dataset", choices=list(DATASETS.keys()))
+    ap.add_argument("--out-root", default="data/processed")
+    ap.add_argument("--n-hvg", type=int, default=2000)
+    ap.add_argument("--n-pca", type=int, default=50)
+    args = ap.parse_args()
+    cfg = dict(DATASETS[args.dataset])
+    raw = cfg.pop("raw")
+    out = os.path.join(args.out_root, cfg["name"])
+    preprocess(raw, out, n_hvg=args.n_hvg, n_pca=args.n_pca, **cfg)