{ "suite_schema": "nmd-vcell-benchmark-suite/1.0", "interface_build": "EA-20260817-57", "resource_release": "v1.2.0-measured-dmd-evidence", "benchmark_release": "repeated-fold-v2.2", "task_count": 8, "executed_or_partial_task_count": 2, "independent_dmd_task_count": 1, "independent_dmd_task_executed_count": 0, "tasks": [ { "task_id": "G0", "name": "Same-context target holdout", "generalization_axis": "unseen target inside the processed HepG2 assay", "state": "EXECUTED_LIMITED", "status_label": "Executed · limited", "input": "Processed HepG2 CRISPRi response deltas", "output": "Held-out 2,000-feature mean response vector", "outcome_required": "Released same-assay target aggregates", "permanent_baselines": "zero change · train mean · ridge", "primary_metrics": "RMSE · raw cosine · residual cosine · strict 16-outcome gate", "current_result": "Small average-error gain; strict gate 0/16 and direction unreliable", "decision_blocker": "No external cell-level or disease-context truth", "route": "/resource/benchmarks/core/" }, { "task_id": "G1", "name": "External unseen perturbation", "generalization_axis": "new perturbation in an external aggregate object", "state": "EXECUTED_PARTIAL_UNSUPPORTED", "status_label": "Partial · unsupported", "input": "HepG2 response substrate plus 55-target external aggregate diagnostic", "output": "Directional-transfer assessment", "outcome_required": "Frozen aggregate target effects", "permanent_baselines": "zero change · train mean · ridge", "primary_metrics": "formula-aligned aggregate directional diagnostics", "current_result": "No directional-transfer support", "decision_blocker": "No harmonized cell-population outcome or matched feature space", "route": "/resource/benchmarks/external-transfer/" }, { "task_id": "G2", "name": "Unseen gene family or pathway", "generalization_axis": "held-out related targets with leakage-safe exposure audit", "state": "REGISTERED_NOT_EXECUTED", "status_label": "Registered · not executed", "input": "Future exposure-audited perturbation training set", "output": "Held-out family response distribution", "outcome_required": "Enough related targets to freeze disjoint train and test families", "permanent_baselines": "zero change · train mean · ridge · nearest neighbour", "primary_metrics": "DES · PDS · MAE · calibration · abstention", "current_result": "No result", "decision_blocker": "Current candidate set is too small for a powered family holdout", "route": "/resource/benchmarks/generalization/" }, { "task_id": "G3", "name": "Unseen donor", "generalization_axis": "entire donor withheld", "state": "LOCKED_NO_TRUTH", "status_label": "Locked · no donor-indexed perturbation outcome", "input": "Future donor-resolved perturbation populations", "output": "Donor-conditioned response and responder distribution", "outcome_required": "Multiple biological donors with repeated perturbations", "permanent_baselines": "population mean · donor-agnostic ridge · nearest donor", "primary_metrics": "calibration · donor replication · distribution distance · abstention", "current_result": "No result", "decision_blocker": "No qualifying donor-resolved DMD perturbation matrix", "route": "/resource/benchmarks/generalization/" }, { "task_id": "G4", "name": "Unseen cell state", "generalization_axis": "train in one myogenic state and test another", "state": "LOCKED_NO_TRUTH", "status_label": "Locked · no perturbed state transition", "input": "Future perturbations sampled across myoblast-to-myotube states", "output": "State-conditioned population transition", "outcome_required": "Matched perturbation identities across physical time and state", "permanent_baselines": "unperturbed trajectory · state mean · additive shift", "primary_metrics": "state occupancy · DES · PDS · MAE · transport distance", "current_result": "No result", "decision_blocker": "Current myogenic trajectory is real but unperturbed", "route": "/resource/trajectory/" }, { "task_id": "G5", "name": "Future batch or external laboratory", "generalization_axis": "prospective time and site shift", "state": "LOCKED_NO_TRUTH", "status_label": "Locked · no prospective batch", "input": "Future frozen prediction plus later independent batch", "output": "Prospective response, uncertainty and abstention", "outcome_required": "Outcome collected after Prediction Card freeze", "permanent_baselines": "historical mean · ridge · expert selection", "primary_metrics": "calibration · AUPRC · hit enrichment · abstention quality", "current_result": "No result", "decision_blocker": "No registered prediction or future outcome", "route": "/resource/prediction_registry" }, { "task_id": "G6", "name": "Independent DMD disease line", "generalization_axis": "new disease line or genotype", "state": "LOCKED_NO_TRUTH", "status_label": "Locked · zero DMD outcomes", "input": "Future matched DMD/control perturbation conditions", "output": "Molecular, functional, toxicity and replication outcomes", "outcome_required": "Independent DMD candidate perturbations across donors and reagents", "permanent_baselines": "zero · train mean · ridge · expert choice · random selection", "primary_metrics": "functional hit rate · AUPRC · calibration · toxicity miss rate · replication", "current_result": "No result", "decision_blocker": "0 independent DMD candidate-perturbation outcomes", "route": "/resource/minimum-perturbome/" }, { "task_id": "G7", "name": "New modality and combination", "generalization_axis": "KO, CRISPRa, drug or double perturbation", "state": "LOCKED_NO_TRUTH", "status_label": "Locked · calibration references only", "input": "Future matched singleton and combination perturbations", "output": "Main effects, interaction, synergy class and uncertainty", "outcome_required": "Observed same-screen singles and doubles or matched genetic–chemical pairs", "permanent_baselines": "additive null · pathway mean · nearest neighbour", "primary_metrics": "interaction error · synergy classification · calibration · toxicity", "current_result": "Only an additive no-interaction preview is available", "decision_blocker": "No observed candidate doubles or matched DMD drug screens", "route": "/resource/perturbation-lab" } ], "global_metric_lens": { "source": "Arc Virtual Cell Challenge official metric explainer", "url": "https://arcinstitute.org/news/behind-the-data-virtual-cell-challenge", "metrics": { "DES": "differential-expression recovery", "PDS": "perturbation discrimination with raw and norm-matched scale diagnostics", "MAE": "global expression error", "DIST": "population geometry and state composition", "CAL": "calibration and abstention", "DMD": "function, replication and toxicity" }, "adoption_rule": "Use as a multi-family panel for future harmonized cell-level response tasks; never select a model from raw PDS alone and do not retrofit these metrics to incompatible aggregate objects.", "governance_object": "/resource/api/v1.1/metric_governance.json" }, "boundary": "Registered and locked tasks contain no performance result. Results on one generalization axis do not upgrade another." }