When is a resolution criterion a real test?

The gate imports these same functions. A test fails if the page and the gate diverge. Data files: the MISO series as ingested (SHA-256, source EIA-930, exported 26 September 2026) and the simulation output.

The file below is lib/insights/criterionPowerMath.ts as it stands today. The thresholds block in it, comments included, is byte for byte as committed in 75b7ae9d9, and a test pins it. Comments elsewhere in the file were revised on 26 September 2026, and a dated note after the block corrects two of its figures. The threshold values are unchanged since 75b7ae9d9.

/**
 * The pure arithmetic of the criterion gate.
 *
 * A resolution criterion of the form "the metric moves k standard deviations
 * away from its baseline" is only informative if that move is rare when nothing
 * has happened. Everything in this file answers that question with closed-form
 * arithmetic over numbers the caller supplies: a normal tail probability, the
 * chance of at least one firing across many checks, the realized firing rates of
 * a pair of criteria, and the pre-registered thresholds the gate judges them
 * against.
 *
 * WHY THIS FILE HAS NO IMPORTS
 * The gate (`criterionPower.ts`) reads the observations table and re-exports
 * everything here. The public methods page at /methods/resolution-criteria
 * imports this file directly and runs it in the reader's browser. One
 * implementation serves both, so the page computes with the same function that
 * decides admissibility, and a test fails if the two diverge
 * (tests/insights/criterionPowerMath.test.ts). Keep it free of imports: adding
 * one would pull server code into a static page.
 *
 * @module lib/insights/criterionPowerMath
 */

/** One side of a supports/falsifies pair. */
export interface NullPowerSide {
  sigma: number;
  direction: "above" | "below";
  /** Observations in the scan window: the number of chances to fire. */
  scanN: number;
  /** How many of them actually crossed the threshold. */
  crossings: number;
  /** crossings / scanN, the REALIZED rate, which is what matters. */
  fireRate: number;
  /** Per-observation p under a normal null. What the "k sigma" label implies. */
  nominalP: number;
  /** 1 - (1 - nominalP)^scanN: the honest probability of >=1 fire on noise. */
  familywiseP: number;
}

export interface NullPowerReport {
  metricKey: string;
  dimensions: Record<string, unknown> | null;
  /** The instant the report describes; both issue date and today are useful. */
  asOf: string;
  baseline: {
    n: number;
    mean: number | null;
    stddev: number | null;
    /** Few distinct values => a sigma is not a rarity measure. */
    distinctValues: number;
  };
  sides: NullPowerSide[];
  /**
   * max(fireRate) / max(min(fireRate), floor). Null when a side is missing.
   * >3 means the pair is structurally biased toward one verdict.
   */
  asymmetryRatio: number | null;
  /** Human-readable observations, safe to publish. */
  notes: string[];
}

/** Standard normal CDF via Abramowitz & Stegun 7.1.26 (|error| < 1.5e-7). */
export function normalCdf(z: number): number {
  const sign = z < 0 ? -1 : 1;
  const x = Math.abs(z) / Math.SQRT2;
  const t = 1 / (1 + 0.3275911 * x);
  const y =
    1 -
    ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) *
      t +
      0.254829592) *
      t *
      Math.exp(-x * x);
  return 0.5 * (1 + sign * y);
}

/** One-sided p under a normal null for a k-sigma threshold. */
export function nominalTailP(sigma: number): number {
  return 1 - normalCdf(Math.abs(sigma));
}

/**
 * P(at least one crossing | null) across `scanN` chances.
 *
 * This is the number that should appear next to any "k sigma" claim. Treating
 * the scans as independent is generous to us. Hourly grid demand is strongly
 * autocorrelated, which REDUCES the effective number of independent trials, so
 * this is an upper bound on the familywise rate and a lower bound on how bad the
 * overstatement is. Stated that way on purpose.
 */
export function familywiseP(sigma: number, scanN: number): number {
  if (scanN <= 0) return 0;
  return 1 - Math.pow(1 - nominalTailP(sigma), scanN);
}

/** Pure core: given values + baseline stats, what would each side have done? */
export function assessCriterionPair(
  scanValues: number[],
  baseline: { mean: number | null; stddev: number | null },
  sides: Array<{ sigma: number; direction: "above" | "below" }>,
): { sides: NullPowerSide[]; asymmetryRatio: number | null } {
  const { mean, stddev } = baseline;
  const out: NullPowerSide[] = sides.map((s) => {
    let crossings = 0;
    if (mean !== null && stddev !== null && stddev > 0) {
      for (const v of scanValues) {
        const z = (v - mean) / stddev;
        if (s.direction === "above" ? z >= s.sigma : z <= -s.sigma) crossings++;
      }
    }
    const scanN = scanValues.length;
    return {
      sigma: s.sigma,
      direction: s.direction,
      scanN,
      crossings,
      fireRate: scanN > 0 ? crossings / scanN : 0,
      nominalP: nominalTailP(s.sigma),
      familywiseP: familywiseP(s.sigma, scanN),
    };
  });

  let asymmetryRatio: number | null = null;
  if (out.length >= 2) {
    const rates = out.map((s) => s.fireRate);
    const hi = Math.max(...rates);
    const lo = Math.min(...rates);
    // Floor the denominator at one observation's worth of rate so a
    // never-fires side yields a large finite ratio rather than Infinity.
    const floor = out[0].scanN > 0 ? 1 / out[0].scanN : 1;
    asymmetryRatio = hi === 0 ? 1 : hi / Math.max(lo, floor);
  }
  return { sides: out, asymmetryRatio };
}

/**
 * PRE-REGISTERED rejection thresholds.
 *
 * Declared here, with a justification each, BEFORE being run against the live
 * population — deliberately, because "tune until it rejects exactly the known-bad
 * set and nothing else" is curve-fitting wearing a gate's clothes, and it is the
 * first thing a sceptical reader would accuse us of. Whatever the gate rejects
 * when it runs is an OUTCOME to be reported, including surprises.
 *
 * Each threshold maps to an observed failure, not to a preference.
 */
export const CRITERION_POWER_THRESHOLDS = {
  /**
   * Realized firing rates of a supports/falsifies pair may not differ by more
   * than this. Observed: the MISO pair sat at ~16:1 at mint and had inverted to
   * ~8.5:1 fifteen days later. NOT a nominal |k|-equality rule — on a skewed,
   * capacity-bounded series equal thresholds give unequal rates by construction
   * (43.5% at or above +1σ against 1.37% at or below −1σ).
   */
  maxPairAsymmetryRatio: 3,
  /**
   * A side firing on more than this share of observations is saturating, not
   * detecting. 5% is one observation in twenty — already far from "anomaly".
   */
  maxSideFireRate: 0.05,
  /**
   * P(at least one fire | null) across the criterion's actual scan count. The
   * single largest error in the original design: a "3σ" criterion scanned hourly
   * for 30 days sits at ~51% here against the 0.135% its label implies.
   */
  maxFamilywiseP: 0.05,
  /** Below this, a sigma is describing a near-constant series, not rarity. */
  minBaselineDistinctValues: 8,
  /** Below this, the baseline statistics are not worth acting on. */
  minBaselineN: 30,
} as const;

/*
 * Note dated 26 September 2026, kept outside the declared block above so that
 * block stays as committed in 75b7ae9d9 (25 July 2026, 22:29 UTC). Two of its
 * justifications are loose: the MISO pair's ~16:1 describes the data as it
 * stood on 10 July 2026, not the moment of issue, and the "3σ" criterion's
 * 30-day window held 532 hourly readings on 25 July 2026 rather than a full
 * 720. The threshold values are unchanged. See /methods/resolution-criteria.
 */

interface CriterionPowerVerdict {
  /** False ⇒ the criterion should not be issued (in `enforce`). */
  admissible: boolean;
  /** Machine-readable rejection codes, empty when admissible. */
  violations: string[];
  report: NullPowerReport;
}

/**
 * Judge a criterion (or a pair) against the pre-registered thresholds.
 *
 * Fail-OPEN on missing data: a report with no scan observations cannot show a
 * criterion is bad, and refusing to issue on absent data would silently empty the
 * portfolio whenever an adapter stalls. Absence of evidence is reported in
 * `notes`, not converted into a rejection.
 */
export function judgeCriterionPower(
  report: NullPowerReport,
): CriterionPowerVerdict {
  const t = CRITERION_POWER_THRESHOLDS;
  const violations: string[] = [];
  const measurable = report.sides.some((s) => s.scanN > 0);

  if (measurable) {
    if (
      report.asymmetryRatio !== null &&
      report.asymmetryRatio > t.maxPairAsymmetryRatio
    ) {
      violations.push(
        `pair_asymmetry:${report.asymmetryRatio.toFixed(1)}x>${t.maxPairAsymmetryRatio}x`,
      );
    }
    for (const s of report.sides) {
      if (s.scanN === 0) continue;
      if (s.fireRate > t.maxSideFireRate) {
        violations.push(
          `saturating:${s.direction}${s.sigma}sigma@${(s.fireRate * 100).toFixed(1)}%`,
        );
      }
      if (s.familywiseP > t.maxFamilywiseP) {
        violations.push(
          `familywise:${s.direction}${s.sigma}sigma@${(s.familywiseP * 100).toFixed(0)}%over${s.scanN}scans`,
        );
      }
    }
    if (report.baseline.n > 0 && report.baseline.n < t.minBaselineN) {
      violations.push(`thin_baseline:n=${report.baseline.n}`);
    }
    if (
      report.baseline.distinctValues > 0 &&
      report.baseline.distinctValues < t.minBaselineDistinctValues
    ) {
      violations.push(
        `low_cardinality:${report.baseline.distinctValues}values`,
      );
    }
  }

  return { admissible: violations.length === 0, violations, report };
}

/**
 * Per-SIDE admissibility within a group report.
 *
 * WHY THIS EXISTS. {@link judgeCriterionPower} answers "is this SERIES' criterion
 * set admissible", which is the right unit for the pair-level checks and the
 * wrong unit for a decision about one row. Several criteria written by
 * DIFFERENT generators can land on the same series. Live example, 2026-07-27:
 * `github.NVIDIA.Megatron-LM.commits_7d` carried a legacy 3σ criterion
 * (familywise 3.5%, admissible) and a new 2.5σ one (familywise 15.0%, not).
 * Judging the group and acting on the group verdict either withdraws the sound
 * criterion along with the unsound one, or keeps both. Neither is correct.
 *
 * Side-scoped checks (`saturating`, `familywise`) are attributed to the side
 * they describe. Baseline checks (`thin_baseline`, `low_cardinality`) and the
 * PAIR check (`pair_asymmetry`) are properties of the series or of the pair as a
 * unit, so they disqualify every side in the group. An asymmetric pair has no
 * innocent half.
 */
export function judgeCriterionSide(
  report: NullPowerReport,
  side: { sigma: number; direction: "above" | "below" },
): { admissible: boolean; violations: string[] } {
  const t = CRITERION_POWER_THRESHOLDS;
  const violations: string[] = [];

  // Group-scoped: shared by every side on this series.
  if (
    report.asymmetryRatio !== null &&
    report.asymmetryRatio > t.maxPairAsymmetryRatio
  ) {
    violations.push(
      `pair_asymmetry:${report.asymmetryRatio.toFixed(1)}x>${t.maxPairAsymmetryRatio}x`,
    );
  }
  const measurable = report.sides.some((s) => s.scanN > 0);
  if (measurable) {
    if (report.baseline.n > 0 && report.baseline.n < t.minBaselineN) {
      violations.push(`thin_baseline:n=${report.baseline.n}`);
    }
    if (
      report.baseline.distinctValues > 0 &&
      report.baseline.distinctValues < t.minBaselineDistinctValues
    ) {
      violations.push(
        `low_cardinality:${report.baseline.distinctValues}values`,
      );
    }
  }

  // Side-scoped: only this side's own realized behaviour.
  const own = report.sides.find(
    (s) => s.direction === side.direction && s.sigma === side.sigma,
  );
  if (own && own.scanN > 0) {
    if (own.fireRate > t.maxSideFireRate) {
      violations.push(
        `saturating:${own.direction}${own.sigma}sigma@${(own.fireRate * 100).toFixed(1)}%`,
      );
    }
    if (own.familywiseP > t.maxFamilywiseP) {
      violations.push(
        `familywise:${own.direction}${own.sigma}sigma@${(own.familywiseP * 100).toFixed(0)}%over${own.scanN}scans`,
      );
    }
  }

  return { admissible: violations.length === 0, violations };
}

/**
 * The sigma BOTH sides of a supports/falsifies pair must use.
 *
 * THE DEFECT THIS CLOSES. All three criterion generators paired a 3σ supporter
 * with a 1σ falsifier. Under a normal null those are not comparable tests: the
 * one-sided tail p is 0.001350 at 3σ against 0.158655 at 1σ, so the falsifier
 * was **117.5x** easier to satisfy than its partner. Against a declared
 * `maxSideFireRate` of 0.05 that is not a rounding error. It is a portfolio
 * biased toward one verdict by construction, which is worse for a published
 * calibration record than being wrong, because the bias survives every honest
 * effort to score it. An earlier fix equalised the two sides' WINDOWS
 * (`validateWindowSymmetry`) and left the sigmas untouched.
 *
 * WHY 3 AND NOT SOMETHING SMALLER. It falls out of the pre-registered
 * `maxFamilywiseP` of 0.05. A criterion scanned once a day for 30 days gets ~30
 * chances, so admissibility needs a per-scan tail p below
 * `1 - 0.95^(1/30)` = 0.00171, i.e. |z| >= 2.93. 3σ clears that; 2.5σ (p =
 * 0.0062, familywise ~17%) does not. A pair set below this is rejected by the
 * gate it was issued under, which is the coherence that was missing.
 */
export const PAIRED_CRITERION_SIGMA = 3;

/**
 * Ratio of the two sides' NOMINAL tail probabilities: the asymmetry that is
 * knowable from the thresholds alone, before any data.
 *
 * Distinct from {@link NullPowerReport.asymmetryRatio}, which is measured from
 * REALIZED firing rates and needs the metric's history. Both matter: nominal
 * asymmetry catches a mis-specified pair at write time (and in a unit test);
 * realized asymmetry catches a nominally-fair pair on a skewed series. Returns 1
 * for a single side (nothing to compare) and Infinity is impossible, since the
 * normal tail is never exactly 0 at finite z.
 */
export function nominalAsymmetryRatio(sides: Array<{ sigma: number }>): number {
  if (sides.length < 2) return 1;
  const ps = sides.map((s) => nominalTailP(s.sigma));
  return Math.max(...ps) / Math.min(...ps);
}

/**
 * The smallest threshold, in standard deviations, whose chance of at least one
 * firing on noise over `scanN` checks does not exceed `ceiling`.
 *
 * Bisection on [0, 8] for 60 iterations: familywiseP is strictly decreasing in
 * sigma for scanN >= 1, so the interval halves onto the boundary. Returns 0 when
 * there are no checks (nothing can fire).
 */
export function minAdmissibleSigma(
  scanN: number,
  ceiling: number = CRITERION_POWER_THRESHOLDS.maxFamilywiseP,
): number {
  if (scanN <= 0) return 0;
  let lo = 0;
  let hi = 8;
  for (let i = 0; i < 60; i++) {
    const mid = (lo + hi) / 2;
    if (familywiseP(mid, scanN) <= ceiling) hi = mid;
    else lo = mid;
  }
  return hi;
}

/**
 * Number of checks a criterion gets: checks per day, times days in the window,
 * times the share of the window that actually has readings.
 */
export function checksFor(
  perDay: number,
  days: number,
  coverage: number,
): number {
  return Math.round(perDay * days * coverage);
}