{
  "generatedAt": "2026-10-03T03:42:31.590Z",
  "headline": "clean",
  "configs": {
    "clean": "Plain Claude Code: no user settings, hooks, plugins, MCP servers or skills.",
    "ambient": "The author's everyday setup: 19 MCP tools (Vercel, Google Drive, Calendar, Gmail and others), 63 skills and 102 slash commands loaded in every session.",
    "clean+verify-md": "Clean, plus a five-line CLAUDE.md: read the rules first, never edit tests, enumerate the edge cases the spec implies and test each one, re-run the tests before stopping."
  },
  "runs": 276,
  "headlineRuns": 132,
  "sets": {
    "coding-v1": {
      "description": "Twenty small, well-specified tasks: off-by-one fixes, one-line edits, single-function features, simple extractions.",
      "manifest": "e9527dd003a0",
      "tasks": [
        "feature/01",
        "feature/02",
        "feature/03",
        "feature/04",
        "feature/05",
        "fix-test/01",
        "fix-test/02",
        "fix-test/03",
        "fix-test/04",
        "fix-test/05",
        "refactor/01",
        "refactor/02",
        "refactor/03",
        "refactor/04",
        "refactor/05",
        "small-edit/01",
        "small-edit/02",
        "small-edit/03",
        "small-edit/04",
        "small-edit/05"
      ],
      "lastRun": "2026-10-03T02:47:15.593Z",
      "tiers": {
        "opus": {
          "n": 20,
          "pass": 20,
          "passRate": 1,
          "cost": 1,
          "costPerTask": 0.061545000000000016,
          "costPerPass": 0.061545000000000016,
          "sec": 12.4646,
          "parity": 1
        },
        "sonnet": {
          "n": 20,
          "pass": 20,
          "passRate": 1,
          "cost": 0.5127142741083758,
          "costPerTask": 0.03155499999999999,
          "costPerPass": 0.03155499999999999,
          "sec": 7.7569,
          "parity": 1
        },
        "haiku": {
          "n": 20,
          "pass": 20,
          "passRate": 1,
          "cost": 0.524819237955967,
          "costPerTask": 0.032299999999999995,
          "costPerPass": 0.032299999999999995,
          "sec": 15.235499999999998,
          "parity": 1
        }
      },
      "types": {
        "fix-test": {
          "opus": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.056459999999999996,
            "costPerPass": 0.056459999999999996,
            "sec": 14.247399999999999,
            "parity": 1
          },
          "sonnet": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.528515763372299,
            "costPerTask": 0.02984,
            "costPerPass": 0.02984,
            "sec": 6.9682,
            "parity": 1
          },
          "haiku": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.46687920651788883,
            "costPerTask": 0.02636,
            "costPerPass": 0.02636,
            "sec": 11.5134,
            "parity": 1
          }
        },
        "small-edit": {
          "opus": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.05956,
            "costPerPass": 0.05956,
            "sec": 10.818999999999999,
            "parity": 1
          },
          "sonnet": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.5271994627266622,
            "costPerTask": 0.0314,
            "costPerPass": 0.0314,
            "sec": 7.4166,
            "parity": 1
          },
          "haiku": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.5335795836131632,
            "costPerTask": 0.031779999999999996,
            "costPerPass": 0.031779999999999996,
            "sec": 14.698599999999999,
            "parity": 1
          }
        },
        "feature": {
          "opus": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.0668,
            "costPerPass": 0.0668,
            "sec": 12.6396,
            "parity": 1
          },
          "sonnet": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.5017964071856288,
            "costPerTask": 0.03352,
            "costPerPass": 0.03352,
            "sec": 9.2596,
            "parity": 1
          },
          "haiku": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.5547904191616766,
            "costPerTask": 0.037059999999999996,
            "costPerPass": 0.037059999999999996,
            "sec": 19.3196,
            "parity": 1
          }
        },
        "refactor": {
          "opus": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.06336,
            "costPerPass": 0.06336,
            "sec": 12.1524,
            "parity": 1
          },
          "sonnet": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.4965277777777778,
            "costPerTask": 0.03146,
            "costPerPass": 0.03146,
            "sec": 7.3831999999999995,
            "parity": 1
          },
          "haiku": {
            "n": 5,
            "pass": 5,
            "passRate": 1,
            "cost": 0.5366161616161617,
            "costPerTask": 0.034,
            "costPerPass": 0.034,
            "sec": 15.410400000000001,
            "parity": 1
          }
        }
      },
      "matrix": {
        "feature/01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/04": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/05": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/04": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/05": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/04": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/05": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/04": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/05": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        }
      },
      "failures": [],
      "corrections": []
    },
    "coding-v2": {
      "description": "Twelve harder tasks: an LRU recency bug, RFC 4180 CSV quoting, a concurrency limiter that leaks slots, prototype-pollution hardening, a leading-edge debounce, cursor pagination, an expression parser, semver ranges, topological sort with cycle detection, and multi-file and nesting refactors.",
      "manifest": "a9190d302477",
      "tasks": [
        "feature/v2-01",
        "feature/v2-02",
        "feature/v2-03",
        "fix-test/v2-01",
        "fix-test/v2-02",
        "fix-test/v2-03",
        "refactor/v2-01",
        "refactor/v2-02",
        "refactor/v2-03",
        "small-edit/v2-01",
        "small-edit/v2-02",
        "small-edit/v2-03"
      ],
      "lastRun": "2026-10-03T02:58:07.306Z",
      "tiers": {
        "opus": {
          "n": 12,
          "pass": 12,
          "passRate": 1,
          "cost": 1,
          "costPerTask": 0.07343333333333334,
          "costPerPass": 0.07343333333333334,
          "sec": 14.871,
          "parity": 1
        },
        "sonnet": {
          "n": 12,
          "pass": 12,
          "passRate": 1,
          "cost": 0.5051066727190195,
          "costPerTask": 0.03709166666666667,
          "costPerPass": 0.03709166666666667,
          "sec": 9.67125,
          "parity": 1
        },
        "haiku": {
          "n": 12,
          "pass": 12,
          "passRate": 1,
          "cost": 0.6404902405810257,
          "costPerTask": 0.04703333333333332,
          "costPerPass": 0.04703333333333332,
          "sec": 29.75425,
          "parity": 1
        }
      },
      "types": {
        "fix-test": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.06906666666666667,
            "costPerPass": 0.06906666666666667,
            "sec": 13.083666666666666,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.48600386100386106,
            "costPerTask": 0.03356666666666667,
            "costPerPass": 0.03356666666666667,
            "sec": 8.17,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.51496138996139,
            "costPerTask": 0.03556666666666666,
            "costPerPass": 0.03556666666666666,
            "sec": 18.526666666666667,
            "parity": 1
          }
        },
        "small-edit": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.0709,
            "costPerPass": 0.0709,
            "sec": 12.016333333333334,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.5049365303244006,
            "costPerTask": 0.0358,
            "costPerPass": 0.0358,
            "sec": 9.039,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.6337564645039961,
            "costPerTask": 0.04493333333333333,
            "costPerPass": 0.04493333333333333,
            "sec": 28.703666666666667,
            "parity": 1
          }
        },
        "feature": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.08679999999999999,
            "costPerPass": 0.08679999999999999,
            "sec": 19.597666666666665,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.47695852534562216,
            "costPerTask": 0.0414,
            "costPerPass": 0.0414,
            "sec": 12.088333333333333,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.8375576036866359,
            "costPerTask": 0.07269999999999999,
            "costPerPass": 0.07269999999999999,
            "sec": 53.63066666666666,
            "parity": 1
          }
        },
        "refactor": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.06696666666666666,
            "costPerPass": 0.06696666666666666,
            "sec": 14.786333333333333,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.5614733698357393,
            "costPerTask": 0.0376,
            "costPerPass": 0.0376,
            "sec": 9.387666666666666,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.5216525634644102,
            "costPerTask": 0.03493333333333334,
            "costPerPass": 0.03493333333333334,
            "sec": 18.156000000000002,
            "parity": 1
          }
        }
      },
      "matrix": {
        "feature/v2-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/v2-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/v2-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/v2-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/v2-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/v2-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v2-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v2-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v2-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/v2-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/v2-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/v2-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        }
      },
      "failures": [],
      "corrections": []
    },
    "coding-v3": {
      "description": "Twelve tasks with hidden tests, written into the folder only after the session ends: tax rounded once not per line, business days with holidays, async memoization that must not cache failures, globstar matching, a quadratic dedupe made fast without changing results, strict INI errors with line numbers, cron with the day-of-month OR day-of-week rule, a provably minimal diff, a hand-written JSON parser with exact error positions, breaking an import cycle, a dual callback/promise API, and a long function split up and checked against the original on 3,000 random inputs.",
      "manifest": "3ffc6a672c57",
      "tasks": [
        "feature/v3-01",
        "feature/v3-02",
        "feature/v3-03",
        "fix-test/v3-01",
        "fix-test/v3-02",
        "fix-test/v3-03",
        "refactor/v3-01",
        "refactor/v3-02",
        "refactor/v3-03",
        "small-edit/v3-01",
        "small-edit/v3-02",
        "small-edit/v3-03"
      ],
      "lastRun": "2026-10-03T03:23:54.367Z",
      "tiers": {
        "opus": {
          "n": 12,
          "pass": 12,
          "passRate": 1,
          "cost": 1,
          "costPerTask": 0.12286666666666664,
          "costPerPass": 0.12286666666666664,
          "sec": 29.485,
          "parity": 1
        },
        "sonnet": {
          "n": 12,
          "pass": 12,
          "passRate": 1,
          "cost": 0.40918339663591974,
          "costPerTask": 0.05027499999999999,
          "costPerPass": 0.05027499999999999,
          "sec": 16.688583333333334,
          "parity": 1
        },
        "haiku": {
          "n": 12,
          "pass": 9,
          "passRate": 0.75,
          "cost": 1.0754205100379817,
          "costPerTask": 0.09910000000000001,
          "costPerPass": 0.13213333333333332,
          "sec": 82.73066666666666,
          "parity": 0.75
        }
      },
      "types": {
        "fix-test": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.08583333333333333,
            "costPerPass": 0.08583333333333333,
            "sec": 20.028666666666666,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.43766990291262137,
            "costPerTask": 0.037566666666666665,
            "costPerPass": 0.037566666666666665,
            "sec": 11.259,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.6093203883495145,
            "costPerTask": 0.05229999999999999,
            "costPerPass": 0.05229999999999999,
            "sec": 34.34366666666667,
            "parity": 1
          }
        },
        "small-edit": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.10773333333333335,
            "costPerPass": 0.10773333333333335,
            "sec": 26.40733333333333,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.4808168316831682,
            "costPerTask": 0.05179999999999999,
            "costPerPass": 0.05179999999999999,
            "sec": 16.939666666666668,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 1,
            "passRate": 0.3333333333333333,
            "cost": 1.5426980198019802,
            "costPerTask": 0.055400000000000005,
            "costPerPass": 0.16620000000000001,
            "sec": 37.708333333333336,
            "parity": 0.3333333333333333
          }
        },
        "feature": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.20526666666666668,
            "costPerPass": 0.20526666666666668,
            "sec": 52.69866666666667,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.3329002923026957,
            "costPerTask": 0.06833333333333334,
            "costPerPass": 0.06833333333333334,
            "sec": 23.479,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 2,
            "passRate": 0.6666666666666666,
            "cost": 1.7085092562520299,
            "costPerTask": 0.2338,
            "costPerPass": 0.3507,
            "sec": 217.29066666666665,
            "parity": 0.6666666666666666
          }
        },
        "refactor": {
          "opus": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 1,
            "costPerTask": 0.09263333333333335,
            "costPerPass": 0.09263333333333335,
            "sec": 18.805333333333333,
            "parity": 1
          },
          "sonnet": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.468513853904282,
            "costPerTask": 0.043399999999999994,
            "costPerPass": 0.043399999999999994,
            "sec": 15.076666666666666,
            "parity": 1
          },
          "haiku": {
            "n": 3,
            "pass": 3,
            "passRate": 1,
            "cost": 0.5926592299388269,
            "costPerTask": 0.054900000000000004,
            "costPerPass": 0.054900000000000004,
            "sec": 41.58,
            "parity": 1
          }
        }
      },
      "matrix": {
        "feature/v3-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/v3-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "feature/v3-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "fail"
        },
        "fix-test/v3-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/v3-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "fix-test/v3-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v3-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v3-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "refactor/v3-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        },
        "small-edit/v3-01": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "fail"
        },
        "small-edit/v3-02": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "fail"
        },
        "small-edit/v3-03": {
          "opus": "pass",
          "sonnet": "pass",
          "haiku": "pass"
        }
      },
      "failures": [
        {
          "task": "small-edit/v3-01",
          "tier": "haiku",
          "reason": "an escaped * produced an invalid regular expression; **/ did not match zero directories"
        },
        {
          "task": "small-edit/v3-02",
          "tier": "haiku",
          "reason": "counted NaN as a duplicate (Set semantics, not ===)"
        },
        {
          "task": "feature/v3-03",
          "tier": "haiku",
          "reason": "lost an own __proto__ key and reported wrong error positions (hit the 6-minute cap)"
        }
      ],
      "corrections": []
    }
  },
  "overall": {
    "opus": {
      "n": 44,
      "pass": 44,
      "passRate": 1,
      "cost": 1,
      "costPerTask": 0.08151136363636363,
      "costPerPass": 0.08151136363636363,
      "sec": 17.76281818181818,
      "parity": 1
    },
    "sonnet": {
      "n": 44,
      "pass": 44,
      "passRate": 1,
      "cost": 0.4682838421859752,
      "costPerTask": 0.03817045454545454,
      "costPerPass": 0.03817045454545454,
      "sec": 10.71490909090909,
      "parity": 1
    },
    "haiku": {
      "n": 44,
      "pass": 41,
      "passRate": 0.9318181818181818,
      "cost": 0.7180204901170718,
      "costPerTask": 0.05453636363636363,
      "costPerPass": 0.05852682926829268,
      "sec": 37.602931818181816,
      "parity": 0.9318181818181818
    }
  },
  "compare": [
    {
      "config": "ambient",
      "tier": "opus",
      "tasks": 44,
      "sets": [
        "coding-v1",
        "coding-v2",
        "coding-v3"
      ],
      "clean": {
        "pass": 44,
        "costPerTask": 0.08151136363636363,
        "tokens": 73088.84090909091
      },
      "other": {
        "pass": 44,
        "costPerTask": 0.17210909090909088,
        "tokens": 103356.72727272728
      }
    },
    {
      "config": "ambient",
      "tier": "sonnet",
      "tasks": 44,
      "sets": [
        "coding-v1",
        "coding-v2",
        "coding-v3"
      ],
      "clean": {
        "pass": 44,
        "costPerTask": 0.03817045454545454,
        "tokens": 58444.84090909091
      },
      "other": {
        "pass": 44,
        "costPerTask": 0.09145227272727273,
        "tokens": 102974.38636363637
      }
    },
    {
      "config": "ambient",
      "tier": "haiku",
      "tasks": 44,
      "sets": [
        "coding-v1",
        "coding-v2",
        "coding-v3"
      ],
      "clean": {
        "pass": 41,
        "costPerTask": 0.05453636363636363,
        "tokens": 145930.95454545456
      },
      "other": {
        "pass": 40,
        "costPerTask": 0.06153636363636364,
        "tokens": 163904.1590909091
      }
    },
    {
      "config": "clean+verify-md",
      "tier": "haiku",
      "tasks": 12,
      "sets": [
        "coding-v3"
      ],
      "clean": {
        "pass": 9,
        "costPerTask": 0.09910000000000001,
        "tokens": 217059.08333333334
      },
      "other": {
        "pass": 10,
        "costPerTask": 0.10879166666666666,
        "tokens": 295934.9166666667
      }
    }
  ]
}
