From 1a3916ce1a10c667397714849c7536bac8f5bc32 Mon Sep 17 00:00:00 2001 From: YannAhlgrim Date: Tue, 7 Jul 2026 09:43:28 +0200 Subject: [PATCH] min seeds --- tools/aggregate_seeds.py | 109 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/tools/aggregate_seeds.py b/tools/aggregate_seeds.py index 143cf36..bfe9aa6 100644 --- a/tools/aggregate_seeds.py +++ b/tools/aggregate_seeds.py @@ -22,6 +22,7 @@ Outputs: Usage: python3 tools/aggregate_seeds.py --root experiment_logs/eval-wilds python3 tools/aggregate_seeds.py --primary F1-macro_all --acc acc_avg + python3 tools/aggregate_seeds.py --root experiment_logs/eval-wilds --min-seeds 5 """ import argparse import csv @@ -60,6 +61,67 @@ def _strip_seed(run_name): return _SEED_SUFFIX_RE.sub("", run_name) +def _extract_run_info(*metrics_objs): + """Find a run_info dict inside any metrics object (dict- or list-form). + + Eval-only runs write metrics as a list ([{metrics}, "summary"]) with no + run_info; training runs write a dict with a top-level run_info. Search both. + """ + + def _search(obj): + if isinstance(obj, dict): + ri = obj.get("run_info") + if isinstance(ri, dict): + return ri + for v in obj.values(): + found = _search(v) + if found is not None: + return found + elif isinstance(obj, list): + for item in obj: + found = _search(item) + if found is not None: + return found + return None + + for m in metrics_objs: + found = _search(m) + if found is not None: + return found + return {} + + +def _seed_from_name(run_name): + """Parse a trailing -seedN from a run folder name; None if absent.""" + m = re.search(r"-seed(\d+)$", run_name) + return int(m.group(1)) if m else None + + +def _seed_from_params(run_dir): + """Read `seed:` from params-eval.yaml without requiring PyYAML.""" + path = os.path.join(run_dir, "params-eval.yaml") + try: + with open(path, "r") as f: + for line in f: + m = re.match(r"\s*seed\s*:\s*(\d+)\s*$", line) + if m: + return int(m.group(1)) + except OSError: + pass + return None + + +def _resolve_seed(run_info, run_name, run_dir): + """Seed detection chain: run_info -> folder name -> params-eval.yaml.""" + seed = run_info.get("seed") + if seed is not None: + return seed + seed = _seed_from_name(run_name) + if seed is not None: + return seed + return _seed_from_params(run_dir) + + def _load_json(path): try: with open(path, "r") as f: @@ -134,6 +196,12 @@ def main(): default="acc_avg", help="average-accuracy metric key (default: %(default)s)", ) + parser.add_argument( + "--min-seeds", + type=int, + default=1, + help="only report models with at least this many seeds (default: %(default)s)", + ) args = parser.parse_args() if not os.path.isdir(args.root): @@ -151,17 +219,13 @@ def main(): if id_metrics is None and ood_metrics is None: continue - run_info = {} - for m in (ood_metrics, id_metrics): - if isinstance(m, dict) and isinstance(m.get("run_info"), dict): - run_info = m["run_info"] - break + run_info = _extract_run_info(ood_metrics, id_metrics) group = _strip_seed(entry) groups.setdefault(group, []).append( { "run_name": entry, - "seed": run_info.get("seed"), + "seed": _resolve_seed(run_info, entry, run_dir), "id_metrics": id_metrics, "ood_metrics": ood_metrics, "run_info": run_info, @@ -172,6 +236,26 @@ def main(): print(f"No metrics found under {args.root}") return + if args.min_seeds > 1: + dropped = { + name: len(recs) + for name, recs in groups.items() + if len(recs) < args.min_seeds + } + groups = { + name: recs for name, recs in groups.items() if len(recs) >= args.min_seeds + } + if dropped: + print( + f"Skipping {len(dropped)} model(s) with fewer than " + f"{args.min_seeds} seed(s):" + ) + for name in sorted(dropped): + print(f" {name} ({dropped[name]} seed(s))") + if not groups: + print(f"No models have >= {args.min_seeds} seeds under {args.root}") + return + os.makedirs(args.out, exist_ok=True) csv_rows = [] @@ -259,10 +343,21 @@ def main(): gpu_reserved_vals = [r["run_info"].get("peak_gpu_reserved_gb") for r in records] time_mean, time_std, _ = _mean_std(time_vals) epoch_mean, epoch_std, _ = _mean_std(epoch_vals) - host_ram_mean, host_ram_std, _ = _mean_std(host_ram_vals) + host_ram_mean, host_ram_std, host_ram_n = _mean_std(host_ram_vals) gpu_alloc_mean, gpu_alloc_std, _ = _mean_std(gpu_alloc_vals) gpu_reserved_mean, gpu_reserved_std, _ = _mean_std(gpu_reserved_vals) + if host_ram_n == 0: + # No usable peak_host_ram_gb in any seed's run_info. This usually means + # the metrics JSONs were (re)generated by a standalone eval run that did + # not fold in train_supervised.py's run_info, or run_info is absent. + missing_run_info = sum(1 for r in records if not r["run_info"]) + print( + f"[warn] {group_name}: RAM was not recorded for these eval runs " + f"(no peak_host_ram_gb across {len(records)} seed(s); " + f"{missing_run_info} missing run_info entirely). RAM column stays blank." + ) + # Headline (leaderboard) numbers. id_f1_mean, id_f1_std, _ = _mean_std(metric_values("id_metrics", args.primary)) id_acc_mean, id_acc_std, _ = _mean_std(metric_values("id_metrics", args.acc))