diff --git a/README.md b/README.md index 376989d..0f24ecb 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,39 @@ Outputs: - `experiment_logs/seed-runs/summary_all.csv` (one row per model, paper-ready; includes `peak_host_ram_gb_mean/std` and `peak_gpu_alloc_gb_mean/std`) +## Label-efficiency experiments + +To measure how well the frozen representations work with fewer labels, train +linear probes on 1%, 10%, 50%, and 100% of the labeled Source split. The subset +is stratified by class and deterministic per seed (so every class is represented +even at 1%). + +Grids for all supervised models are generated under `configs/grids/label_efficiency/`. +Launch the full sweep: + +``` +bash tools/run_label_efficiency.sh --partition $slurm_partition --time $time +``` + +Run a subset of models or fractions: + +``` +bash tools/run_label_efficiency.sh --partition $slurm_partition \ + --models "vith14_224_in22k vitg16_224_in22k" \ + --fractions "0.01 0.10 0.50" +``` + +Each grid submits one submitit job per seed (5 seeds per fraction). After the +jobs finish, aggregate into a paper-style Table 4 CSV: + +``` +python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds +``` + +Outputs: +- `experiment_logs/label-efficiency/summary.csv` (columns: 1%, 10%, 50%, 100% OOD F1-Macro) +- `experiment_logs/label-efficiency//summary.json` + ## License See the `LICENSE` file for details about the license under which this code is made available. diff --git a/configs/grids/label_efficiency/vitb16_448_frac0.01.yaml b/configs/grids/label_efficiency/vitb16_448_frac0.01.yaml new file mode 100644 index 0000000..c22552d --- /dev/null +++ b/configs/grids/label_efficiency/vitb16_448_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vitb16_448 +# Generated from configs/grids/seeds/vitb16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitb16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitb16_448_frac0.10.yaml b/configs/grids/label_efficiency/vitb16_448_frac0.10.yaml new file mode 100644 index 0000000..ec8f83a --- /dev/null +++ b/configs/grids/label_efficiency/vitb16_448_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vitb16_448 +# Generated from configs/grids/seeds/vitb16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitb16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitb16_448_frac0.50.yaml b/configs/grids/label_efficiency/vitb16_448_frac0.50.yaml new file mode 100644 index 0000000..9991824 --- /dev/null +++ b/configs/grids/label_efficiency/vitb16_448_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vitb16_448 +# Generated from configs/grids/seeds/vitb16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitb16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitb16_448_frac1.00.yaml b/configs/grids/label_efficiency/vitb16_448_frac1.00.yaml new file mode 100644 index 0000000..90bb4c0 --- /dev/null +++ b/configs/grids/label_efficiency/vitb16_448_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vitb16_448 +# Generated from configs/grids/seeds/vitb16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitb16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitg16_224_in22k_frac0.01.yaml b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.01.yaml new file mode 100644 index 0000000..449d4ae --- /dev/null +++ b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vitg16_224_in22k +# Generated from configs/grids/seeds/vitg16_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitg16_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitg16_224_in22k_frac0.10.yaml b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.10.yaml new file mode 100644 index 0000000..3250dd5 --- /dev/null +++ b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vitg16_224_in22k +# Generated from configs/grids/seeds/vitg16_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitg16_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitg16_224_in22k_frac0.50.yaml b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.50.yaml new file mode 100644 index 0000000..75e2dc0 --- /dev/null +++ b/configs/grids/label_efficiency/vitg16_224_in22k_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vitg16_224_in22k +# Generated from configs/grids/seeds/vitg16_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitg16_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vitg16_224_in22k_frac1.00.yaml b/configs/grids/label_efficiency/vitg16_224_in22k_frac1.00.yaml new file mode 100644 index 0000000..9918f4e --- /dev/null +++ b/configs/grids/label_efficiency/vitg16_224_in22k_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vitg16_224_in22k +# Generated from configs/grids/seeds/vitg16_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vitg16_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_frac0.01.yaml b/configs/grids/label_efficiency/vith14_224_frac0.01.yaml new file mode 100644 index 0000000..93e92ab --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vith14_224 +# Generated from configs/grids/seeds/vith14_224.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_frac0.10.yaml b/configs/grids/label_efficiency/vith14_224_frac0.10.yaml new file mode 100644 index 0000000..9bb2be0 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vith14_224 +# Generated from configs/grids/seeds/vith14_224.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_frac0.50.yaml b/configs/grids/label_efficiency/vith14_224_frac0.50.yaml new file mode 100644 index 0000000..221b871 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vith14_224 +# Generated from configs/grids/seeds/vith14_224.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_frac1.00.yaml b/configs/grids/label_efficiency/vith14_224_frac1.00.yaml new file mode 100644 index 0000000..08f2766 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vith14_224 +# Generated from configs/grids/seeds/vith14_224.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in1k_frac0.01.yaml b/configs/grids/label_efficiency/vith14_224_in1k_frac0.01.yaml new file mode 100644 index 0000000..7ae034d --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in1k_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vith14_224_in1k +# Generated from configs/grids/seeds/vith14_224_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in1k_frac0.10.yaml b/configs/grids/label_efficiency/vith14_224_in1k_frac0.10.yaml new file mode 100644 index 0000000..f6c8d79 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in1k_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vith14_224_in1k +# Generated from configs/grids/seeds/vith14_224_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in1k_frac0.50.yaml b/configs/grids/label_efficiency/vith14_224_in1k_frac0.50.yaml new file mode 100644 index 0000000..bdde56d --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in1k_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vith14_224_in1k +# Generated from configs/grids/seeds/vith14_224_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in1k_frac1.00.yaml b/configs/grids/label_efficiency/vith14_224_in1k_frac1.00.yaml new file mode 100644 index 0000000..3140883 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in1k_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vith14_224_in1k +# Generated from configs/grids/seeds/vith14_224_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in22k_frac0.01.yaml b/configs/grids/label_efficiency/vith14_224_in22k_frac0.01.yaml new file mode 100644 index 0000000..c8f51c9 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in22k_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vith14_224_in22k +# Generated from configs/grids/seeds/vith14_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in22k_frac0.10.yaml b/configs/grids/label_efficiency/vith14_224_in22k_frac0.10.yaml new file mode 100644 index 0000000..4ece025 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in22k_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vith14_224_in22k +# Generated from configs/grids/seeds/vith14_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in22k_frac0.50.yaml b/configs/grids/label_efficiency/vith14_224_in22k_frac0.50.yaml new file mode 100644 index 0000000..4069b50 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in22k_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vith14_224_in22k +# Generated from configs/grids/seeds/vith14_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith14_224_in22k_frac1.00.yaml b/configs/grids/label_efficiency/vith14_224_in22k_frac1.00.yaml new file mode 100644 index 0000000..80a5517 --- /dev/null +++ b/configs/grids/label_efficiency/vith14_224_in22k_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vith14_224_in22k +# Generated from configs/grids/seeds/vith14_224_in22k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith14_224_in22k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_frac0.01.yaml b/configs/grids/label_efficiency/vith16_448_frac0.01.yaml new file mode 100644 index 0000000..196af85 --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vith16_448 +# Generated from configs/grids/seeds/vith16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_frac0.10.yaml b/configs/grids/label_efficiency/vith16_448_frac0.10.yaml new file mode 100644 index 0000000..9119b90 --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vith16_448 +# Generated from configs/grids/seeds/vith16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_frac0.50.yaml b/configs/grids/label_efficiency/vith16_448_frac0.50.yaml new file mode 100644 index 0000000..d597731 --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vith16_448 +# Generated from configs/grids/seeds/vith16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_frac1.00.yaml b/configs/grids/label_efficiency/vith16_448_frac1.00.yaml new file mode 100644 index 0000000..655ea3e --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vith16_448 +# Generated from configs/grids/seeds/vith16_448.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_in1k_frac0.01.yaml b/configs/grids/label_efficiency/vith16_448_in1k_frac0.01.yaml new file mode 100644 index 0000000..8984a7b --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_in1k_frac0.01.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (1%) for vith16_448_in1k +# Generated from configs/grids/seeds/vith16_448_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.01 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_in1k_frac0.10.yaml b/configs/grids/label_efficiency/vith16_448_in1k_frac0.10.yaml new file mode 100644 index 0000000..d019c46 --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_in1k_frac0.10.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (10%) for vith16_448_in1k +# Generated from configs/grids/seeds/vith16_448_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.1 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_in1k_frac0.50.yaml b/configs/grids/label_efficiency/vith16_448_in1k_frac0.50.yaml new file mode 100644 index 0000000..155849a --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_in1k_frac0.50.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (50%) for vith16_448_in1k +# Generated from configs/grids/seeds/vith16_448_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 0.5 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/configs/grids/label_efficiency/vith16_448_in1k_frac1.00.yaml b/configs/grids/label_efficiency/vith16_448_in1k_frac1.00.yaml new file mode 100644 index 0000000..5aa7fb7 --- /dev/null +++ b/configs/grids/label_efficiency/vith16_448_in1k_frac1.00.yaml @@ -0,0 +1,20 @@ +# Label-efficiency sweep (100%) for vith16_448_in1k +# Generated from configs/grids/seeds/vith16_448_in1k.yaml +# Each seed becomes a separate submitit job via tools/run_grid.py. +base_config: configs/supervised_vith16_448_in1k.yaml +constants: + logging.write_tag: linear_probe + data.label_fraction: 1.0 +grid: + meta.seed: + - 0 + - 1 + - 2 + - 3 + - 4 +launch: + folder: submitit_logs/ + partition: gpu1 + nodes: 1 + tasks_per_node: 1 + time: 4300 diff --git a/src/datasets/wilds.py b/src/datasets/wilds.py index a6f34d2..edf0580 100644 --- a/src/datasets/wilds.py +++ b/src/datasets/wilds.py @@ -1,11 +1,51 @@ -import torch from logging import getLogger + +import numpy as np +import torch from wilds import get_dataset from wilds.common.data_loaders import get_eval_loader logger = getLogger() +def _stratified_subset_indices(labels, fraction, seed): + """Return a deterministic, seed-dependent stratified subset of indices. + + For each class, keep ceil(count * fraction) samples, guaranteeing every + class is represented even for very small fractions. + + Parameters + ---------- + labels : array-like of int + Per-sample class labels. + fraction : float + Fraction of samples to keep (0 < fraction <= 1). + seed : int + Seed for NumPy's RNG; different seeds give different subsets. + + Returns + ------- + indices : np.ndarray + Sorted array of selected global indices. + """ + if not (0 < fraction <= 1.0): + raise ValueError(f"label_fraction must be in (0, 1], got {fraction}") + + rng = np.random.default_rng(seed) + labels = np.asarray(labels).reshape(-1) + classes = np.unique(labels) + + selected = [] + for cls in classes: + cls_idx = np.nonzero(labels == cls)[0] + n_keep = max(1, int(np.ceil(len(cls_idx) * fraction))) + n_keep = min(n_keep, len(cls_idx)) + selected.extend(rng.choice(cls_idx, size=n_keep, replace=False).tolist()) + + indices = np.array(sorted(selected), dtype=np.int64) + return indices + + def make_iwildcam( transform, batch_size, @@ -18,6 +58,8 @@ def make_iwildcam( download=True, pin_mem=True, drop_last=True, + label_fraction=None, + seed=0, ): unlabeled = True if split == "extra_unlabeled" else False shuffle = True if split == "extra_unlabeled" or split == "train" else False @@ -28,6 +70,18 @@ def make_iwildcam( dataset = full_dataset.get_subset(split, transform=transform) + # Subset the labeled training data for label-efficiency experiments. + if label_fraction is not None and split == "train": + labels = dataset.y_array + indices = _stratified_subset_indices( + labels=labels, fraction=float(label_fraction), seed=int(seed) + ) + dataset = torch.utils.data.Subset(dataset, indices) + logger.info( + f"iWildCam {split} subset created with {len(dataset)} samples " + f"(label_fraction={label_fraction}, seed={seed})" + ) + # Use the unified wrapper that mimics the ImageNet structure dataset = WildsToTorchWrapper(dataset, is_unlabeled=unlabeled) diff --git a/src/train_supervised.py b/src/train_supervised.py index f87b99e..8d67c0e 100644 --- a/src/train_supervised.py +++ b/src/train_supervised.py @@ -366,6 +366,12 @@ def main(args, resume_preempt=False): crop_size=d_args["crop_size"], ) + label_fraction = d_args.get("label_fraction", None) + if label_fraction is not None: + logger.info( + f"Using label_fraction={label_fraction} for the training split (seed={seed})" + ) + _, train_loader, train_sampler = make_iwildcam( transform=train_transform, split="train", @@ -377,6 +383,8 @@ def main(args, resume_preempt=False): num_workers=d_args["num_workers"], pin_mem=d_args["pin_mem"], drop_last=True, + label_fraction=label_fraction, + seed=seed, ) _, val_loader, val_sampler = make_iwildcam( @@ -643,6 +651,7 @@ def main(args, resume_preempt=False): # Common run-level info folded into every metrics JSON + the params summary. run_info = { "seed": seed, + "label_fraction": label_fraction if label_fraction is not None else 1.0, "train_time_seconds": float(train_time_seconds), "train_time_hms": _format_hms(train_time_seconds), "epochs_run": int(epochs_run), diff --git a/src/utils/logging.py b/src/utils/logging.py index 02e2eb5..6203fc3 100644 --- a/src/utils/logging.py +++ b/src/utils/logging.py @@ -87,6 +87,7 @@ def build_run_name(args): add("leps", opt_args.get("lars_eps")) add("ipe", opt_args.get("ipe_scale")) add("cs", data_args.get("crop_scale")) + add("lf", data_args.get("label_fraction")) add("eval", val_args.get("eval_every")) add("seed", meta_args.get("seed")) diff --git a/tools/aggregate_label_efficiency.py b/tools/aggregate_label_efficiency.py new file mode 100755 index 0000000..81046bf --- /dev/null +++ b/tools/aggregate_label_efficiency.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +""" +Aggregate label-efficiency supervised runs into a paper-style Table 4 CSV. + +For each model, this collects the OOD F1-Macro across seeds and label +fractions (1%, 10%, 50%, 100%) and outputs mean +/- std in a wide CSV table. + +It expects the run_info written by train_supervised.py to contain +`seed` and `data.label_fraction` (or the eval folder name to contain +`-lf-seed`). The 100% runs are the full-data runs already +configured for the paper. + +Outputs: + experiment_logs/label-efficiency/summary.csv + experiment_logs/label-efficiency//summary.json + +Usage: + python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds + python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds --metric F1-macro_all +""" +import argparse +import csv +import json +import math +import os +import re +from collections import defaultdict + +import yaml + +OOD_METRICS_FILE = "iwildcam_test_metrics.json" + +# Match trailing -lf0.01-seed0 or -seed0. +_LF_SEED_SUFFIX_RE = re.compile(r"-lf(?P0\.\d+|[1-9]\d*\.?\d*)-seed(?P\d+)$") +_SEED_SUFFIX_RE = re.compile(r"-seed(?P\d+)$") + + +FRACTIONS = [0.01, 0.10, 0.50, 1.00] + + +def _find_metric(metrics, key): + if isinstance(metrics, dict): + if key in metrics: + return metrics[key] + for value in metrics.values(): + found = _find_metric(value, key) + if found is not None: + return found + elif isinstance(metrics, list): + for item in metrics: + found = _find_metric(item, key) + if found is not None: + return found + return None + + +def _extract_run_info(metrics_obj): + def _search(obj): + if isinstance(obj, dict): + ri = obj.get("run_info") + if isinstance(ri, dict): + return ri + for v in obj.values(): + found = _search(v) + if found is not None: + return found + elif isinstance(obj, list): + for item in obj: + found = _search(item) + if found is not None: + return found + return None + + return _search(metrics_obj) if metrics_obj else {} + + +def _load_json(path): + try: + with open(path, "r") as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return None + + +def _load_yaml(path): + try: + with open(path, "r") as f: + return yaml.load(f, Loader=yaml.FullLoader) + except (OSError, yaml.YAMLError): + return None + + +def _parse_fraction_and_seed(run_name, run_info, run_dir): + """Return (fraction, seed) from run_info, folder name, or params.yaml.""" + fraction = run_info.get("label_fraction") + seed = run_info.get("seed") + + # Try folder name first for both values. + m = _LF_SEED_SUFFIX_RE.search(run_name) + if m: + if fraction is None: + try: + fraction = float(m.group("frac")) + except ValueError: + pass + if seed is None: + try: + seed = int(m.group("seed")) + except ValueError: + pass + else: + m = _SEED_SUFFIX_RE.search(run_name) + if m and seed is None: + try: + seed = int(m.group("seed")) + except ValueError: + pass + + # Fallback to params.yaml. + if fraction is None or seed is None: + params = _load_yaml(os.path.join(run_dir, "params.yaml")) + if params: + if fraction is None: + fraction = _find_metric(params, "label_fraction") + if seed is None: + seed = _find_metric(params, "seed") + + return fraction, seed + + +def _model_key(run_name): + """Strip the label-fraction and seed suffix to obtain a model group key.""" + key = _LF_SEED_SUFFIX_RE.sub("", run_name) + key = _SEED_SUFFIX_RE.sub("", key) + return key + + +def _mean_std(values): + vals = [v for v in values if v is not None and not _is_nan(v)] + if not vals: + return None, None, 0 + n = len(vals) + mean = sum(vals) / n + if n > 1: + std = math.sqrt(sum((v - mean) ** 2 for v in vals) / (n - 1)) + else: + std = 0.0 + return mean, std, n + + +def _is_nan(v): + try: + return math.isnan(float(v)) + except (TypeError, ValueError): + return False + + +def _fmt(mean, std): + if mean is None: + return "" + if std is None or std == 0.0: + return f"{mean:.4f}" + return f"{mean:.4f} +/- {std:.4f}" + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, + ) + parser.add_argument( + "--root", + default="experiment_logs/eval-wilds", + help="root folder holding per-run eval subfolders (default: %(default)s)", + ) + parser.add_argument( + "--out", + default="experiment_logs/label-efficiency", + help="output folder for summaries (default: %(default)s)", + ) + parser.add_argument( + "--metric", + default="F1-macro_all", + help="OOD metric to aggregate (default: %(default)s)", + ) + parser.add_argument( + "--min-seeds", + type=int, + default=1, + help="only report fractions with at least this many seeds (default: %(default)s)", + ) + args = parser.parse_args() + + if not os.path.isdir(args.root): + print(f"Root folder not found: {args.root}") + return + + # model_key -> {fraction: [records]} + groups = defaultdict(lambda: defaultdict(list)) + skipped = [] + + for entry in sorted(os.listdir(args.root)): + run_dir = os.path.join(args.root, entry) + if not os.path.isdir(run_dir): + continue + + ood_metrics = _load_json(os.path.join(run_dir, OOD_METRICS_FILE)) + if ood_metrics is None: + continue + + run_info = _extract_run_info(ood_metrics) + fraction, seed = _parse_fraction_and_seed(entry, run_info, run_dir) + value = _find_metric(ood_metrics, args.metric) + + if fraction is None: + skipped.append((entry, "no label_fraction")) + continue + if value is None: + skipped.append((entry, f"metric {args.metric} missing")) + continue + + key = _model_key(entry) + groups[key][fraction].append( + { + "run_name": entry, + "seed": seed, + "value": float(value), + "run_info": run_info, + } + ) + + if not groups: + print(f"No usable metrics found under {args.root}") + return + + os.makedirs(args.out, exist_ok=True) + + # CSV fieldnames. + fieldnames = ["model"] + for frac in FRACTIONS: + frac_label = f"frac{frac:.2f}" + fieldnames.extend( + [ + f"{frac_label}_mean", + f"{frac_label}_std", + f"{frac_label}_n", + ] + ) + fieldnames.append("seeds") + + csv_rows = [] + + for model_key in sorted(groups.keys()): + fractions = groups[model_key] + row = {"model": model_key} + per_frac = {} + all_seeds = set() + + for frac in FRACTIONS: + records = fractions.get(frac, []) + values = [r["value"] for r in records] + mean, std, n = _mean_std(values) + frac_label = f"frac{frac:.2f}" + row[f"{frac_label}_mean"] = mean + row[f"{frac_label}_std"] = std + row[f"{frac_label}_n"] = n + per_frac[frac] = { + "mean": mean, + "std": std, + "n": n, + "seeds": [r["seed"] for r in records], + "values": values, + "run_names": [r["run_name"] for r in records], + } + for r in records: + if r["seed"] is not None: + all_seeds.add(r["seed"]) + + row["seeds"] = " ".join(str(s) for s in sorted(all_seeds)) + csv_rows.append(row) + + # Write per-model JSON summary. + model_dir = os.path.join(args.out, model_key) + os.makedirs(model_dir, exist_ok=True) + with open(os.path.join(model_dir, "summary.json"), "w") as f: + json.dump( + { + "model": model_key, + "metric": args.metric, + "fractions": {f"{k:.2f}": v for k, v in per_frac.items()}, + }, + f, + indent=2, + sort_keys=True, + ) + + csv_path = os.path.join(args.out, "summary.csv") + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in csv_rows: + writer.writerow(row) + + # Terminal table. + print(f"\nAggregated {len(csv_rows)} model(s). Metric: {args.metric}\n") + header = f"{'model':<50}" + for frac in FRACTIONS: + header += f" {f'{int(frac*100)}%':>18}" + print(header) + print("-" * len(header)) + for row in csv_rows: + line = f"{row['model'][:50]:<50}" + for frac in FRACTIONS: + frac_label = f"frac{frac:.2f}" + mean = row[f"{frac_label}_mean"] + std = row[f"{frac_label}_std"] + line += f" {_fmt(mean, std):>18}" + print(line) + + if skipped: + print(f"\nSkipped {len(skipped)} run(s):") + for run_name, reason in skipped: + print(f" {run_name}: {reason}") + + print(f"\nWrote per-model summaries to: {args.out}//summary.json") + print(f"Wrote combined CSV to: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/run_label_efficiency.sh b/tools/run_label_efficiency.sh new file mode 100755 index 0000000..3b8f331 --- /dev/null +++ b/tools/run_label_efficiency.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# Run label-efficiency supervised experiments for all model grids. +# +# For every grid under configs/grids/label_efficiency/, this submits one +# submitit job per seed (via tools/run_grid.py). Models are launched +# sequentially so you can run them "one by one"; within a grid, the 5 seeds +# are submitted together. +# +# Each run automatically: +# - uses a stratified subset of the Source split (data.label_fraction) +# - seeds training from meta.seed +# - evaluates on id_test (ID) and test (OOD) WILDS splits +# - records WILDS metrics + training time + epochs_run into the metrics JSON +# +# After all jobs finish, aggregate with: +# python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds +# +# Usage: +# bash tools/run_label_efficiency.sh [--partition P] [--time MIN] [--folder DIR] +# [--models "vith14_224_in22k vitg16_224_in22k"] +# [--fractions "0.01 0.10 0.50"] +# +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GRID_DIR="${PROJECT_ROOT}/configs/grids/label_efficiency" + +PARTITION="" +TIME="" +FOLDER="" +MODELS="" +FRACTIONS="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --partition) PARTITION="$2"; shift 2 ;; + --time) TIME="$2"; shift 2 ;; + --folder) FOLDER="$2"; shift 2 ;; + --models) MODELS="$2"; shift 2 ;; + --fractions) FRACTIONS="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +# Resolve the list of grid files to run. +if [[ -n "${MODELS}" ]]; then + GRIDS=() + for m in ${MODELS}; do + if [[ -n "${FRACTIONS}" ]]; then + for frac in ${FRACTIONS}; do + g="${GRID_DIR}/${m}_frac${frac}.yaml" + if [[ ! -f "${g}" ]]; then + echo "Grid not found for model '${m}' fraction '${frac}': ${g}" >&2 + exit 1 + fi + GRIDS+=("${g}") + done + else + for g in "${GRID_DIR}/${m}"_frac*.yaml; do + if [[ -f "${g}" ]]; then + GRIDS+=("${g}") + fi + done + fi + done +else + # All models and fractions, sorted. + GRIDS=() + while IFS= read -r g; do GRIDS+=("${g}"); done < <(ls "${GRID_DIR}"/*.yaml | sort) +fi + +if [[ ${#GRIDS[@]} -eq 0 ]]; then + echo "No grid files found in ${GRID_DIR}" >&2 + exit 1 +fi + +echo "Launching label-efficiency sweeps for ${#GRIDS[@]} grid(s):" +for g in "${GRIDS[@]}"; do echo " - $(basename "${g}")"; done +echo + +for g in "${GRIDS[@]}"; do + echo "==================================================================" + echo "Model grid: $(basename "${g}")" + echo "==================================================================" + + cmd=("${PROJECT_ROOT}/.venv/bin/python" "${PROJECT_ROOT}/tools/run_grid.py" --grid "${g}") + [[ -n "${PARTITION}" ]] && cmd+=(--partition "${PARTITION}") + [[ -n "${TIME}" ]] && cmd+=(--time "${TIME}") + [[ -n "${FOLDER}" ]] && cmd+=(--folder "${FOLDER}") + + echo "+ ${cmd[*]}" + "${cmd[@]}" + echo +done + +echo "All label-efficiency jobs submitted." +echo "When they finish, aggregate results with:" +echo " ${PROJECT_ROOT}/.venv/bin/python tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds"