label efficiency test
This commit is contained in:
YannAhlgrim
@@ -119,6 +119,39 @@ Outputs:
|
||||
- `experiment_logs/seed-runs/summary_all.csv` (one row per model, paper-ready;
|
||||
includes `peak_host_ram_gb_mean/std` and `peak_gpu_alloc_gb_mean/std`)
|
||||
|
||||
## Label-efficiency experiments
|
||||
|
||||
To measure how well the frozen representations work with fewer labels, train
|
||||
linear probes on 1%, 10%, 50%, and 100% of the labeled Source split. The subset
|
||||
is stratified by class and deterministic per seed (so every class is represented
|
||||
even at 1%).
|
||||
|
||||
Grids for all supervised models are generated under `configs/grids/label_efficiency/`.
|
||||
Launch the full sweep:
|
||||
|
||||
```
|
||||
bash tools/run_label_efficiency.sh --partition $slurm_partition --time $time
|
||||
```
|
||||
|
||||
Run a subset of models or fractions:
|
||||
|
||||
```
|
||||
bash tools/run_label_efficiency.sh --partition $slurm_partition \
|
||||
--models "vith14_224_in22k vitg16_224_in22k" \
|
||||
--fractions "0.01 0.10 0.50"
|
||||
```
|
||||
|
||||
Each grid submits one submitit job per seed (5 seeds per fraction). After the
|
||||
jobs finish, aggregate into a paper-style Table 4 CSV:
|
||||
|
||||
```
|
||||
python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds
|
||||
```
|
||||
|
||||
Outputs:
|
||||
- `experiment_logs/label-efficiency/summary.csv` (columns: 1%, 10%, 50%, 100% OOD F1-Macro)
|
||||
- `experiment_logs/label-efficiency/<model>/summary.json`
|
||||
|
||||
## License
|
||||
|
||||
See the `LICENSE` file for details about the license under which this code is made available.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vitb16_448
|
||||
# Generated from configs/grids/seeds/vitb16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitb16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vitb16_448
|
||||
# Generated from configs/grids/seeds/vitb16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitb16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vitb16_448
|
||||
# Generated from configs/grids/seeds/vitb16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitb16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vitb16_448
|
||||
# Generated from configs/grids/seeds/vitb16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitb16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vitg16_224_in22k
|
||||
# Generated from configs/grids/seeds/vitg16_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitg16_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vitg16_224_in22k
|
||||
# Generated from configs/grids/seeds/vitg16_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitg16_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vitg16_224_in22k
|
||||
# Generated from configs/grids/seeds/vitg16_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitg16_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vitg16_224_in22k
|
||||
# Generated from configs/grids/seeds/vitg16_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vitg16_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vith14_224
|
||||
# Generated from configs/grids/seeds/vith14_224.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vith14_224
|
||||
# Generated from configs/grids/seeds/vith14_224.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vith14_224
|
||||
# Generated from configs/grids/seeds/vith14_224.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vith14_224
|
||||
# Generated from configs/grids/seeds/vith14_224.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vith14_224_in1k
|
||||
# Generated from configs/grids/seeds/vith14_224_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vith14_224_in1k
|
||||
# Generated from configs/grids/seeds/vith14_224_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vith14_224_in1k
|
||||
# Generated from configs/grids/seeds/vith14_224_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vith14_224_in1k
|
||||
# Generated from configs/grids/seeds/vith14_224_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vith14_224_in22k
|
||||
# Generated from configs/grids/seeds/vith14_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vith14_224_in22k
|
||||
# Generated from configs/grids/seeds/vith14_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vith14_224_in22k
|
||||
# Generated from configs/grids/seeds/vith14_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vith14_224_in22k
|
||||
# Generated from configs/grids/seeds/vith14_224_in22k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith14_224_in22k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vith16_448
|
||||
# Generated from configs/grids/seeds/vith16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vith16_448
|
||||
# Generated from configs/grids/seeds/vith16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vith16_448
|
||||
# Generated from configs/grids/seeds/vith16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vith16_448
|
||||
# Generated from configs/grids/seeds/vith16_448.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (1%) for vith16_448_in1k
|
||||
# Generated from configs/grids/seeds/vith16_448_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.01
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (10%) for vith16_448_in1k
|
||||
# Generated from configs/grids/seeds/vith16_448_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.1
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (50%) for vith16_448_in1k
|
||||
# Generated from configs/grids/seeds/vith16_448_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 0.5
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
@@ -0,0 +1,20 @@
|
||||
# Label-efficiency sweep (100%) for vith16_448_in1k
|
||||
# Generated from configs/grids/seeds/vith16_448_in1k.yaml
|
||||
# Each seed becomes a separate submitit job via tools/run_grid.py.
|
||||
base_config: configs/supervised_vith16_448_in1k.yaml
|
||||
constants:
|
||||
logging.write_tag: linear_probe
|
||||
data.label_fraction: 1.0
|
||||
grid:
|
||||
meta.seed:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
launch:
|
||||
folder: submitit_logs/
|
||||
partition: gpu1
|
||||
nodes: 1
|
||||
tasks_per_node: 1
|
||||
time: 4300
|
||||
+55
-1
@@ -1,11 +1,51 @@
|
||||
import torch
|
||||
from logging import getLogger
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from wilds import get_dataset
|
||||
from wilds.common.data_loaders import get_eval_loader
|
||||
|
||||
logger = getLogger()
|
||||
|
||||
|
||||
def _stratified_subset_indices(labels, fraction, seed):
|
||||
"""Return a deterministic, seed-dependent stratified subset of indices.
|
||||
|
||||
For each class, keep ceil(count * fraction) samples, guaranteeing every
|
||||
class is represented even for very small fractions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
labels : array-like of int
|
||||
Per-sample class labels.
|
||||
fraction : float
|
||||
Fraction of samples to keep (0 < fraction <= 1).
|
||||
seed : int
|
||||
Seed for NumPy's RNG; different seeds give different subsets.
|
||||
|
||||
Returns
|
||||
-------
|
||||
indices : np.ndarray
|
||||
Sorted array of selected global indices.
|
||||
"""
|
||||
if not (0 < fraction <= 1.0):
|
||||
raise ValueError(f"label_fraction must be in (0, 1], got {fraction}")
|
||||
|
||||
rng = np.random.default_rng(seed)
|
||||
labels = np.asarray(labels).reshape(-1)
|
||||
classes = np.unique(labels)
|
||||
|
||||
selected = []
|
||||
for cls in classes:
|
||||
cls_idx = np.nonzero(labels == cls)[0]
|
||||
n_keep = max(1, int(np.ceil(len(cls_idx) * fraction)))
|
||||
n_keep = min(n_keep, len(cls_idx))
|
||||
selected.extend(rng.choice(cls_idx, size=n_keep, replace=False).tolist())
|
||||
|
||||
indices = np.array(sorted(selected), dtype=np.int64)
|
||||
return indices
|
||||
|
||||
|
||||
def make_iwildcam(
|
||||
transform,
|
||||
batch_size,
|
||||
@@ -18,6 +58,8 @@ def make_iwildcam(
|
||||
download=True,
|
||||
pin_mem=True,
|
||||
drop_last=True,
|
||||
label_fraction=None,
|
||||
seed=0,
|
||||
):
|
||||
unlabeled = True if split == "extra_unlabeled" else False
|
||||
shuffle = True if split == "extra_unlabeled" or split == "train" else False
|
||||
@@ -28,6 +70,18 @@ def make_iwildcam(
|
||||
|
||||
dataset = full_dataset.get_subset(split, transform=transform)
|
||||
|
||||
# Subset the labeled training data for label-efficiency experiments.
|
||||
if label_fraction is not None and split == "train":
|
||||
labels = dataset.y_array
|
||||
indices = _stratified_subset_indices(
|
||||
labels=labels, fraction=float(label_fraction), seed=int(seed)
|
||||
)
|
||||
dataset = torch.utils.data.Subset(dataset, indices)
|
||||
logger.info(
|
||||
f"iWildCam {split} subset created with {len(dataset)} samples "
|
||||
f"(label_fraction={label_fraction}, seed={seed})"
|
||||
)
|
||||
|
||||
# Use the unified wrapper that mimics the ImageNet structure
|
||||
dataset = WildsToTorchWrapper(dataset, is_unlabeled=unlabeled)
|
||||
|
||||
|
||||
@@ -366,6 +366,12 @@ def main(args, resume_preempt=False):
|
||||
crop_size=d_args["crop_size"],
|
||||
)
|
||||
|
||||
label_fraction = d_args.get("label_fraction", None)
|
||||
if label_fraction is not None:
|
||||
logger.info(
|
||||
f"Using label_fraction={label_fraction} for the training split (seed={seed})"
|
||||
)
|
||||
|
||||
_, train_loader, train_sampler = make_iwildcam(
|
||||
transform=train_transform,
|
||||
split="train",
|
||||
@@ -377,6 +383,8 @@ def main(args, resume_preempt=False):
|
||||
num_workers=d_args["num_workers"],
|
||||
pin_mem=d_args["pin_mem"],
|
||||
drop_last=True,
|
||||
label_fraction=label_fraction,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
_, val_loader, val_sampler = make_iwildcam(
|
||||
@@ -643,6 +651,7 @@ def main(args, resume_preempt=False):
|
||||
# Common run-level info folded into every metrics JSON + the params summary.
|
||||
run_info = {
|
||||
"seed": seed,
|
||||
"label_fraction": label_fraction if label_fraction is not None else 1.0,
|
||||
"train_time_seconds": float(train_time_seconds),
|
||||
"train_time_hms": _format_hms(train_time_seconds),
|
||||
"epochs_run": int(epochs_run),
|
||||
|
||||
@@ -87,6 +87,7 @@ def build_run_name(args):
|
||||
add("leps", opt_args.get("lars_eps"))
|
||||
add("ipe", opt_args.get("ipe_scale"))
|
||||
add("cs", data_args.get("crop_scale"))
|
||||
add("lf", data_args.get("label_fraction"))
|
||||
add("eval", val_args.get("eval_every"))
|
||||
add("seed", meta_args.get("seed"))
|
||||
|
||||
|
||||
Executable
+330
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate label-efficiency supervised runs into a paper-style Table 4 CSV.
|
||||
|
||||
For each model, this collects the OOD F1-Macro across seeds and label
|
||||
fractions (1%, 10%, 50%, 100%) and outputs mean +/- std in a wide CSV table.
|
||||
|
||||
It expects the run_info written by train_supervised.py to contain
|
||||
`seed` and `data.label_fraction` (or the eval folder name to contain
|
||||
`-lf<FRACTION>-seed<SEED>`). The 100% runs are the full-data runs already
|
||||
configured for the paper.
|
||||
|
||||
Outputs:
|
||||
experiment_logs/label-efficiency/summary.csv
|
||||
experiment_logs/label-efficiency/<model>/summary.json
|
||||
|
||||
Usage:
|
||||
python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds
|
||||
python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds --metric F1-macro_all
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
import yaml
|
||||
|
||||
OOD_METRICS_FILE = "iwildcam_test_metrics.json"
|
||||
|
||||
# Match trailing -lf0.01-seed0 or -seed0.
|
||||
_LF_SEED_SUFFIX_RE = re.compile(r"-lf(?P<frac>0\.\d+|[1-9]\d*\.?\d*)-seed(?P<seed>\d+)$")
|
||||
_SEED_SUFFIX_RE = re.compile(r"-seed(?P<seed>\d+)$")
|
||||
|
||||
|
||||
FRACTIONS = [0.01, 0.10, 0.50, 1.00]
|
||||
|
||||
|
||||
def _find_metric(metrics, key):
|
||||
if isinstance(metrics, dict):
|
||||
if key in metrics:
|
||||
return metrics[key]
|
||||
for value in metrics.values():
|
||||
found = _find_metric(value, key)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(metrics, list):
|
||||
for item in metrics:
|
||||
found = _find_metric(item, key)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _extract_run_info(metrics_obj):
|
||||
def _search(obj):
|
||||
if isinstance(obj, dict):
|
||||
ri = obj.get("run_info")
|
||||
if isinstance(ri, dict):
|
||||
return ri
|
||||
for v in obj.values():
|
||||
found = _search(v)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
found = _search(item)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
return _search(metrics_obj) if metrics_obj else {}
|
||||
|
||||
|
||||
def _load_json(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _load_yaml(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
return yaml.load(f, Loader=yaml.FullLoader)
|
||||
except (OSError, yaml.YAMLError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_fraction_and_seed(run_name, run_info, run_dir):
|
||||
"""Return (fraction, seed) from run_info, folder name, or params.yaml."""
|
||||
fraction = run_info.get("label_fraction")
|
||||
seed = run_info.get("seed")
|
||||
|
||||
# Try folder name first for both values.
|
||||
m = _LF_SEED_SUFFIX_RE.search(run_name)
|
||||
if m:
|
||||
if fraction is None:
|
||||
try:
|
||||
fraction = float(m.group("frac"))
|
||||
except ValueError:
|
||||
pass
|
||||
if seed is None:
|
||||
try:
|
||||
seed = int(m.group("seed"))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
m = _SEED_SUFFIX_RE.search(run_name)
|
||||
if m and seed is None:
|
||||
try:
|
||||
seed = int(m.group("seed"))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Fallback to params.yaml.
|
||||
if fraction is None or seed is None:
|
||||
params = _load_yaml(os.path.join(run_dir, "params.yaml"))
|
||||
if params:
|
||||
if fraction is None:
|
||||
fraction = _find_metric(params, "label_fraction")
|
||||
if seed is None:
|
||||
seed = _find_metric(params, "seed")
|
||||
|
||||
return fraction, seed
|
||||
|
||||
|
||||
def _model_key(run_name):
|
||||
"""Strip the label-fraction and seed suffix to obtain a model group key."""
|
||||
key = _LF_SEED_SUFFIX_RE.sub("", run_name)
|
||||
key = _SEED_SUFFIX_RE.sub("", key)
|
||||
return key
|
||||
|
||||
|
||||
def _mean_std(values):
|
||||
vals = [v for v in values if v is not None and not _is_nan(v)]
|
||||
if not vals:
|
||||
return None, None, 0
|
||||
n = len(vals)
|
||||
mean = sum(vals) / n
|
||||
if n > 1:
|
||||
std = math.sqrt(sum((v - mean) ** 2 for v in vals) / (n - 1))
|
||||
else:
|
||||
std = 0.0
|
||||
return mean, std, n
|
||||
|
||||
|
||||
def _is_nan(v):
|
||||
try:
|
||||
return math.isnan(float(v))
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _fmt(mean, std):
|
||||
if mean is None:
|
||||
return ""
|
||||
if std is None or std == 0.0:
|
||||
return f"{mean:.4f}"
|
||||
return f"{mean:.4f} +/- {std:.4f}"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description=__doc__,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
default="experiment_logs/eval-wilds",
|
||||
help="root folder holding per-run eval subfolders (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
default="experiment_logs/label-efficiency",
|
||||
help="output folder for summaries (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--metric",
|
||||
default="F1-macro_all",
|
||||
help="OOD metric to aggregate (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-seeds",
|
||||
type=int,
|
||||
default=1,
|
||||
help="only report fractions with at least this many seeds (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.root):
|
||||
print(f"Root folder not found: {args.root}")
|
||||
return
|
||||
|
||||
# model_key -> {fraction: [records]}
|
||||
groups = defaultdict(lambda: defaultdict(list))
|
||||
skipped = []
|
||||
|
||||
for entry in sorted(os.listdir(args.root)):
|
||||
run_dir = os.path.join(args.root, entry)
|
||||
if not os.path.isdir(run_dir):
|
||||
continue
|
||||
|
||||
ood_metrics = _load_json(os.path.join(run_dir, OOD_METRICS_FILE))
|
||||
if ood_metrics is None:
|
||||
continue
|
||||
|
||||
run_info = _extract_run_info(ood_metrics)
|
||||
fraction, seed = _parse_fraction_and_seed(entry, run_info, run_dir)
|
||||
value = _find_metric(ood_metrics, args.metric)
|
||||
|
||||
if fraction is None:
|
||||
skipped.append((entry, "no label_fraction"))
|
||||
continue
|
||||
if value is None:
|
||||
skipped.append((entry, f"metric {args.metric} missing"))
|
||||
continue
|
||||
|
||||
key = _model_key(entry)
|
||||
groups[key][fraction].append(
|
||||
{
|
||||
"run_name": entry,
|
||||
"seed": seed,
|
||||
"value": float(value),
|
||||
"run_info": run_info,
|
||||
}
|
||||
)
|
||||
|
||||
if not groups:
|
||||
print(f"No usable metrics found under {args.root}")
|
||||
return
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
# CSV fieldnames.
|
||||
fieldnames = ["model"]
|
||||
for frac in FRACTIONS:
|
||||
frac_label = f"frac{frac:.2f}"
|
||||
fieldnames.extend(
|
||||
[
|
||||
f"{frac_label}_mean",
|
||||
f"{frac_label}_std",
|
||||
f"{frac_label}_n",
|
||||
]
|
||||
)
|
||||
fieldnames.append("seeds")
|
||||
|
||||
csv_rows = []
|
||||
|
||||
for model_key in sorted(groups.keys()):
|
||||
fractions = groups[model_key]
|
||||
row = {"model": model_key}
|
||||
per_frac = {}
|
||||
all_seeds = set()
|
||||
|
||||
for frac in FRACTIONS:
|
||||
records = fractions.get(frac, [])
|
||||
values = [r["value"] for r in records]
|
||||
mean, std, n = _mean_std(values)
|
||||
frac_label = f"frac{frac:.2f}"
|
||||
row[f"{frac_label}_mean"] = mean
|
||||
row[f"{frac_label}_std"] = std
|
||||
row[f"{frac_label}_n"] = n
|
||||
per_frac[frac] = {
|
||||
"mean": mean,
|
||||
"std": std,
|
||||
"n": n,
|
||||
"seeds": [r["seed"] for r in records],
|
||||
"values": values,
|
||||
"run_names": [r["run_name"] for r in records],
|
||||
}
|
||||
for r in records:
|
||||
if r["seed"] is not None:
|
||||
all_seeds.add(r["seed"])
|
||||
|
||||
row["seeds"] = " ".join(str(s) for s in sorted(all_seeds))
|
||||
csv_rows.append(row)
|
||||
|
||||
# Write per-model JSON summary.
|
||||
model_dir = os.path.join(args.out, model_key)
|
||||
os.makedirs(model_dir, exist_ok=True)
|
||||
with open(os.path.join(model_dir, "summary.json"), "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"model": model_key,
|
||||
"metric": args.metric,
|
||||
"fractions": {f"{k:.2f}": v for k, v in per_frac.items()},
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
csv_path = os.path.join(args.out, "summary.csv")
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for row in csv_rows:
|
||||
writer.writerow(row)
|
||||
|
||||
# Terminal table.
|
||||
print(f"\nAggregated {len(csv_rows)} model(s). Metric: {args.metric}\n")
|
||||
header = f"{'model':<50}"
|
||||
for frac in FRACTIONS:
|
||||
header += f" {f'{int(frac*100)}%':>18}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for row in csv_rows:
|
||||
line = f"{row['model'][:50]:<50}"
|
||||
for frac in FRACTIONS:
|
||||
frac_label = f"frac{frac:.2f}"
|
||||
mean = row[f"{frac_label}_mean"]
|
||||
std = row[f"{frac_label}_std"]
|
||||
line += f" {_fmt(mean, std):>18}"
|
||||
print(line)
|
||||
|
||||
if skipped:
|
||||
print(f"\nSkipped {len(skipped)} run(s):")
|
||||
for run_name, reason in skipped:
|
||||
print(f" {run_name}: {reason}")
|
||||
|
||||
print(f"\nWrote per-model summaries to: {args.out}/<model>/summary.json")
|
||||
print(f"Wrote combined CSV to: {csv_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run label-efficiency supervised experiments for all model grids.
|
||||
#
|
||||
# For every grid under configs/grids/label_efficiency/, this submits one
|
||||
# submitit job per seed (via tools/run_grid.py). Models are launched
|
||||
# sequentially so you can run them "one by one"; within a grid, the 5 seeds
|
||||
# are submitted together.
|
||||
#
|
||||
# Each run automatically:
|
||||
# - uses a stratified subset of the Source split (data.label_fraction)
|
||||
# - seeds training from meta.seed
|
||||
# - evaluates on id_test (ID) and test (OOD) WILDS splits
|
||||
# - records WILDS metrics + training time + epochs_run into the metrics JSON
|
||||
#
|
||||
# After all jobs finish, aggregate with:
|
||||
# python3 tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds
|
||||
#
|
||||
# Usage:
|
||||
# bash tools/run_label_efficiency.sh [--partition P] [--time MIN] [--folder DIR]
|
||||
# [--models "vith14_224_in22k vitg16_224_in22k"]
|
||||
# [--fractions "0.01 0.10 0.50"]
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
GRID_DIR="${PROJECT_ROOT}/configs/grids/label_efficiency"
|
||||
|
||||
PARTITION=""
|
||||
TIME=""
|
||||
FOLDER=""
|
||||
MODELS=""
|
||||
FRACTIONS=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--partition) PARTITION="$2"; shift 2 ;;
|
||||
--time) TIME="$2"; shift 2 ;;
|
||||
--folder) FOLDER="$2"; shift 2 ;;
|
||||
--models) MODELS="$2"; shift 2 ;;
|
||||
--fractions) FRACTIONS="$2"; shift 2 ;;
|
||||
*) echo "Unknown argument: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve the list of grid files to run.
|
||||
if [[ -n "${MODELS}" ]]; then
|
||||
GRIDS=()
|
||||
for m in ${MODELS}; do
|
||||
if [[ -n "${FRACTIONS}" ]]; then
|
||||
for frac in ${FRACTIONS}; do
|
||||
g="${GRID_DIR}/${m}_frac${frac}.yaml"
|
||||
if [[ ! -f "${g}" ]]; then
|
||||
echo "Grid not found for model '${m}' fraction '${frac}': ${g}" >&2
|
||||
exit 1
|
||||
fi
|
||||
GRIDS+=("${g}")
|
||||
done
|
||||
else
|
||||
for g in "${GRID_DIR}/${m}"_frac*.yaml; do
|
||||
if [[ -f "${g}" ]]; then
|
||||
GRIDS+=("${g}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
else
|
||||
# All models and fractions, sorted.
|
||||
GRIDS=()
|
||||
while IFS= read -r g; do GRIDS+=("${g}"); done < <(ls "${GRID_DIR}"/*.yaml | sort)
|
||||
fi
|
||||
|
||||
if [[ ${#GRIDS[@]} -eq 0 ]]; then
|
||||
echo "No grid files found in ${GRID_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Launching label-efficiency sweeps for ${#GRIDS[@]} grid(s):"
|
||||
for g in "${GRIDS[@]}"; do echo " - $(basename "${g}")"; done
|
||||
echo
|
||||
|
||||
for g in "${GRIDS[@]}"; do
|
||||
echo "=================================================================="
|
||||
echo "Model grid: $(basename "${g}")"
|
||||
echo "=================================================================="
|
||||
|
||||
cmd=("${PROJECT_ROOT}/.venv/bin/python" "${PROJECT_ROOT}/tools/run_grid.py" --grid "${g}")
|
||||
[[ -n "${PARTITION}" ]] && cmd+=(--partition "${PARTITION}")
|
||||
[[ -n "${TIME}" ]] && cmd+=(--time "${TIME}")
|
||||
[[ -n "${FOLDER}" ]] && cmd+=(--folder "${FOLDER}")
|
||||
|
||||
echo "+ ${cmd[*]}"
|
||||
"${cmd[@]}"
|
||||
echo
|
||||
done
|
||||
|
||||
echo "All label-efficiency jobs submitted."
|
||||
echo "When they finish, aggregate results with:"
|
||||
echo " ${PROJECT_ROOT}/.venv/bin/python tools/aggregate_label_efficiency.py --root experiment_logs/eval-wilds"
|
||||
Reference in New Issue
Block a user