From d053e0a8e0d6a900f449c9574b150335239c0010 Mon Sep 17 00:00:00 2001 From: YannAhlgrim Date: Mon, 6 Jul 2026 11:53:54 +0200 Subject: [PATCH] clean up configs --- README.md | 6 +-- ...600-lp.yaml => supervised_vitb16_448.yaml} | 22 ++------ ....yaml => supervised_vitg16_224_in22k.yaml} | 18 ++----- configs/supervised_vith14_224.yaml | 48 ++++++++++++++++++ configs/supervised_vith14_224_in1k.yaml | 49 ++++++++++++++++++ ....yaml => supervised_vith14_224_in22k.yaml} | 26 +++------- ...300-lp.yaml => supervised_vith16_448.yaml} | 24 +++------ configs/supervised_vith16_448_in1k.yaml | 50 +++++++++++++++++++ 8 files changed, 172 insertions(+), 71 deletions(-) rename configs/{supervised_wilds_vitb16-448_ep600-lp.yaml => supervised_vitb16_448.yaml} (63%) rename configs/{supervised_wilds_vitg16_ep300-lp.yaml => supervised_vitg16_224_in22k.yaml} (72%) create mode 100644 configs/supervised_vith14_224.yaml create mode 100644 configs/supervised_vith14_224_in1k.yaml rename configs/{supervised_wilds_vith14_ep300-lp.yaml => supervised_vith14_224_in22k.yaml} (58%) rename configs/{supervised_wilds_vith16-448_ep300-lp.yaml => supervised_vith16_448.yaml} (61%) create mode 100644 configs/supervised_vith16_448_in1k.yaml diff --git a/README.md b/README.md index 661a510..b890df6 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Reference: official I-JEPA README https://github.com/facebookresearch/ijepa/blob ## Models - ViT-H, 14x14 patches, 224x224 resolution (trained) -- ViT-H, 16x16 patches, 448x448 resolution (planned) +- ViT-H, 16x16 patches, 448x448 resolution (trained) - Plan: add a graph comparing models with the WILDS leaderboard https://wilds.stanford.edu/leaderboard/#with-unlabeled-data-1 @@ -25,7 +25,7 @@ Reference: official I-JEPA README https://github.com/facebookresearch/ijepa/blob - `src/train_supervised.py`: supervised training loop - `configs/`: training configs - `configs/wilds_vith14_ep300.yaml`: SSL config used here -- `configs/supervised_wilds_vith14_ep300.yaml`: supervised config used here +- `configs/supervised_vith14_224.yaml`: supervised config used here (see `configs/` for all supervised linear-probe configs) - `main_distributed.py`: entrypoint for distributed SSL training - `main_distributed_supervised.py`: entrypoint for distributed supervised training - `requirements.txt`: dependencies @@ -50,7 +50,7 @@ python3 main_distributed.py --fname configs/wilds_vith14_ep300.yaml --folder $su Supervised fine-tuning: ``` -python3 main_distributed_supervised.py --fname configs/supervised_wilds_vith14_ep300.yaml --folder $submitit_folder --partition $slurm_partition --nodes $nodes --tasks-per-node $tasks_per_node --time $time +python3 main_distributed_supervised.py --fname configs/supervised_vith14_224.yaml --folder $submitit_folder --partition $slurm_partition --nodes $nodes --tasks-per-node $tasks_per_node --time $time ``` Evaluation on iWildCam test split: diff --git a/configs/supervised_wilds_vitb16-448_ep600-lp.yaml b/configs/supervised_vitb16_448.yaml similarity index 63% rename from configs/supervised_wilds_vitb16-448_ep600-lp.yaml rename to configs/supervised_vitb16_448.yaml index c775619..80f0e5a 100644 --- a/configs/supervised_wilds_vitb16-448_ep600-lp.yaml +++ b/configs/supervised_vitb16_448.yaml @@ -10,12 +10,11 @@ meta: head_type: linear data: - batch_size: 1024 + batch_size: 128 root_path: ./wilds_data num_workers: 10 pin_mem: true crop_size: 448 - crop_scale: [1.0, 1.0] use_random_resized_crop: false use_horizontal_flip: false use_color_distortion: false @@ -26,25 +25,14 @@ mask: patch_size: 16 optimization: - optimizer: adamw # 'adamw', 'sgd', or 'lars' - freeze_weights: true # true for linear probing, false for full fine-tuning + optimizer: adamw + freeze_weights: true epochs: 300 lr: 0.01 - weight_decay: 0 + weight_decay: 5.0e-4 lr_schedule: cosine - step_milestones: [15, 30, 45] - step_gamma: 0.1 - start_lr: 0.0 + gradient_accumulation_steps: 1 final_lr: 0.0 - warmup: 0 - momentum: 0.9 - lars_eta: 0.001 - lars_eps: 1.0e-8 - lars_exclude_bias_and_norm: true - ipe_scale: 1.0 - # Sweep suggestions (manual edits): - # - lr: 0.01 | 0.05 | 0.001 - # - weight_decay: 5.0e-4 | 0.0 early_stopping: enabled: true patience: 10 diff --git a/configs/supervised_wilds_vitg16_ep300-lp.yaml b/configs/supervised_vitg16_224_in22k.yaml similarity index 72% rename from configs/supervised_wilds_vitg16_ep300-lp.yaml rename to configs/supervised_vitg16_224_in22k.yaml index b672c03..fc58d0f 100644 --- a/configs/supervised_wilds_vitg16_ep300-lp.yaml +++ b/configs/supervised_vitg16_224_in22k.yaml @@ -3,12 +3,12 @@ meta: tag: in22k embed_dim: 1408 load_checkpoint: true - checkpoint_folder: experiment_logs/imagenet-vith16.448/ + checkpoint_folder: experiment_logs/imagenet/ read_checkpoint: IN22K-vit.g.16-600e.pth.tar use_bfloat16: true num_classes: 182 representation_type: last_avgpool - head_type: linear + head_type: bn_linear use_gradient_checkpointing: true data: @@ -17,7 +17,6 @@ data: num_workers: 10 pin_mem: true crop_size: 224 - crop_scale: [1.0, 1.0] use_random_resized_crop: false use_horizontal_flip: false use_color_distortion: false @@ -31,20 +30,11 @@ optimization: optimizer: adamw freeze_weights: true epochs: 300 - lr: 0.01 + lr: 0.001 weight_decay: 5.0e-4 lr_schedule: cosine - step_milestones: [15, 30, 45] - step_gamma: 0.1 - start_lr: 0.0 + gradient_accumulation_steps: 16 final_lr: 0.0 - warmup: 0 - momentum: 0.99 - lars_eta: 0.001 - lars_eps: 1.0e-8 - lars_exclude_bias_and_norm: true - ipe_scale: 1.0 - gradient_accumulation_steps: 1 early_stopping: enabled: true patience: 10 diff --git a/configs/supervised_vith14_224.yaml b/configs/supervised_vith14_224.yaml new file mode 100644 index 0000000..b0d312f --- /dev/null +++ b/configs/supervised_vith14_224.yaml @@ -0,0 +1,48 @@ +meta: + model_name: vit_huge + embed_dim: 1280 + load_checkpoint: true + checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/ + read_checkpoint: jepa-ep300.pth.tar + use_bfloat16: true + num_classes: 182 + representation_type: last4_avgpool_concat + head_type: bn_linear + +data: + batch_size: 16 + root_path: ./wilds_data + num_workers: 10 + pin_mem: true + crop_size: 224 + use_random_resized_crop: false + use_horizontal_flip: false + use_color_distortion: false + color_jitter_strength: 0.0 + use_gaussian_blur: false + +mask: + patch_size: 14 + +optimization: + optimizer: adamw + freeze_weights: true + epochs: 300 + lr: 0.001 + weight_decay: 5.0e-4 + lr_schedule: cosine + gradient_accumulation_steps: 32 + final_lr: 0.0 + early_stopping: + enabled: true + patience: 10 + min_delta: 1.0e-4 + min_epochs: 15 + restore_best_weights: true + +validation: + eval_every: 1 + +logging: + write_tag: linear_probe + auto_folder: true diff --git a/configs/supervised_vith14_224_in1k.yaml b/configs/supervised_vith14_224_in1k.yaml new file mode 100644 index 0000000..b1de2af --- /dev/null +++ b/configs/supervised_vith14_224_in1k.yaml @@ -0,0 +1,49 @@ +meta: + model_name: vit_huge + tag: in1k + embed_dim: 1280 + load_checkpoint: true + checkpoint_folder: experiment_logs/imagenet/ + read_checkpoint: IN1K-vit.h.14-300e.pth.tar + use_bfloat16: true + num_classes: 182 + representation_type: last_avgpool + head_type: linear + +data: + batch_size: 16 + root_path: ./wilds_data + num_workers: 10 + pin_mem: true + crop_size: 224 + use_random_resized_crop: false + use_horizontal_flip: false + use_color_distortion: false + color_jitter_strength: 0.0 + use_gaussian_blur: false + +mask: + patch_size: 14 + +optimization: + optimizer: adamw + freeze_weights: true + epochs: 300 + lr: 0.01 + weight_decay: 0 + lr_schedule: cosine + gradient_accumulation_steps: 64 + final_lr: 0.0 + early_stopping: + enabled: true + patience: 10 + min_delta: 1.0e-4 + min_epochs: 15 + restore_best_weights: true + +validation: + eval_every: 1 + +logging: + write_tag: linear_probe + auto_folder: true diff --git a/configs/supervised_wilds_vith14_ep300-lp.yaml b/configs/supervised_vith14_224_in22k.yaml similarity index 58% rename from configs/supervised_wilds_vith14_ep300-lp.yaml rename to configs/supervised_vith14_224_in22k.yaml index 7f032f7..d9a59e7 100644 --- a/configs/supervised_wilds_vith14_ep300-lp.yaml +++ b/configs/supervised_vith14_224_in22k.yaml @@ -3,20 +3,19 @@ meta: tag: in22k embed_dim: 1280 load_checkpoint: true - checkpoint_folder: experiment_logs/imagenet-vith16.448/ + checkpoint_folder: experiment_logs/imagenet/ read_checkpoint: IN22K-vit.h.14-900e.pth.tar use_bfloat16: true num_classes: 182 representation_type: last_avgpool - head_type: linear + head_type: bn_linear data: - batch_size: 256 + batch_size: 16 root_path: ./wilds_data num_workers: 10 pin_mem: true crop_size: 224 - crop_scale: [1.0, 1.0] use_random_resized_crop: false use_horizontal_flip: false use_color_distortion: false @@ -27,25 +26,14 @@ mask: patch_size: 14 optimization: - optimizer: adamw # 'adamw', 'sgd', or 'lars' - freeze_weights: true # true for linear probing, false for full fine-tuning + optimizer: adamw + freeze_weights: true epochs: 300 - lr: 0.01 + lr: 0.001 weight_decay: 5.0e-4 lr_schedule: cosine - step_milestones: [15, 30, 45] - step_gamma: 0.1 - start_lr: 0.0 + gradient_accumulation_steps: 32 final_lr: 0.0 - warmup: 0 - momentum: 0.99 - lars_eta: 0.001 - lars_eps: 1.0e-8 - lars_exclude_bias_and_norm: true - ipe_scale: 1.0 - # Sweep suggestions (manual edits): - # - lr: 0.01 | 0.05 | 0.001 - # - weight_decay: 5.0e-4 | 0.0 early_stopping: enabled: true patience: 10 diff --git a/configs/supervised_wilds_vith16-448_ep300-lp.yaml b/configs/supervised_vith16_448.yaml similarity index 61% rename from configs/supervised_wilds_vith16-448_ep300-lp.yaml rename to configs/supervised_vith16_448.yaml index 098996f..3e4b95f 100644 --- a/configs/supervised_wilds_vith16-448_ep300-lp.yaml +++ b/configs/supervised_vith16_448.yaml @@ -7,15 +7,14 @@ meta: use_bfloat16: true num_classes: 182 representation_type: last_avgpool - head_type: linear + head_type: bn_linear data: - batch_size: 1024 + batch_size: 16 root_path: ./wilds_data num_workers: 10 pin_mem: true crop_size: 448 - crop_scale: [1.0, 1.0] use_random_resized_crop: false use_horizontal_flip: false use_color_distortion: false @@ -26,28 +25,17 @@ mask: patch_size: 16 optimization: - optimizer: adamw # 'adamw', 'sgd', or 'lars' - freeze_weights: true # true for linear probing, false for full fine-tuning + optimizer: adamw + freeze_weights: true epochs: 300 lr: 0.01 weight_decay: 5.0e-4 lr_schedule: cosine - step_milestones: [15, 30, 45] - step_gamma: 0.1 - start_lr: 0.0 + gradient_accumulation_steps: 64 final_lr: 0.0 - warmup: 0 - momentum: 0.99 - lars_eta: 0.001 - lars_eps: 1.0e-8 - lars_exclude_bias_and_norm: true - ipe_scale: 1.0 - # Sweep suggestions (manual edits): - # - lr: 0.01 | 0.05 | 0.001 - # - weight_decay: 5.0e-4 | 0.0 early_stopping: enabled: true - patience: 15 + patience: 10 min_delta: 1.0e-4 min_epochs: 15 restore_best_weights: true diff --git a/configs/supervised_vith16_448_in1k.yaml b/configs/supervised_vith16_448_in1k.yaml new file mode 100644 index 0000000..5eae316 --- /dev/null +++ b/configs/supervised_vith16_448_in1k.yaml @@ -0,0 +1,50 @@ +meta: + model_name: vit_huge + tag: in1k + embed_dim: 1280 + load_checkpoint: true + checkpoint_folder: experiment_logs/imagenet/ + read_checkpoint: IN1K-vit.h.16-448px-300e.pth.tar + use_bfloat16: true + num_classes: 182 + representation_type: last4_avgpool_concat + head_type: bn_linear + use_gradient_checkpointing: true + +data: + batch_size: 64 + root_path: ./wilds_data + num_workers: 10 + pin_mem: true + crop_size: 448 + use_random_resized_crop: false + use_horizontal_flip: false + use_color_distortion: false + color_jitter_strength: 0.0 + use_gaussian_blur: false + +mask: + patch_size: 16 + +optimization: + optimizer: adamw + freeze_weights: true + epochs: 300 + lr: 0.001 + weight_decay: 0 + lr_schedule: cosine + gradient_accumulation_steps: 8 + final_lr: 0.0 + early_stopping: + enabled: true + patience: 10 + min_delta: 1.0e-4 + min_epochs: 15 + restore_best_weights: true + +validation: + eval_every: 1 + +logging: + write_tag: linear_probe + auto_folder: true