add lars opt and configs

configs/supervised_wilds_vith14_ep300-lp.yaml

@@ -8,7 +8,7 @@ meta:
   num_classes: 182
 
 data:
-  batch_size: 128
+  batch_size: 16384
   root_path: ./wilds_data
   num_workers: 10
   pin_mem: true
@@ -24,26 +24,33 @@ mask:
   patch_size: 14
 
 optimization:
-  optimizer: adamw # 'adamw' or 'sgd'
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
   freeze_weights: true # true for linear probing, false for full fine-tuning
-  epochs: 300 # can be set higher if early_stopping
-  lr: 5.0e-4
-  weight_decay: 1.0e-2
-  use_cosine_schedule: true
-  start_lr: 0.0001
-  final_lr: 1.0e-06
-  warmup: 5
+  epochs: 50
+  lr: 0.01
+  weight_decay: 5.0e-4
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
   ipe_scale: 1.0
   early_stopping:
-    enabled: true
+    enabled: false
     patience: 10
     min_delta: 1.0e-4
-    min_epochs: 15
+    min_epochs: 50
     restore_best_weights: true
 
 validation:
   eval_every: 1
 
 logging:
-  folder: experiment_logs/supervised-vith14.224-bs.128-ep.300-lp-0crop/
-  write_tag: linear_probe
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.01-wd0.0005/
+  write_tag: linear_probe-lars-lr0.01-wd0.0005

configs/supervised_wilds_vith14_ep50-lp-lars-lr0.001-wd0.0.yaml

@@ -0,0 +1,56 @@
+meta:
+  model_name: vit_huge
+  embed_dim: 1280
+  load_checkpoint: true
+  checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/
+  read_checkpoint: jepa-ep300.pth.tar
+  use_bfloat16: true
+  num_classes: 182
+
+data:
+  batch_size: 16384
+  root_path: ./wilds_data
+  num_workers: 10
+  pin_mem: true
+  crop_size: 224
+  crop_scale: [1.0, 1.0]
+  use_random_resized_crop: false
+  use_horizontal_flip: false
+  use_color_distortion: false
+  color_jitter_strength: 0.0
+  use_gaussian_blur: false
+
+mask:
+  patch_size: 14
+
+optimization:
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
+  freeze_weights: true # true for linear probing, false for full fine-tuning
+  epochs: 50
+  lr: 0.001
+  weight_decay: 0.0
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
+  ipe_scale: 1.0
+  early_stopping:
+    enabled: false
+    patience: 10
+    min_delta: 1.0e-4
+    min_epochs: 50
+    restore_best_weights: true
+
+validation:
+  eval_every: 1
+
+logging:
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.001-wd0.0/
+  write_tag: linear_probe-lars-lr0.001-wd0.0

configs/supervised_wilds_vith14_ep50-lp-lars-lr0.001-wd0.0005.yaml

@@ -0,0 +1,56 @@
+meta:
+  model_name: vit_huge
+  embed_dim: 1280
+  load_checkpoint: true
+  checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/
+  read_checkpoint: jepa-ep300.pth.tar
+  use_bfloat16: true
+  num_classes: 182
+
+data:
+  batch_size: 16384
+  root_path: ./wilds_data
+  num_workers: 10
+  pin_mem: true
+  crop_size: 224
+  crop_scale: [1.0, 1.0]
+  use_random_resized_crop: false
+  use_horizontal_flip: false
+  use_color_distortion: false
+  color_jitter_strength: 0.0
+  use_gaussian_blur: false
+
+mask:
+  patch_size: 14
+
+optimization:
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
+  freeze_weights: true # true for linear probing, false for full fine-tuning
+  epochs: 50
+  lr: 0.001
+  weight_decay: 5.0e-4
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
+  ipe_scale: 1.0
+  early_stopping:
+    enabled: false
+    patience: 10
+    min_delta: 1.0e-4
+    min_epochs: 50
+    restore_best_weights: true
+
+validation:
+  eval_every: 1
+
+logging:
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.001-wd0.0005/
+  write_tag: linear_probe-lars-lr0.001-wd0.0005

configs/supervised_wilds_vith14_ep50-lp-lars-lr0.01-wd0.0.yaml

@@ -0,0 +1,56 @@
+meta:
+  model_name: vit_huge
+  embed_dim: 1280
+  load_checkpoint: true
+  checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/
+  read_checkpoint: jepa-ep300.pth.tar
+  use_bfloat16: true
+  num_classes: 182
+
+data:
+  batch_size: 16384
+  root_path: ./wilds_data
+  num_workers: 10
+  pin_mem: true
+  crop_size: 224
+  crop_scale: [1.0, 1.0]
+  use_random_resized_crop: false
+  use_horizontal_flip: false
+  use_color_distortion: false
+  color_jitter_strength: 0.0
+  use_gaussian_blur: false
+
+mask:
+  patch_size: 14
+
+optimization:
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
+  freeze_weights: true # true for linear probing, false for full fine-tuning
+  epochs: 50
+  lr: 0.01
+  weight_decay: 0.0
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
+  ipe_scale: 1.0
+  early_stopping:
+    enabled: false
+    patience: 10
+    min_delta: 1.0e-4
+    min_epochs: 50
+    restore_best_weights: true
+
+validation:
+  eval_every: 1
+
+logging:
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.01-wd0.0/
+  write_tag: linear_probe-lars-lr0.01-wd0.0

configs/supervised_wilds_vith14_ep50-lp-lars-lr0.05-wd0.0.yaml

@@ -0,0 +1,56 @@
+meta:
+  model_name: vit_huge
+  embed_dim: 1280
+  load_checkpoint: true
+  checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/
+  read_checkpoint: jepa-ep300.pth.tar
+  use_bfloat16: true
+  num_classes: 182
+
+data:
+  batch_size: 16384
+  root_path: ./wilds_data
+  num_workers: 10
+  pin_mem: true
+  crop_size: 224
+  crop_scale: [1.0, 1.0]
+  use_random_resized_crop: false
+  use_horizontal_flip: false
+  use_color_distortion: false
+  color_jitter_strength: 0.0
+  use_gaussian_blur: false
+
+mask:
+  patch_size: 14
+
+optimization:
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
+  freeze_weights: true # true for linear probing, false for full fine-tuning
+  epochs: 50
+  lr: 0.05
+  weight_decay: 0.0
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
+  ipe_scale: 1.0
+  early_stopping:
+    enabled: false
+    patience: 10
+    min_delta: 1.0e-4
+    min_epochs: 50
+    restore_best_weights: true
+
+validation:
+  eval_every: 1
+
+logging:
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.05-wd0.0/
+  write_tag: linear_probe-lars-lr0.05-wd0.0

configs/supervised_wilds_vith14_ep50-lp-lars-lr0.05-wd0.0005.yaml

@@ -0,0 +1,56 @@
+meta:
+  model_name: vit_huge
+  embed_dim: 1280
+  load_checkpoint: true
+  checkpoint_folder: experiment_logs/vith14.224-bs.128-ep.300/
+  read_checkpoint: jepa-ep300.pth.tar
+  use_bfloat16: true
+  num_classes: 182
+
+data:
+  batch_size: 16384
+  root_path: ./wilds_data
+  num_workers: 10
+  pin_mem: true
+  crop_size: 224
+  crop_scale: [1.0, 1.0]
+  use_random_resized_crop: false
+  use_horizontal_flip: false
+  use_color_distortion: false
+  color_jitter_strength: 0.0
+  use_gaussian_blur: false
+
+mask:
+  patch_size: 14
+
+optimization:
+  optimizer: lars # 'adamw', 'sgd', or 'lars'
+  freeze_weights: true # true for linear probing, false for full fine-tuning
+  epochs: 50
+  lr: 0.05
+  weight_decay: 5.0e-4
+  use_cosine_schedule: false
+  lr_schedule: step
+  step_milestones: [15, 30, 45]
+  step_gamma: 0.1
+  start_lr: 0.0
+  final_lr: 0.0
+  warmup: 0
+  momentum: 0.9
+  lars_eta: 0.001
+  lars_eps: 1.0e-8
+  lars_exclude_bias_and_norm: true
+  ipe_scale: 1.0
+  early_stopping:
+    enabled: false
+    patience: 10
+    min_delta: 1.0e-4
+    min_epochs: 50
+    restore_best_weights: true
+
+validation:
+  eval_every: 1
+
+logging:
+  folder: experiment_logs/supervised-vith14.224-bs.16384-ep.50-lp-lars-lr0.05-wd0.0005/
+  write_tag: linear_probe-lars-lr0.05-wd0.0005